In [1]:
"""
We use following lines because we are running on Google Colab
If you are running notebook on a local computer, you don't need this cell
"""
from google.colab import drive
drive.mount('/content/gdrive')
import os
os.chdir('/content/gdrive/My Drive/finch/tensorflow1/spoken_language_understanding/atis/main')
Drive already mounted at /content/gdrive; to attempt to forcibly remount, call drive.mount("/content/gdrive", force_remount=True).
In [2]:
import tensorflow as tf
import pprint
import logging
import time
import numpy as np

from sklearn.metrics import classification_report, f1_score
from pathlib import Path

print("TensorFlow Version", tf.__version__)
print('GPU Enabled:', tf.test.is_gpu_available())
TensorFlow Version 1.14.0
GPU Enabled: True
In [0]:
def get_vocab(vocab_path):
  word2idx = {}
  with open(vocab_path) as f:
    for i, line in enumerate(f):
      line = line.rstrip()
      word2idx[line] = i
  return word2idx
In [0]:
def data_generator(f_path, params):
  print('Reading', f_path)
  with open(f_path) as f:
    for line in f:
      line = line.rstrip()
      text, slot_intent = line.split('\t')
      words = text.split()[1:-1]
      slot_intent = slot_intent.split()
      slots, intent = slot_intent[1:-1], slot_intent[-1]
      assert len(words) == len(slots)
      
      yield (words, (intent, slots))
In [0]:
def dataset(is_training, params):
  _shapes = ([None], ((), [None]))
  _types = (tf.string, (tf.string, tf.string))
  _pads = ('<pad>', ('_', 'O'))
  
  if is_training:
    ds = tf.data.Dataset.from_generator(
      lambda: data_generator(params['train_path'], params),
      output_shapes = _shapes,
      output_types = _types,)
    ds = ds.shuffle(params['num_samples'])
    ds = ds.padded_batch(params['batch_size'], _shapes, _pads)
    ds = ds.prefetch(tf.data.experimental.AUTOTUNE)
  else:
    ds = tf.data.Dataset.from_generator(
      lambda: data_generator(params['test_path'], params),
      output_shapes = _shapes,
      output_types = _types,)
    ds = ds.padded_batch(params['batch_size'], _shapes, _pads)
    ds = ds.prefetch(tf.data.experimental.AUTOTUNE)
  
  return ds
In [0]:
def model_fn(features, labels, mode, params):
  is_training = (mode == tf.estimator.ModeKeys.TRAIN)
  
  vocab = tf.contrib.lookup.index_table_from_file(
    params['word_path'], num_oov_buckets=1)
  words = vocab.lookup(features)
  seq_len = tf.count_nonzero(words, 1, dtype=tf.int32)
  
  embedding = np.load(params['vocab_path'])
  embedding = tf.Variable(embedding, name='embedding', dtype=tf.float32)
  x = tf.nn.embedding_lookup(embedding, words)
  x = tf.layers.dropout(x, params['dropout_rate'], training=is_training)
  
  cell_fw = tf.nn.rnn_cell.GRUCell(params['rnn_units'])
  cell_bw = tf.nn.rnn_cell.GRUCell(params['rnn_units'])
  o, _ = tf.nn.bidirectional_dynamic_rnn(cell_fw, cell_bw, x, seq_len, dtype=tf.float32)
  x = tf.concat(o, -1)
  
  y_intent = tf.layers.dense(tf.reduce_max(x, 1), params['intent_size'])
  y_slots = tf.layers.dense(x, params['slot_size'])

  crf_params = tf.get_variable(
    "transitions", [params['slot_size'], params['slot_size']], dtype=tf.float32)

  pred_ids, _ = tf.contrib.crf.crf_decode(y_slots, crf_params, seq_len)
  
  
  if labels is not None:
    intent, slots = labels
    vocab = tf.contrib.lookup.index_table_from_file(
      params['intent_path'], num_oov_buckets=1)
    intent = vocab.lookup(intent)
    vocab = tf.contrib.lookup.index_table_from_file(
      params['slot_path'], num_oov_buckets=1)
    slots = vocab.lookup(slots)
    
    loss_intent = tf.nn.sparse_softmax_cross_entropy_with_logits(
      labels=intent, logits=y_intent)
    loss_intent = tf.reduce_mean(loss_intent)
    
    log_likelihood, _ = tf.contrib.crf.crf_log_likelihood(
      y_slots, slots, seq_len, crf_params)
    loss_slots = tf.reduce_mean(-log_likelihood)
    
    loss_op = 10 * loss_intent + loss_slots

  
  if mode == tf.estimator.ModeKeys.TRAIN:
    variables = tf.trainable_variables()
    tf.logging.info('\n'+pprint.pformat(variables))
    
    grads = tf.gradients(loss_op, variables)
    grads, _ = tf.clip_by_global_norm(grads, params['clip_norm'])
    
    global_step=tf.train.get_or_create_global_step()
    decay_lr = tf.train.exponential_decay(
        params['lr'], global_step, 1000, .9)
    hook = tf.train.LoggingTensorHook({'lr': decay_lr}, every_n_iter=100)
    
    optim = tf.train.AdamOptimizer(decay_lr)
    train_op = optim.apply_gradients(
      zip(grads, variables), global_step=global_step)
    
    return tf.estimator.EstimatorSpec(mode=mode,
                                      loss=loss_op,
                                      train_op=train_op,
                                      training_hooks=[hook],)
  
  
  if mode == tf.estimator.ModeKeys.EVAL:
    return tf.estimator.EstimatorSpec(mode=mode,
                                      loss=loss_op)
  
  
  if mode == tf.estimator.ModeKeys.PREDICT:
    return tf.estimator.EstimatorSpec(mode,
      predictions={'intent': tf.argmax(y_intent, -1),
                   'slots': pred_ids})
In [0]:
params = {
  'model_dir': '../model/bigru_crf',
  'log_path': '../log/bigru_crf.txt',
  'train_path': '../data/atis.train.w-intent.iob',
  'test_path': '../data/atis.test.w-intent.iob',
  'word_path': '../vocab/word.txt',
  'vocab_path': '../vocab/word.npy',
  'intent_path': '../vocab/intent.txt',
  'slot_path': '../vocab/slot.txt',
  'batch_size': 16,
  'num_samples': 4978,
  'rnn_units': 300,
  'dropout_rate': 0.2,
  'clip_norm': 5.0,
  'lr': 3e-4,
  'num_patience': 3,
}
In [0]:
params['word2idx'] = get_vocab(params['word_path'])
params['intent2idx'] = get_vocab(params['intent_path'])
params['slot2idx'] = get_vocab(params['slot_path'])

params['word_size'] = len(params['word2idx']) + 1
params['intent_size'] = len(params['intent2idx']) + 1
params['slot_size'] = len(params['slot2idx']) + 1
In [0]:
def is_descending(history: list):
  history = history[-(params['num_patience']+1):]
  for i in range(1, len(history)):
    if history[i-1] <= history[i]:
      return False
  return True  
In [10]:
# Create directory if not exist
Path(os.path.dirname(params['log_path'])).mkdir(exist_ok=True)
Path(params['model_dir']).mkdir(exist_ok=True, parents=True)

# Logging
logger = logging.getLogger('tensorflow')
logger.setLevel(logging.INFO)
fh = logging.FileHandler(params['log_path'])
logger.addHandler(fh)

# Create an estimator
_eval_steps = params['num_samples']//params['batch_size'] + 1
config = tf.estimator.RunConfig(
  save_checkpoints_steps=_eval_steps,)

estimator = tf.estimator.Estimator(
  model_fn=model_fn,
  model_dir=params['model_dir'],
  config=config,
  params=params)

# Train on training data and Evaluate on testing data
train_spec = tf.estimator.TrainSpec(
  input_fn=lambda: dataset(is_training=True, params=params),)

eval_spec = tf.estimator.EvalSpec(
  input_fn=lambda: dataset(is_training=False, params=params),
  steps=None,
  throttle_secs=10,)

best_f1 = .0
history_f1 = []
tf.enable_eager_execution()

while True:
  tf.estimator.train_and_evaluate(estimator, train_spec, eval_spec)
  
  intent = []
  slots = []
  for w, (i, s) in dataset(is_training=False, params=params):
    intent.append(i.numpy())
    slots.append(s.numpy())
  intent = [i for batch in intent for i in batch]
  intent = [params['intent2idx'].get(str(t, 'utf-8'), len(params['intent2idx'])) for t in intent]
  slots = [j for batch in slots for i in batch for j in i]
  slots = [params['slot2idx'].get(str(s, 'utf-8'), len(params['slot2idx'])) for s in slots]

  predicted = list(estimator.predict(input_fn=lambda: dataset(is_training=False, params=params)))
  y_slots = [j for i in predicted for j in i['slots']]
  y_intent = [i['intent'] for i in predicted]
  
  logger.info('\n'+classification_report(y_true = intent,
                                         y_pred = y_intent,
                                         labels = list(params['intent2idx'].values()),
                                         target_names = list(params['intent2idx'].keys()),
                                         digits=3))

  logger.info('\n'+classification_report(y_true = slots,
                                         y_pred = y_slots,
                                         labels = list(params['slot2idx'].values()),
                                         target_names = list(params['slot2idx'].keys()),
                                         sample_weight = np.sign(slots),
                                         digits=3))
  
  f1_slots = f1_score(y_true = slots,
                      y_pred = y_slots,
                      labels = list(params['slot2idx'].values()),
                      sample_weight = np.sign(slots),
                      average='micro',)
  history_f1.append(f1_slots)

  if f1_slots > best_f1:
    best_f1 = f1_slots
  logger.info("Best Slot F1: {:.3f}".format(best_f1))
  
  if len(history_f1) > params['num_patience'] and is_descending(history_f1):
    logger.info("Testing Slot F1 not improved over {} epochs, Early Stop".format(params['num_patience']))
    break
WARNING: Logging before flag parsing goes to stderr.
I0716 00:47:41.587580 140076727973760 estimator.py:209] Using config: {'_model_dir': '../model/bigru_crf', '_tf_random_seed': None, '_save_summary_steps': 100, '_save_checkpoints_steps': 312, '_save_checkpoints_secs': None, '_session_config': allow_soft_placement: true
graph_options {
  rewrite_options {
    meta_optimizer_iterations: ONE
  }
}
, '_keep_checkpoint_max': 5, '_keep_checkpoint_every_n_hours': 10000, '_log_step_count_steps': 100, '_train_distribute': None, '_device_fn': None, '_protocol': None, '_eval_distribute': None, '_experimental_distribute': None, '_experimental_max_worker_delay_secs': None, '_service': None, '_cluster_spec': <tensorflow.python.training.server_lib.ClusterSpec object at 0x7f65d01917b8>, '_task_type': 'worker', '_task_id': 0, '_global_id_in_cluster': 0, '_master': '', '_evaluation_master': '', '_is_chief': True, '_num_ps_replicas': 0, '_num_worker_replicas': 1}
I0716 00:47:41.594257 140076727973760 estimator_training.py:186] Not using Distribute Coordinator.
I0716 00:47:41.599280 140076727973760 training.py:612] Running training and evaluation locally (non-distributed).
I0716 00:47:41.602160 140076727973760 training.py:700] Start train and evaluate loop. The evaluate will happen after every checkpoint. Checkpoint frequency is determined based on RunConfig arguments: save_checkpoints_steps 312 or save_checkpoints_secs None.
W0716 00:47:41.614618 140076727973760 deprecation.py:323] From /usr/local/lib/python3.6/dist-packages/tensorflow/python/training/training_util.py:236: Variable.initialized_value (from tensorflow.python.ops.variables) is deprecated and will be removed in a future version.
Instructions for updating:
Use Variable.read_value. Variables in 2.X are initialized automatically both in eager and graph (inside tf.defun) contexts.
W0716 00:47:41.645507 140076727973760 deprecation.py:323] From /usr/local/lib/python3.6/dist-packages/tensorflow/python/data/ops/dataset_ops.py:494: py_func (from tensorflow.python.ops.script_ops) is deprecated and will be removed in a future version.
Instructions for updating:
tf.py_func is deprecated in TF V2. Instead, there are two
    options available in V2.
    - tf.py_function takes a python function which manipulates tf eager
    tensors instead of numpy arrays. It's easy to convert a tf eager tensor to
    an ndarray (just call tensor.numpy()) but having access to eager tensors
    means `tf.py_function`s can use accelerators such as GPUs as well as
    being differentiable using a gradient tape.
    - tf.numpy_function maintains the semantics of the deprecated tf.py_func
    (it is not differentiable, and manipulates numpy arrays). It drops the
    stateful argument making all functions stateful.
    
I0716 00:47:41.701436 140076727973760 estimator.py:1145] Calling model_fn.
W0716 00:47:42.255606 140076727973760 lazy_loader.py:50] 
The TensorFlow contrib module will not be included in TensorFlow 2.0.
For more information, please see:
  * https://github.com/tensorflow/community/blob/master/rfcs/20180907-contrib-sunset.md
  * https://github.com/tensorflow/addons
  * https://github.com/tensorflow/io (for I/O related ops)
If you depend on functionality not listed there, please file an issue.

W0716 00:47:42.267921 140076727973760 deprecation.py:323] From /usr/local/lib/python3.6/dist-packages/tensorflow/python/ops/lookup_ops.py:978: add_dispatch_support.<locals>.wrapper (from tensorflow.python.ops.array_ops) is deprecated and will be removed in a future version.
Instructions for updating:
Use tf.where in 2.0, which has the same broadcast rule as np.where
W0716 00:47:42.270508 140076727973760 deprecation.py:506] From /usr/local/lib/python3.6/dist-packages/tensorflow/python/util/deprecation.py:507: calling count_nonzero (from tensorflow.python.ops.math_ops) with axis is deprecated and will be removed in a future version.
Instructions for updating:
reduction_indices is deprecated, use axis instead
W0716 00:47:42.294181 140076727973760 deprecation.py:323] From <ipython-input-6-0a85a3fd693f>:12: dropout (from tensorflow.python.layers.core) is deprecated and will be removed in a future version.
Instructions for updating:
Use keras.layers.dropout instead.
W0716 00:47:42.366486 140076727973760 deprecation.py:323] From <ipython-input-6-0a85a3fd693f>:14: GRUCell.__init__ (from tensorflow.python.ops.rnn_cell_impl) is deprecated and will be removed in a future version.
Instructions for updating:
This class is equivalent as tf.keras.layers.GRUCell, and will be replaced by that in Tensorflow 2.0.
W0716 00:47:42.368409 140076727973760 deprecation.py:323] From <ipython-input-6-0a85a3fd693f>:16: bidirectional_dynamic_rnn (from tensorflow.python.ops.rnn) is deprecated and will be removed in a future version.
Instructions for updating:
Please use `keras.layers.Bidirectional(keras.layers.RNN(cell))`, which is equivalent to this API
W0716 00:47:42.370395 140076727973760 deprecation.py:323] From /usr/local/lib/python3.6/dist-packages/tensorflow/python/ops/rnn.py:464: dynamic_rnn (from tensorflow.python.ops.rnn) is deprecated and will be removed in a future version.
Instructions for updating:
Please use `keras.layers.RNN(cell)`, which is equivalent to this API
W0716 00:47:42.460355 140076727973760 deprecation.py:506] From /usr/local/lib/python3.6/dist-packages/tensorflow/python/ops/init_ops.py:1251: calling VarianceScaling.__init__ (from tensorflow.python.ops.init_ops) with dtype is deprecated and will be removed in a future version.
Instructions for updating:
Call initializer instance with the dtype argument instead of passing it to the constructor
W0716 00:47:42.474985 140076727973760 deprecation.py:506] From /usr/local/lib/python3.6/dist-packages/tensorflow/python/ops/rnn_cell_impl.py:564: calling Constant.__init__ (from tensorflow.python.ops.init_ops) with dtype is deprecated and will be removed in a future version.
Instructions for updating:
Call initializer instance with the dtype argument instead of passing it to the constructor
W0716 00:47:42.491813 140076727973760 deprecation.py:506] From /usr/local/lib/python3.6/dist-packages/tensorflow/python/ops/rnn_cell_impl.py:574: calling Zeros.__init__ (from tensorflow.python.ops.init_ops) with dtype is deprecated and will be removed in a future version.
Instructions for updating:
Call initializer instance with the dtype argument instead of passing it to the constructor
W0716 00:47:43.021589 140076727973760 deprecation.py:323] From <ipython-input-6-0a85a3fd693f>:19: dense (from tensorflow.python.layers.core) is deprecated and will be removed in a future version.
Instructions for updating:
Use keras.layers.dense instead.
I0716 00:47:43.924069 140076727973760 <ipython-input-6-0a85a3fd693f>:50] 
[<tf.Variable 'embedding:0' shape=(750, 300) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/gates/kernel:0' shape=(600, 600) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/gates/bias:0' shape=(600,) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/candidate/kernel:0' shape=(600, 300) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/candidate/bias:0' shape=(300,) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/gates/kernel:0' shape=(600, 600) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/gates/bias:0' shape=(600,) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/candidate/kernel:0' shape=(600, 300) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/candidate/bias:0' shape=(300,) dtype=float32_ref>,
 <tf.Variable 'dense/kernel:0' shape=(600, 23) dtype=float32_ref>,
 <tf.Variable 'dense/bias:0' shape=(23,) dtype=float32_ref>,
 <tf.Variable 'dense_1/kernel:0' shape=(600, 122) dtype=float32_ref>,
 <tf.Variable 'dense_1/bias:0' shape=(122,) dtype=float32_ref>,
 <tf.Variable 'transitions:0' shape=(122, 122) dtype=float32_ref>]
I0716 00:47:45.399207 140076727973760 estimator.py:1147] Done calling model_fn.
I0716 00:47:45.403253 140076727973760 basic_session_run_hooks.py:541] Create CheckpointSaverHook.
I0716 00:47:46.071866 140076727973760 monitored_session.py:240] Graph was finalized.
I0716 00:47:46.747561 140076727973760 session_manager.py:500] Running local_init_op.
I0716 00:47:46.785521 140076727973760 session_manager.py:502] Done running local_init_op.
I0716 00:47:48.168303 140076727973760 basic_session_run_hooks.py:606] Saving checkpoints for 0 into ../model/bigru_crf/model.ckpt.
Reading ../data/atis.train.w-intent.iob
I0716 00:47:50.547693 140076727973760 basic_session_run_hooks.py:262] loss = 87.49991, step = 0
I0716 00:47:50.549657 140076727973760 basic_session_run_hooks.py:262] lr = 0.0003
I0716 00:48:02.059066 140076727973760 basic_session_run_hooks.py:692] global_step/sec: 8.68664
I0716 00:48:02.065382 140076727973760 basic_session_run_hooks.py:260] loss = 18.045454, step = 100 (11.518 sec)
I0716 00:48:02.068305 140076727973760 basic_session_run_hooks.py:260] lr = 0.00029685578 (11.519 sec)
I0716 00:48:12.927720 140076727973760 basic_session_run_hooks.py:692] global_step/sec: 9.20081
I0716 00:48:12.934930 140076727973760 basic_session_run_hooks.py:260] loss = 17.184025, step = 200 (10.869 sec)
I0716 00:48:12.938170 140076727973760 basic_session_run_hooks.py:260] lr = 0.00029374452 (10.870 sec)
I0716 00:48:23.584062 140076727973760 basic_session_run_hooks.py:692] global_step/sec: 9.38412
I0716 00:48:23.586608 140076727973760 basic_session_run_hooks.py:260] loss = 4.921689, step = 300 (10.652 sec)
I0716 00:48:23.588835 140076727973760 basic_session_run_hooks.py:260] lr = 0.00029066586 (10.651 sec)
I0716 00:48:24.686854 140076727973760 basic_session_run_hooks.py:606] Saving checkpoints for 312 into ../model/bigru_crf/model.ckpt.
I0716 00:48:25.032292 140076727973760 estimator.py:1145] Calling model_fn.
I0716 00:48:25.983808 140076727973760 estimator.py:1147] Done calling model_fn.
I0716 00:48:26.013410 140076727973760 evaluation.py:255] Starting evaluation at 2019-07-16T00:48:26Z
I0716 00:48:26.323209 140076727973760 monitored_session.py:240] Graph was finalized.
W0716 00:48:26.336217 140076727973760 deprecation.py:323] From /usr/local/lib/python3.6/dist-packages/tensorflow/python/training/saver.py:1276: checkpoint_exists (from tensorflow.python.training.checkpoint_management) is deprecated and will be removed in a future version.
Instructions for updating:
Use standard file APIs to check for files with this prefix.
I0716 00:48:26.346000 140076727973760 saver.py:1280] Restoring parameters from ../model/bigru_crf/model.ckpt-312
I0716 00:48:26.456235 140076727973760 session_manager.py:500] Running local_init_op.
I0716 00:48:26.481973 140076727973760 session_manager.py:502] Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
I0716 00:48:28.744487 140076727973760 evaluation.py:275] Finished evaluation at 2019-07-16-00:48:28
I0716 00:48:28.746257 140076727973760 estimator.py:2039] Saving dict for global step 312: global_step = 312, loss = 9.044307
I0716 00:48:28.938807 140076727973760 estimator.py:2099] Saving 'checkpoint_path' summary for global step 312: ../model/bigru_crf/model.ckpt-312
I0716 00:48:28.991008 140076727973760 estimator.py:368] Loss for final step: 14.758665.
Reading ../data/atis.test.w-intent.iob
I0716 00:48:29.316013 140076727973760 estimator.py:1145] Calling model_fn.
I0716 00:48:29.964270 140076727973760 estimator.py:1147] Done calling model_fn.
I0716 00:48:30.074452 140076727973760 monitored_session.py:240] Graph was finalized.
I0716 00:48:30.089570 140076727973760 saver.py:1280] Restoring parameters from ../model/bigru_crf/model.ckpt-312
I0716 00:48:30.188294 140076727973760 session_manager.py:500] Running local_init_op.
I0716 00:48:30.205507 140076727973760 session_manager.py:502] Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.
  'precision', 'predicted', average, warn_for)
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1439: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples.
  'recall', 'true', average, warn_for)
I0716 00:48:33.260554 140076727973760 interactiveshell.py:2882] 
                                          precision    recall  f1-score   support

                             atis_flight      0.935     0.994     0.963       632
                            atis_airfare      0.825     0.979     0.895        48
                     atis_ground_service      0.655     1.000     0.791        36
                            atis_airline      0.844     1.000     0.916        38
                       atis_abbreviation      0.733     1.000     0.846        33
                           atis_aircraft      0.474     1.000     0.643         9
                        atis_flight_time      0.000     0.000     0.000         1
                           atis_quantity      0.000     0.000     0.000         3
                atis_flight#atis_airfare      0.000     0.000     0.000        12
                            atis_airport      0.000     0.000     0.000        18
                           atis_distance      0.000     0.000     0.000        10
                               atis_city      0.000     0.000     0.000         6
                        atis_ground_fare      0.000     0.000     0.000         7
                           atis_capacity      0.000     0.000     0.000        21
                          atis_flight_no      0.000     0.000     0.000         8
                               atis_meal      0.000     0.000     0.000         6
                        atis_restriction      0.000     0.000     0.000         0
             atis_airline#atis_flight_no      0.000     0.000     0.000         0
    atis_ground_service#atis_ground_fare      0.000     0.000     0.000         0
           atis_airfare#atis_flight_time      0.000     0.000     0.000         0
                           atis_cheapest      0.000     0.000     0.000         0
atis_aircraft#atis_flight#atis_flight_no      0.000     0.000     0.000         0

                               micro avg      0.886     0.891     0.888       888
                               macro avg      0.203     0.271     0.230       888
                            weighted avg      0.804     0.891     0.843       888

I0716 00:48:33.309482 140076727973760 interactiveshell.py:2882] 
                              precision    recall  f1-score   support

                           O      0.000     0.000     0.000       0.0
           B-toloc.city_name      0.951     0.986     0.968     716.0
         B-fromloc.city_name      0.921     0.994     0.956     704.0
           I-toloc.city_name      0.818     0.981     0.892     265.0
      B-depart_date.day_name      0.914     1.000     0.955     212.0
              B-airline_name      0.906     0.861     0.883     101.0
         I-fromloc.city_name      0.859     0.898     0.878     177.0
 B-depart_time.period_of_day      0.799     0.885     0.839     130.0
              I-airline_name      1.000     0.862     0.926      65.0
    B-depart_date.day_number      0.883     0.964     0.922      55.0
    B-depart_date.month_name      0.900     0.964     0.931      56.0
          B-depart_time.time      0.548     1.000     0.708      57.0
                B-round_trip      1.000     0.781     0.877      73.0
             B-cost_relative      0.919     0.919     0.919      37.0
                I-round_trip      0.966     0.789     0.868      71.0
                B-flight_mod      0.875     0.875     0.875      24.0
 B-depart_time.time_relative      0.797     0.908     0.849      65.0
          I-depart_time.time      0.619     1.000     0.765      52.0
         B-stoploc.city_name      0.762     0.800     0.780      20.0
                 B-city_name      0.392     0.509     0.443      57.0
                B-class_type      0.957     0.917     0.936      24.0
          B-arrive_time.time      0.731     0.559     0.633      34.0
 B-arrive_time.time_relative      1.000     0.387     0.558      31.0
                I-class_type      1.000     1.000     1.000      17.0
               B-flight_stop      1.000     1.000     1.000      21.0
          I-arrive_time.time      0.864     0.543     0.667      35.0
              B-airline_code      0.938     0.441     0.600      34.0
    I-depart_date.day_number      0.824     0.933     0.875      15.0
      I-fromloc.airport_name      0.333     0.200     0.250      15.0
      B-fromloc.airport_name      0.000     0.000     0.000      12.0
      B-arrive_date.day_name      0.000     0.000     0.000      11.0
          B-toloc.state_code      0.900     1.000     0.947      18.0
B-depart_date.today_relative      0.000     0.000     0.000       9.0
             B-flight_number      0.833     0.455     0.588      11.0
 B-depart_date.date_relative      0.750     0.176     0.286      17.0
          B-toloc.state_name      0.000     0.000     0.000      28.0
           B-fare_basis_code      0.356     0.941     0.516      17.0
               B-flight_time      0.000     0.000     0.000       1.0
                        B-or      0.000     0.000     0.000       3.0
 B-arrive_time.period_of_day      0.000     0.000     0.000       6.0
          B-meal_description      1.000     0.200     0.333      10.0
             I-cost_relative      0.000     0.000     0.000       3.0
              I-airport_name      0.778     0.241     0.368      29.0
               B-fare_amount      0.000     0.000     0.000       2.0
               I-fare_amount      1.000     0.500     0.667       2.0
                 I-city_name      0.000     0.000     0.000      30.0
        I-toloc.airport_name      0.000     0.000     0.000       3.0
            B-transport_type      0.000     0.000     0.000      10.0
    B-arrive_date.month_name      0.000     0.000     0.000       6.0
    B-arrive_date.day_number      0.000     0.000     0.000       6.0
         I-stoploc.city_name      0.000     0.000     0.000      10.0
                      B-meal      1.000     0.062     0.118      16.0
        B-fromloc.state_code      1.000     0.826     0.905      23.0
    B-depart_time.period_mod      0.000     0.000     0.000       5.0
                   B-connect      0.000     0.000     0.000       6.0
               B-flight_days      0.000     0.000     0.000      10.0
        B-toloc.airport_name      0.000     0.000     0.000       3.0
        B-fromloc.state_name      0.000     0.000     0.000      17.0
              B-airport_name      0.000     0.000     0.000      21.0
                   B-economy      0.000     0.000     0.000       6.0
               I-flight_time      0.000     0.000     0.000       1.0
             B-aircraft_code      0.000     0.000     0.000      33.0
                       B-mod      0.000     0.000     0.000       2.0
              B-airport_code      0.000     0.000     0.000       9.0
    B-depart_time.start_time      0.000     0.000     0.000       3.0
      B-depart_time.end_time      0.000     0.000     0.000       3.0
          B-depart_date.year      0.000     0.000     0.000       3.0
            I-transport_type      0.000     0.000     0.000       1.0
          B-restriction_code      0.000     0.000     0.000       4.0
    B-arrive_time.start_time      0.000     0.000     0.000       8.0
        B-toloc.airport_code      0.000     0.000     0.000       4.0
      B-arrive_time.end_time      0.000     0.000     0.000       8.0
      I-arrive_time.end_time      0.000     0.000     0.000       8.0
      I-depart_time.end_time      0.000     0.000     0.000       3.0
               I-flight_stop      0.000     0.000     0.000       0.0
      B-fromloc.airport_code      0.000     0.000     0.000       5.0
          I-restriction_code      0.000     0.000     0.000       3.0
    I-depart_time.start_time      0.000     0.000     0.000       1.0
          I-toloc.state_name      0.000     0.000     0.000       1.0
I-depart_date.today_relative      0.000     0.000     0.000       0.0
 B-arrive_date.date_relative      0.000     0.000     0.000       2.0
                I-flight_mod      0.000     0.000     0.000       6.0
                   I-economy      0.000     0.000     0.000       0.0
 B-return_date.date_relative      0.000     0.000     0.000       3.0
        I-fromloc.state_name      0.000     0.000     0.000       1.0
                B-state_code      0.000     0.000     0.000       1.0
    I-arrive_time.start_time      0.000     0.000     0.000       1.0
    I-arrive_date.day_number      0.000     0.000     0.000       0.0
                 B-meal_code      0.000     0.000     0.000       1.0
 I-depart_time.period_of_day      0.000     0.000     0.000       1.0
                  B-day_name      0.000     0.000     0.000       2.0
             B-period_of_day      0.000     0.000     0.000       4.0
        B-stoploc.state_code      0.000     0.000     0.000       0.0
    B-return_date.month_name      0.000     0.000     0.000       0.0
    B-return_date.day_number      0.000     0.000     0.000       0.0
    B-arrive_time.period_mod      0.000     0.000     0.000       0.0
                 I-meal_code      0.000     0.000     0.000       0.0
        B-toloc.country_name      0.000     0.000     0.000       1.0
                 B-days_code      0.000     0.000     0.000       1.0
 I-arrive_time.period_of_day      0.000     0.000     0.000       0.0
            I-today_relative      0.000     0.000     0.000       0.0
 B-return_time.period_of_day      0.000     0.000     0.000       0.0
                      B-time      0.000     0.000     0.000       0.0
           I-fare_basis_code      0.000     0.000     0.000       0.0
 I-arrive_time.time_relative      0.000     0.000     0.000       4.0
 I-depart_time.time_relative      0.000     0.000     0.000       1.0
            B-today_relative      0.000     0.000     0.000       0.0
                B-state_name      0.000     0.000     0.000       9.0
B-arrive_date.today_relative      0.000     0.000     0.000       0.0
    B-return_time.period_mod      0.000     0.000     0.000       0.0
                B-month_name      0.000     0.000     0.000       0.0
                B-day_number      0.000     0.000     0.000       0.0
 I-return_date.date_relative      0.000     0.000     0.000       3.0
I-return_date.today_relative      0.000     0.000     0.000       0.0
      B-stoploc.airport_name      0.000     0.000     0.000       0.0
             B-time_relative      0.000     0.000     0.000       0.0
                      I-time      0.000     0.000     0.000       0.0
    I-return_date.day_number      0.000     0.000     0.000       0.0
          I-meal_description      0.000     0.000     0.000       0.0
B-return_date.today_relative      0.000     0.000     0.000       0.0
      B-return_date.day_name      0.000     0.000     0.000       2.0

                   micro avg      0.818     0.820     0.819    3657.0
                   macro avg      0.257     0.226     0.227    3657.0
                weighted avg      0.797     0.820     0.797    3657.0

I0716 00:48:33.337860 140076727973760 interactiveshell.py:2882] Best Slot F1: 0.819
I0716 00:48:33.339370 140076727973760 estimator_training.py:186] Not using Distribute Coordinator.
I0716 00:48:33.342253 140076727973760 training.py:612] Running training and evaluation locally (non-distributed).
I0716 00:48:33.346786 140076727973760 training.py:700] Start train and evaluate loop. The evaluate will happen after every checkpoint. Checkpoint frequency is determined based on RunConfig arguments: save_checkpoints_steps 312 or save_checkpoints_secs None.
I0716 00:48:33.408508 140076727973760 estimator.py:1145] Calling model_fn.
I0716 00:48:34.660151 140076727973760 <ipython-input-6-0a85a3fd693f>:50] 
[<tf.Variable 'embedding:0' shape=(750, 300) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/gates/kernel:0' shape=(600, 600) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/gates/bias:0' shape=(600,) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/candidate/kernel:0' shape=(600, 300) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/candidate/bias:0' shape=(300,) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/gates/kernel:0' shape=(600, 600) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/gates/bias:0' shape=(600,) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/candidate/kernel:0' shape=(600, 300) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/candidate/bias:0' shape=(300,) dtype=float32_ref>,
 <tf.Variable 'dense/kernel:0' shape=(600, 23) dtype=float32_ref>,
 <tf.Variable 'dense/bias:0' shape=(23,) dtype=float32_ref>,
 <tf.Variable 'dense_1/kernel:0' shape=(600, 122) dtype=float32_ref>,
 <tf.Variable 'dense_1/bias:0' shape=(122,) dtype=float32_ref>,
 <tf.Variable 'transitions:0' shape=(122, 122) dtype=float32_ref>]
I0716 00:48:36.176086 140076727973760 estimator.py:1147] Done calling model_fn.
I0716 00:48:36.180043 140076727973760 basic_session_run_hooks.py:541] Create CheckpointSaverHook.
I0716 00:48:36.378660 140076727973760 monitored_session.py:240] Graph was finalized.
I0716 00:48:36.398118 140076727973760 saver.py:1280] Restoring parameters from ../model/bigru_crf/model.ckpt-312
W0716 00:48:36.559721 140076727973760 deprecation.py:323] From /usr/local/lib/python3.6/dist-packages/tensorflow/python/training/saver.py:1066: get_checkpoint_mtimes (from tensorflow.python.training.checkpoint_management) is deprecated and will be removed in a future version.
Instructions for updating:
Use standard file utilities to get mtimes.
I0716 00:48:36.641471 140076727973760 session_manager.py:500] Running local_init_op.
I0716 00:48:36.679799 140076727973760 session_manager.py:502] Done running local_init_op.
I0716 00:48:38.155996 140076727973760 basic_session_run_hooks.py:606] Saving checkpoints for 312 into ../model/bigru_crf/model.ckpt.
Reading ../data/atis.train.w-intent.iob
I0716 00:48:40.395767 140076727973760 basic_session_run_hooks.py:262] loss = 6.800891, step = 312
I0716 00:48:40.397950 140076727973760 basic_session_run_hooks.py:262] lr = 0.0002902986
I0716 00:48:51.700778 140076727973760 basic_session_run_hooks.py:692] global_step/sec: 8.84518
I0716 00:48:51.708012 140076727973760 basic_session_run_hooks.py:260] loss = 6.5419207, step = 412 (11.312 sec)
I0716 00:48:51.709680 140076727973760 basic_session_run_hooks.py:260] lr = 0.00028725603 (11.312 sec)
I0716 00:49:02.394297 140076727973760 basic_session_run_hooks.py:692] global_step/sec: 9.35144
I0716 00:49:02.402181 140076727973760 basic_session_run_hooks.py:260] loss = 7.228915, step = 512 (10.694 sec)
I0716 00:49:02.404015 140076727973760 basic_session_run_hooks.py:260] lr = 0.00028424538 (10.694 sec)
I0716 00:49:13.167154 140076727973760 basic_session_run_hooks.py:692] global_step/sec: 9.28271
I0716 00:49:13.169937 140076727973760 basic_session_run_hooks.py:260] loss = 1.3466052, step = 612 (10.768 sec)
I0716 00:49:13.172691 140076727973760 basic_session_run_hooks.py:260] lr = 0.00028126626 (10.769 sec)
I0716 00:49:14.309535 140076727973760 basic_session_run_hooks.py:606] Saving checkpoints for 624 into ../model/bigru_crf/model.ckpt.
I0716 00:49:14.678862 140076727973760 estimator.py:1145] Calling model_fn.
I0716 00:49:15.832185 140076727973760 estimator.py:1147] Done calling model_fn.
I0716 00:49:15.862119 140076727973760 evaluation.py:255] Starting evaluation at 2019-07-16T00:49:15Z
I0716 00:49:15.979552 140076727973760 monitored_session.py:240] Graph was finalized.
I0716 00:49:15.994215 140076727973760 saver.py:1280] Restoring parameters from ../model/bigru_crf/model.ckpt-624
I0716 00:49:16.117654 140076727973760 session_manager.py:500] Running local_init_op.
I0716 00:49:16.145142 140076727973760 session_manager.py:502] Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
I0716 00:49:18.489275 140076727973760 evaluation.py:275] Finished evaluation at 2019-07-16-00:49:18
I0716 00:49:18.490964 140076727973760 estimator.py:2039] Saving dict for global step 624: global_step = 624, loss = 5.336379
I0716 00:49:18.498535 140076727973760 estimator.py:2099] Saving 'checkpoint_path' summary for global step 624: ../model/bigru_crf/model.ckpt-624
I0716 00:49:18.561314 140076727973760 estimator.py:368] Loss for final step: 1.6356039.
Reading ../data/atis.test.w-intent.iob
I0716 00:49:18.900227 140076727973760 estimator.py:1145] Calling model_fn.
I0716 00:49:19.567500 140076727973760 estimator.py:1147] Done calling model_fn.
I0716 00:49:19.680938 140076727973760 monitored_session.py:240] Graph was finalized.
I0716 00:49:19.695843 140076727973760 saver.py:1280] Restoring parameters from ../model/bigru_crf/model.ckpt-624
I0716 00:49:19.793823 140076727973760 session_manager.py:500] Running local_init_op.
I0716 00:49:19.814134 140076727973760 session_manager.py:502] Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.
  'precision', 'predicted', average, warn_for)
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1439: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples.
  'recall', 'true', average, warn_for)
I0716 00:49:22.979851 140076727973760 interactiveshell.py:2882] 
                                          precision    recall  f1-score   support

                             atis_flight      0.950     0.995     0.972       632
                            atis_airfare      0.960     1.000     0.980        48
                     atis_ground_service      0.667     1.000     0.800        36
                            atis_airline      0.950     1.000     0.974        38
                       atis_abbreviation      0.917     1.000     0.957        33
                           atis_aircraft      0.450     1.000     0.621         9
                        atis_flight_time      1.000     1.000     1.000         1
                           atis_quantity      0.600     1.000     0.750         3
                atis_flight#atis_airfare      0.500     0.083     0.143        12
                            atis_airport      1.000     0.556     0.714        18
                           atis_distance      1.000     0.400     0.571        10
                               atis_city      0.000     0.000     0.000         6
                        atis_ground_fare      0.000     0.000     0.000         7
                           atis_capacity      1.000     0.429     0.600        21
                          atis_flight_no      0.000     0.000     0.000         8
                               atis_meal      0.000     0.000     0.000         6
                        atis_restriction      0.000     0.000     0.000         0
             atis_airline#atis_flight_no      0.000     0.000     0.000         0
    atis_ground_service#atis_ground_fare      0.000     0.000     0.000         0
           atis_airfare#atis_flight_time      0.000     0.000     0.000         0
                           atis_cheapest      0.000     0.000     0.000         0
atis_aircraft#atis_flight#atis_flight_no      0.000     0.000     0.000         0

                               micro avg      0.919     0.925     0.922       888
                               macro avg      0.454     0.430     0.413       888
                            weighted avg      0.900     0.925     0.902       888

I0716 00:49:23.031128 140076727973760 interactiveshell.py:2882] 
                              precision    recall  f1-score   support

                           O      0.000     0.000     0.000       0.0
           B-toloc.city_name      0.966     0.989     0.977     716.0
         B-fromloc.city_name      0.955     0.999     0.976     704.0
           I-toloc.city_name      0.933     0.992     0.962     265.0
      B-depart_date.day_name      0.946     1.000     0.972     212.0
              B-airline_name      0.943     0.980     0.961     101.0
         I-fromloc.city_name      0.910     0.972     0.940     177.0
 B-depart_time.period_of_day      0.919     0.869     0.893     130.0
              I-airline_name      1.000     0.969     0.984      65.0
    B-depart_date.day_number      0.846     1.000     0.917      55.0
    B-depart_date.month_name      0.902     0.982     0.940      56.0
          B-depart_time.time      0.737     0.982     0.842      57.0
                B-round_trip      1.000     0.945     0.972      73.0
             B-cost_relative      1.000     0.973     0.986      37.0
                I-round_trip      1.000     0.958     0.978      71.0
                B-flight_mod      1.000     0.875     0.933      24.0
 B-depart_time.time_relative      0.984     0.938     0.961      65.0
          I-depart_time.time      0.810     0.981     0.887      52.0
         B-stoploc.city_name      0.900     0.900     0.900      20.0
                 B-city_name      0.857     0.526     0.652      57.0
                B-class_type      0.960     1.000     0.980      24.0
          B-arrive_time.time      0.633     0.912     0.747      34.0
 B-arrive_time.time_relative      0.875     0.903     0.889      31.0
                I-class_type      1.000     1.000     1.000      17.0
               B-flight_stop      1.000     1.000     1.000      21.0
          I-arrive_time.time      0.744     0.914     0.821      35.0
              B-airline_code      0.778     0.824     0.800      34.0
    I-depart_date.day_number      0.833     1.000     0.909      15.0
      I-fromloc.airport_name      0.393     0.733     0.512      15.0
      B-fromloc.airport_name      0.278     0.417     0.333      12.0
      B-arrive_date.day_name      1.000     0.364     0.533      11.0
          B-toloc.state_code      0.947     1.000     0.973      18.0
B-depart_date.today_relative      1.000     0.778     0.875       9.0
             B-flight_number      0.714     0.909     0.800      11.0
 B-depart_date.date_relative      0.895     1.000     0.944      17.0
          B-toloc.state_name      0.864     0.679     0.760      28.0
           B-fare_basis_code      0.842     0.941     0.889      17.0
               B-flight_time      1.000     1.000     1.000       1.0
                        B-or      1.000     1.000     1.000       3.0
 B-arrive_time.period_of_day      0.500     0.333     0.400       6.0
          B-meal_description      1.000     0.800     0.889      10.0
             I-cost_relative      1.000     0.667     0.800       3.0
              I-airport_name      1.000     0.310     0.474      29.0
               B-fare_amount      0.667     1.000     0.800       2.0
               I-fare_amount      1.000     1.000     1.000       2.0
                 I-city_name      0.800     0.533     0.640      30.0
        I-toloc.airport_name      1.000     0.333     0.500       3.0
            B-transport_type      1.000     0.800     0.889      10.0
    B-arrive_date.month_name      0.000     0.000     0.000       6.0
    B-arrive_date.day_number      0.000     0.000     0.000       6.0
         I-stoploc.city_name      0.600     0.300     0.400      10.0
                      B-meal      0.786     0.688     0.733      16.0
        B-fromloc.state_code      1.000     0.913     0.955      23.0
    B-depart_time.period_mod      0.750     0.600     0.667       5.0
                   B-connect      1.000     0.667     0.800       6.0
               B-flight_days      1.000     1.000     1.000      10.0
        B-toloc.airport_name      1.000     0.333     0.500       3.0
        B-fromloc.state_name      0.700     0.412     0.519      17.0
              B-airport_name      0.667     0.286     0.400      21.0
                   B-economy      1.000     1.000     1.000       6.0
               I-flight_time      1.000     1.000     1.000       1.0
             B-aircraft_code      1.000     0.576     0.731      33.0
                       B-mod      0.000     0.000     0.000       2.0
              B-airport_code      1.000     0.111     0.200       9.0
    B-depart_time.start_time      0.000     0.000     0.000       3.0
      B-depart_time.end_time      0.000     0.000     0.000       3.0
          B-depart_date.year      0.000     0.000     0.000       3.0
            I-transport_type      0.000     0.000     0.000       1.0
          B-restriction_code      0.500     0.750     0.600       4.0
    B-arrive_time.start_time      0.000     0.000     0.000       8.0
        B-toloc.airport_code      0.000     0.000     0.000       4.0
      B-arrive_time.end_time      0.000     0.000     0.000       8.0
      I-arrive_time.end_time      0.000     0.000     0.000       8.0
      I-depart_time.end_time      0.000     0.000     0.000       3.0
               I-flight_stop      0.000     0.000     0.000       0.0
      B-fromloc.airport_code      0.000     0.000     0.000       5.0
          I-restriction_code      0.000     0.000     0.000       3.0
    I-depart_time.start_time      0.000     0.000     0.000       1.0
          I-toloc.state_name      0.000     0.000     0.000       1.0
I-depart_date.today_relative      0.000     0.000     0.000       0.0
 B-arrive_date.date_relative      0.000     0.000     0.000       2.0
                I-flight_mod      0.000     0.000     0.000       6.0
                   I-economy      0.000     0.000     0.000       0.0
 B-return_date.date_relative      0.000     0.000     0.000       3.0
        I-fromloc.state_name      0.000     0.000     0.000       1.0
                B-state_code      0.000     0.000     0.000       1.0
    I-arrive_time.start_time      0.000     0.000     0.000       1.0
    I-arrive_date.day_number      0.000     0.000     0.000       0.0
                 B-meal_code      0.000     0.000     0.000       1.0
 I-depart_time.period_of_day      0.000     0.000     0.000       1.0
                  B-day_name      0.000     0.000     0.000       2.0
             B-period_of_day      0.000     0.000     0.000       4.0
        B-stoploc.state_code      0.000     0.000     0.000       0.0
    B-return_date.month_name      0.000     0.000     0.000       0.0
    B-return_date.day_number      0.000     0.000     0.000       0.0
    B-arrive_time.period_mod      0.000     0.000     0.000       0.0
                 I-meal_code      0.000     0.000     0.000       0.0
        B-toloc.country_name      0.000     0.000     0.000       1.0
                 B-days_code      0.000     0.000     0.000       1.0
 I-arrive_time.period_of_day      0.000     0.000     0.000       0.0
            I-today_relative      0.000     0.000     0.000       0.0
 B-return_time.period_of_day      0.000     0.000     0.000       0.0
                      B-time      0.000     0.000     0.000       0.0
           I-fare_basis_code      0.000     0.000     0.000       0.0
 I-arrive_time.time_relative      0.000     0.000     0.000       4.0
 I-depart_time.time_relative      0.000     0.000     0.000       1.0
            B-today_relative      0.000     0.000     0.000       0.0
                B-state_name      0.000     0.000     0.000       9.0
B-arrive_date.today_relative      0.000     0.000     0.000       0.0
    B-return_time.period_mod      0.000     0.000     0.000       0.0
                B-month_name      0.000     0.000     0.000       0.0
                B-day_number      0.000     0.000     0.000       0.0
 I-return_date.date_relative      0.000     0.000     0.000       3.0
I-return_date.today_relative      0.000     0.000     0.000       0.0
      B-stoploc.airport_name      0.000     0.000     0.000       0.0
             B-time_relative      0.000     0.000     0.000       0.0
                      I-time      0.000     0.000     0.000       0.0
    I-return_date.day_number      0.000     0.000     0.000       0.0
          I-meal_description      0.000     0.000     0.000       0.0
B-return_date.today_relative      0.000     0.000     0.000       0.0
      B-return_date.day_name      0.000     0.000     0.000       2.0

                   micro avg      0.903     0.904     0.903    3657.0
                   macro avg      0.441     0.402     0.407    3657.0
                weighted avg      0.898     0.904     0.894    3657.0

I0716 00:49:23.058561 140076727973760 interactiveshell.py:2882] Best Slot F1: 0.903
I0716 00:49:23.060357 140076727973760 estimator_training.py:186] Not using Distribute Coordinator.
I0716 00:49:23.063817 140076727973760 training.py:612] Running training and evaluation locally (non-distributed).
I0716 00:49:23.066178 140076727973760 training.py:700] Start train and evaluate loop. The evaluate will happen after every checkpoint. Checkpoint frequency is determined based on RunConfig arguments: save_checkpoints_steps 312 or save_checkpoints_secs None.
I0716 00:49:23.131276 140076727973760 estimator.py:1145] Calling model_fn.
I0716 00:49:24.409299 140076727973760 <ipython-input-6-0a85a3fd693f>:50] 
[<tf.Variable 'embedding:0' shape=(750, 300) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/gates/kernel:0' shape=(600, 600) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/gates/bias:0' shape=(600,) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/candidate/kernel:0' shape=(600, 300) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/candidate/bias:0' shape=(300,) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/gates/kernel:0' shape=(600, 600) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/gates/bias:0' shape=(600,) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/candidate/kernel:0' shape=(600, 300) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/candidate/bias:0' shape=(300,) dtype=float32_ref>,
 <tf.Variable 'dense/kernel:0' shape=(600, 23) dtype=float32_ref>,
 <tf.Variable 'dense/bias:0' shape=(23,) dtype=float32_ref>,
 <tf.Variable 'dense_1/kernel:0' shape=(600, 122) dtype=float32_ref>,
 <tf.Variable 'dense_1/bias:0' shape=(122,) dtype=float32_ref>,
 <tf.Variable 'transitions:0' shape=(122, 122) dtype=float32_ref>]
I0716 00:49:25.941650 140076727973760 estimator.py:1147] Done calling model_fn.
I0716 00:49:25.945616 140076727973760 basic_session_run_hooks.py:541] Create CheckpointSaverHook.
I0716 00:49:26.142787 140076727973760 monitored_session.py:240] Graph was finalized.
I0716 00:49:26.163645 140076727973760 saver.py:1280] Restoring parameters from ../model/bigru_crf/model.ckpt-624
I0716 00:49:26.411920 140076727973760 session_manager.py:500] Running local_init_op.
I0716 00:49:26.458307 140076727973760 session_manager.py:502] Done running local_init_op.
I0716 00:49:27.992535 140076727973760 basic_session_run_hooks.py:606] Saving checkpoints for 624 into ../model/bigru_crf/model.ckpt.
Reading ../data/atis.train.w-intent.iob
I0716 00:49:30.287108 140076727973760 basic_session_run_hooks.py:262] loss = 1.1525946, step = 624
I0716 00:49:30.290117 140076727973760 basic_session_run_hooks.py:262] lr = 0.00028091087
I0716 00:49:42.025858 140076727973760 basic_session_run_hooks.py:692] global_step/sec: 8.51829
I0716 00:49:42.028457 140076727973760 basic_session_run_hooks.py:260] loss = 1.4876823, step = 724 (11.741 sec)
I0716 00:49:42.033679 140076727973760 basic_session_run_hooks.py:260] lr = 0.00027796673 (11.744 sec)
I0716 00:49:52.962687 140076727973760 basic_session_run_hooks.py:692] global_step/sec: 9.14341
I0716 00:49:52.971317 140076727973760 basic_session_run_hooks.py:260] loss = 2.2452643, step = 824 (10.943 sec)
I0716 00:49:52.972865 140076727973760 basic_session_run_hooks.py:260] lr = 0.00027505343 (10.939 sec)
I0716 00:50:03.513283 140076727973760 basic_session_run_hooks.py:692] global_step/sec: 9.47812
I0716 00:50:03.521591 140076727973760 basic_session_run_hooks.py:260] loss = 1.1489712, step = 924 (10.550 sec)
I0716 00:50:03.523039 140076727973760 basic_session_run_hooks.py:260] lr = 0.00027217067 (10.550 sec)
I0716 00:50:04.538872 140076727973760 basic_session_run_hooks.py:606] Saving checkpoints for 936 into ../model/bigru_crf/model.ckpt.
I0716 00:50:04.897520 140076727973760 estimator.py:1145] Calling model_fn.
I0716 00:50:06.068701 140076727973760 estimator.py:1147] Done calling model_fn.
I0716 00:50:06.097920 140076727973760 evaluation.py:255] Starting evaluation at 2019-07-16T00:50:06Z
I0716 00:50:06.214033 140076727973760 monitored_session.py:240] Graph was finalized.
I0716 00:50:06.229865 140076727973760 saver.py:1280] Restoring parameters from ../model/bigru_crf/model.ckpt-936
I0716 00:50:06.342840 140076727973760 session_manager.py:500] Running local_init_op.
I0716 00:50:06.369131 140076727973760 session_manager.py:502] Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
I0716 00:50:08.675725 140076727973760 evaluation.py:275] Finished evaluation at 2019-07-16-00:50:08
I0716 00:50:08.677827 140076727973760 estimator.py:2039] Saving dict for global step 936: global_step = 936, loss = 4.4151998
I0716 00:50:08.683806 140076727973760 estimator.py:2099] Saving 'checkpoint_path' summary for global step 936: ../model/bigru_crf/model.ckpt-936
I0716 00:50:08.747270 140076727973760 estimator.py:368] Loss for final step: 0.18660742.
Reading ../data/atis.test.w-intent.iob
I0716 00:50:09.071233 140076727973760 estimator.py:1145] Calling model_fn.
I0716 00:50:09.730399 140076727973760 estimator.py:1147] Done calling model_fn.
I0716 00:50:09.843429 140076727973760 monitored_session.py:240] Graph was finalized.
I0716 00:50:09.859822 140076727973760 saver.py:1280] Restoring parameters from ../model/bigru_crf/model.ckpt-936
I0716 00:50:09.958875 140076727973760 session_manager.py:500] Running local_init_op.
I0716 00:50:09.976560 140076727973760 session_manager.py:502] Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.
  'precision', 'predicted', average, warn_for)
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1439: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples.
  'recall', 'true', average, warn_for)
I0716 00:50:13.089282 140076727973760 interactiveshell.py:2882] 
                                          precision    recall  f1-score   support

                             atis_flight      0.956     0.992     0.974       632
                            atis_airfare      0.906     1.000     0.950        48
                     atis_ground_service      0.837     1.000     0.911        36
                            atis_airline      0.974     1.000     0.987        38
                       atis_abbreviation      0.917     1.000     0.957        33
                           atis_aircraft      1.000     1.000     1.000         9
                        atis_flight_time      1.000     1.000     1.000         1
                           atis_quantity      0.333     1.000     0.500         3
                atis_flight#atis_airfare      1.000     0.083     0.154        12
                            atis_airport      0.938     0.833     0.882        18
                           atis_distance      1.000     0.400     0.571        10
                               atis_city      1.000     0.500     0.667         6
                        atis_ground_fare      1.000     0.571     0.727         7
                           atis_capacity      1.000     0.905     0.950        21
                          atis_flight_no      0.000     0.000     0.000         8
                               atis_meal      0.000     0.000     0.000         6
                        atis_restriction      0.000     0.000     0.000         0
             atis_airline#atis_flight_no      0.000     0.000     0.000         0
    atis_ground_service#atis_ground_fare      0.000     0.000     0.000         0
           atis_airfare#atis_flight_time      0.000     0.000     0.000         0
                           atis_cheapest      0.000     0.000     0.000         0
atis_aircraft#atis_flight#atis_flight_no      0.000     0.000     0.000         0

                               micro avg      0.942     0.947     0.944       888
                               macro avg      0.585     0.513     0.510       888
                            weighted avg      0.933     0.947     0.931       888

I0716 00:50:13.138233 140076727973760 interactiveshell.py:2882] 
                              precision    recall  f1-score   support

                           O      0.000     0.000     0.000       0.0
           B-toloc.city_name      0.970     0.992     0.981     716.0
         B-fromloc.city_name      0.975     0.999     0.987     704.0
           I-toloc.city_name      0.974     0.985     0.979     265.0
      B-depart_date.day_name      0.990     0.972     0.981     212.0
              B-airline_name      0.962     0.990     0.976     101.0
         I-fromloc.city_name      0.989     0.983     0.986     177.0
 B-depart_time.period_of_day      0.974     0.869     0.919     130.0
              I-airline_name      0.970     1.000     0.985      65.0
    B-depart_date.day_number      0.965     1.000     0.982      55.0
    B-depart_date.month_name      0.982     0.982     0.982      56.0
          B-depart_time.time      0.794     0.877     0.833      57.0
                B-round_trip      1.000     0.973     0.986      73.0
             B-cost_relative      1.000     0.973     0.986      37.0
                I-round_trip      1.000     0.986     0.993      71.0
                B-flight_mod      0.958     0.958     0.958      24.0
 B-depart_time.time_relative      0.983     0.877     0.927      65.0
          I-depart_time.time      0.900     0.865     0.882      52.0
         B-stoploc.city_name      0.905     0.950     0.927      20.0
                 B-city_name      0.846     0.579     0.688      57.0
                B-class_type      0.960     1.000     0.980      24.0
          B-arrive_time.time      0.767     0.971     0.857      34.0
 B-arrive_time.time_relative      0.784     0.935     0.853      31.0
                I-class_type      1.000     1.000     1.000      17.0
               B-flight_stop      1.000     1.000     1.000      21.0
          I-arrive_time.time      0.723     0.971     0.829      35.0
              B-airline_code      0.935     0.853     0.892      34.0
    I-depart_date.day_number      1.000     1.000     1.000      15.0
      I-fromloc.airport_name      0.438     0.933     0.596      15.0
      B-fromloc.airport_name      0.400     0.833     0.541      12.0
      B-arrive_date.day_name      0.579     1.000     0.733      11.0
          B-toloc.state_code      0.947     1.000     0.973      18.0
B-depart_date.today_relative      1.000     0.889     0.941       9.0
             B-flight_number      0.714     0.909     0.800      11.0
 B-depart_date.date_relative      0.895     1.000     0.944      17.0
          B-toloc.state_name      0.862     0.893     0.877      28.0
           B-fare_basis_code      0.895     1.000     0.944      17.0
               B-flight_time      1.000     1.000     1.000       1.0
                        B-or      1.000     1.000     1.000       3.0
 B-arrive_time.period_of_day      0.667     1.000     0.800       6.0
          B-meal_description      1.000     0.900     0.947      10.0
             I-cost_relative      1.000     0.667     0.800       3.0
              I-airport_name      0.923     0.414     0.571      29.0
               B-fare_amount      1.000     1.000     1.000       2.0
               I-fare_amount      1.000     1.000     1.000       2.0
                 I-city_name      0.905     0.633     0.745      30.0
        I-toloc.airport_name      1.000     0.667     0.800       3.0
            B-transport_type      1.000     0.900     0.947      10.0
    B-arrive_date.month_name      0.833     0.833     0.833       6.0
    B-arrive_date.day_number      1.000     0.833     0.909       6.0
         I-stoploc.city_name      0.727     0.800     0.762      10.0
                      B-meal      1.000     1.000     1.000      16.0
        B-fromloc.state_code      1.000     0.913     0.955      23.0
    B-depart_time.period_mod      0.750     0.600     0.667       5.0
                   B-connect      1.000     0.667     0.800       6.0
               B-flight_days      1.000     1.000     1.000      10.0
        B-toloc.airport_name      1.000     0.333     0.500       3.0
        B-fromloc.state_name      0.812     0.765     0.788      17.0
              B-airport_name      0.750     0.286     0.414      21.0
                   B-economy      1.000     1.000     1.000       6.0
               I-flight_time      1.000     1.000     1.000       1.0
             B-aircraft_code      1.000     0.667     0.800      33.0
                       B-mod      0.000     0.000     0.000       2.0
              B-airport_code      0.250     0.111     0.154       9.0
    B-depart_time.start_time      1.000     0.333     0.500       3.0
      B-depart_time.end_time      1.000     0.333     0.500       3.0
          B-depart_date.year      1.000     0.667     0.800       3.0
            I-transport_type      0.000     0.000     0.000       1.0
          B-restriction_code      0.429     0.750     0.545       4.0
    B-arrive_time.start_time      0.889     1.000     0.941       8.0
        B-toloc.airport_code      0.000     0.000     0.000       4.0
      B-arrive_time.end_time      0.857     0.750     0.800       8.0
      I-arrive_time.end_time      0.857     0.750     0.800       8.0
      I-depart_time.end_time      1.000     0.333     0.500       3.0
               I-flight_stop      0.000     0.000     0.000       0.0
      B-fromloc.airport_code      1.000     0.800     0.889       5.0
          I-restriction_code      0.750     1.000     0.857       3.0
    I-depart_time.start_time      1.000     1.000     1.000       1.0
          I-toloc.state_name      1.000     1.000     1.000       1.0
I-depart_date.today_relative      0.000     0.000     0.000       0.0
 B-arrive_date.date_relative      0.000     0.000     0.000       2.0
                I-flight_mod      0.000     0.000     0.000       6.0
                   I-economy      0.000     0.000     0.000       0.0
 B-return_date.date_relative      0.000     0.000     0.000       3.0
        I-fromloc.state_name      0.000     0.000     0.000       1.0
                B-state_code      0.000     0.000     0.000       1.0
    I-arrive_time.start_time      0.000     0.000     0.000       1.0
    I-arrive_date.day_number      0.000     0.000     0.000       0.0
                 B-meal_code      0.000     0.000     0.000       1.0
 I-depart_time.period_of_day      0.000     0.000     0.000       1.0
                  B-day_name      0.000     0.000     0.000       2.0
             B-period_of_day      0.000     0.000     0.000       4.0
        B-stoploc.state_code      0.000     0.000     0.000       0.0
    B-return_date.month_name      0.000     0.000     0.000       0.0
    B-return_date.day_number      0.000     0.000     0.000       0.0
    B-arrive_time.period_mod      0.000     0.000     0.000       0.0
                 I-meal_code      0.000     0.000     0.000       0.0
        B-toloc.country_name      0.000     0.000     0.000       1.0
                 B-days_code      0.000     0.000     0.000       1.0
 I-arrive_time.period_of_day      0.000     0.000     0.000       0.0
            I-today_relative      0.000     0.000     0.000       0.0
 B-return_time.period_of_day      0.000     0.000     0.000       0.0
                      B-time      0.000     0.000     0.000       0.0
           I-fare_basis_code      0.000     0.000     0.000       0.0
 I-arrive_time.time_relative      0.000     0.000     0.000       4.0
 I-depart_time.time_relative      0.000     0.000     0.000       1.0
            B-today_relative      0.000     0.000     0.000       0.0
                B-state_name      0.000     0.000     0.000       9.0
B-arrive_date.today_relative      0.000     0.000     0.000       0.0
    B-return_time.period_mod      0.000     0.000     0.000       0.0
                B-month_name      0.000     0.000     0.000       0.0
                B-day_number      0.000     0.000     0.000       0.0
 I-return_date.date_relative      0.000     0.000     0.000       3.0
I-return_date.today_relative      0.000     0.000     0.000       0.0
      B-stoploc.airport_name      0.000     0.000     0.000       0.0
             B-time_relative      0.000     0.000     0.000       0.0
                      I-time      0.000     0.000     0.000       0.0
    I-return_date.day_number      0.000     0.000     0.000       0.0
          I-meal_description      0.000     0.000     0.000       0.0
B-return_date.today_relative      0.000     0.000     0.000       0.0
      B-return_date.day_name      0.000     0.000     0.000       2.0

                   micro avg      0.929     0.931     0.930    3657.0
                   macro avg      0.549     0.518     0.521    3657.0
                weighted avg      0.936     0.931     0.929    3657.0

I0716 00:50:13.166728 140076727973760 interactiveshell.py:2882] Best Slot F1: 0.930
I0716 00:50:13.168068 140076727973760 estimator_training.py:186] Not using Distribute Coordinator.
I0716 00:50:13.170693 140076727973760 training.py:612] Running training and evaluation locally (non-distributed).
I0716 00:50:13.176110 140076727973760 training.py:700] Start train and evaluate loop. The evaluate will happen after every checkpoint. Checkpoint frequency is determined based on RunConfig arguments: save_checkpoints_steps 312 or save_checkpoints_secs None.
I0716 00:50:13.237679 140076727973760 estimator.py:1145] Calling model_fn.
I0716 00:50:14.515203 140076727973760 <ipython-input-6-0a85a3fd693f>:50] 
[<tf.Variable 'embedding:0' shape=(750, 300) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/gates/kernel:0' shape=(600, 600) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/gates/bias:0' shape=(600,) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/candidate/kernel:0' shape=(600, 300) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/candidate/bias:0' shape=(300,) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/gates/kernel:0' shape=(600, 600) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/gates/bias:0' shape=(600,) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/candidate/kernel:0' shape=(600, 300) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/candidate/bias:0' shape=(300,) dtype=float32_ref>,
 <tf.Variable 'dense/kernel:0' shape=(600, 23) dtype=float32_ref>,
 <tf.Variable 'dense/bias:0' shape=(23,) dtype=float32_ref>,
 <tf.Variable 'dense_1/kernel:0' shape=(600, 122) dtype=float32_ref>,
 <tf.Variable 'dense_1/bias:0' shape=(122,) dtype=float32_ref>,
 <tf.Variable 'transitions:0' shape=(122, 122) dtype=float32_ref>]
I0716 00:50:16.029022 140076727973760 estimator.py:1147] Done calling model_fn.
I0716 00:50:16.032940 140076727973760 basic_session_run_hooks.py:541] Create CheckpointSaverHook.
I0716 00:50:16.226980 140076727973760 monitored_session.py:240] Graph was finalized.
I0716 00:50:16.246722 140076727973760 saver.py:1280] Restoring parameters from ../model/bigru_crf/model.ckpt-936
I0716 00:50:16.501127 140076727973760 session_manager.py:500] Running local_init_op.
I0716 00:50:16.538402 140076727973760 session_manager.py:502] Done running local_init_op.
I0716 00:50:18.092715 140076727973760 basic_session_run_hooks.py:606] Saving checkpoints for 936 into ../model/bigru_crf/model.ckpt.
Reading ../data/atis.train.w-intent.iob
I0716 00:50:20.375187 140076727973760 basic_session_run_hooks.py:262] loss = 0.44405657, step = 936
I0716 00:50:20.377056 140076727973760 basic_session_run_hooks.py:262] lr = 0.00027182678
I0716 00:50:31.885331 140076727973760 basic_session_run_hooks.py:692] global_step/sec: 8.68759
I0716 00:50:31.891675 140076727973760 basic_session_run_hooks.py:260] loss = 1.2857233, step = 1036 (11.516 sec)
I0716 00:50:31.893631 140076727973760 basic_session_run_hooks.py:260] lr = 0.00026897783 (11.517 sec)
I0716 00:50:42.975480 140076727973760 basic_session_run_hooks.py:692] global_step/sec: 9.01701
I0716 00:50:42.981963 140076727973760 basic_session_run_hooks.py:260] loss = 0.9570614, step = 1136 (11.090 sec)
I0716 00:50:42.985614 140076727973760 basic_session_run_hooks.py:260] lr = 0.00026615875 (11.092 sec)
I0716 00:50:53.415146 140076727973760 basic_session_run_hooks.py:692] global_step/sec: 9.57881
I0716 00:50:53.418401 140076727973760 basic_session_run_hooks.py:260] loss = 5.0473733, step = 1236 (10.436 sec)
I0716 00:50:53.423001 140076727973760 basic_session_run_hooks.py:260] lr = 0.0002633692 (10.437 sec)
I0716 00:50:54.469538 140076727973760 basic_session_run_hooks.py:606] Saving checkpoints for 1248 into ../model/bigru_crf/model.ckpt.
I0716 00:50:54.849996 140076727973760 estimator.py:1145] Calling model_fn.
I0716 00:50:55.995228 140076727973760 estimator.py:1147] Done calling model_fn.
I0716 00:50:56.023944 140076727973760 evaluation.py:255] Starting evaluation at 2019-07-16T00:50:56Z
I0716 00:50:56.137515 140076727973760 monitored_session.py:240] Graph was finalized.
I0716 00:50:56.152611 140076727973760 saver.py:1280] Restoring parameters from ../model/bigru_crf/model.ckpt-1248
I0716 00:50:56.270451 140076727973760 session_manager.py:500] Running local_init_op.
I0716 00:50:56.296618 140076727973760 session_manager.py:502] Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
I0716 00:50:58.578980 140076727973760 evaluation.py:275] Finished evaluation at 2019-07-16-00:50:58
I0716 00:50:58.580543 140076727973760 estimator.py:2039] Saving dict for global step 1248: global_step = 1248, loss = 3.8197186
I0716 00:50:58.587998 140076727973760 estimator.py:2099] Saving 'checkpoint_path' summary for global step 1248: ../model/bigru_crf/model.ckpt-1248
I0716 00:50:58.647767 140076727973760 estimator.py:368] Loss for final step: 3.6961079.
Reading ../data/atis.test.w-intent.iob
I0716 00:50:58.973516 140076727973760 estimator.py:1145] Calling model_fn.
I0716 00:50:59.605149 140076727973760 estimator.py:1147] Done calling model_fn.
I0716 00:50:59.715630 140076727973760 monitored_session.py:240] Graph was finalized.
I0716 00:50:59.732222 140076727973760 saver.py:1280] Restoring parameters from ../model/bigru_crf/model.ckpt-1248
I0716 00:50:59.834825 140076727973760 session_manager.py:500] Running local_init_op.
I0716 00:50:59.853385 140076727973760 session_manager.py:502] Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.
  'precision', 'predicted', average, warn_for)
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1439: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples.
  'recall', 'true', average, warn_for)
I0716 00:51:02.935676 140076727973760 interactiveshell.py:2882] 
                                          precision    recall  f1-score   support

                             atis_flight      0.957     0.992     0.974       632
                            atis_airfare      0.960     1.000     0.980        48
                     atis_ground_service      0.947     1.000     0.973        36
                            atis_airline      1.000     1.000     1.000        38
                       atis_abbreviation      0.943     1.000     0.971        33
                           atis_aircraft      1.000     1.000     1.000         9
                        atis_flight_time      0.250     1.000     0.400         1
                           atis_quantity      0.375     1.000     0.545         3
                atis_flight#atis_airfare      0.800     0.333     0.471        12
                            atis_airport      0.933     0.778     0.848        18
                           atis_distance      1.000     0.700     0.824        10
                               atis_city      1.000     0.500     0.667         6
                        atis_ground_fare      1.000     0.714     0.833         7
                           atis_capacity      0.952     0.952     0.952        21
                          atis_flight_no      0.000     0.000     0.000         8
                               atis_meal      0.000     0.000     0.000         6
                        atis_restriction      0.000     0.000     0.000         0
             atis_airline#atis_flight_no      0.000     0.000     0.000         0
    atis_ground_service#atis_ground_fare      0.000     0.000     0.000         0
           atis_airfare#atis_flight_time      0.000     0.000     0.000         0
                           atis_cheapest      0.000     0.000     0.000         0
atis_aircraft#atis_flight#atis_flight_no      0.000     0.000     0.000         0

                               micro avg      0.950     0.955     0.952       888
                               macro avg      0.551     0.544     0.520       888
                            weighted avg      0.939     0.955     0.944       888

I0716 00:51:02.986434 140076727973760 interactiveshell.py:2882] 
                              precision    recall  f1-score   support

                           O      0.000     0.000     0.000       0.0
           B-toloc.city_name      0.974     0.990     0.982     716.0
         B-fromloc.city_name      0.980     0.999     0.989     704.0
           I-toloc.city_name      0.978     0.992     0.985     265.0
      B-depart_date.day_name      0.991     0.995     0.993     212.0
              B-airline_name      0.971     1.000     0.985     101.0
         I-fromloc.city_name      0.983     0.994     0.989     177.0
 B-depart_time.period_of_day      0.975     0.892     0.932     130.0
              I-airline_name      0.970     1.000     0.985      65.0
    B-depart_date.day_number      0.964     0.982     0.973      55.0
    B-depart_date.month_name      0.982     0.982     0.982      56.0
          B-depart_time.time      0.806     0.947     0.871      57.0
                B-round_trip      1.000     0.973     0.986      73.0
             B-cost_relative      1.000     0.973     0.986      37.0
                I-round_trip      1.000     0.958     0.978      71.0
                B-flight_mod      0.957     0.917     0.936      24.0
 B-depart_time.time_relative      0.968     0.938     0.953      65.0
          I-depart_time.time      0.891     0.942     0.916      52.0
         B-stoploc.city_name      0.950     0.950     0.950      20.0
                 B-city_name      0.912     0.544     0.681      57.0
                B-class_type      0.960     1.000     0.980      24.0
          B-arrive_time.time      0.868     0.971     0.917      34.0
 B-arrive_time.time_relative      0.875     0.903     0.889      31.0
                I-class_type      1.000     1.000     1.000      17.0
               B-flight_stop      1.000     1.000     1.000      21.0
          I-arrive_time.time      0.850     0.971     0.907      35.0
              B-airline_code      1.000     0.794     0.885      34.0
    I-depart_date.day_number      1.000     1.000     1.000      15.0
      I-fromloc.airport_name      0.395     1.000     0.566      15.0
      B-fromloc.airport_name      0.357     0.833     0.500      12.0
      B-arrive_date.day_name      0.846     1.000     0.917      11.0
          B-toloc.state_code      0.947     1.000     0.973      18.0
B-depart_date.today_relative      1.000     0.889     0.941       9.0
             B-flight_number      0.714     0.909     0.800      11.0
 B-depart_date.date_relative      0.850     1.000     0.919      17.0
          B-toloc.state_name      0.833     0.893     0.862      28.0
           B-fare_basis_code      0.895     1.000     0.944      17.0
               B-flight_time      1.000     1.000     1.000       1.0
                        B-or      1.000     1.000     1.000       3.0
 B-arrive_time.period_of_day      0.667     1.000     0.800       6.0
          B-meal_description      1.000     0.900     0.947      10.0
             I-cost_relative      1.000     0.667     0.800       3.0
              I-airport_name      0.909     0.345     0.500      29.0
               B-fare_amount      1.000     1.000     1.000       2.0
               I-fare_amount      1.000     1.000     1.000       2.0
                 I-city_name      1.000     0.567     0.723      30.0
        I-toloc.airport_name      1.000     1.000     1.000       3.0
            B-transport_type      1.000     0.900     0.947      10.0
    B-arrive_date.month_name      0.833     0.833     0.833       6.0
    B-arrive_date.day_number      0.833     0.833     0.833       6.0
         I-stoploc.city_name      0.769     1.000     0.870      10.0
                      B-meal      1.000     1.000     1.000      16.0
        B-fromloc.state_code      1.000     0.957     0.978      23.0
    B-depart_time.period_mod      0.750     0.600     0.667       5.0
                   B-connect      1.000     1.000     1.000       6.0
               B-flight_days      1.000     1.000     1.000      10.0
        B-toloc.airport_name      1.000     0.667     0.800       3.0
        B-fromloc.state_name      0.938     0.882     0.909      17.0
              B-airport_name      0.750     0.286     0.414      21.0
                   B-economy      1.000     1.000     1.000       6.0
               I-flight_time      1.000     1.000     1.000       1.0
             B-aircraft_code      1.000     0.909     0.952      33.0
                       B-mod      0.000     0.000     0.000       2.0
              B-airport_code      0.000     0.000     0.000       9.0
    B-depart_time.start_time      1.000     0.333     0.500       3.0
      B-depart_time.end_time      1.000     0.333     0.500       3.0
          B-depart_date.year      1.000     1.000     1.000       3.0
            I-transport_type      0.000     0.000     0.000       1.0
          B-restriction_code      0.333     0.750     0.462       4.0
    B-arrive_time.start_time      0.889     1.000     0.941       8.0
        B-toloc.airport_code      0.750     0.750     0.750       4.0
      B-arrive_time.end_time      0.857     0.750     0.800       8.0
      I-arrive_time.end_time      0.857     0.750     0.800       8.0
      I-depart_time.end_time      1.000     0.333     0.500       3.0
               I-flight_stop      0.000     0.000     0.000       0.0
      B-fromloc.airport_code      0.833     1.000     0.909       5.0
          I-restriction_code      0.750     1.000     0.857       3.0
    I-depart_time.start_time      1.000     1.000     1.000       1.0
          I-toloc.state_name      1.000     1.000     1.000       1.0
I-depart_date.today_relative      0.000     0.000     0.000       0.0
 B-arrive_date.date_relative      1.000     0.500     0.667       2.0
                I-flight_mod      0.000     0.000     0.000       6.0
                   I-economy      0.000     0.000     0.000       0.0
 B-return_date.date_relative      0.000     0.000     0.000       3.0
        I-fromloc.state_name      1.000     1.000     1.000       1.0
                B-state_code      1.000     1.000     1.000       1.0
    I-arrive_time.start_time      0.000     0.000     0.000       1.0
    I-arrive_date.day_number      0.000     0.000     0.000       0.0
                 B-meal_code      0.000     0.000     0.000       1.0
 I-depart_time.period_of_day      0.000     0.000     0.000       1.0
                  B-day_name      0.500     0.500     0.500       2.0
             B-period_of_day      1.000     0.250     0.400       4.0
        B-stoploc.state_code      0.000     0.000     0.000       0.0
    B-return_date.month_name      0.000     0.000     0.000       0.0
    B-return_date.day_number      0.000     0.000     0.000       0.0
    B-arrive_time.period_mod      0.000     0.000     0.000       0.0
                 I-meal_code      0.000     0.000     0.000       0.0
        B-toloc.country_name      0.000     0.000     0.000       1.0
                 B-days_code      0.000     0.000     0.000       1.0
 I-arrive_time.period_of_day      0.000     0.000     0.000       0.0
            I-today_relative      0.000     0.000     0.000       0.0
 B-return_time.period_of_day      0.000     0.000     0.000       0.0
                      B-time      0.000     0.000     0.000       0.0
           I-fare_basis_code      0.000     0.000     0.000       0.0
 I-arrive_time.time_relative      0.000     0.000     0.000       4.0
 I-depart_time.time_relative      0.000     0.000     0.000       1.0
            B-today_relative      0.000     0.000     0.000       0.0
                B-state_name      0.000     0.000     0.000       9.0
B-arrive_date.today_relative      0.000     0.000     0.000       0.0
    B-return_time.period_mod      0.000     0.000     0.000       0.0
                B-month_name      0.000     0.000     0.000       0.0
                B-day_number      0.000     0.000     0.000       0.0
 I-return_date.date_relative      0.000     0.000     0.000       3.0
I-return_date.today_relative      0.000     0.000     0.000       0.0
      B-stoploc.airport_name      0.000     0.000     0.000       0.0
             B-time_relative      0.000     0.000     0.000       0.0
                      I-time      0.000     0.000     0.000       0.0
    I-return_date.day_number      0.000     0.000     0.000       0.0
          I-meal_description      0.000     0.000     0.000       0.0
B-return_date.today_relative      0.000     0.000     0.000       0.0
      B-return_date.day_name      0.000     0.000     0.000       2.0

                   micro avg      0.940     0.941     0.940    3657.0
                   macro avg      0.594     0.568     0.567    3657.0
                weighted avg      0.946     0.941     0.938    3657.0

I0716 00:51:03.019695 140076727973760 interactiveshell.py:2882] Best Slot F1: 0.940
I0716 00:51:03.021811 140076727973760 estimator_training.py:186] Not using Distribute Coordinator.
I0716 00:51:03.027317 140076727973760 training.py:612] Running training and evaluation locally (non-distributed).
I0716 00:51:03.030241 140076727973760 training.py:700] Start train and evaluate loop. The evaluate will happen after every checkpoint. Checkpoint frequency is determined based on RunConfig arguments: save_checkpoints_steps 312 or save_checkpoints_secs None.
I0716 00:51:03.094722 140076727973760 estimator.py:1145] Calling model_fn.
I0716 00:51:04.346454 140076727973760 <ipython-input-6-0a85a3fd693f>:50] 
[<tf.Variable 'embedding:0' shape=(750, 300) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/gates/kernel:0' shape=(600, 600) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/gates/bias:0' shape=(600,) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/candidate/kernel:0' shape=(600, 300) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/candidate/bias:0' shape=(300,) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/gates/kernel:0' shape=(600, 600) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/gates/bias:0' shape=(600,) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/candidate/kernel:0' shape=(600, 300) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/candidate/bias:0' shape=(300,) dtype=float32_ref>,
 <tf.Variable 'dense/kernel:0' shape=(600, 23) dtype=float32_ref>,
 <tf.Variable 'dense/bias:0' shape=(23,) dtype=float32_ref>,
 <tf.Variable 'dense_1/kernel:0' shape=(600, 122) dtype=float32_ref>,
 <tf.Variable 'dense_1/bias:0' shape=(122,) dtype=float32_ref>,
 <tf.Variable 'transitions:0' shape=(122, 122) dtype=float32_ref>]
I0716 00:51:05.827634 140076727973760 estimator.py:1147] Done calling model_fn.
I0716 00:51:05.832204 140076727973760 basic_session_run_hooks.py:541] Create CheckpointSaverHook.
I0716 00:51:06.030192 140076727973760 monitored_session.py:240] Graph was finalized.
I0716 00:51:06.048776 140076727973760 saver.py:1280] Restoring parameters from ../model/bigru_crf/model.ckpt-1248
I0716 00:51:06.305366 140076727973760 session_manager.py:500] Running local_init_op.
I0716 00:51:06.343114 140076727973760 session_manager.py:502] Done running local_init_op.
I0716 00:51:08.014221 140076727973760 basic_session_run_hooks.py:606] Saving checkpoints for 1248 into ../model/bigru_crf/model.ckpt.
Reading ../data/atis.train.w-intent.iob
I0716 00:51:10.322084 140076727973760 basic_session_run_hooks.py:262] loss = 0.4459946, step = 1248
I0716 00:51:10.323844 140076727973760 basic_session_run_hooks.py:262] lr = 0.00026303643
I0716 00:51:21.938597 140076727973760 basic_session_run_hooks.py:692] global_step/sec: 8.608
I0716 00:51:21.944592 140076727973760 basic_session_run_hooks.py:260] loss = 0.68877256, step = 1348 (11.622 sec)
I0716 00:51:21.947494 140076727973760 basic_session_run_hooks.py:260] lr = 0.00026027963 (11.624 sec)
I0716 00:51:32.471386 140076727973760 basic_session_run_hooks.py:692] global_step/sec: 9.4942
I0716 00:51:32.474540 140076727973760 basic_session_run_hooks.py:260] loss = 0.36192447, step = 1448 (10.530 sec)
I0716 00:51:32.478130 140076727973760 basic_session_run_hooks.py:260] lr = 0.00025755167 (10.531 sec)
I0716 00:51:43.429825 140076727973760 basic_session_run_hooks.py:692] global_step/sec: 9.12531
I0716 00:51:43.436366 140076727973760 basic_session_run_hooks.py:260] loss = 1.3301551, step = 1548 (10.962 sec)
I0716 00:51:43.438526 140076727973760 basic_session_run_hooks.py:260] lr = 0.00025485238 (10.960 sec)
I0716 00:51:44.629787 140076727973760 basic_session_run_hooks.py:606] Saving checkpoints for 1560 into ../model/bigru_crf/model.ckpt.
W0716 00:51:44.723216 140076727973760 deprecation.py:323] From /usr/local/lib/python3.6/dist-packages/tensorflow/python/training/saver.py:960: remove_checkpoint (from tensorflow.python.training.checkpoint_management) is deprecated and will be removed in a future version.
Instructions for updating:
Use standard file APIs to delete files with this prefix.
I0716 00:51:45.011709 140076727973760 estimator.py:1145] Calling model_fn.
I0716 00:51:45.963483 140076727973760 estimator.py:1147] Done calling model_fn.
I0716 00:51:45.992590 140076727973760 evaluation.py:255] Starting evaluation at 2019-07-16T00:51:45Z
I0716 00:51:46.105449 140076727973760 monitored_session.py:240] Graph was finalized.
I0716 00:51:46.122810 140076727973760 saver.py:1280] Restoring parameters from ../model/bigru_crf/model.ckpt-1560
I0716 00:51:46.250744 140076727973760 session_manager.py:500] Running local_init_op.
I0716 00:51:46.276480 140076727973760 session_manager.py:502] Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
I0716 00:51:48.555608 140076727973760 evaluation.py:275] Finished evaluation at 2019-07-16-00:51:48
I0716 00:51:48.557240 140076727973760 estimator.py:2039] Saving dict for global step 1560: global_step = 1560, loss = 3.5510614
I0716 00:51:48.564484 140076727973760 estimator.py:2099] Saving 'checkpoint_path' summary for global step 1560: ../model/bigru_crf/model.ckpt-1560
I0716 00:51:48.625926 140076727973760 estimator.py:368] Loss for final step: 0.13305828.
Reading ../data/atis.test.w-intent.iob
I0716 00:51:48.942385 140076727973760 estimator.py:1145] Calling model_fn.
I0716 00:51:49.856646 140076727973760 estimator.py:1147] Done calling model_fn.
I0716 00:51:49.971741 140076727973760 monitored_session.py:240] Graph was finalized.
I0716 00:51:49.987007 140076727973760 saver.py:1280] Restoring parameters from ../model/bigru_crf/model.ckpt-1560
I0716 00:51:50.088970 140076727973760 session_manager.py:500] Running local_init_op.
I0716 00:51:50.106129 140076727973760 session_manager.py:502] Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.
  'precision', 'predicted', average, warn_for)
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1439: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples.
  'recall', 'true', average, warn_for)
I0716 00:51:53.188315 140076727973760 interactiveshell.py:2882] 
                                          precision    recall  f1-score   support

                             atis_flight      0.974     0.992     0.983       632
                            atis_airfare      0.960     1.000     0.980        48
                     atis_ground_service      1.000     1.000     1.000        36
                            atis_airline      1.000     1.000     1.000        38
                       atis_abbreviation      0.943     1.000     0.971        33
                           atis_aircraft      1.000     1.000     1.000         9
                        atis_flight_time      0.071     1.000     0.133         1
                           atis_quantity      0.375     1.000     0.545         3
                atis_flight#atis_airfare      0.500     0.083     0.143        12
                            atis_airport      1.000     0.944     0.971        18
                           atis_distance      1.000     0.900     0.947        10
                               atis_city      1.000     0.500     0.667         6
                        atis_ground_fare      1.000     0.857     0.923         7
                           atis_capacity      0.952     0.952     0.952        21
                          atis_flight_no      1.000     0.125     0.222         8
                               atis_meal      0.000     0.000     0.000         6
                        atis_restriction      0.000     0.000     0.000         0
             atis_airline#atis_flight_no      0.000     0.000     0.000         0
    atis_ground_service#atis_ground_fare      0.000     0.000     0.000         0
           atis_airfare#atis_flight_time      0.000     0.000     0.000         0
                           atis_cheapest      0.000     0.000     0.000         0
atis_aircraft#atis_flight#atis_flight_no      0.000     0.000     0.000         0

                               micro avg      0.954     0.959     0.957       888
                               macro avg      0.581     0.562     0.520       888
                            weighted avg      0.959     0.959     0.953       888

I0716 00:51:53.239272 140076727973760 interactiveshell.py:2882] 
                              precision    recall  f1-score   support

                           O      0.000     0.000     0.000       0.0
           B-toloc.city_name      0.974     0.993     0.983     716.0
         B-fromloc.city_name      0.978     0.999     0.988     704.0
           I-toloc.city_name      0.974     1.000     0.987     265.0
      B-depart_date.day_name      0.991     0.991     0.991     212.0
              B-airline_name      0.971     1.000     0.985     101.0
         I-fromloc.city_name      0.989     0.989     0.989     177.0
 B-depart_time.period_of_day      0.966     0.869     0.915     130.0
              I-airline_name      0.970     1.000     0.985      65.0
    B-depart_date.day_number      0.981     0.964     0.972      55.0
    B-depart_date.month_name      0.982     0.964     0.973      56.0
          B-depart_time.time      0.800     0.982     0.882      57.0
                B-round_trip      1.000     0.973     0.986      73.0
             B-cost_relative      1.000     0.973     0.986      37.0
                I-round_trip      1.000     0.986     0.993      71.0
                B-flight_mod      0.957     0.917     0.936      24.0
 B-depart_time.time_relative      0.968     0.938     0.953      65.0
          I-depart_time.time      0.862     0.962     0.909      52.0
         B-stoploc.city_name      0.952     1.000     0.976      20.0
                 B-city_name      0.966     0.491     0.651      57.0
                B-class_type      0.960     1.000     0.980      24.0
          B-arrive_time.time      0.943     0.971     0.957      34.0
 B-arrive_time.time_relative      0.903     0.903     0.903      31.0
                I-class_type      1.000     1.000     1.000      17.0
               B-flight_stop      1.000     1.000     1.000      21.0
          I-arrive_time.time      0.944     0.971     0.958      35.0
              B-airline_code      0.963     0.765     0.852      34.0
    I-depart_date.day_number      1.000     1.000     1.000      15.0
      I-fromloc.airport_name      0.405     1.000     0.577      15.0
      B-fromloc.airport_name      0.400     0.833     0.541      12.0
      B-arrive_date.day_name      0.786     1.000     0.880      11.0
          B-toloc.state_code      0.947     1.000     0.973      18.0
B-depart_date.today_relative      1.000     0.889     0.941       9.0
             B-flight_number      0.688     1.000     0.815      11.0
 B-depart_date.date_relative      0.895     1.000     0.944      17.0
          B-toloc.state_name      0.893     0.893     0.893      28.0
           B-fare_basis_code      0.895     1.000     0.944      17.0
               B-flight_time      1.000     1.000     1.000       1.0
                        B-or      1.000     1.000     1.000       3.0
 B-arrive_time.period_of_day      0.750     1.000     0.857       6.0
          B-meal_description      1.000     0.900     0.947      10.0
             I-cost_relative      1.000     0.667     0.800       3.0
              I-airport_name      0.909     0.345     0.500      29.0
               B-fare_amount      1.000     1.000     1.000       2.0
               I-fare_amount      1.000     1.000     1.000       2.0
                 I-city_name      1.000     0.567     0.723      30.0
        I-toloc.airport_name      1.000     1.000     1.000       3.0
            B-transport_type      1.000     1.000     1.000      10.0
    B-arrive_date.month_name      0.714     0.833     0.769       6.0
    B-arrive_date.day_number      0.714     0.833     0.769       6.0
         I-stoploc.city_name      0.833     1.000     0.909      10.0
                      B-meal      1.000     1.000     1.000      16.0
        B-fromloc.state_code      1.000     0.957     0.978      23.0
    B-depart_time.period_mod      0.600     0.600     0.600       5.0
                   B-connect      1.000     1.000     1.000       6.0
               B-flight_days      1.000     1.000     1.000      10.0
        B-toloc.airport_name      1.000     1.000     1.000       3.0
        B-fromloc.state_name      0.882     0.882     0.882      17.0
              B-airport_name      0.778     0.333     0.467      21.0
                   B-economy      1.000     1.000     1.000       6.0
               I-flight_time      1.000     1.000     1.000       1.0
             B-aircraft_code      1.000     0.909     0.952      33.0
                       B-mod      0.000     0.000     0.000       2.0
              B-airport_code      0.000     0.000     0.000       9.0
    B-depart_time.start_time      1.000     0.333     0.500       3.0
      B-depart_time.end_time      1.000     0.333     0.500       3.0
          B-depart_date.year      1.000     1.000     1.000       3.0
            I-transport_type      0.000     0.000     0.000       1.0
          B-restriction_code      0.500     1.000     0.667       4.0
    B-arrive_time.start_time      0.889     1.000     0.941       8.0
        B-toloc.airport_code      0.750     0.750     0.750       4.0
      B-arrive_time.end_time      0.875     0.875     0.875       8.0
      I-arrive_time.end_time      0.875     0.875     0.875       8.0
      I-depart_time.end_time      1.000     0.333     0.500       3.0
               I-flight_stop      0.000     0.000     0.000       0.0
      B-fromloc.airport_code      0.833     1.000     0.909       5.0
          I-restriction_code      1.000     1.000     1.000       3.0
    I-depart_time.start_time      1.000     1.000     1.000       1.0
          I-toloc.state_name      1.000     1.000     1.000       1.0
I-depart_date.today_relative      0.000     0.000     0.000       0.0
 B-arrive_date.date_relative      1.000     1.000     1.000       2.0
                I-flight_mod      0.000     0.000     0.000       6.0
                   I-economy      0.000     0.000     0.000       0.0
 B-return_date.date_relative      0.000     0.000     0.000       3.0
        I-fromloc.state_name      1.000     1.000     1.000       1.0
                B-state_code      1.000     1.000     1.000       1.0
    I-arrive_time.start_time      1.000     1.000     1.000       1.0
    I-arrive_date.day_number      0.000     0.000     0.000       0.0
                 B-meal_code      0.000     0.000     0.000       1.0
 I-depart_time.period_of_day      0.000     0.000     0.000       1.0
                  B-day_name      1.000     0.500     0.667       2.0
             B-period_of_day      1.000     0.500     0.667       4.0
        B-stoploc.state_code      0.000     0.000     0.000       0.0
    B-return_date.month_name      0.000     0.000     0.000       0.0
    B-return_date.day_number      0.000     0.000     0.000       0.0
    B-arrive_time.period_mod      0.000     0.000     0.000       0.0
                 I-meal_code      0.000     0.000     0.000       0.0
        B-toloc.country_name      1.000     1.000     1.000       1.0
                 B-days_code      1.000     1.000     1.000       1.0
 I-arrive_time.period_of_day      0.000     0.000     0.000       0.0
            I-today_relative      0.000     0.000     0.000       0.0
 B-return_time.period_of_day      0.000     0.000     0.000       0.0
                      B-time      0.000     0.000     0.000       0.0
           I-fare_basis_code      0.000     0.000     0.000       0.0
 I-arrive_time.time_relative      0.000     0.000     0.000       4.0
 I-depart_time.time_relative      0.000     0.000     0.000       1.0
            B-today_relative      0.000     0.000     0.000       0.0
                B-state_name      0.000     0.000     0.000       9.0
B-arrive_date.today_relative      0.000     0.000     0.000       0.0
    B-return_time.period_mod      0.000     0.000     0.000       0.0
                B-month_name      0.000     0.000     0.000       0.0
                B-day_number      0.000     0.000     0.000       0.0
 I-return_date.date_relative      0.000     0.000     0.000       3.0
I-return_date.today_relative      0.000     0.000     0.000       0.0
      B-stoploc.airport_name      0.000     0.000     0.000       0.0
             B-time_relative      0.000     0.000     0.000       0.0
                      I-time      0.000     0.000     0.000       0.0
    I-return_date.day_number      0.000     0.000     0.000       0.0
          I-meal_description      0.000     0.000     0.000       0.0
B-return_date.today_relative      0.000     0.000     0.000       0.0
      B-return_date.day_name      0.000     0.000     0.000       2.0

                   micro avg      0.943     0.944     0.943    3657.0
                   macro avg      0.626     0.608     0.605    3657.0
                weighted avg      0.949     0.944     0.941    3657.0

I0716 00:51:53.268521 140076727973760 interactiveshell.py:2882] Best Slot F1: 0.943
I0716 00:51:53.270126 140076727973760 estimator_training.py:186] Not using Distribute Coordinator.
I0716 00:51:53.274030 140076727973760 training.py:612] Running training and evaluation locally (non-distributed).
I0716 00:51:53.275785 140076727973760 training.py:700] Start train and evaluate loop. The evaluate will happen after every checkpoint. Checkpoint frequency is determined based on RunConfig arguments: save_checkpoints_steps 312 or save_checkpoints_secs None.
I0716 00:51:53.335677 140076727973760 estimator.py:1145] Calling model_fn.
I0716 00:51:54.454640 140076727973760 <ipython-input-6-0a85a3fd693f>:50] 
[<tf.Variable 'embedding:0' shape=(750, 300) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/gates/kernel:0' shape=(600, 600) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/gates/bias:0' shape=(600,) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/candidate/kernel:0' shape=(600, 300) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/candidate/bias:0' shape=(300,) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/gates/kernel:0' shape=(600, 600) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/gates/bias:0' shape=(600,) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/candidate/kernel:0' shape=(600, 300) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/candidate/bias:0' shape=(300,) dtype=float32_ref>,
 <tf.Variable 'dense/kernel:0' shape=(600, 23) dtype=float32_ref>,
 <tf.Variable 'dense/bias:0' shape=(23,) dtype=float32_ref>,
 <tf.Variable 'dense_1/kernel:0' shape=(600, 122) dtype=float32_ref>,
 <tf.Variable 'dense_1/bias:0' shape=(122,) dtype=float32_ref>,
 <tf.Variable 'transitions:0' shape=(122, 122) dtype=float32_ref>]
I0716 00:51:55.969718 140076727973760 estimator.py:1147] Done calling model_fn.
I0716 00:51:55.973501 140076727973760 basic_session_run_hooks.py:541] Create CheckpointSaverHook.
I0716 00:51:56.171922 140076727973760 monitored_session.py:240] Graph was finalized.
I0716 00:51:56.190224 140076727973760 saver.py:1280] Restoring parameters from ../model/bigru_crf/model.ckpt-1560
I0716 00:51:56.448569 140076727973760 session_manager.py:500] Running local_init_op.
I0716 00:51:56.490169 140076727973760 session_manager.py:502] Done running local_init_op.
I0716 00:51:57.973837 140076727973760 basic_session_run_hooks.py:606] Saving checkpoints for 1560 into ../model/bigru_crf/model.ckpt.
Reading ../data/atis.train.w-intent.iob
I0716 00:52:00.293659 140076727973760 basic_session_run_hooks.py:262] loss = 0.62887794, step = 1560
I0716 00:52:00.295683 140076727973760 basic_session_run_hooks.py:262] lr = 0.00025453034
I0716 00:52:12.014127 140076727973760 basic_session_run_hooks.py:692] global_step/sec: 8.53166
I0716 00:52:12.023669 140076727973760 basic_session_run_hooks.py:260] loss = 0.37790924, step = 1660 (11.730 sec)
I0716 00:52:12.026660 140076727973760 basic_session_run_hooks.py:260] lr = 0.0002518627 (11.731 sec)
I0716 00:52:22.731035 140076727973760 basic_session_run_hooks.py:692] global_step/sec: 9.33103
I0716 00:52:22.733642 140076727973760 basic_session_run_hooks.py:260] loss = 0.4490533, step = 1760 (10.710 sec)
I0716 00:52:22.739980 140076727973760 basic_session_run_hooks.py:260] lr = 0.00024922297 (10.713 sec)
I0716 00:52:33.562735 140076727973760 basic_session_run_hooks.py:692] global_step/sec: 9.23216
I0716 00:52:33.565537 140076727973760 basic_session_run_hooks.py:260] loss = 0.33208147, step = 1860 (10.832 sec)
I0716 00:52:33.571051 140076727973760 basic_session_run_hooks.py:260] lr = 0.00024661093 (10.831 sec)
I0716 00:52:34.677954 140076727973760 basic_session_run_hooks.py:606] Saving checkpoints for 1872 into ../model/bigru_crf/model.ckpt.
I0716 00:52:35.050499 140076727973760 estimator.py:1145] Calling model_fn.
I0716 00:52:36.197221 140076727973760 estimator.py:1147] Done calling model_fn.
I0716 00:52:36.226628 140076727973760 evaluation.py:255] Starting evaluation at 2019-07-16T00:52:36Z
I0716 00:52:36.343533 140076727973760 monitored_session.py:240] Graph was finalized.
I0716 00:52:36.360372 140076727973760 saver.py:1280] Restoring parameters from ../model/bigru_crf/model.ckpt-1872
I0716 00:52:36.479712 140076727973760 session_manager.py:500] Running local_init_op.
I0716 00:52:36.513121 140076727973760 session_manager.py:502] Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
I0716 00:52:38.787700 140076727973760 evaluation.py:275] Finished evaluation at 2019-07-16-00:52:38
I0716 00:52:38.789341 140076727973760 estimator.py:2039] Saving dict for global step 1872: global_step = 1872, loss = 3.3480728
I0716 00:52:38.797121 140076727973760 estimator.py:2099] Saving 'checkpoint_path' summary for global step 1872: ../model/bigru_crf/model.ckpt-1872
I0716 00:52:38.858462 140076727973760 estimator.py:368] Loss for final step: 0.002657627.
Reading ../data/atis.test.w-intent.iob
I0716 00:52:39.178158 140076727973760 estimator.py:1145] Calling model_fn.
I0716 00:52:39.836250 140076727973760 estimator.py:1147] Done calling model_fn.
I0716 00:52:40.216125 140076727973760 monitored_session.py:240] Graph was finalized.
I0716 00:52:40.233426 140076727973760 saver.py:1280] Restoring parameters from ../model/bigru_crf/model.ckpt-1872
I0716 00:52:40.340087 140076727973760 session_manager.py:500] Running local_init_op.
I0716 00:52:40.357437 140076727973760 session_manager.py:502] Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.
  'precision', 'predicted', average, warn_for)
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1439: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples.
  'recall', 'true', average, warn_for)
I0716 00:52:43.466185 140076727973760 interactiveshell.py:2882] 
                                          precision    recall  f1-score   support

                             atis_flight      0.972     0.992     0.982       632
                            atis_airfare      0.960     1.000     0.980        48
                     atis_ground_service      1.000     1.000     1.000        36
                            atis_airline      1.000     1.000     1.000        38
                       atis_abbreviation      0.943     1.000     0.971        33
                           atis_aircraft      1.000     1.000     1.000         9
                        atis_flight_time      0.333     1.000     0.500         1
                           atis_quantity      0.375     1.000     0.545         3
                atis_flight#atis_airfare      0.500     0.083     0.143        12
                            atis_airport      1.000     0.944     0.971        18
                           atis_distance      1.000     0.900     0.947        10
                               atis_city      1.000     0.500     0.667         6
                        atis_ground_fare      1.000     0.857     0.923         7
                           atis_capacity      1.000     0.952     0.976        21
                          atis_flight_no      0.875     0.875     0.875         8
                               atis_meal      1.000     0.667     0.800         6
                        atis_restriction      0.000     0.000     0.000         0
             atis_airline#atis_flight_no      0.000     0.000     0.000         0
    atis_ground_service#atis_ground_fare      0.000     0.000     0.000         0
           atis_airfare#atis_flight_time      0.000     0.000     0.000         0
                           atis_cheapest      0.000     0.000     0.000         0
atis_aircraft#atis_flight#atis_flight_no      0.000     0.000     0.000         0

                               micro avg      0.965     0.971     0.968       888
                               macro avg      0.634     0.626     0.604       888
                            weighted avg      0.965     0.971     0.964       888

I0716 00:52:43.517219 140076727973760 interactiveshell.py:2882] 
                              precision    recall  f1-score   support

                           O      0.000     0.000     0.000       0.0
           B-toloc.city_name      0.975     0.993     0.984     716.0
         B-fromloc.city_name      0.990     0.999     0.994     704.0
           I-toloc.city_name      0.981     0.989     0.985     265.0
      B-depart_date.day_name      0.991     0.991     0.991     212.0
              B-airline_name      0.971     1.000     0.985     101.0
         I-fromloc.city_name      0.983     0.989     0.986     177.0
 B-depart_time.period_of_day      0.992     0.900     0.944     130.0
              I-airline_name      0.970     1.000     0.985      65.0
    B-depart_date.day_number      0.981     0.964     0.972      55.0
    B-depart_date.month_name      0.982     0.964     0.973      56.0
          B-depart_time.time      0.851     1.000     0.919      57.0
                B-round_trip      1.000     0.973     0.986      73.0
             B-cost_relative      1.000     0.973     0.986      37.0
                I-round_trip      1.000     1.000     1.000      71.0
                B-flight_mod      0.957     0.917     0.936      24.0
 B-depart_time.time_relative      0.969     0.954     0.961      65.0
          I-depart_time.time      0.912     1.000     0.954      52.0
         B-stoploc.city_name      0.909     1.000     0.952      20.0
                 B-city_name      0.868     0.579     0.695      57.0
                B-class_type      0.960     1.000     0.980      24.0
          B-arrive_time.time      1.000     0.971     0.985      34.0
 B-arrive_time.time_relative      0.933     0.903     0.918      31.0
                I-class_type      1.000     1.000     1.000      17.0
               B-flight_stop      0.955     1.000     0.977      21.0
          I-arrive_time.time      0.971     0.971     0.971      35.0
              B-airline_code      1.000     0.765     0.867      34.0
    I-depart_date.day_number      1.000     0.933     0.966      15.0
      I-fromloc.airport_name      0.441     1.000     0.612      15.0
      B-fromloc.airport_name      0.407     0.917     0.564      12.0
      B-arrive_date.day_name      0.786     1.000     0.880      11.0
          B-toloc.state_code      0.947     1.000     0.973      18.0
B-depart_date.today_relative      1.000     0.889     0.941       9.0
             B-flight_number      0.579     1.000     0.733      11.0
 B-depart_date.date_relative      0.895     1.000     0.944      17.0
          B-toloc.state_name      0.862     0.893     0.877      28.0
           B-fare_basis_code      0.850     1.000     0.919      17.0
               B-flight_time      1.000     1.000     1.000       1.0
                        B-or      1.000     1.000     1.000       3.0
 B-arrive_time.period_of_day      0.857     1.000     0.923       6.0
          B-meal_description      1.000     0.900     0.947      10.0
             I-cost_relative      1.000     0.667     0.800       3.0
              I-airport_name      0.917     0.379     0.537      29.0
               B-fare_amount      1.000     1.000     1.000       2.0
               I-fare_amount      1.000     1.000     1.000       2.0
                 I-city_name      0.952     0.667     0.784      30.0
        I-toloc.airport_name      1.000     1.000     1.000       3.0
            B-transport_type      1.000     1.000     1.000      10.0
    B-arrive_date.month_name      0.714     0.833     0.769       6.0
    B-arrive_date.day_number      0.625     0.833     0.714       6.0
         I-stoploc.city_name      0.769     1.000     0.870      10.0
                      B-meal      1.000     1.000     1.000      16.0
        B-fromloc.state_code      1.000     0.957     0.978      23.0
    B-depart_time.period_mod      0.714     1.000     0.833       5.0
                   B-connect      1.000     1.000     1.000       6.0
               B-flight_days      1.000     1.000     1.000      10.0
        B-toloc.airport_name      1.000     1.000     1.000       3.0
        B-fromloc.state_name      0.944     1.000     0.971      17.0
              B-airport_name      0.778     0.333     0.467      21.0
                   B-economy      1.000     1.000     1.000       6.0
               I-flight_time      1.000     1.000     1.000       1.0
             B-aircraft_code      1.000     0.818     0.900      33.0
                       B-mod      0.000     0.000     0.000       2.0
              B-airport_code      0.500     0.111     0.182       9.0
    B-depart_time.start_time      1.000     0.667     0.800       3.0
      B-depart_time.end_time      1.000     0.333     0.500       3.0
          B-depart_date.year      1.000     1.000     1.000       3.0
            I-transport_type      0.000     0.000     0.000       1.0
          B-restriction_code      0.500     1.000     0.667       4.0
    B-arrive_time.start_time      0.889     1.000     0.941       8.0
        B-toloc.airport_code      0.750     0.750     0.750       4.0
      B-arrive_time.end_time      0.889     1.000     0.941       8.0
      I-arrive_time.end_time      0.875     0.875     0.875       8.0
      I-depart_time.end_time      1.000     0.333     0.500       3.0
               I-flight_stop      0.000     0.000     0.000       0.0
      B-fromloc.airport_code      1.000     1.000     1.000       5.0
          I-restriction_code      1.000     1.000     1.000       3.0
    I-depart_time.start_time      1.000     1.000     1.000       1.0
          I-toloc.state_name      1.000     1.000     1.000       1.0
I-depart_date.today_relative      0.000     0.000     0.000       0.0
 B-arrive_date.date_relative      1.000     1.000     1.000       2.0
                I-flight_mod      0.000     0.000     0.000       6.0
                   I-economy      0.000     0.000     0.000       0.0
 B-return_date.date_relative      0.333     0.333     0.333       3.0
        I-fromloc.state_name      1.000     1.000     1.000       1.0
                B-state_code      1.000     1.000     1.000       1.0
    I-arrive_time.start_time      1.000     1.000     1.000       1.0
    I-arrive_date.day_number      0.000     0.000     0.000       0.0
                 B-meal_code      0.000     0.000     0.000       1.0
 I-depart_time.period_of_day      0.000     0.000     0.000       1.0
                  B-day_name      1.000     0.500     0.667       2.0
             B-period_of_day      1.000     1.000     1.000       4.0
        B-stoploc.state_code      0.000     0.000     0.000       0.0
    B-return_date.month_name      0.000     0.000     0.000       0.0
    B-return_date.day_number      0.000     0.000     0.000       0.0
    B-arrive_time.period_mod      0.000     0.000     0.000       0.0
                 I-meal_code      0.000     0.000     0.000       0.0
        B-toloc.country_name      1.000     1.000     1.000       1.0
                 B-days_code      1.000     1.000     1.000       1.0
 I-arrive_time.period_of_day      0.000     0.000     0.000       0.0
            I-today_relative      0.000     0.000     0.000       0.0
 B-return_time.period_of_day      0.000     0.000     0.000       0.0
                      B-time      0.000     0.000     0.000       0.0
           I-fare_basis_code      0.000     0.000     0.000       0.0
 I-arrive_time.time_relative      0.000     0.000     0.000       4.0
 I-depart_time.time_relative      0.000     0.000     0.000       1.0
            B-today_relative      0.000     0.000     0.000       0.0
                B-state_name      0.000     0.000     0.000       9.0
B-arrive_date.today_relative      0.000     0.000     0.000       0.0
    B-return_time.period_mod      0.000     0.000     0.000       0.0
                B-month_name      0.000     0.000     0.000       0.0
                B-day_number      0.000     0.000     0.000       0.0
 I-return_date.date_relative      0.000     0.000     0.000       3.0
I-return_date.today_relative      0.000     0.000     0.000       0.0
      B-stoploc.airport_name      0.000     0.000     0.000       0.0
             B-time_relative      0.000     0.000     0.000       0.0
                      I-time      0.000     0.000     0.000       0.0
    I-return_date.day_number      0.000     0.000     0.000       0.0
          I-meal_description      0.000     0.000     0.000       0.0
B-return_date.today_relative      0.000     0.000     0.000       0.0
      B-return_date.day_name      0.000     0.000     0.000       2.0

                   micro avg      0.949     0.950     0.949    3657.0
                   macro avg      0.635     0.626     0.619    3657.0
                weighted avg      0.955     0.950     0.948    3657.0

I0716 00:52:43.546526 140076727973760 interactiveshell.py:2882] Best Slot F1: 0.949
I0716 00:52:43.548300 140076727973760 estimator_training.py:186] Not using Distribute Coordinator.
I0716 00:52:43.550570 140076727973760 training.py:612] Running training and evaluation locally (non-distributed).
I0716 00:52:43.554519 140076727973760 training.py:700] Start train and evaluate loop. The evaluate will happen after every checkpoint. Checkpoint frequency is determined based on RunConfig arguments: save_checkpoints_steps 312 or save_checkpoints_secs None.
I0716 00:52:43.617251 140076727973760 estimator.py:1145] Calling model_fn.
I0716 00:52:44.579117 140076727973760 <ipython-input-6-0a85a3fd693f>:50] 
[<tf.Variable 'embedding:0' shape=(750, 300) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/gates/kernel:0' shape=(600, 600) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/gates/bias:0' shape=(600,) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/candidate/kernel:0' shape=(600, 300) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/candidate/bias:0' shape=(300,) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/gates/kernel:0' shape=(600, 600) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/gates/bias:0' shape=(600,) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/candidate/kernel:0' shape=(600, 300) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/candidate/bias:0' shape=(300,) dtype=float32_ref>,
 <tf.Variable 'dense/kernel:0' shape=(600, 23) dtype=float32_ref>,
 <tf.Variable 'dense/bias:0' shape=(23,) dtype=float32_ref>,
 <tf.Variable 'dense_1/kernel:0' shape=(600, 122) dtype=float32_ref>,
 <tf.Variable 'dense_1/bias:0' shape=(122,) dtype=float32_ref>,
 <tf.Variable 'transitions:0' shape=(122, 122) dtype=float32_ref>]
I0716 00:52:46.101108 140076727973760 estimator.py:1147] Done calling model_fn.
I0716 00:52:46.104851 140076727973760 basic_session_run_hooks.py:541] Create CheckpointSaverHook.
I0716 00:52:46.299911 140076727973760 monitored_session.py:240] Graph was finalized.
I0716 00:52:46.316984 140076727973760 saver.py:1280] Restoring parameters from ../model/bigru_crf/model.ckpt-1872
I0716 00:52:46.580218 140076727973760 session_manager.py:500] Running local_init_op.
I0716 00:52:46.618957 140076727973760 session_manager.py:502] Done running local_init_op.
I0716 00:52:48.299108 140076727973760 basic_session_run_hooks.py:606] Saving checkpoints for 1872 into ../model/bigru_crf/model.ckpt.
Reading ../data/atis.train.w-intent.iob
I0716 00:52:50.632071 140076727973760 basic_session_run_hooks.py:262] loss = 0.17376733, step = 1872
I0716 00:52:50.633970 140076727973760 basic_session_run_hooks.py:262] lr = 0.0002462993
I0716 00:53:02.274831 140076727973760 basic_session_run_hooks.py:692] global_step/sec: 8.58861
I0716 00:53:02.279967 140076727973760 basic_session_run_hooks.py:260] loss = 0.3174852, step = 1972 (11.648 sec)
I0716 00:53:02.282042 140076727973760 basic_session_run_hooks.py:260] lr = 0.00024371794 (11.648 sec)
I0716 00:53:13.167236 140076727973760 basic_session_run_hooks.py:692] global_step/sec: 9.18068
I0716 00:53:13.174702 140076727973760 basic_session_run_hooks.py:260] loss = 1.4741601, step = 2072 (10.895 sec)
I0716 00:53:13.176987 140076727973760 basic_session_run_hooks.py:260] lr = 0.00024116358 (10.895 sec)
I0716 00:53:23.805697 140076727973760 basic_session_run_hooks.py:692] global_step/sec: 9.39978
I0716 00:53:23.812388 140076727973760 basic_session_run_hooks.py:260] loss = 0.22079225, step = 2172 (10.638 sec)
I0716 00:53:23.815447 140076727973760 basic_session_run_hooks.py:260] lr = 0.00023863598 (10.638 sec)
I0716 00:53:24.994797 140076727973760 basic_session_run_hooks.py:606] Saving checkpoints for 2184 into ../model/bigru_crf/model.ckpt.
I0716 00:53:25.369107 140076727973760 estimator.py:1145] Calling model_fn.
I0716 00:53:26.342289 140076727973760 estimator.py:1147] Done calling model_fn.
I0716 00:53:26.371496 140076727973760 evaluation.py:255] Starting evaluation at 2019-07-16T00:53:26Z
I0716 00:53:26.488402 140076727973760 monitored_session.py:240] Graph was finalized.
I0716 00:53:26.505404 140076727973760 saver.py:1280] Restoring parameters from ../model/bigru_crf/model.ckpt-2184
I0716 00:53:26.623869 140076727973760 session_manager.py:500] Running local_init_op.
I0716 00:53:26.651631 140076727973760 session_manager.py:502] Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
I0716 00:53:28.981069 140076727973760 evaluation.py:275] Finished evaluation at 2019-07-16-00:53:28
I0716 00:53:28.982563 140076727973760 estimator.py:2039] Saving dict for global step 2184: global_step = 2184, loss = 3.5048676
I0716 00:53:28.987373 140076727973760 estimator.py:2099] Saving 'checkpoint_path' summary for global step 2184: ../model/bigru_crf/model.ckpt-2184
I0716 00:53:29.049600 140076727973760 estimator.py:368] Loss for final step: 0.046680942.
Reading ../data/atis.test.w-intent.iob
I0716 00:53:29.369243 140076727973760 estimator.py:1145] Calling model_fn.
I0716 00:53:30.287421 140076727973760 estimator.py:1147] Done calling model_fn.
I0716 00:53:30.399026 140076727973760 monitored_session.py:240] Graph was finalized.
I0716 00:53:30.416694 140076727973760 saver.py:1280] Restoring parameters from ../model/bigru_crf/model.ckpt-2184
I0716 00:53:30.519573 140076727973760 session_manager.py:500] Running local_init_op.
I0716 00:53:30.538872 140076727973760 session_manager.py:502] Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.
  'precision', 'predicted', average, warn_for)
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1439: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples.
  'recall', 'true', average, warn_for)
I0716 00:53:33.655115 140076727973760 interactiveshell.py:2882] 
                                          precision    recall  f1-score   support

                             atis_flight      0.974     0.992     0.983       632
                            atis_airfare      0.960     1.000     0.980        48
                     atis_ground_service      1.000     1.000     1.000        36
                            atis_airline      1.000     1.000     1.000        38
                       atis_abbreviation      0.943     1.000     0.971        33
                           atis_aircraft      1.000     1.000     1.000         9
                        atis_flight_time      0.333     1.000     0.500         1
                           atis_quantity      0.375     1.000     0.545         3
                atis_flight#atis_airfare      0.500     0.083     0.143        12
                            atis_airport      1.000     0.944     0.971        18
                           atis_distance      1.000     0.900     0.947        10
                               atis_city      1.000     0.500     0.667         6
                        atis_ground_fare      1.000     0.857     0.923         7
                           atis_capacity      1.000     0.952     0.976        21
                          atis_flight_no      0.889     1.000     0.941         8
                               atis_meal      1.000     0.667     0.800         6
                        atis_restriction      0.000     0.000     0.000         0
             atis_airline#atis_flight_no      0.000     0.000     0.000         0
    atis_ground_service#atis_ground_fare      0.000     0.000     0.000         0
           atis_airfare#atis_flight_time      0.000     0.000     0.000         0
                           atis_cheapest      0.000     0.000     0.000         0
atis_aircraft#atis_flight#atis_flight_no      0.000     0.000     0.000         0

                               micro avg      0.966     0.972     0.969       888
                               macro avg      0.635     0.632     0.607       888
                            weighted avg      0.966     0.972     0.965       888

I0716 00:53:33.705924 140076727973760 interactiveshell.py:2882] 
                              precision    recall  f1-score   support

                           O      0.000     0.000     0.000       0.0
           B-toloc.city_name      0.975     0.992     0.983     716.0
         B-fromloc.city_name      0.983     0.999     0.991     704.0
           I-toloc.city_name      0.981     0.977     0.979     265.0
      B-depart_date.day_name      0.991     0.991     0.991     212.0
              B-airline_name      0.981     1.000     0.990     101.0
         I-fromloc.city_name      0.951     0.994     0.972     177.0
 B-depart_time.period_of_day      0.992     0.923     0.956     130.0
              I-airline_name      1.000     1.000     1.000      65.0
    B-depart_date.day_number      0.981     0.964     0.972      55.0
    B-depart_date.month_name      0.982     0.964     0.973      56.0
          B-depart_time.time      0.838     1.000     0.912      57.0
                B-round_trip      1.000     0.973     0.986      73.0
             B-cost_relative      1.000     0.973     0.986      37.0
                I-round_trip      1.000     1.000     1.000      71.0
                B-flight_mod      0.958     0.958     0.958      24.0
 B-depart_time.time_relative      0.953     0.938     0.946      65.0
          I-depart_time.time      0.929     1.000     0.963      52.0
         B-stoploc.city_name      1.000     1.000     1.000      20.0
                 B-city_name      0.909     0.526     0.667      57.0
                B-class_type      0.960     1.000     0.980      24.0
          B-arrive_time.time      0.970     0.941     0.955      34.0
 B-arrive_time.time_relative      0.900     0.871     0.885      31.0
                I-class_type      1.000     1.000     1.000      17.0
               B-flight_stop      1.000     1.000     1.000      21.0
          I-arrive_time.time      1.000     0.943     0.971      35.0
              B-airline_code      1.000     0.882     0.938      34.0
    I-depart_date.day_number      1.000     0.933     0.966      15.0
      I-fromloc.airport_name      0.441     1.000     0.612      15.0
      B-fromloc.airport_name      0.440     0.917     0.595      12.0
      B-arrive_date.day_name      0.786     1.000     0.880      11.0
          B-toloc.state_code      0.947     1.000     0.973      18.0
B-depart_date.today_relative      1.000     0.889     0.941       9.0
             B-flight_number      0.500     1.000     0.667      11.0
 B-depart_date.date_relative      0.895     1.000     0.944      17.0
          B-toloc.state_name      0.862     0.893     0.877      28.0
           B-fare_basis_code      0.850     1.000     0.919      17.0
               B-flight_time      1.000     1.000     1.000       1.0
                        B-or      1.000     1.000     1.000       3.0
 B-arrive_time.period_of_day      0.857     1.000     0.923       6.0
          B-meal_description      1.000     0.900     0.947      10.0
             I-cost_relative      1.000     0.667     0.800       3.0
              I-airport_name      0.929     0.448     0.605      29.0
               B-fare_amount      1.000     1.000     1.000       2.0
               I-fare_amount      1.000     1.000     1.000       2.0
                 I-city_name      1.000     0.567     0.723      30.0
        I-toloc.airport_name      1.000     1.000     1.000       3.0
            B-transport_type      1.000     1.000     1.000      10.0
    B-arrive_date.month_name      0.714     0.833     0.769       6.0
    B-arrive_date.day_number      0.625     0.833     0.714       6.0
         I-stoploc.city_name      0.833     1.000     0.909      10.0
                      B-meal      1.000     1.000     1.000      16.0
        B-fromloc.state_code      1.000     0.957     0.978      23.0
    B-depart_time.period_mod      0.833     1.000     0.909       5.0
                   B-connect      1.000     1.000     1.000       6.0
               B-flight_days      1.000     1.000     1.000      10.0
        B-toloc.airport_name      1.000     1.000     1.000       3.0
        B-fromloc.state_name      0.895     1.000     0.944      17.0
              B-airport_name      0.833     0.476     0.606      21.0
                   B-economy      1.000     1.000     1.000       6.0
               I-flight_time      1.000     1.000     1.000       1.0
             B-aircraft_code      1.000     0.818     0.900      33.0
                       B-mod      0.000     0.000     0.000       2.0
              B-airport_code      0.000     0.000     0.000       9.0
    B-depart_time.start_time      1.000     1.000     1.000       3.0
      B-depart_time.end_time      1.000     0.333     0.500       3.0
          B-depart_date.year      1.000     1.000     1.000       3.0
            I-transport_type      0.000     0.000     0.000       1.0
          B-restriction_code      0.500     1.000     0.667       4.0
    B-arrive_time.start_time      1.000     1.000     1.000       8.0
        B-toloc.airport_code      0.750     0.750     0.750       4.0
      B-arrive_time.end_time      0.889     1.000     0.941       8.0
      I-arrive_time.end_time      0.889     1.000     0.941       8.0
      I-depart_time.end_time      1.000     0.333     0.500       3.0
               I-flight_stop      0.000     0.000     0.000       0.0
      B-fromloc.airport_code      0.833     1.000     0.909       5.0
          I-restriction_code      1.000     1.000     1.000       3.0
    I-depart_time.start_time      1.000     1.000     1.000       1.0
          I-toloc.state_name      1.000     1.000     1.000       1.0
I-depart_date.today_relative      0.000     0.000     0.000       0.0
 B-arrive_date.date_relative      1.000     1.000     1.000       2.0
                I-flight_mod      0.000     0.000     0.000       6.0
                   I-economy      0.000     0.000     0.000       0.0
 B-return_date.date_relative      0.000     0.000     0.000       3.0
        I-fromloc.state_name      1.000     1.000     1.000       1.0
                B-state_code      1.000     1.000     1.000       1.0
    I-arrive_time.start_time      1.000     1.000     1.000       1.0
    I-arrive_date.day_number      0.000     0.000     0.000       0.0
                 B-meal_code      0.000     0.000     0.000       1.0
 I-depart_time.period_of_day      0.000     0.000     0.000       1.0
                  B-day_name      1.000     0.500     0.667       2.0
             B-period_of_day      1.000     1.000     1.000       4.0
        B-stoploc.state_code      0.000     0.000     0.000       0.0
    B-return_date.month_name      0.000     0.000     0.000       0.0
    B-return_date.day_number      0.000     0.000     0.000       0.0
    B-arrive_time.period_mod      0.000     0.000     0.000       0.0
                 I-meal_code      0.000     0.000     0.000       0.0
        B-toloc.country_name      1.000     1.000     1.000       1.0
                 B-days_code      1.000     1.000     1.000       1.0
 I-arrive_time.period_of_day      0.000     0.000     0.000       0.0
            I-today_relative      0.000     0.000     0.000       0.0
 B-return_time.period_of_day      0.000     0.000     0.000       0.0
                      B-time      0.000     0.000     0.000       0.0
           I-fare_basis_code      0.000     0.000     0.000       0.0
 I-arrive_time.time_relative      0.000     0.000     0.000       4.0
 I-depart_time.time_relative      0.000     0.000     0.000       1.0
            B-today_relative      0.000     0.000     0.000       0.0
                B-state_name      0.000     0.000     0.000       9.0
B-arrive_date.today_relative      0.000     0.000     0.000       0.0
    B-return_time.period_mod      0.000     0.000     0.000       0.0
                B-month_name      0.000     0.000     0.000       0.0
                B-day_number      0.000     0.000     0.000       0.0
 I-return_date.date_relative      0.000     0.000     0.000       3.0
I-return_date.today_relative      0.000     0.000     0.000       0.0
      B-stoploc.airport_name      0.000     0.000     0.000       0.0
             B-time_relative      0.000     0.000     0.000       0.0
                      I-time      0.000     0.000     0.000       0.0
    I-return_date.day_number      0.000     0.000     0.000       0.0
          I-meal_description      0.000     0.000     0.000       0.0
B-return_date.today_relative      0.000     0.000     0.000       0.0
      B-return_date.day_name      0.000     0.000     0.000       2.0

                   micro avg      0.949     0.950     0.949    3657.0
                   macro avg      0.631     0.627     0.619    3657.0
                weighted avg      0.953     0.950     0.947    3657.0

I0716 00:53:33.735055 140076727973760 interactiveshell.py:2882] Best Slot F1: 0.949
I0716 00:53:33.736518 140076727973760 estimator_training.py:186] Not using Distribute Coordinator.
I0716 00:53:33.738923 140076727973760 training.py:612] Running training and evaluation locally (non-distributed).
I0716 00:53:33.741752 140076727973760 training.py:700] Start train and evaluate loop. The evaluate will happen after every checkpoint. Checkpoint frequency is determined based on RunConfig arguments: save_checkpoints_steps 312 or save_checkpoints_secs None.
I0716 00:53:33.807803 140076727973760 estimator.py:1145] Calling model_fn.
I0716 00:53:34.952826 140076727973760 <ipython-input-6-0a85a3fd693f>:50] 
[<tf.Variable 'embedding:0' shape=(750, 300) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/gates/kernel:0' shape=(600, 600) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/gates/bias:0' shape=(600,) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/candidate/kernel:0' shape=(600, 300) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/candidate/bias:0' shape=(300,) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/gates/kernel:0' shape=(600, 600) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/gates/bias:0' shape=(600,) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/candidate/kernel:0' shape=(600, 300) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/candidate/bias:0' shape=(300,) dtype=float32_ref>,
 <tf.Variable 'dense/kernel:0' shape=(600, 23) dtype=float32_ref>,
 <tf.Variable 'dense/bias:0' shape=(23,) dtype=float32_ref>,
 <tf.Variable 'dense_1/kernel:0' shape=(600, 122) dtype=float32_ref>,
 <tf.Variable 'dense_1/bias:0' shape=(122,) dtype=float32_ref>,
 <tf.Variable 'transitions:0' shape=(122, 122) dtype=float32_ref>]
I0716 00:53:36.465239 140076727973760 estimator.py:1147] Done calling model_fn.
I0716 00:53:36.469094 140076727973760 basic_session_run_hooks.py:541] Create CheckpointSaverHook.
I0716 00:53:36.665792 140076727973760 monitored_session.py:240] Graph was finalized.
I0716 00:53:36.686483 140076727973760 saver.py:1280] Restoring parameters from ../model/bigru_crf/model.ckpt-2184
I0716 00:53:36.944722 140076727973760 session_manager.py:500] Running local_init_op.
I0716 00:53:36.981242 140076727973760 session_manager.py:502] Done running local_init_op.
I0716 00:53:38.447672 140076727973760 basic_session_run_hooks.py:606] Saving checkpoints for 2184 into ../model/bigru_crf/model.ckpt.
Reading ../data/atis.train.w-intent.iob
I0716 00:53:40.730842 140076727973760 basic_session_run_hooks.py:262] loss = 0.7028024, step = 2184
I0716 00:53:40.733004 140076727973760 basic_session_run_hooks.py:262] lr = 0.00023833448
I0716 00:53:51.756081 140076727973760 basic_session_run_hooks.py:692] global_step/sec: 9.06967
I0716 00:53:51.763741 140076727973760 basic_session_run_hooks.py:260] loss = 1.0234358, step = 2284 (11.033 sec)
I0716 00:53:51.766004 140076727973760 basic_session_run_hooks.py:260] lr = 0.00023583656 (11.033 sec)
I0716 00:54:02.797284 140076727973760 basic_session_run_hooks.py:692] global_step/sec: 9.05689
I0716 00:54:02.800950 140076727973760 basic_session_run_hooks.py:260] loss = 0.46780574, step = 2384 (11.037 sec)
I0716 00:54:02.805921 140076727973760 basic_session_run_hooks.py:260] lr = 0.00023336482 (11.040 sec)
I0716 00:54:13.926261 140076727973760 basic_session_run_hooks.py:692] global_step/sec: 8.98556
I0716 00:54:13.929260 140076727973760 basic_session_run_hooks.py:260] loss = 0.023807462, step = 2484 (11.128 sec)
I0716 00:54:13.931284 140076727973760 basic_session_run_hooks.py:260] lr = 0.00023091899 (11.125 sec)
I0716 00:54:15.247632 140076727973760 basic_session_run_hooks.py:606] Saving checkpoints for 2496 into ../model/bigru_crf/model.ckpt.
I0716 00:54:15.630003 140076727973760 estimator.py:1145] Calling model_fn.
I0716 00:54:16.794045 140076727973760 estimator.py:1147] Done calling model_fn.
I0716 00:54:16.824453 140076727973760 evaluation.py:255] Starting evaluation at 2019-07-16T00:54:16Z
I0716 00:54:16.946480 140076727973760 monitored_session.py:240] Graph was finalized.
I0716 00:54:16.963471 140076727973760 saver.py:1280] Restoring parameters from ../model/bigru_crf/model.ckpt-2496
I0716 00:54:17.081112 140076727973760 session_manager.py:500] Running local_init_op.
I0716 00:54:17.108100 140076727973760 session_manager.py:502] Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
I0716 00:54:19.417138 140076727973760 evaluation.py:275] Finished evaluation at 2019-07-16-00:54:19
I0716 00:54:19.418688 140076727973760 estimator.py:2039] Saving dict for global step 2496: global_step = 2496, loss = 3.451709
I0716 00:54:19.425845 140076727973760 estimator.py:2099] Saving 'checkpoint_path' summary for global step 2496: ../model/bigru_crf/model.ckpt-2496
I0716 00:54:19.484352 140076727973760 estimator.py:368] Loss for final step: 0.06776102.
Reading ../data/atis.test.w-intent.iob
I0716 00:54:19.817067 140076727973760 estimator.py:1145] Calling model_fn.
I0716 00:54:20.792238 140076727973760 estimator.py:1147] Done calling model_fn.
I0716 00:54:20.909548 140076727973760 monitored_session.py:240] Graph was finalized.
I0716 00:54:20.924972 140076727973760 saver.py:1280] Restoring parameters from ../model/bigru_crf/model.ckpt-2496
I0716 00:54:21.038371 140076727973760 session_manager.py:500] Running local_init_op.
I0716 00:54:21.056823 140076727973760 session_manager.py:502] Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.
  'precision', 'predicted', average, warn_for)
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1439: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples.
  'recall', 'true', average, warn_for)
I0716 00:54:24.257605 140076727973760 interactiveshell.py:2882] 
                                          precision    recall  f1-score   support

                             atis_flight      0.974     0.992     0.983       632
                            atis_airfare      0.960     1.000     0.980        48
                     atis_ground_service      1.000     1.000     1.000        36
                            atis_airline      1.000     1.000     1.000        38
                       atis_abbreviation      0.943     1.000     0.971        33
                           atis_aircraft      1.000     1.000     1.000         9
                        atis_flight_time      0.500     1.000     0.667         1
                           atis_quantity      0.375     1.000     0.545         3
                atis_flight#atis_airfare      0.500     0.083     0.143        12
                            atis_airport      1.000     0.944     0.971        18
                           atis_distance      1.000     1.000     1.000        10
                               atis_city      1.000     0.500     0.667         6
                        atis_ground_fare      1.000     0.857     0.923         7
                           atis_capacity      1.000     0.952     0.976        21
                          atis_flight_no      0.889     1.000     0.941         8
                               atis_meal      1.000     0.667     0.800         6
                        atis_restriction      0.000     0.000     0.000         0
             atis_airline#atis_flight_no      0.000     0.000     0.000         0
    atis_ground_service#atis_ground_fare      0.000     0.000     0.000         0
           atis_airfare#atis_flight_time      0.000     0.000     0.000         0
                           atis_cheapest      0.000     0.000     0.000         0
atis_aircraft#atis_flight#atis_flight_no      0.000     0.000     0.000         0

                               micro avg      0.968     0.973     0.970       888
                               macro avg      0.643     0.636     0.617       888
                            weighted avg      0.966     0.973     0.966       888

I0716 00:54:24.309821 140076727973760 interactiveshell.py:2882] 
                              precision    recall  f1-score   support

                           O      0.000     0.000     0.000       0.0
           B-toloc.city_name      0.975     0.992     0.983     716.0
         B-fromloc.city_name      0.980     0.999     0.989     704.0
           I-toloc.city_name      0.981     0.977     0.979     265.0
      B-depart_date.day_name      0.991     0.991     0.991     212.0
              B-airline_name      0.981     1.000     0.990     101.0
         I-fromloc.city_name      0.957     0.994     0.975     177.0
 B-depart_time.period_of_day      0.992     0.908     0.948     130.0
              I-airline_name      1.000     1.000     1.000      65.0
    B-depart_date.day_number      0.981     0.964     0.972      55.0
    B-depart_date.month_name      0.982     0.964     0.973      56.0
          B-depart_time.time      0.838     1.000     0.912      57.0
                B-round_trip      1.000     0.973     0.986      73.0
             B-cost_relative      1.000     0.973     0.986      37.0
                I-round_trip      1.000     1.000     1.000      71.0
                B-flight_mod      1.000     0.958     0.979      24.0
 B-depart_time.time_relative      0.953     0.938     0.946      65.0
          I-depart_time.time      0.912     1.000     0.954      52.0
         B-stoploc.city_name      0.952     1.000     0.976      20.0
                 B-city_name      0.935     0.509     0.659      57.0
                B-class_type      0.960     1.000     0.980      24.0
          B-arrive_time.time      0.970     0.941     0.955      34.0
 B-arrive_time.time_relative      0.900     0.871     0.885      31.0
                I-class_type      1.000     1.000     1.000      17.0
               B-flight_stop      1.000     1.000     1.000      21.0
          I-arrive_time.time      1.000     0.943     0.971      35.0
              B-airline_code      1.000     0.882     0.938      34.0
    I-depart_date.day_number      1.000     0.933     0.966      15.0
      I-fromloc.airport_name      0.441     1.000     0.612      15.0
      B-fromloc.airport_name      0.462     1.000     0.632      12.0
      B-arrive_date.day_name      0.786     1.000     0.880      11.0
          B-toloc.state_code      0.947     1.000     0.973      18.0
B-depart_date.today_relative      1.000     0.889     0.941       9.0
             B-flight_number      0.500     1.000     0.667      11.0
 B-depart_date.date_relative      0.895     1.000     0.944      17.0
          B-toloc.state_name      0.862     0.893     0.877      28.0
           B-fare_basis_code      0.944     1.000     0.971      17.0
               B-flight_time      1.000     1.000     1.000       1.0
                        B-or      1.000     1.000     1.000       3.0
 B-arrive_time.period_of_day      0.857     1.000     0.923       6.0
          B-meal_description      1.000     0.900     0.947      10.0
             I-cost_relative      1.000     0.667     0.800       3.0
              I-airport_name      0.867     0.448     0.591      29.0
               B-fare_amount      1.000     1.000     1.000       2.0
               I-fare_amount      1.000     1.000     1.000       2.0
                 I-city_name      1.000     0.567     0.723      30.0
        I-toloc.airport_name      1.000     1.000     1.000       3.0
            B-transport_type      1.000     1.000     1.000      10.0
    B-arrive_date.month_name      0.714     0.833     0.769       6.0
    B-arrive_date.day_number      0.625     0.833     0.714       6.0
         I-stoploc.city_name      0.833     1.000     0.909      10.0
                      B-meal      1.000     1.000     1.000      16.0
        B-fromloc.state_code      1.000     0.957     0.978      23.0
    B-depart_time.period_mod      0.714     1.000     0.833       5.0
                   B-connect      1.000     1.000     1.000       6.0
               B-flight_days      1.000     1.000     1.000      10.0
        B-toloc.airport_name      1.000     1.000     1.000       3.0
        B-fromloc.state_name      0.895     1.000     0.944      17.0
              B-airport_name      0.833     0.476     0.606      21.0
                   B-economy      1.000     1.000     1.000       6.0
               I-flight_time      1.000     1.000     1.000       1.0
             B-aircraft_code      0.962     0.758     0.847      33.0
                       B-mod      0.000     0.000     0.000       2.0
              B-airport_code      0.500     0.111     0.182       9.0
    B-depart_time.start_time      1.000     0.333     0.500       3.0
      B-depart_time.end_time      1.000     0.333     0.500       3.0
          B-depart_date.year      1.000     1.000     1.000       3.0
            I-transport_type      0.000     0.000     0.000       1.0
          B-restriction_code      0.333     1.000     0.500       4.0
    B-arrive_time.start_time      0.889     1.000     0.941       8.0
        B-toloc.airport_code      1.000     0.750     0.857       4.0
      B-arrive_time.end_time      0.889     1.000     0.941       8.0
      I-arrive_time.end_time      0.889     1.000     0.941       8.0
      I-depart_time.end_time      1.000     0.333     0.500       3.0
               I-flight_stop      0.000     0.000     0.000       0.0
      B-fromloc.airport_code      1.000     1.000     1.000       5.0
          I-restriction_code      1.000     1.000     1.000       3.0
    I-depart_time.start_time      1.000     1.000     1.000       1.0
          I-toloc.state_name      1.000     1.000     1.000       1.0
I-depart_date.today_relative      0.000     0.000     0.000       0.0
 B-arrive_date.date_relative      1.000     1.000     1.000       2.0
                I-flight_mod      0.500     0.167     0.250       6.0
                   I-economy      0.000     0.000     0.000       0.0
 B-return_date.date_relative      1.000     0.333     0.500       3.0
        I-fromloc.state_name      1.000     1.000     1.000       1.0
                B-state_code      1.000     1.000     1.000       1.0
    I-arrive_time.start_time      1.000     1.000     1.000       1.0
    I-arrive_date.day_number      0.000     0.000     0.000       0.0
                 B-meal_code      0.000     0.000     0.000       1.0
 I-depart_time.period_of_day      0.000     0.000     0.000       1.0
                  B-day_name      1.000     0.500     0.667       2.0
             B-period_of_day      1.000     0.750     0.857       4.0
        B-stoploc.state_code      0.000     0.000     0.000       0.0
    B-return_date.month_name      0.000     0.000     0.000       0.0
    B-return_date.day_number      0.000     0.000     0.000       0.0
    B-arrive_time.period_mod      0.000     0.000     0.000       0.0
                 I-meal_code      0.000     0.000     0.000       0.0
        B-toloc.country_name      1.000     1.000     1.000       1.0
                 B-days_code      1.000     1.000     1.000       1.0
 I-arrive_time.period_of_day      0.000     0.000     0.000       0.0
            I-today_relative      0.000     0.000     0.000       0.0
 B-return_time.period_of_day      0.000     0.000     0.000       0.0
                      B-time      0.000     0.000     0.000       0.0
           I-fare_basis_code      0.000     0.000     0.000       0.0
 I-arrive_time.time_relative      0.000     0.000     0.000       4.0
 I-depart_time.time_relative      0.000     0.000     0.000       1.0
            B-today_relative      0.000     0.000     0.000       0.0
                B-state_name      0.000     0.000     0.000       9.0
B-arrive_date.today_relative      0.000     0.000     0.000       0.0
    B-return_time.period_mod      0.000     0.000     0.000       0.0
                B-month_name      0.000     0.000     0.000       0.0
                B-day_number      0.000     0.000     0.000       0.0
 I-return_date.date_relative      1.000     0.333     0.500       3.0
I-return_date.today_relative      0.000     0.000     0.000       0.0
      B-stoploc.airport_name      0.000     0.000     0.000       0.0
             B-time_relative      0.000     0.000     0.000       0.0
                      I-time      0.000     0.000     0.000       0.0
    I-return_date.day_number      0.000     0.000     0.000       0.0
          I-meal_description      0.000     0.000     0.000       0.0
B-return_date.today_relative      0.000     0.000     0.000       0.0
      B-return_date.day_name      0.000     0.000     0.000       2.0

                   micro avg      0.948     0.949     0.949    3657.0
                   macro avg      0.656     0.627     0.625    3657.0
                weighted avg      0.956     0.949     0.947    3657.0

I0716 00:54:24.340912 140076727973760 interactiveshell.py:2882] Best Slot F1: 0.949
I0716 00:54:24.342951 140076727973760 estimator_training.py:186] Not using Distribute Coordinator.
I0716 00:54:24.345455 140076727973760 training.py:612] Running training and evaluation locally (non-distributed).
I0716 00:54:24.349974 140076727973760 training.py:700] Start train and evaluate loop. The evaluate will happen after every checkpoint. Checkpoint frequency is determined based on RunConfig arguments: save_checkpoints_steps 312 or save_checkpoints_secs None.
I0716 00:54:24.421319 140076727973760 estimator.py:1145] Calling model_fn.
I0716 00:54:25.589649 140076727973760 <ipython-input-6-0a85a3fd693f>:50] 
[<tf.Variable 'embedding:0' shape=(750, 300) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/gates/kernel:0' shape=(600, 600) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/gates/bias:0' shape=(600,) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/candidate/kernel:0' shape=(600, 300) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/candidate/bias:0' shape=(300,) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/gates/kernel:0' shape=(600, 600) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/gates/bias:0' shape=(600,) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/candidate/kernel:0' shape=(600, 300) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/candidate/bias:0' shape=(300,) dtype=float32_ref>,
 <tf.Variable 'dense/kernel:0' shape=(600, 23) dtype=float32_ref>,
 <tf.Variable 'dense/bias:0' shape=(23,) dtype=float32_ref>,
 <tf.Variable 'dense_1/kernel:0' shape=(600, 122) dtype=float32_ref>,
 <tf.Variable 'dense_1/bias:0' shape=(122,) dtype=float32_ref>,
 <tf.Variable 'transitions:0' shape=(122, 122) dtype=float32_ref>]
I0716 00:54:27.161427 140076727973760 estimator.py:1147] Done calling model_fn.
I0716 00:54:27.165804 140076727973760 basic_session_run_hooks.py:541] Create CheckpointSaverHook.
I0716 00:54:27.364794 140076727973760 monitored_session.py:240] Graph was finalized.
I0716 00:54:27.386221 140076727973760 saver.py:1280] Restoring parameters from ../model/bigru_crf/model.ckpt-2496
I0716 00:54:27.661347 140076727973760 session_manager.py:500] Running local_init_op.
I0716 00:54:27.706003 140076727973760 session_manager.py:502] Done running local_init_op.
I0716 00:54:29.247517 140076727973760 basic_session_run_hooks.py:606] Saving checkpoints for 2496 into ../model/bigru_crf/model.ckpt.
Reading ../data/atis.train.w-intent.iob
I0716 00:54:31.615714 140076727973760 basic_session_run_hooks.py:262] loss = 0.18788771, step = 2496
I0716 00:54:31.617440 140076727973760 basic_session_run_hooks.py:262] lr = 0.0002306272
I0716 00:54:43.258700 140076727973760 basic_session_run_hooks.py:692] global_step/sec: 8.58851
I0716 00:54:43.265310 140076727973760 basic_session_run_hooks.py:260] loss = 0.10380483, step = 2596 (11.650 sec)
I0716 00:54:43.269205 140076727973760 basic_session_run_hooks.py:260] lr = 0.00022821005 (11.652 sec)
I0716 00:54:54.409753 140076727973760 basic_session_run_hooks.py:692] global_step/sec: 8.96772
I0716 00:54:54.416134 140076727973760 basic_session_run_hooks.py:260] loss = 0.31822184, step = 2696 (11.151 sec)
I0716 00:54:54.418413 140076727973760 basic_session_run_hooks.py:260] lr = 0.00022581825 (11.149 sec)
I0716 00:55:05.198757 140076727973760 basic_session_run_hooks.py:692] global_step/sec: 9.26875
I0716 00:55:05.205396 140076727973760 basic_session_run_hooks.py:260] loss = 0.07498835, step = 2796 (10.789 sec)
I0716 00:55:05.208431 140076727973760 basic_session_run_hooks.py:260] lr = 0.00022345151 (10.790 sec)
I0716 00:55:06.296332 140076727973760 basic_session_run_hooks.py:606] Saving checkpoints for 2808 into ../model/bigru_crf/model.ckpt.
I0716 00:55:06.691539 140076727973760 estimator.py:1145] Calling model_fn.
I0716 00:55:07.863752 140076727973760 estimator.py:1147] Done calling model_fn.
I0716 00:55:07.892094 140076727973760 evaluation.py:255] Starting evaluation at 2019-07-16T00:55:07Z
I0716 00:55:08.009044 140076727973760 monitored_session.py:240] Graph was finalized.
I0716 00:55:08.024304 140076727973760 saver.py:1280] Restoring parameters from ../model/bigru_crf/model.ckpt-2808
I0716 00:55:08.142532 140076727973760 session_manager.py:500] Running local_init_op.
I0716 00:55:08.170197 140076727973760 session_manager.py:502] Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
I0716 00:55:10.470638 140076727973760 evaluation.py:275] Finished evaluation at 2019-07-16-00:55:10
I0716 00:55:10.472292 140076727973760 estimator.py:2039] Saving dict for global step 2808: global_step = 2808, loss = 3.445656
I0716 00:55:10.479767 140076727973760 estimator.py:2099] Saving 'checkpoint_path' summary for global step 2808: ../model/bigru_crf/model.ckpt-2808
I0716 00:55:10.543423 140076727973760 estimator.py:368] Loss for final step: 0.26786232.
Reading ../data/atis.test.w-intent.iob
I0716 00:55:10.875840 140076727973760 estimator.py:1145] Calling model_fn.
I0716 00:55:11.514529 140076727973760 estimator.py:1147] Done calling model_fn.
I0716 00:55:11.630997 140076727973760 monitored_session.py:240] Graph was finalized.
I0716 00:55:11.651287 140076727973760 saver.py:1280] Restoring parameters from ../model/bigru_crf/model.ckpt-2808
I0716 00:55:11.749472 140076727973760 session_manager.py:500] Running local_init_op.
I0716 00:55:11.767871 140076727973760 session_manager.py:502] Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.
  'precision', 'predicted', average, warn_for)
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1439: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples.
  'recall', 'true', average, warn_for)
I0716 00:55:14.877284 140076727973760 interactiveshell.py:2882] 
                                          precision    recall  f1-score   support

                             atis_flight      0.977     0.992     0.984       632
                            atis_airfare      0.960     1.000     0.980        48
                     atis_ground_service      1.000     1.000     1.000        36
                            atis_airline      1.000     1.000     1.000        38
                       atis_abbreviation      0.943     1.000     0.971        33
                           atis_aircraft      1.000     1.000     1.000         9
                        atis_flight_time      1.000     1.000     1.000         1
                           atis_quantity      0.375     1.000     0.545         3
                atis_flight#atis_airfare      0.750     0.250     0.375        12
                            atis_airport      1.000     0.944     0.971        18
                           atis_distance      1.000     1.000     1.000        10
                               atis_city      1.000     0.500     0.667         6
                        atis_ground_fare      1.000     0.857     0.923         7
                           atis_capacity      1.000     0.952     0.976        21
                          atis_flight_no      0.889     1.000     0.941         8
                               atis_meal      1.000     0.833     0.909         6
                        atis_restriction      0.000     0.000     0.000         0
             atis_airline#atis_flight_no      0.000     0.000     0.000         0
    atis_ground_service#atis_ground_fare      0.000     0.000     0.000         0
           atis_airfare#atis_flight_time      0.000     0.000     0.000         0
                           atis_cheapest      0.000     0.000     0.000         0
atis_aircraft#atis_flight#atis_flight_no      0.000     0.000     0.000         0

                               micro avg      0.971     0.976     0.974       888
                               macro avg      0.677     0.651     0.647       888
                            weighted avg      0.973     0.976     0.971       888

I0716 00:55:14.928322 140076727973760 interactiveshell.py:2882] 
                              precision    recall  f1-score   support

                           O      0.000     0.000     0.000       0.0
           B-toloc.city_name      0.975     0.992     0.983     716.0
         B-fromloc.city_name      0.983     0.999     0.991     704.0
           I-toloc.city_name      0.981     0.977     0.979     265.0
      B-depart_date.day_name      0.986     0.991     0.988     212.0
              B-airline_name      0.981     1.000     0.990     101.0
         I-fromloc.city_name      0.957     0.994     0.975     177.0
 B-depart_time.period_of_day      0.992     0.923     0.956     130.0
              I-airline_name      1.000     1.000     1.000      65.0
    B-depart_date.day_number      0.981     0.964     0.972      55.0
    B-depart_date.month_name      0.982     0.964     0.973      56.0
          B-depart_time.time      0.838     1.000     0.912      57.0
                B-round_trip      1.000     0.973     0.986      73.0
             B-cost_relative      1.000     0.973     0.986      37.0
                I-round_trip      1.000     1.000     1.000      71.0
                B-flight_mod      1.000     0.958     0.979      24.0
 B-depart_time.time_relative      0.954     0.954     0.954      65.0
          I-depart_time.time      0.945     1.000     0.972      52.0
         B-stoploc.city_name      1.000     1.000     1.000      20.0
                 B-city_name      0.914     0.561     0.696      57.0
                B-class_type      0.960     1.000     0.980      24.0
          B-arrive_time.time      1.000     0.971     0.985      34.0
 B-arrive_time.time_relative      0.935     0.935     0.935      31.0
                I-class_type      1.000     1.000     1.000      17.0
               B-flight_stop      1.000     1.000     1.000      21.0
          I-arrive_time.time      1.000     0.971     0.986      35.0
              B-airline_code      1.000     0.882     0.938      34.0
    I-depart_date.day_number      1.000     1.000     1.000      15.0
      I-fromloc.airport_name      0.429     1.000     0.600      15.0
      B-fromloc.airport_name      0.440     0.917     0.595      12.0
      B-arrive_date.day_name      0.846     1.000     0.917      11.0
          B-toloc.state_code      0.947     1.000     0.973      18.0
B-depart_date.today_relative      1.000     0.889     0.941       9.0
             B-flight_number      0.550     1.000     0.710      11.0
 B-depart_date.date_relative      0.895     1.000     0.944      17.0
          B-toloc.state_name      0.862     0.893     0.877      28.0
           B-fare_basis_code      0.944     1.000     0.971      17.0
               B-flight_time      1.000     1.000     1.000       1.0
                        B-or      1.000     1.000     1.000       3.0
 B-arrive_time.period_of_day      0.857     1.000     0.923       6.0
          B-meal_description      1.000     0.900     0.947      10.0
             I-cost_relative      1.000     0.667     0.800       3.0
              I-airport_name      0.929     0.448     0.605      29.0
               B-fare_amount      1.000     1.000     1.000       2.0
               I-fare_amount      1.000     1.000     1.000       2.0
                 I-city_name      1.000     0.600     0.750      30.0
        I-toloc.airport_name      1.000     1.000     1.000       3.0
            B-transport_type      1.000     1.000     1.000      10.0
    B-arrive_date.month_name      0.714     0.833     0.769       6.0
    B-arrive_date.day_number      0.714     0.833     0.769       6.0
         I-stoploc.city_name      0.833     1.000     0.909      10.0
                      B-meal      1.000     1.000     1.000      16.0
        B-fromloc.state_code      1.000     0.957     0.978      23.0
    B-depart_time.period_mod      0.833     1.000     0.909       5.0
                   B-connect      1.000     1.000     1.000       6.0
               B-flight_days      1.000     1.000     1.000      10.0
        B-toloc.airport_name      1.000     1.000     1.000       3.0
        B-fromloc.state_name      0.895     1.000     0.944      17.0
              B-airport_name      0.833     0.476     0.606      21.0
                   B-economy      1.000     1.000     1.000       6.0
               I-flight_time      1.000     1.000     1.000       1.0
             B-aircraft_code      1.000     0.818     0.900      33.0
                       B-mod      0.000     0.000     0.000       2.0
              B-airport_code      0.500     0.111     0.182       9.0
    B-depart_time.start_time      1.000     1.000     1.000       3.0
      B-depart_time.end_time      1.000     0.667     0.800       3.0
          B-depart_date.year      1.000     1.000     1.000       3.0
            I-transport_type      0.000     0.000     0.000       1.0
          B-restriction_code      0.400     1.000     0.571       4.0
    B-arrive_time.start_time      1.000     1.000     1.000       8.0
        B-toloc.airport_code      1.000     0.750     0.857       4.0
      B-arrive_time.end_time      1.000     1.000     1.000       8.0
      I-arrive_time.end_time      1.000     1.000     1.000       8.0
      I-depart_time.end_time      1.000     0.667     0.800       3.0
               I-flight_stop      0.000     0.000     0.000       0.0
      B-fromloc.airport_code      1.000     1.000     1.000       5.0
          I-restriction_code      1.000     1.000     1.000       3.0
    I-depart_time.start_time      1.000     1.000     1.000       1.0
          I-toloc.state_name      1.000     1.000     1.000       1.0
I-depart_date.today_relative      0.000     0.000     0.000       0.0
 B-arrive_date.date_relative      1.000     1.000     1.000       2.0
                I-flight_mod      1.000     0.167     0.286       6.0
                   I-economy      0.000     0.000     0.000       0.0
 B-return_date.date_relative      1.000     0.333     0.500       3.0
        I-fromloc.state_name      1.000     1.000     1.000       1.0
                B-state_code      1.000     1.000     1.000       1.0
    I-arrive_time.start_time      1.000     1.000     1.000       1.0
    I-arrive_date.day_number      0.000     0.000     0.000       0.0
                 B-meal_code      0.000     0.000     0.000       1.0
 I-depart_time.period_of_day      0.000     0.000     0.000       1.0
                  B-day_name      1.000     0.500     0.667       2.0
             B-period_of_day      1.000     0.750     0.857       4.0
        B-stoploc.state_code      0.000     0.000     0.000       0.0
    B-return_date.month_name      0.000     0.000     0.000       0.0
    B-return_date.day_number      0.000     0.000     0.000       0.0
    B-arrive_time.period_mod      0.000     0.000     0.000       0.0
                 I-meal_code      0.000     0.000     0.000       0.0
        B-toloc.country_name      1.000     1.000     1.000       1.0
                 B-days_code      1.000     1.000     1.000       1.0
 I-arrive_time.period_of_day      0.000     0.000     0.000       0.0
            I-today_relative      0.000     0.000     0.000       0.0
 B-return_time.period_of_day      0.000     0.000     0.000       0.0
                      B-time      0.000     0.000     0.000       0.0
           I-fare_basis_code      0.000     0.000     0.000       0.0
 I-arrive_time.time_relative      0.000     0.000     0.000       4.0
 I-depart_time.time_relative      0.000     0.000     0.000       1.0
            B-today_relative      0.000     0.000     0.000       0.0
                B-state_name      0.000     0.000     0.000       9.0
B-arrive_date.today_relative      0.000     0.000     0.000       0.0
    B-return_time.period_mod      0.000     0.000     0.000       0.0
                B-month_name      0.000     0.000     0.000       0.0
                B-day_number      0.000     0.000     0.000       0.0
 I-return_date.date_relative      1.000     0.667     0.800       3.0
I-return_date.today_relative      0.000     0.000     0.000       0.0
      B-stoploc.airport_name      0.000     0.000     0.000       0.0
             B-time_relative      0.000     0.000     0.000       0.0
                      I-time      0.000     0.000     0.000       0.0
    I-return_date.day_number      0.000     0.000     0.000       0.0
          I-meal_description      0.000     0.000     0.000       0.0
B-return_date.today_relative      0.000     0.000     0.000       0.0
      B-return_date.day_name      0.000     0.000     0.000       2.0

                   micro avg      0.953     0.954     0.954    3657.0
                   macro avg      0.668     0.643     0.643    3657.0
                weighted avg      0.960     0.954     0.953    3657.0

I0716 00:55:14.955730 140076727973760 interactiveshell.py:2882] Best Slot F1: 0.954
I0716 00:55:14.957202 140076727973760 estimator_training.py:186] Not using Distribute Coordinator.
I0716 00:55:14.962939 140076727973760 training.py:612] Running training and evaluation locally (non-distributed).
I0716 00:55:14.964698 140076727973760 training.py:700] Start train and evaluate loop. The evaluate will happen after every checkpoint. Checkpoint frequency is determined based on RunConfig arguments: save_checkpoints_steps 312 or save_checkpoints_secs None.
I0716 00:55:15.026914 140076727973760 estimator.py:1145] Calling model_fn.
I0716 00:55:16.314570 140076727973760 <ipython-input-6-0a85a3fd693f>:50] 
[<tf.Variable 'embedding:0' shape=(750, 300) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/gates/kernel:0' shape=(600, 600) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/gates/bias:0' shape=(600,) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/candidate/kernel:0' shape=(600, 300) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/candidate/bias:0' shape=(300,) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/gates/kernel:0' shape=(600, 600) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/gates/bias:0' shape=(600,) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/candidate/kernel:0' shape=(600, 300) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/candidate/bias:0' shape=(300,) dtype=float32_ref>,
 <tf.Variable 'dense/kernel:0' shape=(600, 23) dtype=float32_ref>,
 <tf.Variable 'dense/bias:0' shape=(23,) dtype=float32_ref>,
 <tf.Variable 'dense_1/kernel:0' shape=(600, 122) dtype=float32_ref>,
 <tf.Variable 'dense_1/bias:0' shape=(122,) dtype=float32_ref>,
 <tf.Variable 'transitions:0' shape=(122, 122) dtype=float32_ref>]
I0716 00:55:17.652620 140076727973760 estimator.py:1147] Done calling model_fn.
I0716 00:55:17.656292 140076727973760 basic_session_run_hooks.py:541] Create CheckpointSaverHook.
I0716 00:55:18.034195 140076727973760 monitored_session.py:240] Graph was finalized.
I0716 00:55:18.052452 140076727973760 saver.py:1280] Restoring parameters from ../model/bigru_crf/model.ckpt-2808
I0716 00:55:18.314191 140076727973760 session_manager.py:500] Running local_init_op.
I0716 00:55:18.353224 140076727973760 session_manager.py:502] Done running local_init_op.
I0716 00:55:19.874680 140076727973760 basic_session_run_hooks.py:606] Saving checkpoints for 2808 into ../model/bigru_crf/model.ckpt.
Reading ../data/atis.train.w-intent.iob
I0716 00:55:22.212169 140076727973760 basic_session_run_hooks.py:262] loss = 0.7743803, step = 2808
I0716 00:55:22.214075 140076727973760 basic_session_run_hooks.py:262] lr = 0.00022316918
I0716 00:55:34.037479 140076727973760 basic_session_run_hooks.py:692] global_step/sec: 8.45608
I0716 00:55:34.040411 140076727973760 basic_session_run_hooks.py:260] loss = 0.2911498, step = 2908 (11.828 sec)
I0716 00:55:34.043696 140076727973760 basic_session_run_hooks.py:260] lr = 0.00022083019 (11.830 sec)
I0716 00:55:44.647716 140076727973760 basic_session_run_hooks.py:692] global_step/sec: 9.4248
I0716 00:55:44.655600 140076727973760 basic_session_run_hooks.py:260] loss = 0.32517436, step = 3008 (10.615 sec)
I0716 00:55:44.657164 140076727973760 basic_session_run_hooks.py:260] lr = 0.00021851574 (10.614 sec)
I0716 00:55:55.491967 140076727973760 basic_session_run_hooks.py:692] global_step/sec: 9.2215
I0716 00:55:55.498619 140076727973760 basic_session_run_hooks.py:260] loss = 0.08429532, step = 3108 (10.843 sec)
I0716 00:55:55.502708 140076727973760 basic_session_run_hooks.py:260] lr = 0.00021622553 (10.846 sec)
I0716 00:55:56.634205 140076727973760 basic_session_run_hooks.py:606] Saving checkpoints for 3120 into ../model/bigru_crf/model.ckpt.
I0716 00:55:57.033125 140076727973760 estimator.py:1145] Calling model_fn.
I0716 00:55:58.000150 140076727973760 estimator.py:1147] Done calling model_fn.
I0716 00:55:58.029387 140076727973760 evaluation.py:255] Starting evaluation at 2019-07-16T00:55:58Z
I0716 00:55:58.353736 140076727973760 monitored_session.py:240] Graph was finalized.
I0716 00:55:58.371020 140076727973760 saver.py:1280] Restoring parameters from ../model/bigru_crf/model.ckpt-3120
I0716 00:55:58.487563 140076727973760 session_manager.py:500] Running local_init_op.
I0716 00:55:58.515425 140076727973760 session_manager.py:502] Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
I0716 00:56:00.796310 140076727973760 evaluation.py:275] Finished evaluation at 2019-07-16-00:56:00
I0716 00:56:00.797751 140076727973760 estimator.py:2039] Saving dict for global step 3120: global_step = 3120, loss = 3.473759
I0716 00:56:00.805703 140076727973760 estimator.py:2099] Saving 'checkpoint_path' summary for global step 3120: ../model/bigru_crf/model.ckpt-3120
I0716 00:56:00.868404 140076727973760 estimator.py:368] Loss for final step: 0.09935654.
Reading ../data/atis.test.w-intent.iob
I0716 00:56:01.201792 140076727973760 estimator.py:1145] Calling model_fn.
I0716 00:56:01.851432 140076727973760 estimator.py:1147] Done calling model_fn.
I0716 00:56:01.964150 140076727973760 monitored_session.py:240] Graph was finalized.
I0716 00:56:01.979500 140076727973760 saver.py:1280] Restoring parameters from ../model/bigru_crf/model.ckpt-3120
I0716 00:56:02.133183 140076727973760 session_manager.py:500] Running local_init_op.
I0716 00:56:02.152128 140076727973760 session_manager.py:502] Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.
  'precision', 'predicted', average, warn_for)
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1439: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples.
  'recall', 'true', average, warn_for)
I0716 00:56:05.230934 140076727973760 interactiveshell.py:2882] 
                                          precision    recall  f1-score   support

                             atis_flight      0.977     0.992     0.984       632
                            atis_airfare      0.980     1.000     0.990        48
                     atis_ground_service      1.000     1.000     1.000        36
                            atis_airline      1.000     1.000     1.000        38
                       atis_abbreviation      0.943     1.000     0.971        33
                           atis_aircraft      1.000     1.000     1.000         9
                        atis_flight_time      1.000     1.000     1.000         1
                           atis_quantity      0.375     1.000     0.545         3
                atis_flight#atis_airfare      0.750     0.250     0.375        12
                            atis_airport      1.000     0.944     0.971        18
                           atis_distance      1.000     1.000     1.000        10
                               atis_city      1.000     0.500     0.667         6
                        atis_ground_fare      1.000     1.000     1.000         7
                           atis_capacity      1.000     0.952     0.976        21
                          atis_flight_no      0.889     1.000     0.941         8
                               atis_meal      1.000     0.833     0.909         6
                        atis_restriction      0.000     0.000     0.000         0
             atis_airline#atis_flight_no      0.000     0.000     0.000         0
    atis_ground_service#atis_ground_fare      0.000     0.000     0.000         0
           atis_airfare#atis_flight_time      0.000     0.000     0.000         0
                           atis_cheapest      0.000     0.000     0.000         0
atis_aircraft#atis_flight#atis_flight_no      0.000     0.000     0.000         0

                               micro avg      0.972     0.977     0.975       888
                               macro avg      0.678     0.658     0.651       888
                            weighted avg      0.974     0.977     0.973       888

I0716 00:56:05.278249 140076727973760 interactiveshell.py:2882] 
                              precision    recall  f1-score   support

                           O      0.000     0.000     0.000       0.0
           B-toloc.city_name      0.975     0.992     0.983     716.0
         B-fromloc.city_name      0.983     0.997     0.990     704.0
           I-toloc.city_name      0.977     0.981     0.979     265.0
      B-depart_date.day_name      0.991     0.991     0.991     212.0
              B-airline_name      1.000     1.000     1.000     101.0
         I-fromloc.city_name      0.972     0.994     0.983     177.0
 B-depart_time.period_of_day      0.992     0.915     0.952     130.0
              I-airline_name      1.000     1.000     1.000      65.0
    B-depart_date.day_number      0.981     0.964     0.972      55.0
    B-depart_date.month_name      0.982     0.964     0.973      56.0
          B-depart_time.time      0.836     0.982     0.903      57.0
                B-round_trip      1.000     0.986     0.993      73.0
             B-cost_relative      1.000     0.973     0.986      37.0
                I-round_trip      1.000     1.000     1.000      71.0
                B-flight_mod      1.000     0.958     0.979      24.0
 B-depart_time.time_relative      0.969     0.969     0.969      65.0
          I-depart_time.time      0.944     0.981     0.962      52.0
         B-stoploc.city_name      0.952     1.000     0.976      20.0
                 B-city_name      0.912     0.544     0.681      57.0
                B-class_type      0.960     1.000     0.980      24.0
          B-arrive_time.time      1.000     0.971     0.985      34.0
 B-arrive_time.time_relative      0.935     0.935     0.935      31.0
                I-class_type      1.000     1.000     1.000      17.0
               B-flight_stop      1.000     1.000     1.000      21.0
          I-arrive_time.time      1.000     0.971     0.986      35.0
              B-airline_code      1.000     0.941     0.970      34.0
    I-depart_date.day_number      1.000     1.000     1.000      15.0
      I-fromloc.airport_name      0.441     1.000     0.612      15.0
      B-fromloc.airport_name      0.462     1.000     0.632      12.0
      B-arrive_date.day_name      0.786     1.000     0.880      11.0
          B-toloc.state_code      0.947     1.000     0.973      18.0
B-depart_date.today_relative      1.000     0.889     0.941       9.0
             B-flight_number      0.579     1.000     0.733      11.0
 B-depart_date.date_relative      0.895     1.000     0.944      17.0
          B-toloc.state_name      0.781     0.893     0.833      28.0
           B-fare_basis_code      0.944     1.000     0.971      17.0
               B-flight_time      1.000     1.000     1.000       1.0
                        B-or      1.000     1.000     1.000       3.0
 B-arrive_time.period_of_day      0.857     1.000     0.923       6.0
          B-meal_description      1.000     0.900     0.947      10.0
             I-cost_relative      1.000     0.667     0.800       3.0
              I-airport_name      0.929     0.448     0.605      29.0
               B-fare_amount      1.000     1.000     1.000       2.0
               I-fare_amount      1.000     1.000     1.000       2.0
                 I-city_name      1.000     0.600     0.750      30.0
        I-toloc.airport_name      1.000     1.000     1.000       3.0
            B-transport_type      1.000     1.000     1.000      10.0
    B-arrive_date.month_name      0.714     0.833     0.769       6.0
    B-arrive_date.day_number      0.714     0.833     0.769       6.0
         I-stoploc.city_name      0.833     1.000     0.909      10.0
                      B-meal      1.000     1.000     1.000      16.0
        B-fromloc.state_code      1.000     0.957     0.978      23.0
    B-depart_time.period_mod      0.833     1.000     0.909       5.0
                   B-connect      1.000     1.000     1.000       6.0
               B-flight_days      1.000     1.000     1.000      10.0
        B-toloc.airport_name      1.000     1.000     1.000       3.0
        B-fromloc.state_name      0.944     1.000     0.971      17.0
              B-airport_name      0.833     0.476     0.606      21.0
                   B-economy      1.000     1.000     1.000       6.0
               I-flight_time      1.000     1.000     1.000       1.0
             B-aircraft_code      1.000     0.818     0.900      33.0
                       B-mod      1.000     0.500     0.667       2.0
              B-airport_code      0.500     0.111     0.182       9.0
    B-depart_time.start_time      1.000     1.000     1.000       3.0
      B-depart_time.end_time      1.000     0.667     0.800       3.0
          B-depart_date.year      1.000     1.000     1.000       3.0
            I-transport_type      0.000     0.000     0.000       1.0
          B-restriction_code      0.400     1.000     0.571       4.0
    B-arrive_time.start_time      1.000     1.000     1.000       8.0
        B-toloc.airport_code      1.000     0.750     0.857       4.0
      B-arrive_time.end_time      1.000     1.000     1.000       8.0
      I-arrive_time.end_time      1.000     1.000     1.000       8.0
      I-depart_time.end_time      1.000     0.667     0.800       3.0
               I-flight_stop      0.000     0.000     0.000       0.0
      B-fromloc.airport_code      1.000     1.000     1.000       5.0
          I-restriction_code      1.000     1.000     1.000       3.0
    I-depart_time.start_time      1.000     1.000     1.000       1.0
          I-toloc.state_name      1.000     1.000     1.000       1.0
I-depart_date.today_relative      0.000     0.000     0.000       0.0
 B-arrive_date.date_relative      1.000     1.000     1.000       2.0
                I-flight_mod      1.000     0.167     0.286       6.0
                   I-economy      0.000     0.000     0.000       0.0
 B-return_date.date_relative      0.667     0.667     0.667       3.0
        I-fromloc.state_name      1.000     1.000     1.000       1.0
                B-state_code      1.000     1.000     1.000       1.0
    I-arrive_time.start_time      1.000     1.000     1.000       1.0
    I-arrive_date.day_number      0.000     0.000     0.000       0.0
                 B-meal_code      0.000     0.000     0.000       1.0
 I-depart_time.period_of_day      1.000     1.000     1.000       1.0
                  B-day_name      1.000     0.500     0.667       2.0
             B-period_of_day      1.000     0.500     0.667       4.0
        B-stoploc.state_code      0.000     0.000     0.000       0.0
    B-return_date.month_name      0.000     0.000     0.000       0.0
    B-return_date.day_number      0.000     0.000     0.000       0.0
    B-arrive_time.period_mod      0.000     0.000     0.000       0.0
                 I-meal_code      0.000     0.000     0.000       0.0
        B-toloc.country_name      1.000     1.000     1.000       1.0
                 B-days_code      1.000     1.000     1.000       1.0
 I-arrive_time.period_of_day      0.000     0.000     0.000       0.0
            I-today_relative      0.000     0.000     0.000       0.0
 B-return_time.period_of_day      0.000     0.000     0.000       0.0
                      B-time      0.000     0.000     0.000       0.0
           I-fare_basis_code      0.000     0.000     0.000       0.0
 I-arrive_time.time_relative      0.000     0.000     0.000       4.0
 I-depart_time.time_relative      0.000     0.000     0.000       1.0
            B-today_relative      0.000     0.000     0.000       0.0
                B-state_name      0.000     0.000     0.000       9.0
B-arrive_date.today_relative      0.000     0.000     0.000       0.0
    B-return_time.period_mod      0.000     0.000     0.000       0.0
                B-month_name      0.000     0.000     0.000       0.0
                B-day_number      0.000     0.000     0.000       0.0
 I-return_date.date_relative      1.000     0.333     0.500       3.0
I-return_date.today_relative      0.000     0.000     0.000       0.0
      B-stoploc.airport_name      0.000     0.000     0.000       0.0
             B-time_relative      0.000     0.000     0.000       0.0
                      I-time      0.000     0.000     0.000       0.0
    I-return_date.day_number      0.000     0.000     0.000       0.0
          I-meal_description      0.000     0.000     0.000       0.0
B-return_date.today_relative      0.000     0.000     0.000       0.0
      B-return_date.day_name      0.000     0.000     0.000       2.0

                   micro avg      0.953     0.955     0.954    3657.0
                   macro avg      0.681     0.654     0.654    3657.0
                weighted avg      0.962     0.955     0.954    3657.0

I0716 00:56:05.306838 140076727973760 interactiveshell.py:2882] Best Slot F1: 0.954
I0716 00:56:05.308457 140076727973760 estimator_training.py:186] Not using Distribute Coordinator.
I0716 00:56:05.309879 140076727973760 training.py:612] Running training and evaluation locally (non-distributed).
I0716 00:56:05.313166 140076727973760 training.py:700] Start train and evaluate loop. The evaluate will happen after every checkpoint. Checkpoint frequency is determined based on RunConfig arguments: save_checkpoints_steps 312 or save_checkpoints_secs None.
I0716 00:56:05.377611 140076727973760 estimator.py:1145] Calling model_fn.
I0716 00:56:06.324105 140076727973760 <ipython-input-6-0a85a3fd693f>:50] 
[<tf.Variable 'embedding:0' shape=(750, 300) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/gates/kernel:0' shape=(600, 600) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/gates/bias:0' shape=(600,) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/candidate/kernel:0' shape=(600, 300) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/candidate/bias:0' shape=(300,) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/gates/kernel:0' shape=(600, 600) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/gates/bias:0' shape=(600,) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/candidate/kernel:0' shape=(600, 300) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/candidate/bias:0' shape=(300,) dtype=float32_ref>,
 <tf.Variable 'dense/kernel:0' shape=(600, 23) dtype=float32_ref>,
 <tf.Variable 'dense/bias:0' shape=(23,) dtype=float32_ref>,
 <tf.Variable 'dense_1/kernel:0' shape=(600, 122) dtype=float32_ref>,
 <tf.Variable 'dense_1/bias:0' shape=(122,) dtype=float32_ref>,
 <tf.Variable 'transitions:0' shape=(122, 122) dtype=float32_ref>]
I0716 00:56:07.998188 140076727973760 estimator.py:1147] Done calling model_fn.
I0716 00:56:08.002126 140076727973760 basic_session_run_hooks.py:541] Create CheckpointSaverHook.
I0716 00:56:08.209464 140076727973760 monitored_session.py:240] Graph was finalized.
I0716 00:56:08.232329 140076727973760 saver.py:1280] Restoring parameters from ../model/bigru_crf/model.ckpt-3120
I0716 00:56:08.488075 140076727973760 session_manager.py:500] Running local_init_op.
I0716 00:56:08.526196 140076727973760 session_manager.py:502] Done running local_init_op.
I0716 00:56:10.205197 140076727973760 basic_session_run_hooks.py:606] Saving checkpoints for 3120 into ../model/bigru_crf/model.ckpt.
Reading ../data/atis.train.w-intent.iob
I0716 00:56:12.599461 140076727973760 basic_session_run_hooks.py:262] loss = 0.55648494, step = 3120
I0716 00:56:12.601621 140076727973760 basic_session_run_hooks.py:262] lr = 0.00021595233
I0716 00:56:24.031906 140076727973760 basic_session_run_hooks.py:692] global_step/sec: 8.74655
I0716 00:56:24.039638 140076727973760 basic_session_run_hooks.py:260] loss = 0.56351316, step = 3220 (11.440 sec)
I0716 00:56:24.042310 140076727973760 basic_session_run_hooks.py:260] lr = 0.00021368897 (11.441 sec)
I0716 00:56:35.186319 140076727973760 basic_session_run_hooks.py:692] global_step/sec: 8.96507
I0716 00:56:35.189258 140076727973760 basic_session_run_hooks.py:260] loss = 0.5290354, step = 3320 (11.150 sec)
I0716 00:56:35.192775 140076727973760 basic_session_run_hooks.py:260] lr = 0.00021144934 (11.150 sec)
I0716 00:56:45.994630 140076727973760 basic_session_run_hooks.py:692] global_step/sec: 9.2521
I0716 00:56:46.001238 140076727973760 basic_session_run_hooks.py:260] loss = 0.23298821, step = 3420 (10.812 sec)
I0716 00:56:46.004105 140076727973760 basic_session_run_hooks.py:260] lr = 0.0002092332 (10.811 sec)
I0716 00:56:47.097135 140076727973760 basic_session_run_hooks.py:606] Saving checkpoints for 3432 into ../model/bigru_crf/model.ckpt.
I0716 00:56:47.470667 140076727973760 estimator.py:1145] Calling model_fn.
I0716 00:56:48.425848 140076727973760 estimator.py:1147] Done calling model_fn.
I0716 00:56:48.454968 140076727973760 evaluation.py:255] Starting evaluation at 2019-07-16T00:56:48Z
I0716 00:56:48.569559 140076727973760 monitored_session.py:240] Graph was finalized.
I0716 00:56:48.586678 140076727973760 saver.py:1280] Restoring parameters from ../model/bigru_crf/model.ckpt-3432
I0716 00:56:48.703516 140076727973760 session_manager.py:500] Running local_init_op.
I0716 00:56:48.731134 140076727973760 session_manager.py:502] Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
I0716 00:56:51.003590 140076727973760 evaluation.py:275] Finished evaluation at 2019-07-16-00:56:51
I0716 00:56:51.005765 140076727973760 estimator.py:2039] Saving dict for global step 3432: global_step = 3432, loss = 3.515367
I0716 00:56:51.014944 140076727973760 estimator.py:2099] Saving 'checkpoint_path' summary for global step 3432: ../model/bigru_crf/model.ckpt-3432
I0716 00:56:51.077053 140076727973760 estimator.py:368] Loss for final step: 0.6626626.
Reading ../data/atis.test.w-intent.iob
I0716 00:56:51.398154 140076727973760 estimator.py:1145] Calling model_fn.
I0716 00:56:52.337465 140076727973760 estimator.py:1147] Done calling model_fn.
I0716 00:56:52.452007 140076727973760 monitored_session.py:240] Graph was finalized.
I0716 00:56:52.470469 140076727973760 saver.py:1280] Restoring parameters from ../model/bigru_crf/model.ckpt-3432
I0716 00:56:52.571833 140076727973760 session_manager.py:500] Running local_init_op.
I0716 00:56:52.590081 140076727973760 session_manager.py:502] Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.
  'precision', 'predicted', average, warn_for)
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1439: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples.
  'recall', 'true', average, warn_for)
I0716 00:56:55.649140 140076727973760 interactiveshell.py:2882] 
                                          precision    recall  f1-score   support

                             atis_flight      0.978     0.992     0.985       632
                            atis_airfare      0.941     1.000     0.970        48
                     atis_ground_service      1.000     1.000     1.000        36
                            atis_airline      1.000     1.000     1.000        38
                       atis_abbreviation      0.943     1.000     0.971        33
                           atis_aircraft      1.000     1.000     1.000         9
                        atis_flight_time      1.000     1.000     1.000         1
                           atis_quantity      0.375     1.000     0.545         3
                atis_flight#atis_airfare      0.750     0.250     0.375        12
                            atis_airport      1.000     0.944     0.971        18
                           atis_distance      1.000     1.000     1.000        10
                               atis_city      1.000     0.500     0.667         6
                        atis_ground_fare      1.000     0.857     0.923         7
                           atis_capacity      1.000     0.952     0.976        21
                          atis_flight_no      0.889     1.000     0.941         8
                               atis_meal      1.000     0.833     0.909         6
                        atis_restriction      0.000     0.000     0.000         0
             atis_airline#atis_flight_no      0.000     0.000     0.000         0
    atis_ground_service#atis_ground_fare      0.000     0.000     0.000         0
           atis_airfare#atis_flight_time      0.000     0.000     0.000         0
                           atis_cheapest      0.000     0.000     0.000         0
atis_aircraft#atis_flight#atis_flight_no      0.000     0.000     0.000         0

                               micro avg      0.971     0.976     0.974       888
                               macro avg      0.676     0.651     0.647       888
                            weighted avg      0.973     0.976     0.972       888

I0716 00:56:55.697136 140076727973760 interactiveshell.py:2882] 
                              precision    recall  f1-score   support

                           O      0.000     0.000     0.000       0.0
           B-toloc.city_name      0.975     0.996     0.985     716.0
         B-fromloc.city_name      0.978     0.997     0.987     704.0
           I-toloc.city_name      0.981     0.992     0.987     265.0
      B-depart_date.day_name      0.981     0.991     0.986     212.0
              B-airline_name      1.000     1.000     1.000     101.0
         I-fromloc.city_name      0.983     0.994     0.989     177.0
 B-depart_time.period_of_day      0.992     0.915     0.952     130.0
              I-airline_name      1.000     1.000     1.000      65.0
    B-depart_date.day_number      0.981     0.964     0.972      55.0
    B-depart_date.month_name      0.982     0.964     0.973      56.0
          B-depart_time.time      0.826     1.000     0.905      57.0
                B-round_trip      1.000     0.973     0.986      73.0
             B-cost_relative      1.000     0.973     0.986      37.0
                I-round_trip      1.000     1.000     1.000      71.0
                B-flight_mod      1.000     0.958     0.979      24.0
 B-depart_time.time_relative      0.954     0.954     0.954      65.0
          I-depart_time.time      0.929     1.000     0.963      52.0
         B-stoploc.city_name      0.952     1.000     0.976      20.0
                 B-city_name      0.938     0.526     0.674      57.0
                B-class_type      0.960     1.000     0.980      24.0
          B-arrive_time.time      1.000     0.941     0.970      34.0
 B-arrive_time.time_relative      0.935     0.935     0.935      31.0
                I-class_type      1.000     1.000     1.000      17.0
               B-flight_stop      1.000     1.000     1.000      21.0
          I-arrive_time.time      1.000     0.943     0.971      35.0
              B-airline_code      1.000     0.941     0.970      34.0
    I-depart_date.day_number      1.000     1.000     1.000      15.0
      I-fromloc.airport_name      0.417     1.000     0.588      15.0
      B-fromloc.airport_name      0.462     1.000     0.632      12.0
      B-arrive_date.day_name      0.833     0.909     0.870      11.0
          B-toloc.state_code      0.947     1.000     0.973      18.0
B-depart_date.today_relative      1.000     0.889     0.941       9.0
             B-flight_number      0.579     1.000     0.733      11.0
 B-depart_date.date_relative      0.895     1.000     0.944      17.0
          B-toloc.state_name      0.806     0.893     0.847      28.0
           B-fare_basis_code      0.944     1.000     0.971      17.0
               B-flight_time      1.000     1.000     1.000       1.0
                        B-or      1.000     1.000     1.000       3.0
 B-arrive_time.period_of_day      0.857     1.000     0.923       6.0
          B-meal_description      1.000     0.900     0.947      10.0
             I-cost_relative      1.000     0.667     0.800       3.0
              I-airport_name      0.917     0.379     0.537      29.0
               B-fare_amount      1.000     1.000     1.000       2.0
               I-fare_amount      1.000     1.000     1.000       2.0
                 I-city_name      1.000     0.600     0.750      30.0
        I-toloc.airport_name      1.000     1.000     1.000       3.0
            B-transport_type      1.000     1.000     1.000      10.0
    B-arrive_date.month_name      0.714     0.833     0.769       6.0
    B-arrive_date.day_number      0.714     0.833     0.769       6.0
         I-stoploc.city_name      0.833     1.000     0.909      10.0
                      B-meal      1.000     1.000     1.000      16.0
        B-fromloc.state_code      1.000     0.957     0.978      23.0
    B-depart_time.period_mod      0.714     1.000     0.833       5.0
                   B-connect      1.000     1.000     1.000       6.0
               B-flight_days      1.000     1.000     1.000      10.0
        B-toloc.airport_name      1.000     1.000     1.000       3.0
        B-fromloc.state_name      0.895     1.000     0.944      17.0
              B-airport_name      0.800     0.381     0.516      21.0
                   B-economy      1.000     1.000     1.000       6.0
               I-flight_time      1.000     1.000     1.000       1.0
             B-aircraft_code      1.000     0.758     0.862      33.0
                       B-mod      1.000     0.500     0.667       2.0
              B-airport_code      0.500     0.111     0.182       9.0
    B-depart_time.start_time      1.000     1.000     1.000       3.0
      B-depart_time.end_time      1.000     0.667     0.800       3.0
          B-depart_date.year      1.000     1.000     1.000       3.0
            I-transport_type      0.000     0.000     0.000       1.0
          B-restriction_code      0.333     1.000     0.500       4.0
    B-arrive_time.start_time      1.000     1.000     1.000       8.0
        B-toloc.airport_code      1.000     0.750     0.857       4.0
      B-arrive_time.end_time      1.000     1.000     1.000       8.0
      I-arrive_time.end_time      1.000     1.000     1.000       8.0
      I-depart_time.end_time      1.000     0.667     0.800       3.0
               I-flight_stop      0.000     0.000     0.000       0.0
      B-fromloc.airport_code      1.000     1.000     1.000       5.0
          I-restriction_code      1.000     1.000     1.000       3.0
    I-depart_time.start_time      1.000     1.000     1.000       1.0
          I-toloc.state_name      1.000     1.000     1.000       1.0
I-depart_date.today_relative      0.000     0.000     0.000       0.0
 B-arrive_date.date_relative      1.000     1.000     1.000       2.0
                I-flight_mod      1.000     0.167     0.286       6.0
                   I-economy      0.000     0.000     0.000       0.0
 B-return_date.date_relative      1.000     0.667     0.800       3.0
        I-fromloc.state_name      1.000     1.000     1.000       1.0
                B-state_code      1.000     1.000     1.000       1.0
    I-arrive_time.start_time      1.000     1.000     1.000       1.0
    I-arrive_date.day_number      0.000     0.000     0.000       0.0
                 B-meal_code      0.000     0.000     0.000       1.0
 I-depart_time.period_of_day      1.000     1.000     1.000       1.0
                  B-day_name      1.000     0.500     0.667       2.0
             B-period_of_day      1.000     0.500     0.667       4.0
        B-stoploc.state_code      0.000     0.000     0.000       0.0
    B-return_date.month_name      0.000     0.000     0.000       0.0
    B-return_date.day_number      0.000     0.000     0.000       0.0
    B-arrive_time.period_mod      0.000     0.000     0.000       0.0
                 I-meal_code      0.000     0.000     0.000       0.0
        B-toloc.country_name      1.000     1.000     1.000       1.0
                 B-days_code      1.000     1.000     1.000       1.0
 I-arrive_time.period_of_day      0.000     0.000     0.000       0.0
            I-today_relative      0.000     0.000     0.000       0.0
 B-return_time.period_of_day      0.000     0.000     0.000       0.0
                      B-time      0.000     0.000     0.000       0.0
           I-fare_basis_code      0.000     0.000     0.000       0.0
 I-arrive_time.time_relative      0.000     0.000     0.000       4.0
 I-depart_time.time_relative      0.000     0.000     0.000       1.0
            B-today_relative      0.000     0.000     0.000       0.0
                B-state_name      0.000     0.000     0.000       9.0
B-arrive_date.today_relative      0.000     0.000     0.000       0.0
    B-return_time.period_mod      0.000     0.000     0.000       0.0
                B-month_name      0.000     0.000     0.000       0.0
                B-day_number      0.000     0.000     0.000       0.0
 I-return_date.date_relative      1.000     0.667     0.800       3.0
I-return_date.today_relative      0.000     0.000     0.000       0.0
      B-stoploc.airport_name      0.000     0.000     0.000       0.0
             B-time_relative      0.000     0.000     0.000       0.0
                      I-time      0.000     0.000     0.000       0.0
    I-return_date.day_number      0.000     0.000     0.000       0.0
          I-meal_description      0.000     0.000     0.000       0.0
B-return_date.today_relative      0.000     0.000     0.000       0.0
      B-return_date.day_name      0.000     0.000     0.000       2.0

                   micro avg      0.952     0.954     0.953    3657.0
                   macro avg      0.682     0.654     0.654    3657.0
                weighted avg      0.961     0.954     0.952    3657.0

I0716 00:56:55.724939 140076727973760 interactiveshell.py:2882] Best Slot F1: 0.954
I0716 00:56:55.726587 140076727973760 estimator_training.py:186] Not using Distribute Coordinator.
I0716 00:56:55.729643 140076727973760 training.py:612] Running training and evaluation locally (non-distributed).
I0716 00:56:55.733313 140076727973760 training.py:700] Start train and evaluate loop. The evaluate will happen after every checkpoint. Checkpoint frequency is determined based on RunConfig arguments: save_checkpoints_steps 312 or save_checkpoints_secs None.
I0716 00:56:55.796110 140076727973760 estimator.py:1145] Calling model_fn.
I0716 00:56:56.921801 140076727973760 <ipython-input-6-0a85a3fd693f>:50] 
[<tf.Variable 'embedding:0' shape=(750, 300) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/gates/kernel:0' shape=(600, 600) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/gates/bias:0' shape=(600,) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/candidate/kernel:0' shape=(600, 300) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/candidate/bias:0' shape=(300,) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/gates/kernel:0' shape=(600, 600) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/gates/bias:0' shape=(600,) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/candidate/kernel:0' shape=(600, 300) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/candidate/bias:0' shape=(300,) dtype=float32_ref>,
 <tf.Variable 'dense/kernel:0' shape=(600, 23) dtype=float32_ref>,
 <tf.Variable 'dense/bias:0' shape=(23,) dtype=float32_ref>,
 <tf.Variable 'dense_1/kernel:0' shape=(600, 122) dtype=float32_ref>,
 <tf.Variable 'dense_1/bias:0' shape=(122,) dtype=float32_ref>,
 <tf.Variable 'transitions:0' shape=(122, 122) dtype=float32_ref>]
I0716 00:56:58.483569 140076727973760 estimator.py:1147] Done calling model_fn.
I0716 00:56:58.487568 140076727973760 basic_session_run_hooks.py:541] Create CheckpointSaverHook.
I0716 00:56:58.685906 140076727973760 monitored_session.py:240] Graph was finalized.
I0716 00:56:58.703882 140076727973760 saver.py:1280] Restoring parameters from ../model/bigru_crf/model.ckpt-3432
I0716 00:56:58.959684 140076727973760 session_manager.py:500] Running local_init_op.
I0716 00:56:58.994523 140076727973760 session_manager.py:502] Done running local_init_op.
I0716 00:57:00.448671 140076727973760 basic_session_run_hooks.py:606] Saving checkpoints for 3432 into ../model/bigru_crf/model.ckpt.
Reading ../data/atis.train.w-intent.iob
I0716 00:57:02.769141 140076727973760 basic_session_run_hooks.py:262] loss = 0.23062772, step = 3432
I0716 00:57:02.771006 140076727973760 basic_session_run_hooks.py:262] lr = 0.00020896882
I0716 00:57:14.077147 140076727973760 basic_session_run_hooks.py:692] global_step/sec: 8.84288
I0716 00:57:14.085288 140076727973760 basic_session_run_hooks.py:260] loss = 0.16647926, step = 3532 (11.316 sec)
I0716 00:57:14.087406 140076727973760 basic_session_run_hooks.py:260] lr = 0.00020677868 (11.316 sec)
I0716 00:57:25.013280 140076727973760 basic_session_run_hooks.py:692] global_step/sec: 9.14403
I0716 00:57:25.018351 140076727973760 basic_session_run_hooks.py:260] loss = 0.050607484, step = 3632 (10.933 sec)
I0716 00:57:25.022794 140076727973760 basic_session_run_hooks.py:260] lr = 0.00020461148 (10.935 sec)
I0716 00:57:36.042223 140076727973760 basic_session_run_hooks.py:692] global_step/sec: 9.06703
I0716 00:57:36.051251 140076727973760 basic_session_run_hooks.py:260] loss = 0.023925621, step = 3732 (11.033 sec)
I0716 00:57:36.052862 140076727973760 basic_session_run_hooks.py:260] lr = 0.00020246702 (11.030 sec)
I0716 00:57:37.152967 140076727973760 basic_session_run_hooks.py:606] Saving checkpoints for 3744 into ../model/bigru_crf/model.ckpt.
I0716 00:57:37.526447 140076727973760 estimator.py:1145] Calling model_fn.
I0716 00:57:38.704951 140076727973760 estimator.py:1147] Done calling model_fn.
I0716 00:57:38.734515 140076727973760 evaluation.py:255] Starting evaluation at 2019-07-16T00:57:38Z
I0716 00:57:38.852867 140076727973760 monitored_session.py:240] Graph was finalized.
I0716 00:57:38.868181 140076727973760 saver.py:1280] Restoring parameters from ../model/bigru_crf/model.ckpt-3744
I0716 00:57:38.985413 140076727973760 session_manager.py:500] Running local_init_op.
I0716 00:57:39.012212 140076727973760 session_manager.py:502] Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
I0716 00:57:41.287212 140076727973760 evaluation.py:275] Finished evaluation at 2019-07-16-00:57:41
I0716 00:57:41.289004 140076727973760 estimator.py:2039] Saving dict for global step 3744: global_step = 3744, loss = 3.6199257
I0716 00:57:41.292555 140076727973760 estimator.py:2099] Saving 'checkpoint_path' summary for global step 3744: ../model/bigru_crf/model.ckpt-3744
I0716 00:57:41.359728 140076727973760 estimator.py:368] Loss for final step: 0.05703994.
Reading ../data/atis.test.w-intent.iob
I0716 00:57:41.686021 140076727973760 estimator.py:1145] Calling model_fn.
I0716 00:57:42.331563 140076727973760 estimator.py:1147] Done calling model_fn.
I0716 00:57:42.754971 140076727973760 monitored_session.py:240] Graph was finalized.
I0716 00:57:42.769835 140076727973760 saver.py:1280] Restoring parameters from ../model/bigru_crf/model.ckpt-3744
I0716 00:57:42.877285 140076727973760 session_manager.py:500] Running local_init_op.
I0716 00:57:42.894069 140076727973760 session_manager.py:502] Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.
  'precision', 'predicted', average, warn_for)
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1439: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples.
  'recall', 'true', average, warn_for)
I0716 00:57:45.945155 140076727973760 interactiveshell.py:2882] 
                                          precision    recall  f1-score   support

                             atis_flight      0.975     0.992     0.984       632
                            atis_airfare      0.960     1.000     0.980        48
                     atis_ground_service      1.000     1.000     1.000        36
                            atis_airline      1.000     1.000     1.000        38
                       atis_abbreviation      0.943     1.000     0.971        33
                           atis_aircraft      1.000     1.000     1.000         9
                        atis_flight_time      1.000     1.000     1.000         1
                           atis_quantity      0.375     1.000     0.545         3
                atis_flight#atis_airfare      0.667     0.167     0.267        12
                            atis_airport      1.000     0.944     0.971        18
                           atis_distance      1.000     1.000     1.000        10
                               atis_city      1.000     0.500     0.667         6
                        atis_ground_fare      1.000     0.857     0.923         7
                           atis_capacity      1.000     0.952     0.976        21
                          atis_flight_no      0.889     1.000     0.941         8
                               atis_meal      1.000     0.833     0.909         6
                        atis_restriction      0.000     0.000     0.000         0
             atis_airline#atis_flight_no      0.000     0.000     0.000         0
    atis_ground_service#atis_ground_fare      0.000     0.000     0.000         0
           atis_airfare#atis_flight_time      0.000     0.000     0.000         0
                           atis_cheapest      0.000     0.000     0.000         0
atis_aircraft#atis_flight#atis_flight_no      0.000     0.000     0.000         0

                               micro avg      0.970     0.975     0.972       888
                               macro avg      0.673     0.648     0.642       888
                            weighted avg      0.970     0.975     0.969       888

I0716 00:57:45.995500 140076727973760 interactiveshell.py:2882] 
                              precision    recall  f1-score   support

                           O      0.000     0.000     0.000       0.0
           B-toloc.city_name      0.977     0.992     0.984     716.0
         B-fromloc.city_name      0.982     0.997     0.989     704.0
           I-toloc.city_name      0.977     0.981     0.979     265.0
      B-depart_date.day_name      0.986     0.991     0.988     212.0
              B-airline_name      1.000     1.000     1.000     101.0
         I-fromloc.city_name      0.967     0.989     0.978     177.0
 B-depart_time.period_of_day      1.000     0.923     0.960     130.0
              I-airline_name      1.000     1.000     1.000      65.0
    B-depart_date.day_number      0.981     0.964     0.972      55.0
    B-depart_date.month_name      0.982     0.964     0.973      56.0
          B-depart_time.time      0.864     1.000     0.927      57.0
                B-round_trip      1.000     0.973     0.986      73.0
             B-cost_relative      1.000     0.973     0.986      37.0
                I-round_trip      1.000     1.000     1.000      71.0
                B-flight_mod      1.000     1.000     1.000      24.0
 B-depart_time.time_relative      0.969     0.954     0.961      65.0
          I-depart_time.time      0.945     1.000     0.972      52.0
         B-stoploc.city_name      0.833     1.000     0.909      20.0
                 B-city_name      0.944     0.596     0.731      57.0
                B-class_type      0.960     1.000     0.980      24.0
          B-arrive_time.time      0.971     0.971     0.971      34.0
 B-arrive_time.time_relative      0.906     0.935     0.921      31.0
                I-class_type      1.000     1.000     1.000      17.0
               B-flight_stop      1.000     1.000     1.000      21.0
          I-arrive_time.time      1.000     0.971     0.986      35.0
              B-airline_code      1.000     0.882     0.938      34.0
    I-depart_date.day_number      1.000     0.933     0.966      15.0
      I-fromloc.airport_name      0.441     1.000     0.612      15.0
      B-fromloc.airport_name      0.500     1.000     0.667      12.0
      B-arrive_date.day_name      0.846     1.000     0.917      11.0
          B-toloc.state_code      0.947     1.000     0.973      18.0
B-depart_date.today_relative      1.000     0.889     0.941       9.0
             B-flight_number      0.524     1.000     0.688      11.0
 B-depart_date.date_relative      0.944     1.000     0.971      17.0
          B-toloc.state_name      0.781     0.893     0.833      28.0
           B-fare_basis_code      0.944     1.000     0.971      17.0
               B-flight_time      1.000     1.000     1.000       1.0
                        B-or      1.000     1.000     1.000       3.0
 B-arrive_time.period_of_day      0.857     1.000     0.923       6.0
          B-meal_description      1.000     0.900     0.947      10.0
             I-cost_relative      1.000     0.667     0.800       3.0
              I-airport_name      0.929     0.448     0.605      29.0
               B-fare_amount      1.000     1.000     1.000       2.0
               I-fare_amount      1.000     1.000     1.000       2.0
                 I-city_name      1.000     0.667     0.800      30.0
        I-toloc.airport_name      1.000     1.000     1.000       3.0
            B-transport_type      1.000     1.000     1.000      10.0
    B-arrive_date.month_name      0.714     0.833     0.769       6.0
    B-arrive_date.day_number      0.625     0.833     0.714       6.0
         I-stoploc.city_name      0.769     1.000     0.870      10.0
                      B-meal      1.000     1.000     1.000      16.0
        B-fromloc.state_code      1.000     0.957     0.978      23.0
    B-depart_time.period_mod      1.000     1.000     1.000       5.0
                   B-connect      1.000     1.000     1.000       6.0
               B-flight_days      1.000     1.000     1.000      10.0
        B-toloc.airport_name      1.000     1.000     1.000       3.0
        B-fromloc.state_name      0.944     1.000     0.971      17.0
              B-airport_name      0.833     0.476     0.606      21.0
                   B-economy      1.000     1.000     1.000       6.0
               I-flight_time      1.000     1.000     1.000       1.0
             B-aircraft_code      0.933     0.848     0.889      33.0
                       B-mod      1.000     0.500     0.667       2.0
              B-airport_code      1.000     0.111     0.200       9.0
    B-depart_time.start_time      1.000     0.667     0.800       3.0
      B-depart_time.end_time      1.000     0.333     0.500       3.0
          B-depart_date.year      1.000     1.000     1.000       3.0
            I-transport_type      0.000     0.000     0.000       1.0
          B-restriction_code      0.400     1.000     0.571       4.0
    B-arrive_time.start_time      0.889     1.000     0.941       8.0
        B-toloc.airport_code      1.000     0.750     0.857       4.0
      B-arrive_time.end_time      0.889     1.000     0.941       8.0
      I-arrive_time.end_time      0.889     1.000     0.941       8.0
      I-depart_time.end_time      1.000     0.333     0.500       3.0
               I-flight_stop      0.000     0.000     0.000       0.0
      B-fromloc.airport_code      1.000     1.000     1.000       5.0
          I-restriction_code      1.000     1.000     1.000       3.0
    I-depart_time.start_time      1.000     1.000     1.000       1.0
          I-toloc.state_name      1.000     1.000     1.000       1.0
I-depart_date.today_relative      0.000     0.000     0.000       0.0
 B-arrive_date.date_relative      1.000     1.000     1.000       2.0
                I-flight_mod      1.000     0.167     0.286       6.0
                   I-economy      0.000     0.000     0.000       0.0
 B-return_date.date_relative      0.667     0.667     0.667       3.0
        I-fromloc.state_name      1.000     1.000     1.000       1.0
                B-state_code      1.000     1.000     1.000       1.0
    I-arrive_time.start_time      1.000     1.000     1.000       1.0
    I-arrive_date.day_number      0.000     0.000     0.000       0.0
                 B-meal_code      0.000     0.000     0.000       1.0
 I-depart_time.period_of_day      1.000     1.000     1.000       1.0
                  B-day_name      1.000     0.500     0.667       2.0
             B-period_of_day      1.000     1.000     1.000       4.0
        B-stoploc.state_code      0.000     0.000     0.000       0.0
    B-return_date.month_name      0.000     0.000     0.000       0.0
    B-return_date.day_number      0.000     0.000     0.000       0.0
    B-arrive_time.period_mod      0.000     0.000     0.000       0.0
                 I-meal_code      0.000     0.000     0.000       0.0
        B-toloc.country_name      1.000     1.000     1.000       1.0
                 B-days_code      1.000     1.000     1.000       1.0
 I-arrive_time.period_of_day      0.000     0.000     0.000       0.0
            I-today_relative      0.000     0.000     0.000       0.0
 B-return_time.period_of_day      0.000     0.000     0.000       0.0
                      B-time      0.000     0.000     0.000       0.0
           I-fare_basis_code      0.000     0.000     0.000       0.0
 I-arrive_time.time_relative      0.000     0.000     0.000       4.0
 I-depart_time.time_relative      0.000     0.000     0.000       1.0
            B-today_relative      0.000     0.000     0.000       0.0
                B-state_name      0.000     0.000     0.000       9.0
B-arrive_date.today_relative      0.000     0.000     0.000       0.0
    B-return_time.period_mod      0.000     0.000     0.000       0.0
                B-month_name      0.000     0.000     0.000       0.0
                B-day_number      0.000     0.000     0.000       0.0
 I-return_date.date_relative      1.000     0.667     0.800       3.0
I-return_date.today_relative      0.000     0.000     0.000       0.0
      B-stoploc.airport_name      0.000     0.000     0.000       0.0
             B-time_relative      0.000     0.000     0.000       0.0
                      I-time      0.000     0.000     0.000       0.0
    I-return_date.day_number      0.000     0.000     0.000       0.0
          I-meal_description      0.000     0.000     0.000       0.0
B-return_date.today_relative      0.000     0.000     0.000       0.0
      B-return_date.day_name      0.000     0.000     0.000       2.0

                   micro avg      0.954     0.956     0.955    3657.0
                   macro avg      0.682     0.654     0.652    3657.0
                weighted avg      0.962     0.956     0.954    3657.0

I0716 00:57:46.022464 140076727973760 interactiveshell.py:2882] Best Slot F1: 0.955
I0716 00:57:46.023936 140076727973760 estimator_training.py:186] Not using Distribute Coordinator.
I0716 00:57:46.027518 140076727973760 training.py:612] Running training and evaluation locally (non-distributed).
I0716 00:57:46.029837 140076727973760 training.py:700] Start train and evaluate loop. The evaluate will happen after every checkpoint. Checkpoint frequency is determined based on RunConfig arguments: save_checkpoints_steps 312 or save_checkpoints_secs None.
I0716 00:57:46.093193 140076727973760 estimator.py:1145] Calling model_fn.
I0716 00:57:47.049592 140076727973760 <ipython-input-6-0a85a3fd693f>:50] 
[<tf.Variable 'embedding:0' shape=(750, 300) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/gates/kernel:0' shape=(600, 600) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/gates/bias:0' shape=(600,) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/candidate/kernel:0' shape=(600, 300) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/candidate/bias:0' shape=(300,) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/gates/kernel:0' shape=(600, 600) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/gates/bias:0' shape=(600,) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/candidate/kernel:0' shape=(600, 300) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/candidate/bias:0' shape=(300,) dtype=float32_ref>,
 <tf.Variable 'dense/kernel:0' shape=(600, 23) dtype=float32_ref>,
 <tf.Variable 'dense/bias:0' shape=(23,) dtype=float32_ref>,
 <tf.Variable 'dense_1/kernel:0' shape=(600, 122) dtype=float32_ref>,
 <tf.Variable 'dense_1/bias:0' shape=(122,) dtype=float32_ref>,
 <tf.Variable 'transitions:0' shape=(122, 122) dtype=float32_ref>]
I0716 00:57:48.631585 140076727973760 estimator.py:1147] Done calling model_fn.
I0716 00:57:48.635245 140076727973760 basic_session_run_hooks.py:541] Create CheckpointSaverHook.
I0716 00:57:48.859775 140076727973760 monitored_session.py:240] Graph was finalized.
I0716 00:57:48.881602 140076727973760 saver.py:1280] Restoring parameters from ../model/bigru_crf/model.ckpt-3744
I0716 00:57:49.138931 140076727973760 session_manager.py:500] Running local_init_op.
I0716 00:57:49.174760 140076727973760 session_manager.py:502] Done running local_init_op.
I0716 00:57:50.840440 140076727973760 basic_session_run_hooks.py:606] Saving checkpoints for 3744 into ../model/bigru_crf/model.ckpt.
Reading ../data/atis.train.w-intent.iob
I0716 00:57:53.155730 140076727973760 basic_session_run_hooks.py:262] loss = 5.220269, step = 3744
I0716 00:57:53.157506 140076727973760 basic_session_run_hooks.py:262] lr = 0.00020221119
I0716 00:58:04.713745 140076727973760 basic_session_run_hooks.py:692] global_step/sec: 8.65168
I0716 00:58:04.721558 140076727973760 basic_session_run_hooks.py:260] loss = 0.1031324, step = 3844 (11.566 sec)
I0716 00:58:04.723284 140076727973760 basic_session_run_hooks.py:260] lr = 0.00020009186 (11.566 sec)
I0716 00:58:15.912352 140076727973760 basic_session_run_hooks.py:692] global_step/sec: 8.9297
I0716 00:58:15.920065 140076727973760 basic_session_run_hooks.py:260] loss = 0.1160329, step = 3944 (11.199 sec)
I0716 00:58:15.921454 140076727973760 basic_session_run_hooks.py:260] lr = 0.00019799476 (11.198 sec)
I0716 00:58:26.746120 140076727973760 basic_session_run_hooks.py:692] global_step/sec: 9.23041
I0716 00:58:26.748639 140076727973760 basic_session_run_hooks.py:260] loss = 0.08712468, step = 4044 (10.829 sec)
I0716 00:58:26.754236 140076727973760 basic_session_run_hooks.py:260] lr = 0.00019591961 (10.833 sec)
I0716 00:58:27.942724 140076727973760 basic_session_run_hooks.py:606] Saving checkpoints for 4056 into ../model/bigru_crf/model.ckpt.
I0716 00:58:28.324351 140076727973760 estimator.py:1145] Calling model_fn.
I0716 00:58:29.275050 140076727973760 estimator.py:1147] Done calling model_fn.
I0716 00:58:29.303761 140076727973760 evaluation.py:255] Starting evaluation at 2019-07-16T00:58:29Z
I0716 00:58:29.419347 140076727973760 monitored_session.py:240] Graph was finalized.
I0716 00:58:29.434463 140076727973760 saver.py:1280] Restoring parameters from ../model/bigru_crf/model.ckpt-4056
I0716 00:58:29.549960 140076727973760 session_manager.py:500] Running local_init_op.
I0716 00:58:29.576397 140076727973760 session_manager.py:502] Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
I0716 00:58:31.869690 140076727973760 evaluation.py:275] Finished evaluation at 2019-07-16-00:58:31
I0716 00:58:31.871307 140076727973760 estimator.py:2039] Saving dict for global step 4056: global_step = 4056, loss = 3.6657448
I0716 00:58:31.880188 140076727973760 estimator.py:2099] Saving 'checkpoint_path' summary for global step 4056: ../model/bigru_crf/model.ckpt-4056
I0716 00:58:31.943043 140076727973760 estimator.py:368] Loss for final step: 0.5630746.
Reading ../data/atis.test.w-intent.iob
I0716 00:58:32.268442 140076727973760 estimator.py:1145] Calling model_fn.
I0716 00:58:33.228807 140076727973760 estimator.py:1147] Done calling model_fn.
I0716 00:58:33.343011 140076727973760 monitored_session.py:240] Graph was finalized.
I0716 00:58:33.359067 140076727973760 saver.py:1280] Restoring parameters from ../model/bigru_crf/model.ckpt-4056
I0716 00:58:33.461308 140076727973760 session_manager.py:500] Running local_init_op.
I0716 00:58:33.478606 140076727973760 session_manager.py:502] Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.
  'precision', 'predicted', average, warn_for)
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1439: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples.
  'recall', 'true', average, warn_for)
I0716 00:58:36.529662 140076727973760 interactiveshell.py:2882] 
                                          precision    recall  f1-score   support

                             atis_flight      0.977     0.992     0.984       632
                            atis_airfare      0.960     1.000     0.980        48
                     atis_ground_service      1.000     1.000     1.000        36
                            atis_airline      1.000     1.000     1.000        38
                       atis_abbreviation      0.943     1.000     0.971        33
                           atis_aircraft      1.000     1.000     1.000         9
                        atis_flight_time      1.000     1.000     1.000         1
                           atis_quantity      0.375     1.000     0.545         3
                atis_flight#atis_airfare      0.750     0.250     0.375        12
                            atis_airport      1.000     0.944     0.971        18
                           atis_distance      1.000     1.000     1.000        10
                               atis_city      1.000     0.500     0.667         6
                        atis_ground_fare      1.000     0.857     0.923         7
                           atis_capacity      1.000     0.952     0.976        21
                          atis_flight_no      0.889     1.000     0.941         8
                               atis_meal      1.000     0.833     0.909         6
                        atis_restriction      0.000     0.000     0.000         0
             atis_airline#atis_flight_no      0.000     0.000     0.000         0
    atis_ground_service#atis_ground_fare      0.000     0.000     0.000         0
           atis_airfare#atis_flight_time      0.000     0.000     0.000         0
                           atis_cheapest      0.000     0.000     0.000         0
atis_aircraft#atis_flight#atis_flight_no      0.000     0.000     0.000         0

                               micro avg      0.971     0.976     0.974       888
                               macro avg      0.677     0.651     0.647       888
                            weighted avg      0.973     0.976     0.971       888

I0716 00:58:36.578217 140076727973760 interactiveshell.py:2882] 
                              precision    recall  f1-score   support

                           O      0.000     0.000     0.000       0.0
           B-toloc.city_name      0.975     0.994     0.985     716.0
         B-fromloc.city_name      0.979     0.997     0.988     704.0
           I-toloc.city_name      0.963     0.992     0.978     265.0
      B-depart_date.day_name      0.986     0.991     0.988     212.0
              B-airline_name      1.000     1.000     1.000     101.0
         I-fromloc.city_name      0.978     0.989     0.983     177.0
 B-depart_time.period_of_day      0.992     0.915     0.952     130.0
              I-airline_name      1.000     1.000     1.000      65.0
    B-depart_date.day_number      0.981     0.964     0.972      55.0
    B-depart_date.month_name      0.982     0.964     0.973      56.0
          B-depart_time.time      0.851     1.000     0.919      57.0
                B-round_trip      1.000     0.986     0.993      73.0
             B-cost_relative      1.000     0.973     0.986      37.0
                I-round_trip      1.000     1.000     1.000      71.0
                B-flight_mod      1.000     1.000     1.000      24.0
 B-depart_time.time_relative      0.969     0.954     0.961      65.0
          I-depart_time.time      0.929     1.000     0.963      52.0
         B-stoploc.city_name      0.909     1.000     0.952      20.0
                 B-city_name      0.912     0.544     0.681      57.0
                B-class_type      0.960     1.000     0.980      24.0
          B-arrive_time.time      0.971     0.971     0.971      34.0
 B-arrive_time.time_relative      0.906     0.935     0.921      31.0
                I-class_type      1.000     1.000     1.000      17.0
               B-flight_stop      1.000     1.000     1.000      21.0
          I-arrive_time.time      0.971     0.971     0.971      35.0
              B-airline_code      1.000     0.882     0.938      34.0
    I-depart_date.day_number      1.000     0.933     0.966      15.0
      I-fromloc.airport_name      0.429     1.000     0.600      15.0
      B-fromloc.airport_name      0.480     1.000     0.649      12.0
      B-arrive_date.day_name      0.846     1.000     0.917      11.0
          B-toloc.state_code      0.947     1.000     0.973      18.0
B-depart_date.today_relative      1.000     0.889     0.941       9.0
             B-flight_number      0.579     1.000     0.733      11.0
 B-depart_date.date_relative      0.944     1.000     0.971      17.0
          B-toloc.state_name      0.781     0.893     0.833      28.0
           B-fare_basis_code      0.944     1.000     0.971      17.0
               B-flight_time      1.000     1.000     1.000       1.0
                        B-or      1.000     1.000     1.000       3.0
 B-arrive_time.period_of_day      0.750     1.000     0.857       6.0
          B-meal_description      1.000     0.900     0.947      10.0
             I-cost_relative      1.000     0.667     0.800       3.0
              I-airport_name      0.923     0.414     0.571      29.0
               B-fare_amount      1.000     1.000     1.000       2.0
               I-fare_amount      1.000     1.000     1.000       2.0
                 I-city_name      1.000     0.500     0.667      30.0
        I-toloc.airport_name      1.000     1.000     1.000       3.0
            B-transport_type      1.000     1.000     1.000      10.0
    B-arrive_date.month_name      0.714     0.833     0.769       6.0
    B-arrive_date.day_number      0.714     0.833     0.769       6.0
         I-stoploc.city_name      0.833     1.000     0.909      10.0
                      B-meal      1.000     1.000     1.000      16.0
        B-fromloc.state_code      1.000     0.957     0.978      23.0
    B-depart_time.period_mod      1.000     1.000     1.000       5.0
                   B-connect      1.000     1.000     1.000       6.0
               B-flight_days      1.000     1.000     1.000      10.0
        B-toloc.airport_name      1.000     1.000     1.000       3.0
        B-fromloc.state_name      0.944     1.000     0.971      17.0
              B-airport_name      0.818     0.429     0.562      21.0
                   B-economy      1.000     1.000     1.000       6.0
               I-flight_time      1.000     1.000     1.000       1.0
             B-aircraft_code      1.000     0.879     0.935      33.0
                       B-mod      1.000     0.500     0.667       2.0
              B-airport_code      1.000     0.111     0.200       9.0
    B-depart_time.start_time      1.000     0.667     0.800       3.0
      B-depart_time.end_time      1.000     0.333     0.500       3.0
          B-depart_date.year      1.000     1.000     1.000       3.0
            I-transport_type      0.000     0.000     0.000       1.0
          B-restriction_code      0.400     1.000     0.571       4.0
    B-arrive_time.start_time      0.889     1.000     0.941       8.0
        B-toloc.airport_code      1.000     0.750     0.857       4.0
      B-arrive_time.end_time      0.889     1.000     0.941       8.0
      I-arrive_time.end_time      0.875     0.875     0.875       8.0
      I-depart_time.end_time      1.000     0.333     0.500       3.0
               I-flight_stop      0.000     0.000     0.000       0.0
      B-fromloc.airport_code      1.000     1.000     1.000       5.0
          I-restriction_code      1.000     1.000     1.000       3.0
    I-depart_time.start_time      1.000     1.000     1.000       1.0
          I-toloc.state_name      1.000     1.000     1.000       1.0
I-depart_date.today_relative      0.000     0.000     0.000       0.0
 B-arrive_date.date_relative      1.000     1.000     1.000       2.0
                I-flight_mod      1.000     0.167     0.286       6.0
                   I-economy      0.000     0.000     0.000       0.0
 B-return_date.date_relative      1.000     0.333     0.500       3.0
        I-fromloc.state_name      1.000     1.000     1.000       1.0
                B-state_code      1.000     1.000     1.000       1.0
    I-arrive_time.start_time      1.000     1.000     1.000       1.0
    I-arrive_date.day_number      0.000     0.000     0.000       0.0
                 B-meal_code      0.000     0.000     0.000       1.0
 I-depart_time.period_of_day      0.000     0.000     0.000       1.0
                  B-day_name      1.000     0.500     0.667       2.0
             B-period_of_day      1.000     0.500     0.667       4.0
        B-stoploc.state_code      0.000     0.000     0.000       0.0
    B-return_date.month_name      0.000     0.000     0.000       0.0
    B-return_date.day_number      0.000     0.000     0.000       0.0
    B-arrive_time.period_mod      0.000     0.000     0.000       0.0
                 I-meal_code      0.000     0.000     0.000       0.0
        B-toloc.country_name      1.000     1.000     1.000       1.0
                 B-days_code      1.000     1.000     1.000       1.0
 I-arrive_time.period_of_day      0.000     0.000     0.000       0.0
            I-today_relative      0.000     0.000     0.000       0.0
 B-return_time.period_of_day      0.000     0.000     0.000       0.0
                      B-time      0.000     0.000     0.000       0.0
           I-fare_basis_code      0.000     0.000     0.000       0.0
 I-arrive_time.time_relative      0.000     0.000     0.000       4.0
 I-depart_time.time_relative      0.000     0.000     0.000       1.0
            B-today_relative      0.000     0.000     0.000       0.0
                B-state_name      0.000     0.000     0.000       9.0
B-arrive_date.today_relative      0.000     0.000     0.000       0.0
    B-return_time.period_mod      0.000     0.000     0.000       0.0
                B-month_name      0.000     0.000     0.000       0.0
                B-day_number      0.000     0.000     0.000       0.0
 I-return_date.date_relative      0.750     1.000     0.857       3.0
I-return_date.today_relative      0.000     0.000     0.000       0.0
      B-stoploc.airport_name      0.000     0.000     0.000       0.0
             B-time_relative      0.000     0.000     0.000       0.0
                      I-time      0.000     0.000     0.000       0.0
    I-return_date.day_number      0.000     0.000     0.000       0.0
          I-meal_description      0.000     0.000     0.000       0.0
B-return_date.today_relative      0.000     0.000     0.000       0.0
      B-return_date.day_name      0.000     0.000     0.000       2.0

                   micro avg      0.952     0.954     0.953    3657.0
                   macro avg      0.675     0.638     0.638    3657.0
                weighted avg      0.960     0.954     0.951    3657.0

I0716 00:58:36.606584 140076727973760 interactiveshell.py:2882] Best Slot F1: 0.955
I0716 00:58:36.608164 140076727973760 estimator_training.py:186] Not using Distribute Coordinator.
I0716 00:58:36.612288 140076727973760 training.py:612] Running training and evaluation locally (non-distributed).
I0716 00:58:36.615278 140076727973760 training.py:700] Start train and evaluate loop. The evaluate will happen after every checkpoint. Checkpoint frequency is determined based on RunConfig arguments: save_checkpoints_steps 312 or save_checkpoints_secs None.
I0716 00:58:36.676509 140076727973760 estimator.py:1145] Calling model_fn.
I0716 00:58:37.828853 140076727973760 <ipython-input-6-0a85a3fd693f>:50] 
[<tf.Variable 'embedding:0' shape=(750, 300) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/gates/kernel:0' shape=(600, 600) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/gates/bias:0' shape=(600,) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/candidate/kernel:0' shape=(600, 300) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/candidate/bias:0' shape=(300,) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/gates/kernel:0' shape=(600, 600) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/gates/bias:0' shape=(600,) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/candidate/kernel:0' shape=(600, 300) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/candidate/bias:0' shape=(300,) dtype=float32_ref>,
 <tf.Variable 'dense/kernel:0' shape=(600, 23) dtype=float32_ref>,
 <tf.Variable 'dense/bias:0' shape=(23,) dtype=float32_ref>,
 <tf.Variable 'dense_1/kernel:0' shape=(600, 122) dtype=float32_ref>,
 <tf.Variable 'dense_1/bias:0' shape=(122,) dtype=float32_ref>,
 <tf.Variable 'transitions:0' shape=(122, 122) dtype=float32_ref>]
I0716 00:58:39.337429 140076727973760 estimator.py:1147] Done calling model_fn.
I0716 00:58:39.341190 140076727973760 basic_session_run_hooks.py:541] Create CheckpointSaverHook.
I0716 00:58:39.537023 140076727973760 monitored_session.py:240] Graph was finalized.
I0716 00:58:39.555569 140076727973760 saver.py:1280] Restoring parameters from ../model/bigru_crf/model.ckpt-4056
I0716 00:58:39.813717 140076727973760 session_manager.py:500] Running local_init_op.
I0716 00:58:39.856701 140076727973760 session_manager.py:502] Done running local_init_op.
I0716 00:58:41.342786 140076727973760 basic_session_run_hooks.py:606] Saving checkpoints for 4056 into ../model/bigru_crf/model.ckpt.
Reading ../data/atis.train.w-intent.iob
I0716 00:58:43.650696 140076727973760 basic_session_run_hooks.py:262] loss = 0.15703025, step = 4056
I0716 00:58:43.652480 140076727973760 basic_session_run_hooks.py:262] lr = 0.00019567205
I0716 00:58:55.064619 140076727973760 basic_session_run_hooks.py:692] global_step/sec: 8.76077
I0716 00:58:55.067614 140076727973760 basic_session_run_hooks.py:260] loss = 0.02928317, step = 4156 (11.417 sec)
I0716 00:58:55.072853 140076727973760 basic_session_run_hooks.py:260] lr = 0.0001936213 (11.420 sec)
I0716 00:59:06.116815 140076727973760 basic_session_run_hooks.py:692] global_step/sec: 9.04798
I0716 00:59:06.119662 140076727973760 basic_session_run_hooks.py:260] loss = 0.019481849, step = 4256 (11.052 sec)
I0716 00:59:06.125678 140076727973760 basic_session_run_hooks.py:260] lr = 0.00019159199 (11.053 sec)
I0716 00:59:16.842295 140076727973760 basic_session_run_hooks.py:692] global_step/sec: 9.32361
I0716 00:59:16.848613 140076727973760 basic_session_run_hooks.py:260] loss = 0.032674883, step = 4356 (10.729 sec)
I0716 00:59:16.851436 140076727973760 basic_session_run_hooks.py:260] lr = 0.00018958394 (10.726 sec)
I0716 00:59:17.950994 140076727973760 basic_session_run_hooks.py:606] Saving checkpoints for 4368 into ../model/bigru_crf/model.ckpt.
I0716 00:59:18.347246 140076727973760 estimator.py:1145] Calling model_fn.
I0716 00:59:19.515303 140076727973760 estimator.py:1147] Done calling model_fn.
I0716 00:59:19.544620 140076727973760 evaluation.py:255] Starting evaluation at 2019-07-16T00:59:19Z
I0716 00:59:19.664596 140076727973760 monitored_session.py:240] Graph was finalized.
I0716 00:59:19.683052 140076727973760 saver.py:1280] Restoring parameters from ../model/bigru_crf/model.ckpt-4368
I0716 00:59:19.804778 140076727973760 session_manager.py:500] Running local_init_op.
I0716 00:59:19.835994 140076727973760 session_manager.py:502] Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
I0716 00:59:22.116920 140076727973760 evaluation.py:275] Finished evaluation at 2019-07-16-00:59:22
I0716 00:59:22.118548 140076727973760 estimator.py:2039] Saving dict for global step 4368: global_step = 4368, loss = 3.9125943
I0716 00:59:22.130839 140076727973760 estimator.py:2099] Saving 'checkpoint_path' summary for global step 4368: ../model/bigru_crf/model.ckpt-4368
I0716 00:59:22.193551 140076727973760 estimator.py:368] Loss for final step: 0.0059517156.
Reading ../data/atis.test.w-intent.iob
I0716 00:59:22.520851 140076727973760 estimator.py:1145] Calling model_fn.
I0716 00:59:23.178224 140076727973760 estimator.py:1147] Done calling model_fn.
I0716 00:59:23.293753 140076727973760 monitored_session.py:240] Graph was finalized.
I0716 00:59:23.313256 140076727973760 saver.py:1280] Restoring parameters from ../model/bigru_crf/model.ckpt-4368
I0716 00:59:23.419706 140076727973760 session_manager.py:500] Running local_init_op.
I0716 00:59:23.441205 140076727973760 session_manager.py:502] Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.
  'precision', 'predicted', average, warn_for)
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1439: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples.
  'recall', 'true', average, warn_for)
I0716 00:59:26.487559 140076727973760 interactiveshell.py:2882] 
                                          precision    recall  f1-score   support

                             atis_flight      0.974     0.992     0.983       632
                            atis_airfare      0.960     1.000     0.980        48
                     atis_ground_service      1.000     1.000     1.000        36
                            atis_airline      1.000     1.000     1.000        38
                       atis_abbreviation      0.943     1.000     0.971        33
                           atis_aircraft      1.000     1.000     1.000         9
                        atis_flight_time      1.000     1.000     1.000         1
                           atis_quantity      0.375     1.000     0.545         3
                atis_flight#atis_airfare      0.500     0.083     0.143        12
                            atis_airport      1.000     0.944     0.971        18
                           atis_distance      1.000     1.000     1.000        10
                               atis_city      1.000     0.500     0.667         6
                        atis_ground_fare      1.000     0.857     0.923         7
                           atis_capacity      1.000     0.952     0.976        21
                          atis_flight_no      0.889     1.000     0.941         8
                               atis_meal      1.000     0.833     0.909         6
                        atis_restriction      0.000     0.000     0.000         0
             atis_airline#atis_flight_no      0.000     0.000     0.000         0
    atis_ground_service#atis_ground_fare      0.000     0.000     0.000         0
           atis_airfare#atis_flight_time      0.000     0.000     0.000         0
                           atis_cheapest      0.000     0.000     0.000         0
atis_aircraft#atis_flight#atis_flight_no      0.000     0.000     0.000         0

                               micro avg      0.969     0.974     0.971       888
                               macro avg      0.665     0.644     0.637       888
                            weighted avg      0.967     0.974     0.967       888

I0716 00:59:26.546691 140076727973760 interactiveshell.py:2882] 
                              precision    recall  f1-score   support

                           O      0.000     0.000     0.000       0.0
           B-toloc.city_name      0.975     0.993     0.984     716.0
         B-fromloc.city_name      0.982     0.997     0.989     704.0
           I-toloc.city_name      0.974     0.981     0.977     265.0
      B-depart_date.day_name      0.991     0.991     0.991     212.0
              B-airline_name      1.000     1.000     1.000     101.0
         I-fromloc.city_name      0.978     0.989     0.983     177.0
 B-depart_time.period_of_day      1.000     0.923     0.960     130.0
              I-airline_name      1.000     1.000     1.000      65.0
    B-depart_date.day_number      0.981     0.964     0.972      55.0
    B-depart_date.month_name      0.982     0.964     0.973      56.0
          B-depart_time.time      0.864     1.000     0.927      57.0
                B-round_trip      1.000     0.986     0.993      73.0
             B-cost_relative      1.000     0.973     0.986      37.0
                I-round_trip      1.000     1.000     1.000      71.0
                B-flight_mod      1.000     0.958     0.979      24.0
 B-depart_time.time_relative      0.969     0.954     0.961      65.0
          I-depart_time.time      0.945     1.000     0.972      52.0
         B-stoploc.city_name      0.833     1.000     0.909      20.0
                 B-city_name      0.939     0.544     0.689      57.0
                B-class_type      0.960     1.000     0.980      24.0
          B-arrive_time.time      0.971     0.971     0.971      34.0
 B-arrive_time.time_relative      0.906     0.935     0.921      31.0
                I-class_type      1.000     1.000     1.000      17.0
               B-flight_stop      1.000     1.000     1.000      21.0
          I-arrive_time.time      1.000     0.971     0.986      35.0
              B-airline_code      1.000     0.912     0.954      34.0
    I-depart_date.day_number      1.000     0.933     0.966      15.0
      I-fromloc.airport_name      0.429     1.000     0.600      15.0
      B-fromloc.airport_name      0.480     1.000     0.649      12.0
      B-arrive_date.day_name      0.786     1.000     0.880      11.0
          B-toloc.state_code      0.947     1.000     0.973      18.0
B-depart_date.today_relative      1.000     0.889     0.941       9.0
             B-flight_number      0.524     1.000     0.688      11.0
 B-depart_date.date_relative      0.895     1.000     0.944      17.0
          B-toloc.state_name      0.781     0.893     0.833      28.0
           B-fare_basis_code      0.944     1.000     0.971      17.0
               B-flight_time      1.000     1.000     1.000       1.0
                        B-or      1.000     1.000     1.000       3.0
 B-arrive_time.period_of_day      0.857     1.000     0.923       6.0
          B-meal_description      1.000     0.900     0.947      10.0
             I-cost_relative      1.000     0.667     0.800       3.0
              I-airport_name      0.923     0.414     0.571      29.0
               B-fare_amount      1.000     1.000     1.000       2.0
               I-fare_amount      1.000     1.000     1.000       2.0
                 I-city_name      1.000     0.600     0.750      30.0
        I-toloc.airport_name      1.000     1.000     1.000       3.0
            B-transport_type      1.000     1.000     1.000      10.0
    B-arrive_date.month_name      0.714     0.833     0.769       6.0
    B-arrive_date.day_number      0.714     0.833     0.769       6.0
         I-stoploc.city_name      0.769     1.000     0.870      10.0
                      B-meal      1.000     1.000     1.000      16.0
        B-fromloc.state_code      1.000     0.957     0.978      23.0
    B-depart_time.period_mod      1.000     1.000     1.000       5.0
                   B-connect      1.000     1.000     1.000       6.0
               B-flight_days      1.000     1.000     1.000      10.0
        B-toloc.airport_name      1.000     1.000     1.000       3.0
        B-fromloc.state_name      0.944     1.000     0.971      17.0
              B-airport_name      0.818     0.429     0.562      21.0
                   B-economy      1.000     1.000     1.000       6.0
               I-flight_time      1.000     1.000     1.000       1.0
             B-aircraft_code      0.966     0.848     0.903      33.0
                       B-mod      1.000     0.500     0.667       2.0
              B-airport_code      1.000     0.111     0.200       9.0
    B-depart_time.start_time      1.000     0.667     0.800       3.0
      B-depart_time.end_time      1.000     0.333     0.500       3.0
          B-depart_date.year      1.000     1.000     1.000       3.0
            I-transport_type      0.000     0.000     0.000       1.0
          B-restriction_code      0.400     1.000     0.571       4.0
    B-arrive_time.start_time      0.889     1.000     0.941       8.0
        B-toloc.airport_code      1.000     0.750     0.857       4.0
      B-arrive_time.end_time      0.889     1.000     0.941       8.0
      I-arrive_time.end_time      0.889     1.000     0.941       8.0
      I-depart_time.end_time      1.000     0.333     0.500       3.0
               I-flight_stop      0.000     0.000     0.000       0.0
      B-fromloc.airport_code      1.000     1.000     1.000       5.0
          I-restriction_code      1.000     1.000     1.000       3.0
    I-depart_time.start_time      1.000     1.000     1.000       1.0
          I-toloc.state_name      1.000     1.000     1.000       1.0
I-depart_date.today_relative      0.000     0.000     0.000       0.0
 B-arrive_date.date_relative      1.000     1.000     1.000       2.0
                I-flight_mod      1.000     0.167     0.286       6.0
                   I-economy      0.000     0.000     0.000       0.0
 B-return_date.date_relative      1.000     0.333     0.500       3.0
        I-fromloc.state_name      1.000     1.000     1.000       1.0
                B-state_code      1.000     1.000     1.000       1.0
    I-arrive_time.start_time      1.000     1.000     1.000       1.0
    I-arrive_date.day_number      0.000     0.000     0.000       0.0
                 B-meal_code      0.000     0.000     0.000       1.0
 I-depart_time.period_of_day      1.000     1.000     1.000       1.0
                  B-day_name      1.000     0.500     0.667       2.0
             B-period_of_day      1.000     0.750     0.857       4.0
        B-stoploc.state_code      0.000     0.000     0.000       0.0
    B-return_date.month_name      0.000     0.000     0.000       0.0
    B-return_date.day_number      0.000     0.000     0.000       0.0
    B-arrive_time.period_mod      0.000     0.000     0.000       0.0
                 I-meal_code      0.000     0.000     0.000       0.0
        B-toloc.country_name      1.000     1.000     1.000       1.0
                 B-days_code      1.000     1.000     1.000       1.0
 I-arrive_time.period_of_day      0.000     0.000     0.000       0.0
            I-today_relative      0.000     0.000     0.000       0.0
 B-return_time.period_of_day      0.000     0.000     0.000       0.0
                      B-time      0.000     0.000     0.000       0.0
           I-fare_basis_code      0.000     0.000     0.000       0.0
 I-arrive_time.time_relative      0.000     0.000     0.000       4.0
 I-depart_time.time_relative      0.000     0.000     0.000       1.0
            B-today_relative      0.000     0.000     0.000       0.0
                B-state_name      0.000     0.000     0.000       9.0
B-arrive_date.today_relative      0.000     0.000     0.000       0.0
    B-return_time.period_mod      0.000     0.000     0.000       0.0
                B-month_name      0.000     0.000     0.000       0.0
                B-day_number      0.000     0.000     0.000       0.0
 I-return_date.date_relative      0.750     1.000     0.857       3.0
I-return_date.today_relative      0.000     0.000     0.000       0.0
      B-stoploc.airport_name      0.000     0.000     0.000       0.0
             B-time_relative      0.000     0.000     0.000       0.0
                      I-time      0.000     0.000     0.000       0.0
    I-return_date.day_number      0.000     0.000     0.000       0.0
          I-meal_description      0.000     0.000     0.000       0.0
B-return_date.today_relative      0.000     0.000     0.000       0.0
      B-return_date.day_name      0.000     0.000     0.000       2.0

                   micro avg      0.953     0.954     0.954    3657.0
                   macro avg      0.682     0.650     0.648    3657.0
                weighted avg      0.962     0.954     0.952    3657.0

I0716 00:59:26.575141 140076727973760 interactiveshell.py:2882] Best Slot F1: 0.955
I0716 00:59:26.576513 140076727973760 estimator_training.py:186] Not using Distribute Coordinator.
I0716 00:59:26.578639 140076727973760 training.py:612] Running training and evaluation locally (non-distributed).
I0716 00:59:26.580174 140076727973760 training.py:700] Start train and evaluate loop. The evaluate will happen after every checkpoint. Checkpoint frequency is determined based on RunConfig arguments: save_checkpoints_steps 312 or save_checkpoints_secs None.
I0716 00:59:26.645828 140076727973760 estimator.py:1145] Calling model_fn.
I0716 00:59:27.971948 140076727973760 <ipython-input-6-0a85a3fd693f>:50] 
[<tf.Variable 'embedding:0' shape=(750, 300) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/gates/kernel:0' shape=(600, 600) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/gates/bias:0' shape=(600,) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/candidate/kernel:0' shape=(600, 300) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/candidate/bias:0' shape=(300,) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/gates/kernel:0' shape=(600, 600) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/gates/bias:0' shape=(600,) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/candidate/kernel:0' shape=(600, 300) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/candidate/bias:0' shape=(300,) dtype=float32_ref>,
 <tf.Variable 'dense/kernel:0' shape=(600, 23) dtype=float32_ref>,
 <tf.Variable 'dense/bias:0' shape=(23,) dtype=float32_ref>,
 <tf.Variable 'dense_1/kernel:0' shape=(600, 122) dtype=float32_ref>,
 <tf.Variable 'dense_1/bias:0' shape=(122,) dtype=float32_ref>,
 <tf.Variable 'transitions:0' shape=(122, 122) dtype=float32_ref>]
I0716 00:59:29.484366 140076727973760 estimator.py:1147] Done calling model_fn.
I0716 00:59:29.489116 140076727973760 basic_session_run_hooks.py:541] Create CheckpointSaverHook.
I0716 00:59:29.693812 140076727973760 monitored_session.py:240] Graph was finalized.
I0716 00:59:29.709770 140076727973760 saver.py:1280] Restoring parameters from ../model/bigru_crf/model.ckpt-4368
I0716 00:59:29.970859 140076727973760 session_manager.py:500] Running local_init_op.
I0716 00:59:30.005634 140076727973760 session_manager.py:502] Done running local_init_op.
I0716 00:59:31.733657 140076727973760 basic_session_run_hooks.py:606] Saving checkpoints for 4368 into ../model/bigru_crf/model.ckpt.
Reading ../data/atis.train.w-intent.iob
I0716 00:59:34.129183 140076727973760 basic_session_run_hooks.py:262] loss = 0.053391717, step = 4368
I0716 00:59:34.131176 140076727973760 basic_session_run_hooks.py:262] lr = 0.00018934443
I0716 00:59:46.137022 140076727973760 basic_session_run_hooks.py:692] global_step/sec: 8.32745
I0716 00:59:46.144650 140076727973760 basic_session_run_hooks.py:260] loss = 0.08647187, step = 4468 (12.015 sec)
I0716 00:59:46.146343 140076727973760 basic_session_run_hooks.py:260] lr = 0.00018735994 (12.015 sec)
I0716 00:59:56.637960 140076727973760 basic_session_run_hooks.py:692] global_step/sec: 9.52288
I0716 00:59:56.648045 140076727973760 basic_session_run_hooks.py:260] loss = 0.010098492, step = 4568 (10.503 sec)
I0716 00:59:56.649462 140076727973760 basic_session_run_hooks.py:260] lr = 0.00018539626 (10.503 sec)
I0716 01:00:07.690558 140076727973760 basic_session_run_hooks.py:692] global_step/sec: 9.04763
I0716 01:00:07.696761 140076727973760 basic_session_run_hooks.py:260] loss = 0.07669906, step = 4668 (11.049 sec)
I0716 01:00:07.698763 140076727973760 basic_session_run_hooks.py:260] lr = 0.00018345319 (11.049 sec)
I0716 01:00:08.745006 140076727973760 basic_session_run_hooks.py:606] Saving checkpoints for 4680 into ../model/bigru_crf/model.ckpt.
I0716 01:00:09.120846 140076727973760 estimator.py:1145] Calling model_fn.
I0716 01:00:10.092061 140076727973760 estimator.py:1147] Done calling model_fn.
I0716 01:00:10.120520 140076727973760 evaluation.py:255] Starting evaluation at 2019-07-16T01:00:10Z
I0716 01:00:10.238031 140076727973760 monitored_session.py:240] Graph was finalized.
I0716 01:00:10.253270 140076727973760 saver.py:1280] Restoring parameters from ../model/bigru_crf/model.ckpt-4680
I0716 01:00:10.369238 140076727973760 session_manager.py:500] Running local_init_op.
I0716 01:00:10.395821 140076727973760 session_manager.py:502] Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
I0716 01:00:12.694943 140076727973760 evaluation.py:275] Finished evaluation at 2019-07-16-01:00:12
I0716 01:00:12.696441 140076727973760 estimator.py:2039] Saving dict for global step 4680: global_step = 4680, loss = 3.904999
I0716 01:00:12.705231 140076727973760 estimator.py:2099] Saving 'checkpoint_path' summary for global step 4680: ../model/bigru_crf/model.ckpt-4680
I0716 01:00:12.774600 140076727973760 estimator.py:368] Loss for final step: 0.16120368.
Reading ../data/atis.test.w-intent.iob
I0716 01:00:13.094168 140076727973760 estimator.py:1145] Calling model_fn.
I0716 01:00:14.032735 140076727973760 estimator.py:1147] Done calling model_fn.
I0716 01:00:14.146283 140076727973760 monitored_session.py:240] Graph was finalized.
I0716 01:00:14.163064 140076727973760 saver.py:1280] Restoring parameters from ../model/bigru_crf/model.ckpt-4680
I0716 01:00:14.263209 140076727973760 session_manager.py:500] Running local_init_op.
I0716 01:00:14.279859 140076727973760 session_manager.py:502] Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.
  'precision', 'predicted', average, warn_for)
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1439: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples.
  'recall', 'true', average, warn_for)
I0716 01:00:17.395745 140076727973760 interactiveshell.py:2882] 
                                          precision    recall  f1-score   support

                             atis_flight      0.974     0.992     0.983       632
                            atis_airfare      0.959     0.979     0.969        48
                     atis_ground_service      1.000     1.000     1.000        36
                            atis_airline      1.000     1.000     1.000        38
                       atis_abbreviation      0.943     1.000     0.971        33
                           atis_aircraft      1.000     1.000     1.000         9
                        atis_flight_time      1.000     1.000     1.000         1
                           atis_quantity      0.375     1.000     0.545         3
                atis_flight#atis_airfare      0.667     0.167     0.267        12
                            atis_airport      1.000     0.944     0.971        18
                           atis_distance      1.000     1.000     1.000        10
                               atis_city      1.000     0.500     0.667         6
                        atis_ground_fare      1.000     0.857     0.923         7
                           atis_capacity      1.000     0.952     0.976        21
                          atis_flight_no      0.889     1.000     0.941         8
                               atis_meal      1.000     0.833     0.909         6
                        atis_restriction      0.000     0.000     0.000         0
             atis_airline#atis_flight_no      0.000     0.000     0.000         0
    atis_ground_service#atis_ground_fare      0.000     0.000     0.000         0
           atis_airfare#atis_flight_time      0.000     0.000     0.000         0
                           atis_cheapest      0.000     0.000     0.000         0
atis_aircraft#atis_flight#atis_flight_no      0.000     0.000     0.000         0

                               micro avg      0.969     0.974     0.971       888
                               macro avg      0.673     0.647     0.642       888
                            weighted avg      0.969     0.974     0.968       888

I0716 01:00:17.449317 140076727973760 interactiveshell.py:2882] 
                              precision    recall  f1-score   support

                           O      0.000     0.000     0.000       0.0
           B-toloc.city_name      0.975     0.994     0.985     716.0
         B-fromloc.city_name      0.980     0.997     0.989     704.0
           I-toloc.city_name      0.970     0.989     0.979     265.0
      B-depart_date.day_name      0.981     0.991     0.986     212.0
              B-airline_name      1.000     1.000     1.000     101.0
         I-fromloc.city_name      0.978     0.989     0.983     177.0
 B-depart_time.period_of_day      1.000     0.923     0.960     130.0
              I-airline_name      1.000     1.000     1.000      65.0
    B-depart_date.day_number      0.981     0.964     0.972      55.0
    B-depart_date.month_name      0.982     0.964     0.973      56.0
          B-depart_time.time      0.851     1.000     0.919      57.0
                B-round_trip      1.000     0.986     0.993      73.0
             B-cost_relative      1.000     0.973     0.986      37.0
                I-round_trip      1.000     1.000     1.000      71.0
                B-flight_mod      1.000     1.000     1.000      24.0
 B-depart_time.time_relative      0.969     0.969     0.969      65.0
          I-depart_time.time      0.945     1.000     0.972      52.0
         B-stoploc.city_name      0.870     1.000     0.930      20.0
                 B-city_name      0.909     0.526     0.667      57.0
                B-class_type      0.960     1.000     0.980      24.0
          B-arrive_time.time      1.000     0.971     0.985      34.0
 B-arrive_time.time_relative      0.935     0.935     0.935      31.0
                I-class_type      1.000     1.000     1.000      17.0
               B-flight_stop      1.000     1.000     1.000      21.0
          I-arrive_time.time      1.000     0.971     0.986      35.0
              B-airline_code      1.000     0.971     0.985      34.0
    I-depart_date.day_number      1.000     1.000     1.000      15.0
      I-fromloc.airport_name      0.429     1.000     0.600      15.0
      B-fromloc.airport_name      0.480     1.000     0.649      12.0
      B-arrive_date.day_name      0.833     0.909     0.870      11.0
          B-toloc.state_code      0.947     1.000     0.973      18.0
B-depart_date.today_relative      1.000     0.889     0.941       9.0
             B-flight_number      0.611     1.000     0.759      11.0
 B-depart_date.date_relative      0.944     1.000     0.971      17.0
          B-toloc.state_name      0.781     0.893     0.833      28.0
           B-fare_basis_code      0.944     1.000     0.971      17.0
               B-flight_time      1.000     1.000     1.000       1.0
                        B-or      1.000     1.000     1.000       3.0
 B-arrive_time.period_of_day      0.857     1.000     0.923       6.0
          B-meal_description      1.000     0.900     0.947      10.0
             I-cost_relative      1.000     0.667     0.800       3.0
              I-airport_name      0.923     0.414     0.571      29.0
               B-fare_amount      1.000     1.000     1.000       2.0
               I-fare_amount      1.000     1.000     1.000       2.0
                 I-city_name      1.000     0.567     0.723      30.0
        I-toloc.airport_name      1.000     1.000     1.000       3.0
            B-transport_type      1.000     1.000     1.000      10.0
    B-arrive_date.month_name      0.714     0.833     0.769       6.0
    B-arrive_date.day_number      0.714     0.833     0.769       6.0
         I-stoploc.city_name      0.833     1.000     0.909      10.0
                      B-meal      1.000     1.000     1.000      16.0
        B-fromloc.state_code      1.000     0.957     0.978      23.0
    B-depart_time.period_mod      1.000     1.000     1.000       5.0
                   B-connect      1.000     1.000     1.000       6.0
               B-flight_days      1.000     1.000     1.000      10.0
        B-toloc.airport_name      1.000     1.000     1.000       3.0
        B-fromloc.state_name      0.944     1.000     0.971      17.0
              B-airport_name      0.818     0.429     0.562      21.0
                   B-economy      1.000     1.000     1.000       6.0
               I-flight_time      1.000     1.000     1.000       1.0
             B-aircraft_code      1.000     0.818     0.900      33.0
                       B-mod      1.000     0.500     0.667       2.0
              B-airport_code      0.500     0.111     0.182       9.0
    B-depart_time.start_time      1.000     0.667     0.800       3.0
      B-depart_time.end_time      1.000     0.333     0.500       3.0
          B-depart_date.year      1.000     1.000     1.000       3.0
            I-transport_type      0.000     0.000     0.000       1.0
          B-restriction_code      0.444     1.000     0.615       4.0
    B-arrive_time.start_time      0.889     1.000     0.941       8.0
        B-toloc.airport_code      1.000     0.750     0.857       4.0
      B-arrive_time.end_time      0.889     1.000     0.941       8.0
      I-arrive_time.end_time      0.889     1.000     0.941       8.0
      I-depart_time.end_time      1.000     0.333     0.500       3.0
               I-flight_stop      0.000     0.000     0.000       0.0
      B-fromloc.airport_code      1.000     1.000     1.000       5.0
          I-restriction_code      1.000     1.000     1.000       3.0
    I-depart_time.start_time      1.000     1.000     1.000       1.0
          I-toloc.state_name      1.000     1.000     1.000       1.0
I-depart_date.today_relative      0.000     0.000     0.000       0.0
 B-arrive_date.date_relative      1.000     1.000     1.000       2.0
                I-flight_mod      1.000     0.167     0.286       6.0
                   I-economy      0.000     0.000     0.000       0.0
 B-return_date.date_relative      1.000     0.333     0.500       3.0
        I-fromloc.state_name      1.000     1.000     1.000       1.0
                B-state_code      1.000     1.000     1.000       1.0
    I-arrive_time.start_time      1.000     1.000     1.000       1.0
    I-arrive_date.day_number      0.000     0.000     0.000       0.0
                 B-meal_code      0.000     0.000     0.000       1.0
 I-depart_time.period_of_day      1.000     1.000     1.000       1.0
                  B-day_name      1.000     0.500     0.667       2.0
             B-period_of_day      1.000     0.750     0.857       4.0
        B-stoploc.state_code      0.000     0.000     0.000       0.0
    B-return_date.month_name      0.000     0.000     0.000       0.0
    B-return_date.day_number      0.000     0.000     0.000       0.0
    B-arrive_time.period_mod      0.000     0.000     0.000       0.0
                 I-meal_code      0.000     0.000     0.000       0.0
        B-toloc.country_name      1.000     1.000     1.000       1.0
                 B-days_code      1.000     1.000     1.000       1.0
 I-arrive_time.period_of_day      0.000     0.000     0.000       0.0
            I-today_relative      0.000     0.000     0.000       0.0
 B-return_time.period_of_day      0.000     0.000     0.000       0.0
                      B-time      0.000     0.000     0.000       0.0
           I-fare_basis_code      0.000     0.000     0.000       0.0
 I-arrive_time.time_relative      0.000     0.000     0.000       4.0
 I-depart_time.time_relative      0.000     0.000     0.000       1.0
            B-today_relative      0.000     0.000     0.000       0.0
                B-state_name      0.000     0.000     0.000       9.0
B-arrive_date.today_relative      0.000     0.000     0.000       0.0
    B-return_time.period_mod      0.000     0.000     0.000       0.0
                B-month_name      0.000     0.000     0.000       0.0
                B-day_number      0.000     0.000     0.000       0.0
 I-return_date.date_relative      0.750     1.000     0.857       3.0
I-return_date.today_relative      0.000     0.000     0.000       0.0
      B-stoploc.airport_name      0.000     0.000     0.000       0.0
             B-time_relative      0.000     0.000     0.000       0.0
                      I-time      0.000     0.000     0.000       0.0
    I-return_date.day_number      0.000     0.000     0.000       0.0
          I-meal_description      0.000     0.000     0.000       0.0
B-return_date.today_relative      0.000     0.000     0.000       0.0
      B-return_date.day_name      0.000     0.000     0.000       2.0

                   micro avg      0.954     0.955     0.955    3657.0
                   macro avg      0.681     0.650     0.650    3657.0
                weighted avg      0.961     0.955     0.953    3657.0

I0716 01:00:17.477100 140076727973760 interactiveshell.py:2882] Best Slot F1: 0.955
I0716 01:00:17.478603 140076727973760 estimator_training.py:186] Not using Distribute Coordinator.
I0716 01:00:17.480601 140076727973760 training.py:612] Running training and evaluation locally (non-distributed).
I0716 01:00:17.483840 140076727973760 training.py:700] Start train and evaluate loop. The evaluate will happen after every checkpoint. Checkpoint frequency is determined based on RunConfig arguments: save_checkpoints_steps 312 or save_checkpoints_secs None.
I0716 01:00:17.550836 140076727973760 estimator.py:1145] Calling model_fn.
I0716 01:00:18.713108 140076727973760 <ipython-input-6-0a85a3fd693f>:50] 
[<tf.Variable 'embedding:0' shape=(750, 300) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/gates/kernel:0' shape=(600, 600) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/gates/bias:0' shape=(600,) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/candidate/kernel:0' shape=(600, 300) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/candidate/bias:0' shape=(300,) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/gates/kernel:0' shape=(600, 600) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/gates/bias:0' shape=(600,) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/candidate/kernel:0' shape=(600, 300) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/candidate/bias:0' shape=(300,) dtype=float32_ref>,
 <tf.Variable 'dense/kernel:0' shape=(600, 23) dtype=float32_ref>,
 <tf.Variable 'dense/bias:0' shape=(23,) dtype=float32_ref>,
 <tf.Variable 'dense_1/kernel:0' shape=(600, 122) dtype=float32_ref>,
 <tf.Variable 'dense_1/bias:0' shape=(122,) dtype=float32_ref>,
 <tf.Variable 'transitions:0' shape=(122, 122) dtype=float32_ref>]
I0716 01:00:20.243809 140076727973760 estimator.py:1147] Done calling model_fn.
I0716 01:00:20.247571 140076727973760 basic_session_run_hooks.py:541] Create CheckpointSaverHook.
I0716 01:00:20.442173 140076727973760 monitored_session.py:240] Graph was finalized.
I0716 01:00:20.460041 140076727973760 saver.py:1280] Restoring parameters from ../model/bigru_crf/model.ckpt-4680
I0716 01:00:20.719270 140076727973760 session_manager.py:500] Running local_init_op.
I0716 01:00:20.757040 140076727973760 session_manager.py:502] Done running local_init_op.
I0716 01:00:22.253049 140076727973760 basic_session_run_hooks.py:606] Saving checkpoints for 4680 into ../model/bigru_crf/model.ckpt.
Reading ../data/atis.train.w-intent.iob
I0716 01:00:24.572495 140076727973760 basic_session_run_hooks.py:262] loss = 0.009538186, step = 4680
I0716 01:00:24.574312 140076727973760 basic_session_run_hooks.py:262] lr = 0.00018322139
I0716 01:00:36.437860 140076727973760 basic_session_run_hooks.py:692] global_step/sec: 8.42759
I0716 01:00:36.445235 140076727973760 basic_session_run_hooks.py:260] loss = 0.22672327, step = 4780 (11.873 sec)
I0716 01:00:36.447107 140076727973760 basic_session_run_hooks.py:260] lr = 0.00018130106 (11.873 sec)
I0716 01:00:46.907118 140076727973760 basic_session_run_hooks.py:692] global_step/sec: 9.55181
I0716 01:00:46.914115 140076727973760 basic_session_run_hooks.py:260] loss = 0.08291147, step = 4880 (10.469 sec)
I0716 01:00:46.916230 140076727973760 basic_session_run_hooks.py:260] lr = 0.00017940091 (10.469 sec)
I0716 01:00:57.822638 140076727973760 basic_session_run_hooks.py:692] global_step/sec: 9.16119
I0716 01:00:57.826313 140076727973760 basic_session_run_hooks.py:260] loss = 0.28881764, step = 4980 (10.912 sec)
I0716 01:00:57.830625 140076727973760 basic_session_run_hooks.py:260] lr = 0.00017752068 (10.914 sec)
I0716 01:00:59.185415 140076727973760 basic_session_run_hooks.py:606] Saving checkpoints for 4992 into ../model/bigru_crf/model.ckpt.
I0716 01:00:59.556320 140076727973760 estimator.py:1145] Calling model_fn.
I0716 01:01:00.715904 140076727973760 estimator.py:1147] Done calling model_fn.
I0716 01:01:00.745298 140076727973760 evaluation.py:255] Starting evaluation at 2019-07-16T01:01:00Z
I0716 01:01:00.860276 140076727973760 monitored_session.py:240] Graph was finalized.
I0716 01:01:00.875753 140076727973760 saver.py:1280] Restoring parameters from ../model/bigru_crf/model.ckpt-4992
I0716 01:01:01.000029 140076727973760 session_manager.py:500] Running local_init_op.
I0716 01:01:01.031710 140076727973760 session_manager.py:502] Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
I0716 01:01:03.304113 140076727973760 evaluation.py:275] Finished evaluation at 2019-07-16-01:01:03
I0716 01:01:03.305734 140076727973760 estimator.py:2039] Saving dict for global step 4992: global_step = 4992, loss = 3.7710655
I0716 01:01:03.312949 140076727973760 estimator.py:2099] Saving 'checkpoint_path' summary for global step 4992: ../model/bigru_crf/model.ckpt-4992
I0716 01:01:03.376156 140076727973760 estimator.py:368] Loss for final step: 0.013114685.
Reading ../data/atis.test.w-intent.iob
I0716 01:01:03.693571 140076727973760 estimator.py:1145] Calling model_fn.
I0716 01:01:04.345659 140076727973760 estimator.py:1147] Done calling model_fn.
I0716 01:01:04.455111 140076727973760 monitored_session.py:240] Graph was finalized.
I0716 01:01:04.471501 140076727973760 saver.py:1280] Restoring parameters from ../model/bigru_crf/model.ckpt-4992
I0716 01:01:04.572194 140076727973760 session_manager.py:500] Running local_init_op.
I0716 01:01:04.590176 140076727973760 session_manager.py:502] Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.
  'precision', 'predicted', average, warn_for)
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1439: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples.
  'recall', 'true', average, warn_for)
I0716 01:01:07.726710 140076727973760 interactiveshell.py:2882] 
                                          precision    recall  f1-score   support

                             atis_flight      0.978     0.992     0.985       632
                            atis_airfare      0.960     1.000     0.980        48
                     atis_ground_service      1.000     1.000     1.000        36
                            atis_airline      1.000     1.000     1.000        38
                       atis_abbreviation      0.943     1.000     0.971        33
                           atis_aircraft      1.000     1.000     1.000         9
                        atis_flight_time      1.000     1.000     1.000         1
                           atis_quantity      0.375     1.000     0.545         3
                atis_flight#atis_airfare      0.800     0.333     0.471        12
                            atis_airport      1.000     0.944     0.971        18
                           atis_distance      1.000     1.000     1.000        10
                               atis_city      1.000     0.500     0.667         6
                        atis_ground_fare      1.000     0.857     0.923         7
                           atis_capacity      1.000     0.952     0.976        21
                          atis_flight_no      0.889     1.000     0.941         8
                               atis_meal      1.000     0.833     0.909         6
                        atis_restriction      0.000     0.000     0.000         0
             atis_airline#atis_flight_no      0.000     0.000     0.000         0
    atis_ground_service#atis_ground_fare      0.000     0.000     0.000         0
           atis_airfare#atis_flight_time      0.000     0.000     0.000         0
                           atis_cheapest      0.000     0.000     0.000         0
atis_aircraft#atis_flight#atis_flight_no      0.000     0.000     0.000         0

                               micro avg      0.972     0.977     0.975       888
                               macro avg      0.679     0.655     0.652       888
                            weighted avg      0.974     0.977     0.973       888

I0716 01:01:07.776107 140076727973760 interactiveshell.py:2882] 
                              precision    recall  f1-score   support

                           O      0.000     0.000     0.000       0.0
           B-toloc.city_name      0.973     0.993     0.983     716.0
         B-fromloc.city_name      0.982     0.997     0.989     704.0
           I-toloc.city_name      0.960     0.989     0.974     265.0
      B-depart_date.day_name      0.986     0.991     0.988     212.0
              B-airline_name      1.000     1.000     1.000     101.0
         I-fromloc.city_name      0.972     0.983     0.978     177.0
 B-depart_time.period_of_day      1.000     0.923     0.960     130.0
              I-airline_name      1.000     1.000     1.000      65.0
    B-depart_date.day_number      0.981     0.964     0.972      55.0
    B-depart_date.month_name      0.982     0.964     0.973      56.0
          B-depart_time.time      0.848     0.982     0.911      57.0
                B-round_trip      1.000     0.973     0.986      73.0
             B-cost_relative      1.000     0.973     0.986      37.0
                I-round_trip      1.000     1.000     1.000      71.0
                B-flight_mod      1.000     0.958     0.979      24.0
 B-depart_time.time_relative      0.968     0.938     0.953      65.0
          I-depart_time.time      0.944     0.981     0.962      52.0
         B-stoploc.city_name      0.952     1.000     0.976      20.0
                 B-city_name      0.909     0.526     0.667      57.0
                B-class_type      0.960     1.000     0.980      24.0
          B-arrive_time.time      0.970     0.941     0.955      34.0
 B-arrive_time.time_relative      0.906     0.935     0.921      31.0
                I-class_type      1.000     1.000     1.000      17.0
               B-flight_stop      1.000     1.000     1.000      21.0
          I-arrive_time.time      1.000     0.971     0.986      35.0
              B-airline_code      1.000     0.971     0.985      34.0
    I-depart_date.day_number      1.000     0.933     0.966      15.0
      I-fromloc.airport_name      0.417     1.000     0.588      15.0
      B-fromloc.airport_name      0.462     1.000     0.632      12.0
      B-arrive_date.day_name      0.846     1.000     0.917      11.0
          B-toloc.state_code      0.947     1.000     0.973      18.0
B-depart_date.today_relative      1.000     0.889     0.941       9.0
             B-flight_number      0.550     1.000     0.710      11.0
 B-depart_date.date_relative      0.895     1.000     0.944      17.0
          B-toloc.state_name      0.781     0.893     0.833      28.0
           B-fare_basis_code      0.944     1.000     0.971      17.0
               B-flight_time      1.000     1.000     1.000       1.0
                        B-or      1.000     1.000     1.000       3.0
 B-arrive_time.period_of_day      0.750     1.000     0.857       6.0
          B-meal_description      1.000     0.900     0.947      10.0
             I-cost_relative      1.000     0.667     0.800       3.0
              I-airport_name      0.917     0.379     0.537      29.0
               B-fare_amount      1.000     1.000     1.000       2.0
               I-fare_amount      1.000     1.000     1.000       2.0
                 I-city_name      1.000     0.500     0.667      30.0
        I-toloc.airport_name      1.000     1.000     1.000       3.0
            B-transport_type      1.000     1.000     1.000      10.0
    B-arrive_date.month_name      0.714     0.833     0.769       6.0
    B-arrive_date.day_number      0.714     0.833     0.769       6.0
         I-stoploc.city_name      0.909     1.000     0.952      10.0
                      B-meal      1.000     1.000     1.000      16.0
        B-fromloc.state_code      1.000     0.957     0.978      23.0
    B-depart_time.period_mod      0.833     1.000     0.909       5.0
                   B-connect      1.000     1.000     1.000       6.0
               B-flight_days      1.000     1.000     1.000      10.0
        B-toloc.airport_name      1.000     1.000     1.000       3.0
        B-fromloc.state_name      0.944     1.000     0.971      17.0
              B-airport_name      0.800     0.381     0.516      21.0
                   B-economy      1.000     1.000     1.000       6.0
               I-flight_time      1.000     1.000     1.000       1.0
             B-aircraft_code      1.000     0.818     0.900      33.0
                       B-mod      1.000     0.500     0.667       2.0
              B-airport_code      0.500     0.111     0.182       9.0
    B-depart_time.start_time      1.000     0.667     0.800       3.0
      B-depart_time.end_time      1.000     0.333     0.500       3.0
          B-depart_date.year      1.000     1.000     1.000       3.0
            I-transport_type      0.000     0.000     0.000       1.0
          B-restriction_code      0.444     1.000     0.615       4.0
    B-arrive_time.start_time      0.889     1.000     0.941       8.0
        B-toloc.airport_code      1.000     0.750     0.857       4.0
      B-arrive_time.end_time      0.889     1.000     0.941       8.0
      I-arrive_time.end_time      0.889     1.000     0.941       8.0
      I-depart_time.end_time      1.000     0.333     0.500       3.0
               I-flight_stop      0.000     0.000     0.000       0.0
      B-fromloc.airport_code      1.000     1.000     1.000       5.0
          I-restriction_code      1.000     1.000     1.000       3.0
    I-depart_time.start_time      1.000     1.000     1.000       1.0
          I-toloc.state_name      1.000     1.000     1.000       1.0
I-depart_date.today_relative      0.000     0.000     0.000       0.0
 B-arrive_date.date_relative      1.000     1.000     1.000       2.0
                I-flight_mod      1.000     0.167     0.286       6.0
                   I-economy      0.000     0.000     0.000       0.0
 B-return_date.date_relative      1.000     0.333     0.500       3.0
        I-fromloc.state_name      1.000     1.000     1.000       1.0
                B-state_code      1.000     1.000     1.000       1.0
    I-arrive_time.start_time      1.000     1.000     1.000       1.0
    I-arrive_date.day_number      0.000     0.000     0.000       0.0
                 B-meal_code      0.000     0.000     0.000       1.0
 I-depart_time.period_of_day      1.000     1.000     1.000       1.0
                  B-day_name      1.000     0.500     0.667       2.0
             B-period_of_day      1.000     0.750     0.857       4.0
        B-stoploc.state_code      0.000     0.000     0.000       0.0
    B-return_date.month_name      0.000     0.000     0.000       0.0
    B-return_date.day_number      0.000     0.000     0.000       0.0
    B-arrive_time.period_mod      0.000     0.000     0.000       0.0
                 I-meal_code      0.000     0.000     0.000       0.0
        B-toloc.country_name      1.000     1.000     1.000       1.0
                 B-days_code      1.000     1.000     1.000       1.0
 I-arrive_time.period_of_day      0.000     0.000     0.000       0.0
            I-today_relative      0.000     0.000     0.000       0.0
 B-return_time.period_of_day      0.000     0.000     0.000       0.0
                      B-time      0.000     0.000     0.000       0.0
           I-fare_basis_code      0.000     0.000     0.000       0.0
 I-arrive_time.time_relative      0.000     0.000     0.000       4.0
 I-depart_time.time_relative      0.000     0.000     0.000       1.0
            B-today_relative      0.000     0.000     0.000       0.0
                B-state_name      0.000     0.000     0.000       9.0
B-arrive_date.today_relative      0.000     0.000     0.000       0.0
    B-return_time.period_mod      0.000     0.000     0.000       0.0
                B-month_name      0.000     0.000     0.000       0.0
                B-day_number      0.000     0.000     0.000       0.0
 I-return_date.date_relative      0.750     1.000     0.857       3.0
I-return_date.today_relative      0.000     0.000     0.000       0.0
      B-stoploc.airport_name      0.000     0.000     0.000       0.0
             B-time_relative      0.000     0.000     0.000       0.0
                      I-time      0.000     0.000     0.000       0.0
    I-return_date.day_number      0.000     0.000     0.000       0.0
          I-meal_description      0.000     0.000     0.000       0.0
B-return_date.today_relative      0.000     0.000     0.000       0.0
      B-return_date.day_name      0.000     0.000     0.000       2.0

                   micro avg      0.950     0.952     0.951    3657.0
                   macro avg      0.678     0.648     0.646    3657.0
                weighted avg      0.959     0.952     0.949    3657.0

I0716 01:01:07.805117 140076727973760 interactiveshell.py:2882] Best Slot F1: 0.955
I0716 01:01:07.807074 140076727973760 estimator_training.py:186] Not using Distribute Coordinator.
I0716 01:01:07.810772 140076727973760 training.py:612] Running training and evaluation locally (non-distributed).
I0716 01:01:07.813260 140076727973760 training.py:700] Start train and evaluate loop. The evaluate will happen after every checkpoint. Checkpoint frequency is determined based on RunConfig arguments: save_checkpoints_steps 312 or save_checkpoints_secs None.
I0716 01:01:07.878082 140076727973760 estimator.py:1145] Calling model_fn.
I0716 01:01:09.174136 140076727973760 <ipython-input-6-0a85a3fd693f>:50] 
[<tf.Variable 'embedding:0' shape=(750, 300) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/gates/kernel:0' shape=(600, 600) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/gates/bias:0' shape=(600,) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/candidate/kernel:0' shape=(600, 300) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/candidate/bias:0' shape=(300,) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/gates/kernel:0' shape=(600, 600) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/gates/bias:0' shape=(600,) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/candidate/kernel:0' shape=(600, 300) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/candidate/bias:0' shape=(300,) dtype=float32_ref>,
 <tf.Variable 'dense/kernel:0' shape=(600, 23) dtype=float32_ref>,
 <tf.Variable 'dense/bias:0' shape=(23,) dtype=float32_ref>,
 <tf.Variable 'dense_1/kernel:0' shape=(600, 122) dtype=float32_ref>,
 <tf.Variable 'dense_1/bias:0' shape=(122,) dtype=float32_ref>,
 <tf.Variable 'transitions:0' shape=(122, 122) dtype=float32_ref>]
I0716 01:01:10.708974 140076727973760 estimator.py:1147] Done calling model_fn.
I0716 01:01:10.712956 140076727973760 basic_session_run_hooks.py:541] Create CheckpointSaverHook.
I0716 01:01:10.909289 140076727973760 monitored_session.py:240] Graph was finalized.
I0716 01:01:10.931616 140076727973760 saver.py:1280] Restoring parameters from ../model/bigru_crf/model.ckpt-4992
I0716 01:01:11.193254 140076727973760 session_manager.py:500] Running local_init_op.
I0716 01:01:11.235330 140076727973760 session_manager.py:502] Done running local_init_op.
I0716 01:01:12.738537 140076727973760 basic_session_run_hooks.py:606] Saving checkpoints for 4992 into ../model/bigru_crf/model.ckpt.
Reading ../data/atis.train.w-intent.iob
I0716 01:01:15.109292 140076727973760 basic_session_run_hooks.py:262] loss = 0.19115537, step = 4992
I0716 01:01:15.111055 140076727973760 basic_session_run_hooks.py:262] lr = 0.00017729634
I0716 01:01:26.491751 140076727973760 basic_session_run_hooks.py:692] global_step/sec: 8.78504
I0716 01:01:26.498357 140076727973760 basic_session_run_hooks.py:260] loss = 0.0359281, step = 5092 (11.389 sec)
I0716 01:01:26.500997 140076727973760 basic_session_run_hooks.py:260] lr = 0.00017543815 (11.390 sec)
I0716 01:01:37.424564 140076727973760 basic_session_run_hooks.py:692] global_step/sec: 9.14681
I0716 01:01:37.432131 140076727973760 basic_session_run_hooks.py:260] loss = 0.14094912, step = 5192 (10.934 sec)
I0716 01:01:37.434056 140076727973760 basic_session_run_hooks.py:260] lr = 0.00017359943 (10.933 sec)
I0716 01:01:48.286976 140076727973760 basic_session_run_hooks.py:692] global_step/sec: 9.20602
I0716 01:01:48.293480 140076727973760 basic_session_run_hooks.py:260] loss = 0.021416321, step = 5292 (10.861 sec)
I0716 01:01:48.295594 140076727973760 basic_session_run_hooks.py:260] lr = 0.00017177999 (10.862 sec)
I0716 01:01:49.614075 140076727973760 basic_session_run_hooks.py:606] Saving checkpoints for 5304 into ../model/bigru_crf/model.ckpt.
I0716 01:01:49.989251 140076727973760 estimator.py:1145] Calling model_fn.
I0716 01:01:51.147652 140076727973760 estimator.py:1147] Done calling model_fn.
I0716 01:01:51.179307 140076727973760 evaluation.py:255] Starting evaluation at 2019-07-16T01:01:51Z
I0716 01:01:51.295485 140076727973760 monitored_session.py:240] Graph was finalized.
I0716 01:01:51.311907 140076727973760 saver.py:1280] Restoring parameters from ../model/bigru_crf/model.ckpt-5304
I0716 01:01:51.434370 140076727973760 session_manager.py:500] Running local_init_op.
I0716 01:01:51.460444 140076727973760 session_manager.py:502] Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
I0716 01:01:53.768049 140076727973760 evaluation.py:275] Finished evaluation at 2019-07-16-01:01:53
I0716 01:01:53.769579 140076727973760 estimator.py:2039] Saving dict for global step 5304: global_step = 5304, loss = 3.7033532
I0716 01:01:53.774826 140076727973760 estimator.py:2099] Saving 'checkpoint_path' summary for global step 5304: ../model/bigru_crf/model.ckpt-5304
I0716 01:01:53.840809 140076727973760 estimator.py:368] Loss for final step: 0.01616584.
Reading ../data/atis.test.w-intent.iob
I0716 01:01:54.162718 140076727973760 estimator.py:1145] Calling model_fn.
I0716 01:01:55.117933 140076727973760 estimator.py:1147] Done calling model_fn.
I0716 01:01:55.231284 140076727973760 monitored_session.py:240] Graph was finalized.
I0716 01:01:55.244961 140076727973760 saver.py:1280] Restoring parameters from ../model/bigru_crf/model.ckpt-5304
I0716 01:01:55.351427 140076727973760 session_manager.py:500] Running local_init_op.
I0716 01:01:55.369205 140076727973760 session_manager.py:502] Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.
  'precision', 'predicted', average, warn_for)
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1439: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples.
  'recall', 'true', average, warn_for)
I0716 01:01:58.507722 140076727973760 interactiveshell.py:2882] 
                                          precision    recall  f1-score   support

                             atis_flight      0.977     0.992     0.984       632
                            atis_airfare      0.960     1.000     0.980        48
                     atis_ground_service      1.000     1.000     1.000        36
                            atis_airline      1.000     1.000     1.000        38
                       atis_abbreviation      0.943     1.000     0.971        33
                           atis_aircraft      1.000     1.000     1.000         9
                        atis_flight_time      1.000     1.000     1.000         1
                           atis_quantity      0.375     1.000     0.545         3
                atis_flight#atis_airfare      0.750     0.250     0.375        12
                            atis_airport      1.000     0.944     0.971        18
                           atis_distance      1.000     1.000     1.000        10
                               atis_city      1.000     0.500     0.667         6
                        atis_ground_fare      1.000     0.857     0.923         7
                           atis_capacity      1.000     0.952     0.976        21
                          atis_flight_no      0.889     1.000     0.941         8
                               atis_meal      1.000     0.833     0.909         6
                        atis_restriction      0.000     0.000     0.000         0
             atis_airline#atis_flight_no      0.000     0.000     0.000         0
    atis_ground_service#atis_ground_fare      0.000     0.000     0.000         0
           atis_airfare#atis_flight_time      0.000     0.000     0.000         0
                           atis_cheapest      0.000     0.000     0.000         0
atis_aircraft#atis_flight#atis_flight_no      0.000     0.000     0.000         0

                               micro avg      0.971     0.976     0.974       888
                               macro avg      0.677     0.651     0.647       888
                            weighted avg      0.973     0.976     0.971       888

I0716 01:01:58.565859 140076727973760 interactiveshell.py:2882] 
                              precision    recall  f1-score   support

                           O      0.000     0.000     0.000       0.0
           B-toloc.city_name      0.975     0.994     0.985     716.0
         B-fromloc.city_name      0.986     0.997     0.992     704.0
           I-toloc.city_name      0.978     0.989     0.983     265.0
      B-depart_date.day_name      0.991     0.991     0.991     212.0
              B-airline_name      1.000     1.000     1.000     101.0
         I-fromloc.city_name      0.972     0.989     0.980     177.0
 B-depart_time.period_of_day      1.000     0.923     0.960     130.0
              I-airline_name      1.000     1.000     1.000      65.0
    B-depart_date.day_number      0.981     0.964     0.972      55.0
    B-depart_date.month_name      0.982     0.964     0.973      56.0
          B-depart_time.time      0.864     1.000     0.927      57.0
                B-round_trip      1.000     0.973     0.986      73.0
             B-cost_relative      1.000     0.973     0.986      37.0
                I-round_trip      1.000     1.000     1.000      71.0
                B-flight_mod      1.000     1.000     1.000      24.0
 B-depart_time.time_relative      0.969     0.954     0.961      65.0
          I-depart_time.time      0.945     1.000     0.972      52.0
         B-stoploc.city_name      0.909     1.000     0.952      20.0
                 B-city_name      0.919     0.596     0.723      57.0
                B-class_type      0.960     1.000     0.980      24.0
          B-arrive_time.time      0.971     0.971     0.971      34.0
 B-arrive_time.time_relative      0.906     0.935     0.921      31.0
                I-class_type      1.000     1.000     1.000      17.0
               B-flight_stop      1.000     1.000     1.000      21.0
          I-arrive_time.time      1.000     0.971     0.986      35.0
              B-airline_code      1.000     0.912     0.954      34.0
    I-depart_date.day_number      1.000     0.933     0.966      15.0
      I-fromloc.airport_name      0.429     1.000     0.600      15.0
      B-fromloc.airport_name      0.480     1.000     0.649      12.0
      B-arrive_date.day_name      0.786     1.000     0.880      11.0
          B-toloc.state_code      0.947     1.000     0.973      18.0
B-depart_date.today_relative      1.000     0.889     0.941       9.0
             B-flight_number      0.524     1.000     0.688      11.0
 B-depart_date.date_relative      0.944     1.000     0.971      17.0
          B-toloc.state_name      0.806     0.893     0.847      28.0
           B-fare_basis_code      0.944     1.000     0.971      17.0
               B-flight_time      1.000     1.000     1.000       1.0
                        B-or      1.000     1.000     1.000       3.0
 B-arrive_time.period_of_day      0.857     1.000     0.923       6.0
          B-meal_description      1.000     0.900     0.947      10.0
             I-cost_relative      1.000     0.667     0.800       3.0
              I-airport_name      0.923     0.414     0.571      29.0
               B-fare_amount      1.000     1.000     1.000       2.0
               I-fare_amount      1.000     1.000     1.000       2.0
                 I-city_name      0.947     0.600     0.735      30.0
        I-toloc.airport_name      1.000     1.000     1.000       3.0
            B-transport_type      1.000     1.000     1.000      10.0
    B-arrive_date.month_name      0.714     0.833     0.769       6.0
    B-arrive_date.day_number      0.714     0.833     0.769       6.0
         I-stoploc.city_name      0.833     1.000     0.909      10.0
                      B-meal      1.000     1.000     1.000      16.0
        B-fromloc.state_code      1.000     0.957     0.978      23.0
    B-depart_time.period_mod      1.000     1.000     1.000       5.0
                   B-connect      1.000     1.000     1.000       6.0
               B-flight_days      1.000     1.000     1.000      10.0
        B-toloc.airport_name      1.000     1.000     1.000       3.0
        B-fromloc.state_name      0.895     1.000     0.944      17.0
              B-airport_name      0.818     0.429     0.562      21.0
                   B-economy      1.000     1.000     1.000       6.0
               I-flight_time      1.000     1.000     1.000       1.0
             B-aircraft_code      0.964     0.818     0.885      33.0
                       B-mod      1.000     0.500     0.667       2.0
              B-airport_code      0.500     0.111     0.182       9.0
    B-depart_time.start_time      0.667     0.667     0.667       3.0
      B-depart_time.end_time      1.000     0.333     0.500       3.0
          B-depart_date.year      1.000     1.000     1.000       3.0
            I-transport_type      0.000     0.000     0.000       1.0
          B-restriction_code      0.444     1.000     0.615       4.0
    B-arrive_time.start_time      0.889     1.000     0.941       8.0
        B-toloc.airport_code      1.000     0.750     0.857       4.0
      B-arrive_time.end_time      0.889     1.000     0.941       8.0
      I-arrive_time.end_time      0.889     1.000     0.941       8.0
      I-depart_time.end_time      1.000     0.333     0.500       3.0
               I-flight_stop      0.000     0.000     0.000       0.0
      B-fromloc.airport_code      1.000     1.000     1.000       5.0
          I-restriction_code      1.000     1.000     1.000       3.0
    I-depart_time.start_time      1.000     1.000     1.000       1.0
          I-toloc.state_name      1.000     1.000     1.000       1.0
I-depart_date.today_relative      0.000     0.000     0.000       0.0
 B-arrive_date.date_relative      1.000     1.000     1.000       2.0
                I-flight_mod      1.000     0.167     0.286       6.0
                   I-economy      0.000     0.000     0.000       0.0
 B-return_date.date_relative      1.000     0.333     0.500       3.0
        I-fromloc.state_name      1.000     1.000     1.000       1.0
                B-state_code      1.000     1.000     1.000       1.0
    I-arrive_time.start_time      1.000     1.000     1.000       1.0
    I-arrive_date.day_number      0.000     0.000     0.000       0.0
                 B-meal_code      0.000     0.000     0.000       1.0
 I-depart_time.period_of_day      1.000     1.000     1.000       1.0
                  B-day_name      1.000     0.500     0.667       2.0
             B-period_of_day      1.000     1.000     1.000       4.0
        B-stoploc.state_code      0.000     0.000     0.000       0.0
    B-return_date.month_name      0.000     0.000     0.000       0.0
    B-return_date.day_number      0.000     0.000     0.000       0.0
    B-arrive_time.period_mod      0.000     0.000     0.000       0.0
                 I-meal_code      0.000     0.000     0.000       0.0
        B-toloc.country_name      1.000     1.000     1.000       1.0
                 B-days_code      1.000     1.000     1.000       1.0
 I-arrive_time.period_of_day      0.000     0.000     0.000       0.0
            I-today_relative      0.000     0.000     0.000       0.0
 B-return_time.period_of_day      0.000     0.000     0.000       0.0
                      B-time      0.000     0.000     0.000       0.0
           I-fare_basis_code      0.000     0.000     0.000       0.0
 I-arrive_time.time_relative      0.000     0.000     0.000       4.0
 I-depart_time.time_relative      0.000     0.000     0.000       1.0
            B-today_relative      0.000     0.000     0.000       0.0
                B-state_name      0.000     0.000     0.000       9.0
B-arrive_date.today_relative      0.000     0.000     0.000       0.0
    B-return_time.period_mod      0.000     0.000     0.000       0.0
                B-month_name      0.000     0.000     0.000       0.0
                B-day_number      0.000     0.000     0.000       0.0
 I-return_date.date_relative      0.750     1.000     0.857       3.0
I-return_date.today_relative      0.000     0.000     0.000       0.0
      B-stoploc.airport_name      0.000     0.000     0.000       0.0
             B-time_relative      0.000     0.000     0.000       0.0
                      I-time      0.000     0.000     0.000       0.0
    I-return_date.day_number      0.000     0.000     0.000       0.0
          I-meal_description      0.000     0.000     0.000       0.0
B-return_date.today_relative      0.000     0.000     0.000       0.0
      B-return_date.day_name      0.000     0.000     0.000       2.0

                   micro avg      0.954     0.956     0.955    3657.0
                   macro avg      0.677     0.653     0.649    3657.0
                weighted avg      0.961     0.956     0.954    3657.0

I0716 01:01:58.594734 140076727973760 interactiveshell.py:2882] Best Slot F1: 0.955
I0716 01:01:58.596222 140076727973760 estimator_training.py:186] Not using Distribute Coordinator.
I0716 01:01:58.599316 140076727973760 training.py:612] Running training and evaluation locally (non-distributed).
I0716 01:01:58.603540 140076727973760 training.py:700] Start train and evaluate loop. The evaluate will happen after every checkpoint. Checkpoint frequency is determined based on RunConfig arguments: save_checkpoints_steps 312 or save_checkpoints_secs None.
I0716 01:01:58.664796 140076727973760 estimator.py:1145] Calling model_fn.
I0716 01:01:59.808325 140076727973760 <ipython-input-6-0a85a3fd693f>:50] 
[<tf.Variable 'embedding:0' shape=(750, 300) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/gates/kernel:0' shape=(600, 600) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/gates/bias:0' shape=(600,) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/candidate/kernel:0' shape=(600, 300) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/candidate/bias:0' shape=(300,) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/gates/kernel:0' shape=(600, 600) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/gates/bias:0' shape=(600,) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/candidate/kernel:0' shape=(600, 300) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/candidate/bias:0' shape=(300,) dtype=float32_ref>,
 <tf.Variable 'dense/kernel:0' shape=(600, 23) dtype=float32_ref>,
 <tf.Variable 'dense/bias:0' shape=(23,) dtype=float32_ref>,
 <tf.Variable 'dense_1/kernel:0' shape=(600, 122) dtype=float32_ref>,
 <tf.Variable 'dense_1/bias:0' shape=(122,) dtype=float32_ref>,
 <tf.Variable 'transitions:0' shape=(122, 122) dtype=float32_ref>]
I0716 01:02:01.334798 140076727973760 estimator.py:1147] Done calling model_fn.
I0716 01:02:01.338650 140076727973760 basic_session_run_hooks.py:541] Create CheckpointSaverHook.
I0716 01:02:01.542550 140076727973760 monitored_session.py:240] Graph was finalized.
I0716 01:02:01.559430 140076727973760 saver.py:1280] Restoring parameters from ../model/bigru_crf/model.ckpt-5304
I0716 01:02:01.827495 140076727973760 session_manager.py:500] Running local_init_op.
I0716 01:02:01.865019 140076727973760 session_manager.py:502] Done running local_init_op.
I0716 01:02:03.378782 140076727973760 basic_session_run_hooks.py:606] Saving checkpoints for 5304 into ../model/bigru_crf/model.ckpt.
Reading ../data/atis.train.w-intent.iob
I0716 01:02:05.717416 140076727973760 basic_session_run_hooks.py:262] loss = 0.043646056, step = 5304
I0716 01:02:05.720234 140076727973760 basic_session_run_hooks.py:262] lr = 0.00017156293
I0716 01:02:17.494239 140076727973760 basic_session_run_hooks.py:692] global_step/sec: 8.49069
I0716 01:02:17.499503 140076727973760 basic_session_run_hooks.py:260] loss = 0.033989035, step = 5404 (11.782 sec)
I0716 01:02:17.502146 140076727973760 basic_session_run_hooks.py:260] lr = 0.00016976481 (11.782 sec)
I0716 01:02:28.082748 140076727973760 basic_session_run_hooks.py:692] global_step/sec: 9.4442
I0716 01:02:28.090416 140076727973760 basic_session_run_hooks.py:260] loss = 0.063261576, step = 5504 (10.591 sec)
I0716 01:02:28.092814 140076727973760 basic_session_run_hooks.py:260] lr = 0.00016798556 (10.591 sec)
I0716 01:02:39.193169 140076727973760 basic_session_run_hooks.py:692] global_step/sec: 9.00056
I0716 01:02:39.198879 140076727973760 basic_session_run_hooks.py:260] loss = 0.18737945, step = 5604 (11.108 sec)
I0716 01:02:39.202608 140076727973760 basic_session_run_hooks.py:260] lr = 0.00016622494 (11.110 sec)
I0716 01:02:40.261487 140076727973760 basic_session_run_hooks.py:606] Saving checkpoints for 5616 into ../model/bigru_crf/model.ckpt.
I0716 01:02:40.636910 140076727973760 estimator.py:1145] Calling model_fn.
I0716 01:02:41.826051 140076727973760 estimator.py:1147] Done calling model_fn.
I0716 01:02:41.855344 140076727973760 evaluation.py:255] Starting evaluation at 2019-07-16T01:02:41Z
I0716 01:02:41.973451 140076727973760 monitored_session.py:240] Graph was finalized.
I0716 01:02:41.989606 140076727973760 saver.py:1280] Restoring parameters from ../model/bigru_crf/model.ckpt-5616
I0716 01:02:42.113978 140076727973760 session_manager.py:500] Running local_init_op.
I0716 01:02:42.142760 140076727973760 session_manager.py:502] Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
I0716 01:02:44.442548 140076727973760 evaluation.py:275] Finished evaluation at 2019-07-16-01:02:44
I0716 01:02:44.444120 140076727973760 estimator.py:2039] Saving dict for global step 5616: global_step = 5616, loss = 4.152492
I0716 01:02:44.451269 140076727973760 estimator.py:2099] Saving 'checkpoint_path' summary for global step 5616: ../model/bigru_crf/model.ckpt-5616
I0716 01:02:44.514392 140076727973760 estimator.py:368] Loss for final step: 0.0037684888.
Reading ../data/atis.test.w-intent.iob
I0716 01:02:44.845145 140076727973760 estimator.py:1145] Calling model_fn.
I0716 01:02:45.491397 140076727973760 estimator.py:1147] Done calling model_fn.
I0716 01:02:45.603129 140076727973760 monitored_session.py:240] Graph was finalized.
I0716 01:02:45.619039 140076727973760 saver.py:1280] Restoring parameters from ../model/bigru_crf/model.ckpt-5616
I0716 01:02:45.720300 140076727973760 session_manager.py:500] Running local_init_op.
I0716 01:02:45.739734 140076727973760 session_manager.py:502] Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.
  'precision', 'predicted', average, warn_for)
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1439: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples.
  'recall', 'true', average, warn_for)
I0716 01:02:48.888703 140076727973760 interactiveshell.py:2882] 
                                          precision    recall  f1-score   support

                             atis_flight      0.974     0.992     0.983       632
                            atis_airfare      0.959     0.979     0.969        48
                     atis_ground_service      1.000     1.000     1.000        36
                            atis_airline      1.000     1.000     1.000        38
                       atis_abbreviation      0.943     1.000     0.971        33
                           atis_aircraft      1.000     1.000     1.000         9
                        atis_flight_time      1.000     1.000     1.000         1
                           atis_quantity      0.375     1.000     0.545         3
                atis_flight#atis_airfare      0.667     0.167     0.267        12
                            atis_airport      1.000     0.944     0.971        18
                           atis_distance      1.000     1.000     1.000        10
                               atis_city      1.000     0.500     0.667         6
                        atis_ground_fare      1.000     0.857     0.923         7
                           atis_capacity      1.000     0.952     0.976        21
                          atis_flight_no      0.889     1.000     0.941         8
                               atis_meal      1.000     0.833     0.909         6
                        atis_restriction      0.000     0.000     0.000         0
             atis_airline#atis_flight_no      0.000     0.000     0.000         0
    atis_ground_service#atis_ground_fare      0.000     0.000     0.000         0
           atis_airfare#atis_flight_time      0.000     0.000     0.000         0
                           atis_cheapest      0.000     0.000     0.000         0
atis_aircraft#atis_flight#atis_flight_no      0.000     0.000     0.000         0

                               micro avg      0.969     0.974     0.971       888
                               macro avg      0.673     0.647     0.642       888
                            weighted avg      0.969     0.974     0.968       888

I0716 01:02:48.942149 140076727973760 interactiveshell.py:2882] 
                              precision    recall  f1-score   support

                           O      0.000     0.000     0.000       0.0
           B-toloc.city_name      0.975     0.992     0.983     716.0
         B-fromloc.city_name      0.979     0.997     0.988     704.0
           I-toloc.city_name      0.967     0.981     0.974     265.0
      B-depart_date.day_name      0.991     0.991     0.991     212.0
              B-airline_name      1.000     1.000     1.000     101.0
         I-fromloc.city_name      0.972     0.994     0.983     177.0
 B-depart_time.period_of_day      0.992     0.923     0.956     130.0
              I-airline_name      1.000     1.000     1.000      65.0
    B-depart_date.day_number      0.981     0.964     0.972      55.0
    B-depart_date.month_name      0.982     0.964     0.973      56.0
          B-depart_time.time      0.851     1.000     0.919      57.0
                B-round_trip      1.000     0.973     0.986      73.0
             B-cost_relative      1.000     0.973     0.986      37.0
                I-round_trip      1.000     1.000     1.000      71.0
                B-flight_mod      0.923     1.000     0.960      24.0
 B-depart_time.time_relative      0.969     0.954     0.961      65.0
          I-depart_time.time      0.944     0.981     0.962      52.0
         B-stoploc.city_name      0.909     1.000     0.952      20.0
                 B-city_name      0.889     0.561     0.688      57.0
                B-class_type      0.960     1.000     0.980      24.0
          B-arrive_time.time      1.000     0.971     0.985      34.0
 B-arrive_time.time_relative      0.935     0.935     0.935      31.0
                I-class_type      1.000     1.000     1.000      17.0
               B-flight_stop      1.000     1.000     1.000      21.0
          I-arrive_time.time      1.000     0.971     0.986      35.0
              B-airline_code      1.000     0.971     0.985      34.0
    I-depart_date.day_number      1.000     1.000     1.000      15.0
      I-fromloc.airport_name      0.417     1.000     0.588      15.0
      B-fromloc.airport_name      0.462     1.000     0.632      12.0
      B-arrive_date.day_name      0.786     1.000     0.880      11.0
          B-toloc.state_code      0.947     1.000     0.973      18.0
B-depart_date.today_relative      1.000     0.889     0.941       9.0
             B-flight_number      0.579     1.000     0.733      11.0
 B-depart_date.date_relative      0.944     1.000     0.971      17.0
          B-toloc.state_name      0.788     0.929     0.852      28.0
           B-fare_basis_code      0.944     1.000     0.971      17.0
               B-flight_time      1.000     1.000     1.000       1.0
                        B-or      1.000     1.000     1.000       3.0
 B-arrive_time.period_of_day      0.750     1.000     0.857       6.0
          B-meal_description      1.000     0.900     0.947      10.0
             I-cost_relative      1.000     0.667     0.800       3.0
              I-airport_name      0.917     0.379     0.537      29.0
               B-fare_amount      1.000     1.000     1.000       2.0
               I-fare_amount      1.000     1.000     1.000       2.0
                 I-city_name      0.938     0.500     0.652      30.0
        I-toloc.airport_name      1.000     1.000     1.000       3.0
            B-transport_type      1.000     1.000     1.000      10.0
    B-arrive_date.month_name      0.714     0.833     0.769       6.0
    B-arrive_date.day_number      0.714     0.833     0.769       6.0
         I-stoploc.city_name      0.833     1.000     0.909      10.0
                      B-meal      1.000     1.000     1.000      16.0
        B-fromloc.state_code      1.000     0.957     0.978      23.0
    B-depart_time.period_mod      0.833     1.000     0.909       5.0
                   B-connect      1.000     1.000     1.000       6.0
               B-flight_days      1.000     1.000     1.000      10.0
        B-toloc.airport_name      1.000     1.000     1.000       3.0
        B-fromloc.state_name      0.944     1.000     0.971      17.0
              B-airport_name      0.800     0.381     0.516      21.0
                   B-economy      1.000     1.000     1.000       6.0
               I-flight_time      1.000     1.000     1.000       1.0
             B-aircraft_code      1.000     0.758     0.862      33.0
                       B-mod      1.000     0.500     0.667       2.0
              B-airport_code      0.500     0.111     0.182       9.0
    B-depart_time.start_time      1.000     1.000     1.000       3.0
      B-depart_time.end_time      1.000     0.333     0.500       3.0
          B-depart_date.year      1.000     1.000     1.000       3.0
            I-transport_type      0.000     0.000     0.000       1.0
          B-restriction_code      0.364     1.000     0.533       4.0
    B-arrive_time.start_time      1.000     1.000     1.000       8.0
        B-toloc.airport_code      1.000     0.750     0.857       4.0
      B-arrive_time.end_time      0.889     1.000     0.941       8.0
      I-arrive_time.end_time      0.889     1.000     0.941       8.0
      I-depart_time.end_time      1.000     0.333     0.500       3.0
               I-flight_stop      0.000     0.000     0.000       0.0
      B-fromloc.airport_code      1.000     1.000     1.000       5.0
          I-restriction_code      1.000     1.000     1.000       3.0
    I-depart_time.start_time      1.000     1.000     1.000       1.0
          I-toloc.state_name      1.000     1.000     1.000       1.0
I-depart_date.today_relative      0.000     0.000     0.000       0.0
 B-arrive_date.date_relative      1.000     1.000     1.000       2.0
                I-flight_mod      1.000     0.167     0.286       6.0
                   I-economy      0.000     0.000     0.000       0.0
 B-return_date.date_relative      1.000     0.333     0.500       3.0
        I-fromloc.state_name      1.000     1.000     1.000       1.0
                B-state_code      1.000     1.000     1.000       1.0
    I-arrive_time.start_time      1.000     1.000     1.000       1.0
    I-arrive_date.day_number      0.000     0.000     0.000       0.0
                 B-meal_code      0.000     0.000     0.000       1.0
 I-depart_time.period_of_day      1.000     1.000     1.000       1.0
                  B-day_name      1.000     0.500     0.667       2.0
             B-period_of_day      1.000     0.500     0.667       4.0
        B-stoploc.state_code      0.000     0.000     0.000       0.0
    B-return_date.month_name      0.000     0.000     0.000       0.0
    B-return_date.day_number      0.000     0.000     0.000       0.0
    B-arrive_time.period_mod      0.000     0.000     0.000       0.0
                 I-meal_code      0.000     0.000     0.000       0.0
        B-toloc.country_name      1.000     1.000     1.000       1.0
                 B-days_code      1.000     1.000     1.000       1.0
 I-arrive_time.period_of_day      0.000     0.000     0.000       0.0
            I-today_relative      0.000     0.000     0.000       0.0
 B-return_time.period_of_day      0.000     0.000     0.000       0.0
                      B-time      0.000     0.000     0.000       0.0
           I-fare_basis_code      0.000     0.000     0.000       0.0
 I-arrive_time.time_relative      0.000     0.000     0.000       4.0
 I-depart_time.time_relative      0.000     0.000     0.000       1.0
            B-today_relative      0.000     0.000     0.000       0.0
                B-state_name      0.000     0.000     0.000       9.0
B-arrive_date.today_relative      0.000     0.000     0.000       0.0
    B-return_time.period_mod      0.000     0.000     0.000       0.0
                B-month_name      0.000     0.000     0.000       0.0
                B-day_number      0.000     0.000     0.000       0.0
 I-return_date.date_relative      0.750     1.000     0.857       3.0
I-return_date.today_relative      0.000     0.000     0.000       0.0
      B-stoploc.airport_name      0.000     0.000     0.000       0.0
             B-time_relative      0.000     0.000     0.000       0.0
                      I-time      0.000     0.000     0.000       0.0
    I-return_date.day_number      0.000     0.000     0.000       0.0
          I-meal_description      0.000     0.000     0.000       0.0
B-return_date.today_relative      0.000     0.000     0.000       0.0
      B-return_date.day_name      0.000     0.000     0.000       2.0

                   micro avg      0.952     0.953     0.952    3657.0
                   macro avg      0.677     0.650     0.646    3657.0
                weighted avg      0.958     0.953     0.950    3657.0

I0716 01:02:48.971376 140076727973760 interactiveshell.py:2882] Best Slot F1: 0.955
I0716 01:02:48.972994 140076727973760 estimator_training.py:186] Not using Distribute Coordinator.
I0716 01:02:48.974836 140076727973760 training.py:612] Running training and evaluation locally (non-distributed).
I0716 01:02:48.977099 140076727973760 training.py:700] Start train and evaluate loop. The evaluate will happen after every checkpoint. Checkpoint frequency is determined based on RunConfig arguments: save_checkpoints_steps 312 or save_checkpoints_secs None.
I0716 01:02:49.045064 140076727973760 estimator.py:1145] Calling model_fn.
I0716 01:02:50.341112 140076727973760 <ipython-input-6-0a85a3fd693f>:50] 
[<tf.Variable 'embedding:0' shape=(750, 300) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/gates/kernel:0' shape=(600, 600) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/gates/bias:0' shape=(600,) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/candidate/kernel:0' shape=(600, 300) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/candidate/bias:0' shape=(300,) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/gates/kernel:0' shape=(600, 600) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/gates/bias:0' shape=(600,) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/candidate/kernel:0' shape=(600, 300) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/candidate/bias:0' shape=(300,) dtype=float32_ref>,
 <tf.Variable 'dense/kernel:0' shape=(600, 23) dtype=float32_ref>,
 <tf.Variable 'dense/bias:0' shape=(23,) dtype=float32_ref>,
 <tf.Variable 'dense_1/kernel:0' shape=(600, 122) dtype=float32_ref>,
 <tf.Variable 'dense_1/bias:0' shape=(122,) dtype=float32_ref>,
 <tf.Variable 'transitions:0' shape=(122, 122) dtype=float32_ref>]
I0716 01:02:51.878337 140076727973760 estimator.py:1147] Done calling model_fn.
I0716 01:02:51.882407 140076727973760 basic_session_run_hooks.py:541] Create CheckpointSaverHook.
I0716 01:02:52.090596 140076727973760 monitored_session.py:240] Graph was finalized.
I0716 01:02:52.107488 140076727973760 saver.py:1280] Restoring parameters from ../model/bigru_crf/model.ckpt-5616
I0716 01:02:52.367353 140076727973760 session_manager.py:500] Running local_init_op.
I0716 01:02:52.406087 140076727973760 session_manager.py:502] Done running local_init_op.
I0716 01:02:53.931659 140076727973760 basic_session_run_hooks.py:606] Saving checkpoints for 5616 into ../model/bigru_crf/model.ckpt.
Reading ../data/atis.train.w-intent.iob
I0716 01:02:56.237925 140076727973760 basic_session_run_hooks.py:262] loss = 0.026439372, step = 5616
I0716 01:02:56.239771 140076727973760 basic_session_run_hooks.py:262] lr = 0.00016601493
I0716 01:03:08.023643 140076727973760 basic_session_run_hooks.py:692] global_step/sec: 8.48437
I0716 01:03:08.027216 140076727973760 basic_session_run_hooks.py:260] loss = 0.05999034, step = 5716 (11.789 sec)
I0716 01:03:08.031708 140076727973760 basic_session_run_hooks.py:260] lr = 0.00016427496 (11.792 sec)
I0716 01:03:18.887612 140076727973760 basic_session_run_hooks.py:692] global_step/sec: 9.20475
I0716 01:03:18.894004 140076727973760 basic_session_run_hooks.py:260] loss = 0.033993505, step = 5816 (10.867 sec)
I0716 01:03:18.896204 140076727973760 basic_session_run_hooks.py:260] lr = 0.00016255325 (10.865 sec)
I0716 01:03:29.570792 140076727973760 basic_session_run_hooks.py:692] global_step/sec: 9.36048
I0716 01:03:29.578460 140076727973760 basic_session_run_hooks.py:260] loss = 0.014479409, step = 5916 (10.684 sec)
I0716 01:03:29.580967 140076727973760 basic_session_run_hooks.py:260] lr = 0.00016084957 (10.685 sec)
I0716 01:03:30.664566 140076727973760 basic_session_run_hooks.py:606] Saving checkpoints for 5928 into ../model/bigru_crf/model.ckpt.
I0716 01:03:31.062140 140076727973760 estimator.py:1145] Calling model_fn.
I0716 01:03:32.242047 140076727973760 estimator.py:1147] Done calling model_fn.
I0716 01:03:32.273462 140076727973760 evaluation.py:255] Starting evaluation at 2019-07-16T01:03:32Z
I0716 01:03:32.391228 140076727973760 monitored_session.py:240] Graph was finalized.
I0716 01:03:32.415827 140076727973760 saver.py:1280] Restoring parameters from ../model/bigru_crf/model.ckpt-5928
I0716 01:03:32.535620 140076727973760 session_manager.py:500] Running local_init_op.
I0716 01:03:32.562114 140076727973760 session_manager.py:502] Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
I0716 01:03:34.852986 140076727973760 evaluation.py:275] Finished evaluation at 2019-07-16-01:03:34
I0716 01:03:34.854787 140076727973760 estimator.py:2039] Saving dict for global step 5928: global_step = 5928, loss = 3.983097
I0716 01:03:34.862925 140076727973760 estimator.py:2099] Saving 'checkpoint_path' summary for global step 5928: ../model/bigru_crf/model.ckpt-5928
I0716 01:03:34.922513 140076727973760 estimator.py:368] Loss for final step: 0.08415053.
Reading ../data/atis.test.w-intent.iob
I0716 01:03:35.259230 140076727973760 estimator.py:1145] Calling model_fn.
I0716 01:03:35.912316 140076727973760 estimator.py:1147] Done calling model_fn.
I0716 01:03:36.024791 140076727973760 monitored_session.py:240] Graph was finalized.
I0716 01:03:36.042314 140076727973760 saver.py:1280] Restoring parameters from ../model/bigru_crf/model.ckpt-5928
I0716 01:03:36.143950 140076727973760 session_manager.py:500] Running local_init_op.
I0716 01:03:36.165206 140076727973760 session_manager.py:502] Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.
  'precision', 'predicted', average, warn_for)
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1439: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples.
  'recall', 'true', average, warn_for)
I0716 01:03:39.404024 140076727973760 interactiveshell.py:2882] 
                                          precision    recall  f1-score   support

                             atis_flight      0.977     0.992     0.984       632
                            atis_airfare      0.960     1.000     0.980        48
                     atis_ground_service      1.000     1.000     1.000        36
                            atis_airline      1.000     1.000     1.000        38
                       atis_abbreviation      0.943     1.000     0.971        33
                           atis_aircraft      1.000     1.000     1.000         9
                        atis_flight_time      1.000     1.000     1.000         1
                           atis_quantity      0.375     1.000     0.545         3
                atis_flight#atis_airfare      0.750     0.250     0.375        12
                            atis_airport      1.000     0.944     0.971        18
                           atis_distance      1.000     1.000     1.000        10
                               atis_city      1.000     0.500     0.667         6
                        atis_ground_fare      1.000     0.857     0.923         7
                           atis_capacity      1.000     0.952     0.976        21
                          atis_flight_no      0.889     1.000     0.941         8
                               atis_meal      1.000     0.833     0.909         6
                        atis_restriction      0.000     0.000     0.000         0
             atis_airline#atis_flight_no      0.000     0.000     0.000         0
    atis_ground_service#atis_ground_fare      0.000     0.000     0.000         0
           atis_airfare#atis_flight_time      0.000     0.000     0.000         0
                           atis_cheapest      0.000     0.000     0.000         0
atis_aircraft#atis_flight#atis_flight_no      0.000     0.000     0.000         0

                               micro avg      0.971     0.976     0.974       888
                               macro avg      0.677     0.651     0.647       888
                            weighted avg      0.973     0.976     0.971       888

I0716 01:03:39.454313 140076727973760 interactiveshell.py:2882] 
                              precision    recall  f1-score   support

                           O      0.000     0.000     0.000       0.0
           B-toloc.city_name      0.975     0.996     0.985     716.0
         B-fromloc.city_name      0.985     0.997     0.991     704.0
           I-toloc.city_name      0.974     0.996     0.985     265.0
      B-depart_date.day_name      0.991     0.991     0.991     212.0
              B-airline_name      1.000     1.000     1.000     101.0
         I-fromloc.city_name      0.983     0.989     0.986     177.0
 B-depart_time.period_of_day      1.000     0.915     0.956     130.0
              I-airline_name      1.000     1.000     1.000      65.0
    B-depart_date.day_number      0.981     0.964     0.972      55.0
    B-depart_date.month_name      0.982     0.964     0.973      56.0
          B-depart_time.time      0.862     0.982     0.918      57.0
                B-round_trip      1.000     0.973     0.986      73.0
             B-cost_relative      1.000     0.973     0.986      37.0
                I-round_trip      1.000     1.000     1.000      71.0
                B-flight_mod      0.960     1.000     0.980      24.0
 B-depart_time.time_relative      0.969     0.954     0.961      65.0
          I-depart_time.time      0.944     0.981     0.962      52.0
         B-stoploc.city_name      0.870     1.000     0.930      20.0
                 B-city_name      0.917     0.579     0.710      57.0
                B-class_type      0.960     1.000     0.980      24.0
          B-arrive_time.time      0.971     0.971     0.971      34.0
 B-arrive_time.time_relative      0.906     0.935     0.921      31.0
                I-class_type      1.000     1.000     1.000      17.0
               B-flight_stop      1.000     1.000     1.000      21.0
          I-arrive_time.time      1.000     0.971     0.986      35.0
              B-airline_code      1.000     0.971     0.985      34.0
    I-depart_date.day_number      1.000     0.933     0.966      15.0
      I-fromloc.airport_name      0.441     1.000     0.612      15.0
      B-fromloc.airport_name      0.462     1.000     0.632      12.0
      B-arrive_date.day_name      0.786     1.000     0.880      11.0
          B-toloc.state_code      0.947     1.000     0.973      18.0
B-depart_date.today_relative      1.000     0.889     0.941       9.0
             B-flight_number      0.611     1.000     0.759      11.0
 B-depart_date.date_relative      0.944     1.000     0.971      17.0
          B-toloc.state_name      0.781     0.893     0.833      28.0
           B-fare_basis_code      0.944     1.000     0.971      17.0
               B-flight_time      1.000     1.000     1.000       1.0
                        B-or      1.000     1.000     1.000       3.0
 B-arrive_time.period_of_day      0.857     1.000     0.923       6.0
          B-meal_description      1.000     0.900     0.947      10.0
             I-cost_relative      1.000     0.667     0.800       3.0
              I-airport_name      0.923     0.414     0.571      29.0
               B-fare_amount      1.000     1.000     1.000       2.0
               I-fare_amount      1.000     1.000     1.000       2.0
                 I-city_name      1.000     0.600     0.750      30.0
        I-toloc.airport_name      1.000     1.000     1.000       3.0
            B-transport_type      1.000     1.000     1.000      10.0
    B-arrive_date.month_name      0.714     0.833     0.769       6.0
    B-arrive_date.day_number      0.714     0.833     0.769       6.0
         I-stoploc.city_name      0.833     1.000     0.909      10.0
                      B-meal      1.000     1.000     1.000      16.0
        B-fromloc.state_code      1.000     0.957     0.978      23.0
    B-depart_time.period_mod      0.833     1.000     0.909       5.0
                   B-connect      1.000     1.000     1.000       6.0
               B-flight_days      1.000     1.000     1.000      10.0
        B-toloc.airport_name      1.000     1.000     1.000       3.0
        B-fromloc.state_name      0.944     1.000     0.971      17.0
              B-airport_name      0.818     0.429     0.562      21.0
                   B-economy      1.000     1.000     1.000       6.0
               I-flight_time      1.000     1.000     1.000       1.0
             B-aircraft_code      0.964     0.818     0.885      33.0
                       B-mod      1.000     0.500     0.667       2.0
              B-airport_code      0.500     0.111     0.182       9.0
    B-depart_time.start_time      1.000     0.667     0.800       3.0
      B-depart_time.end_time      1.000     0.333     0.500       3.0
          B-depart_date.year      1.000     1.000     1.000       3.0
            I-transport_type      0.000     0.000     0.000       1.0
          B-restriction_code      0.444     1.000     0.615       4.0
    B-arrive_time.start_time      0.889     1.000     0.941       8.0
        B-toloc.airport_code      1.000     0.750     0.857       4.0
      B-arrive_time.end_time      0.889     1.000     0.941       8.0
      I-arrive_time.end_time      0.889     1.000     0.941       8.0
      I-depart_time.end_time      1.000     0.333     0.500       3.0
               I-flight_stop      0.000     0.000     0.000       0.0
      B-fromloc.airport_code      1.000     1.000     1.000       5.0
          I-restriction_code      1.000     1.000     1.000       3.0
    I-depart_time.start_time      1.000     1.000     1.000       1.0
          I-toloc.state_name      1.000     1.000     1.000       1.0
I-depart_date.today_relative      0.000     0.000     0.000       0.0
 B-arrive_date.date_relative      1.000     1.000     1.000       2.0
                I-flight_mod      1.000     0.167     0.286       6.0
                   I-economy      0.000     0.000     0.000       0.0
 B-return_date.date_relative      1.000     0.333     0.500       3.0
        I-fromloc.state_name      1.000     1.000     1.000       1.0
                B-state_code      1.000     1.000     1.000       1.0
    I-arrive_time.start_time      1.000     1.000     1.000       1.0
    I-arrive_date.day_number      0.000     0.000     0.000       0.0
                 B-meal_code      0.000     0.000     0.000       1.0
 I-depart_time.period_of_day      1.000     1.000     1.000       1.0
                  B-day_name      1.000     0.500     0.667       2.0
             B-period_of_day      1.000     1.000     1.000       4.0
        B-stoploc.state_code      0.000     0.000     0.000       0.0
    B-return_date.month_name      0.000     0.000     0.000       0.0
    B-return_date.day_number      0.000     0.000     0.000       0.0
    B-arrive_time.period_mod      0.000     0.000     0.000       0.0
                 I-meal_code      0.000     0.000     0.000       0.0
        B-toloc.country_name      1.000     1.000     1.000       1.0
                 B-days_code      1.000     1.000     1.000       1.0
 I-arrive_time.period_of_day      0.000     0.000     0.000       0.0
            I-today_relative      0.000     0.000     0.000       0.0
 B-return_time.period_of_day      0.000     0.000     0.000       0.0
                      B-time      0.000     0.000     0.000       0.0
           I-fare_basis_code      0.000     0.000     0.000       0.0
 I-arrive_time.time_relative      0.000     0.000     0.000       4.0
 I-depart_time.time_relative      0.000     0.000     0.000       1.0
            B-today_relative      0.000     0.000     0.000       0.0
                B-state_name      0.000     0.000     0.000       9.0
B-arrive_date.today_relative      0.000     0.000     0.000       0.0
    B-return_time.period_mod      0.000     0.000     0.000       0.0
                B-month_name      0.000     0.000     0.000       0.0
                B-day_number      0.000     0.000     0.000       0.0
 I-return_date.date_relative      0.750     1.000     0.857       3.0
I-return_date.today_relative      0.000     0.000     0.000       0.0
      B-stoploc.airport_name      0.000     0.000     0.000       0.0
             B-time_relative      0.000     0.000     0.000       0.0
                      I-time      0.000     0.000     0.000       0.0
    I-return_date.day_number      0.000     0.000     0.000       0.0
          I-meal_description      0.000     0.000     0.000       0.0
B-return_date.today_relative      0.000     0.000     0.000       0.0
      B-return_date.day_name      0.000     0.000     0.000       2.0

                   micro avg      0.955     0.956     0.955    3657.0
                   macro avg      0.679     0.653     0.650    3657.0
                weighted avg      0.961     0.956     0.954    3657.0

I0716 01:03:39.483549 140076727973760 interactiveshell.py:2882] Best Slot F1: 0.955
I0716 01:03:39.485133 140076727973760 estimator_training.py:186] Not using Distribute Coordinator.
I0716 01:03:39.487146 140076727973760 training.py:612] Running training and evaluation locally (non-distributed).
I0716 01:03:39.492234 140076727973760 training.py:700] Start train and evaluate loop. The evaluate will happen after every checkpoint. Checkpoint frequency is determined based on RunConfig arguments: save_checkpoints_steps 312 or save_checkpoints_secs None.
I0716 01:03:39.555081 140076727973760 estimator.py:1145] Calling model_fn.
I0716 01:03:40.882702 140076727973760 <ipython-input-6-0a85a3fd693f>:50] 
[<tf.Variable 'embedding:0' shape=(750, 300) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/gates/kernel:0' shape=(600, 600) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/gates/bias:0' shape=(600,) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/candidate/kernel:0' shape=(600, 300) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/candidate/bias:0' shape=(300,) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/gates/kernel:0' shape=(600, 600) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/gates/bias:0' shape=(600,) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/candidate/kernel:0' shape=(600, 300) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/candidate/bias:0' shape=(300,) dtype=float32_ref>,
 <tf.Variable 'dense/kernel:0' shape=(600, 23) dtype=float32_ref>,
 <tf.Variable 'dense/bias:0' shape=(23,) dtype=float32_ref>,
 <tf.Variable 'dense_1/kernel:0' shape=(600, 122) dtype=float32_ref>,
 <tf.Variable 'dense_1/bias:0' shape=(122,) dtype=float32_ref>,
 <tf.Variable 'transitions:0' shape=(122, 122) dtype=float32_ref>]
I0716 01:03:42.232350 140076727973760 estimator.py:1147] Done calling model_fn.
I0716 01:03:42.236456 140076727973760 basic_session_run_hooks.py:541] Create CheckpointSaverHook.
I0716 01:03:42.634238 140076727973760 monitored_session.py:240] Graph was finalized.
I0716 01:03:42.653472 140076727973760 saver.py:1280] Restoring parameters from ../model/bigru_crf/model.ckpt-5928
I0716 01:03:42.915189 140076727973760 session_manager.py:500] Running local_init_op.
I0716 01:03:42.953917 140076727973760 session_manager.py:502] Done running local_init_op.
I0716 01:03:44.462702 140076727973760 basic_session_run_hooks.py:606] Saving checkpoints for 5928 into ../model/bigru_crf/model.ckpt.
Reading ../data/atis.train.w-intent.iob
I0716 01:03:46.792754 140076727973760 basic_session_run_hooks.py:262] loss = 0.02714134, step = 5928
I0716 01:03:46.794650 140076727973760 basic_session_run_hooks.py:262] lr = 0.0001606463
I0716 01:03:58.851994 140076727973760 basic_session_run_hooks.py:692] global_step/sec: 8.29203
I0716 01:03:58.858697 140076727973760 basic_session_run_hooks.py:260] loss = 0.031558156, step = 6028 (12.066 sec)
I0716 01:03:58.861077 140076727973760 basic_session_run_hooks.py:260] lr = 0.00015896263 (12.066 sec)
I0716 01:04:09.311504 140076727973760 basic_session_run_hooks.py:692] global_step/sec: 9.56062
I0716 01:04:09.317739 140076727973760 basic_session_run_hooks.py:260] loss = 0.043938663, step = 6128 (10.459 sec)
I0716 01:04:09.319713 140076727973760 basic_session_run_hooks.py:260] lr = 0.00015729658 (10.459 sec)
I0716 01:04:20.074655 140076727973760 basic_session_run_hooks.py:692] global_step/sec: 9.29096
I0716 01:04:20.081131 140076727973760 basic_session_run_hooks.py:260] loss = 0.005411994, step = 6228 (10.763 sec)
I0716 01:04:20.082824 140076727973760 basic_session_run_hooks.py:260] lr = 0.00015564801 (10.763 sec)
I0716 01:04:21.373323 140076727973760 basic_session_run_hooks.py:606] Saving checkpoints for 6240 into ../model/bigru_crf/model.ckpt.
I0716 01:04:21.771438 140076727973760 estimator.py:1145] Calling model_fn.
I0716 01:04:22.745292 140076727973760 estimator.py:1147] Done calling model_fn.
I0716 01:04:22.774259 140076727973760 evaluation.py:255] Starting evaluation at 2019-07-16T01:04:22Z
I0716 01:04:23.118162 140076727973760 monitored_session.py:240] Graph was finalized.
I0716 01:04:23.137000 140076727973760 saver.py:1280] Restoring parameters from ../model/bigru_crf/model.ckpt-6240
I0716 01:04:23.256116 140076727973760 session_manager.py:500] Running local_init_op.
I0716 01:04:23.283829 140076727973760 session_manager.py:502] Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
I0716 01:04:25.624753 140076727973760 evaluation.py:275] Finished evaluation at 2019-07-16-01:04:25
I0716 01:04:25.626775 140076727973760 estimator.py:2039] Saving dict for global step 6240: global_step = 6240, loss = 4.040451
I0716 01:04:25.635725 140076727973760 estimator.py:2099] Saving 'checkpoint_path' summary for global step 6240: ../model/bigru_crf/model.ckpt-6240
I0716 01:04:25.702409 140076727973760 estimator.py:368] Loss for final step: 0.034968104.
Reading ../data/atis.test.w-intent.iob
I0716 01:04:26.029177 140076727973760 estimator.py:1145] Calling model_fn.
I0716 01:04:26.687433 140076727973760 estimator.py:1147] Done calling model_fn.
I0716 01:04:26.804757 140076727973760 monitored_session.py:240] Graph was finalized.
I0716 01:04:26.822222 140076727973760 saver.py:1280] Restoring parameters from ../model/bigru_crf/model.ckpt-6240
I0716 01:04:26.934127 140076727973760 session_manager.py:500] Running local_init_op.
I0716 01:04:26.956567 140076727973760 session_manager.py:502] Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.
  'precision', 'predicted', average, warn_for)
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1439: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples.
  'recall', 'true', average, warn_for)
I0716 01:04:30.142128 140076727973760 interactiveshell.py:2882] 
                                          precision    recall  f1-score   support

                             atis_flight      0.975     0.992     0.984       632
                            atis_airfare      0.959     0.979     0.969        48
                     atis_ground_service      1.000     1.000     1.000        36
                            atis_airline      1.000     1.000     1.000        38
                       atis_abbreviation      0.943     1.000     0.971        33
                           atis_aircraft      1.000     1.000     1.000         9
                        atis_flight_time      1.000     1.000     1.000         1
                           atis_quantity      0.375     1.000     0.545         3
                atis_flight#atis_airfare      0.750     0.250     0.375        12
                            atis_airport      1.000     0.944     0.971        18
                           atis_distance      1.000     1.000     1.000        10
                               atis_city      1.000     0.500     0.667         6
                        atis_ground_fare      1.000     0.857     0.923         7
                           atis_capacity      1.000     0.952     0.976        21
                          atis_flight_no      0.889     1.000     0.941         8
                               atis_meal      1.000     0.833     0.909         6
                        atis_restriction      0.000     0.000     0.000         0
             atis_airline#atis_flight_no      0.000     0.000     0.000         0
    atis_ground_service#atis_ground_fare      0.000     0.000     0.000         0
           atis_airfare#atis_flight_time      0.000     0.000     0.000         0
                           atis_cheapest      0.000     0.000     0.000         0
atis_aircraft#atis_flight#atis_flight_no      0.000     0.000     0.000         0

                               micro avg      0.970     0.975     0.972       888
                               macro avg      0.677     0.650     0.647       888
                            weighted avg      0.971     0.975     0.970       888

I0716 01:04:30.194278 140076727973760 interactiveshell.py:2882] 
                              precision    recall  f1-score   support

                           O      0.000     0.000     0.000       0.0
           B-toloc.city_name      0.975     0.994     0.985     716.0
         B-fromloc.city_name      0.985     0.997     0.991     704.0
           I-toloc.city_name      0.974     0.992     0.983     265.0
      B-depart_date.day_name      0.986     0.991     0.988     212.0
              B-airline_name      1.000     1.000     1.000     101.0
         I-fromloc.city_name      0.978     0.989     0.983     177.0
 B-depart_time.period_of_day      1.000     0.923     0.960     130.0
              I-airline_name      1.000     1.000     1.000      65.0
    B-depart_date.day_number      0.981     0.964     0.972      55.0
    B-depart_date.month_name      0.982     0.964     0.973      56.0
          B-depart_time.time      0.864     1.000     0.927      57.0
                B-round_trip      1.000     0.986     0.993      73.0
             B-cost_relative      1.000     0.973     0.986      37.0
                I-round_trip      1.000     1.000     1.000      71.0
                B-flight_mod      1.000     1.000     1.000      24.0
 B-depart_time.time_relative      0.969     0.954     0.961      65.0
          I-depart_time.time      0.945     1.000     0.972      52.0
         B-stoploc.city_name      0.870     1.000     0.930      20.0
                 B-city_name      0.919     0.596     0.723      57.0
                B-class_type      0.960     1.000     0.980      24.0
          B-arrive_time.time      0.971     0.971     0.971      34.0
 B-arrive_time.time_relative      0.906     0.935     0.921      31.0
                I-class_type      1.000     1.000     1.000      17.0
               B-flight_stop      1.000     1.000     1.000      21.0
          I-arrive_time.time      1.000     0.971     0.986      35.0
              B-airline_code      1.000     0.971     0.985      34.0
    I-depart_date.day_number      1.000     0.933     0.966      15.0
      I-fromloc.airport_name      0.441     1.000     0.612      15.0
      B-fromloc.airport_name      0.500     1.000     0.667      12.0
      B-arrive_date.day_name      0.846     1.000     0.917      11.0
          B-toloc.state_code      0.947     1.000     0.973      18.0
B-depart_date.today_relative      1.000     0.889     0.941       9.0
             B-flight_number      0.688     1.000     0.815      11.0
 B-depart_date.date_relative      0.944     1.000     0.971      17.0
          B-toloc.state_name      0.781     0.893     0.833      28.0
           B-fare_basis_code      0.944     1.000     0.971      17.0
               B-flight_time      1.000     1.000     1.000       1.0
                        B-or      1.000     1.000     1.000       3.0
 B-arrive_time.period_of_day      0.857     1.000     0.923       6.0
          B-meal_description      1.000     0.900     0.947      10.0
             I-cost_relative      1.000     0.667     0.800       3.0
              I-airport_name      0.929     0.448     0.605      29.0
               B-fare_amount      1.000     1.000     1.000       2.0
               I-fare_amount      1.000     1.000     1.000       2.0
                 I-city_name      1.000     0.600     0.750      30.0
        I-toloc.airport_name      1.000     1.000     1.000       3.0
            B-transport_type      1.000     1.000     1.000      10.0
    B-arrive_date.month_name      0.714     0.833     0.769       6.0
    B-arrive_date.day_number      0.714     0.833     0.769       6.0
         I-stoploc.city_name      0.833     1.000     0.909      10.0
                      B-meal      1.000     1.000     1.000      16.0
        B-fromloc.state_code      1.000     0.957     0.978      23.0
    B-depart_time.period_mod      1.000     1.000     1.000       5.0
                   B-connect      1.000     1.000     1.000       6.0
               B-flight_days      1.000     1.000     1.000      10.0
        B-toloc.airport_name      1.000     1.000     1.000       3.0
        B-fromloc.state_name      0.944     1.000     0.971      17.0
              B-airport_name      0.833     0.476     0.606      21.0
                   B-economy      1.000     1.000     1.000       6.0
               I-flight_time      1.000     1.000     1.000       1.0
             B-aircraft_code      0.935     0.879     0.906      33.0
                       B-mod      1.000     0.500     0.667       2.0
              B-airport_code      0.500     0.111     0.182       9.0
    B-depart_time.start_time      1.000     0.667     0.800       3.0
      B-depart_time.end_time      1.000     0.333     0.500       3.0
          B-depart_date.year      1.000     1.000     1.000       3.0
            I-transport_type      0.000     0.000     0.000       1.0
          B-restriction_code      0.400     1.000     0.571       4.0
    B-arrive_time.start_time      0.889     1.000     0.941       8.0
        B-toloc.airport_code      1.000     0.750     0.857       4.0
      B-arrive_time.end_time      0.889     1.000     0.941       8.0
      I-arrive_time.end_time      0.889     1.000     0.941       8.0
      I-depart_time.end_time      1.000     0.333     0.500       3.0
               I-flight_stop      0.000     0.000     0.000       0.0
      B-fromloc.airport_code      1.000     1.000     1.000       5.0
          I-restriction_code      1.000     1.000     1.000       3.0
    I-depart_time.start_time      1.000     1.000     1.000       1.0
          I-toloc.state_name      1.000     1.000     1.000       1.0
I-depart_date.today_relative      0.000     0.000     0.000       0.0
 B-arrive_date.date_relative      1.000     1.000     1.000       2.0
                I-flight_mod      1.000     0.167     0.286       6.0
                   I-economy      0.000     0.000     0.000       0.0
 B-return_date.date_relative      1.000     0.333     0.500       3.0
        I-fromloc.state_name      1.000     1.000     1.000       1.0
                B-state_code      1.000     1.000     1.000       1.0
    I-arrive_time.start_time      1.000     1.000     1.000       1.0
    I-arrive_date.day_number      0.000     0.000     0.000       0.0
                 B-meal_code      0.000     0.000     0.000       1.0
 I-depart_time.period_of_day      1.000     1.000     1.000       1.0
                  B-day_name      1.000     0.500     0.667       2.0
             B-period_of_day      1.000     1.000     1.000       4.0
        B-stoploc.state_code      0.000     0.000     0.000       0.0
    B-return_date.month_name      0.000     0.000     0.000       0.0
    B-return_date.day_number      0.000     0.000     0.000       0.0
    B-arrive_time.period_mod      0.000     0.000     0.000       0.0
                 I-meal_code      0.000     0.000     0.000       0.0
        B-toloc.country_name      1.000     1.000     1.000       1.0
                 B-days_code      1.000     1.000     1.000       1.0
 I-arrive_time.period_of_day      0.000     0.000     0.000       0.0
            I-today_relative      0.000     0.000     0.000       0.0
 B-return_time.period_of_day      0.000     0.000     0.000       0.0
                      B-time      0.000     0.000     0.000       0.0
           I-fare_basis_code      0.000     0.000     0.000       0.0
 I-arrive_time.time_relative      0.000     0.000     0.000       4.0
 I-depart_time.time_relative      0.000     0.000     0.000       1.0
            B-today_relative      0.000     0.000     0.000       0.0
                B-state_name      0.000     0.000     0.000       9.0
B-arrive_date.today_relative      0.000     0.000     0.000       0.0
    B-return_time.period_mod      0.000     0.000     0.000       0.0
                B-month_name      0.000     0.000     0.000       0.0
                B-day_number      0.000     0.000     0.000       0.0
 I-return_date.date_relative      0.750     1.000     0.857       3.0
I-return_date.today_relative      0.000     0.000     0.000       0.0
      B-stoploc.airport_name      0.000     0.000     0.000       0.0
             B-time_relative      0.000     0.000     0.000       0.0
                      I-time      0.000     0.000     0.000       0.0
    I-return_date.day_number      0.000     0.000     0.000       0.0
          I-meal_description      0.000     0.000     0.000       0.0
B-return_date.today_relative      0.000     0.000     0.000       0.0
      B-return_date.day_name      0.000     0.000     0.000       2.0

                   micro avg      0.957     0.958     0.957    3657.0
                   macro avg      0.681     0.654     0.653    3657.0
                weighted avg      0.962     0.958     0.955    3657.0

I0716 01:04:30.223034 140076727973760 interactiveshell.py:2882] Best Slot F1: 0.957
I0716 01:04:30.224660 140076727973760 estimator_training.py:186] Not using Distribute Coordinator.
I0716 01:04:30.228838 140076727973760 training.py:612] Running training and evaluation locally (non-distributed).
I0716 01:04:30.232335 140076727973760 training.py:700] Start train and evaluate loop. The evaluate will happen after every checkpoint. Checkpoint frequency is determined based on RunConfig arguments: save_checkpoints_steps 312 or save_checkpoints_secs None.
I0716 01:04:30.302150 140076727973760 estimator.py:1145] Calling model_fn.
I0716 01:04:31.275996 140076727973760 <ipython-input-6-0a85a3fd693f>:50] 
[<tf.Variable 'embedding:0' shape=(750, 300) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/gates/kernel:0' shape=(600, 600) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/gates/bias:0' shape=(600,) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/candidate/kernel:0' shape=(600, 300) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/candidate/bias:0' shape=(300,) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/gates/kernel:0' shape=(600, 600) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/gates/bias:0' shape=(600,) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/candidate/kernel:0' shape=(600, 300) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/candidate/bias:0' shape=(300,) dtype=float32_ref>,
 <tf.Variable 'dense/kernel:0' shape=(600, 23) dtype=float32_ref>,
 <tf.Variable 'dense/bias:0' shape=(23,) dtype=float32_ref>,
 <tf.Variable 'dense_1/kernel:0' shape=(600, 122) dtype=float32_ref>,
 <tf.Variable 'dense_1/bias:0' shape=(122,) dtype=float32_ref>,
 <tf.Variable 'transitions:0' shape=(122, 122) dtype=float32_ref>]
I0716 01:04:33.008837 140076727973760 estimator.py:1147] Done calling model_fn.
I0716 01:04:33.012751 140076727973760 basic_session_run_hooks.py:541] Create CheckpointSaverHook.
I0716 01:04:33.211349 140076727973760 monitored_session.py:240] Graph was finalized.
I0716 01:04:33.230817 140076727973760 saver.py:1280] Restoring parameters from ../model/bigru_crf/model.ckpt-6240
I0716 01:04:33.490716 140076727973760 session_manager.py:500] Running local_init_op.
I0716 01:04:33.536263 140076727973760 session_manager.py:502] Done running local_init_op.
I0716 01:04:35.267706 140076727973760 basic_session_run_hooks.py:606] Saving checkpoints for 6240 into ../model/bigru_crf/model.ckpt.
Reading ../data/atis.train.w-intent.iob
I0716 01:04:37.611247 140076727973760 basic_session_run_hooks.py:262] loss = 0.031662557, step = 6240
I0716 01:04:37.613147 140076727973760 basic_session_run_hooks.py:262] lr = 0.00015545134
I0716 01:04:49.298227 140076727973760 basic_session_run_hooks.py:692] global_step/sec: 8.55609
I0716 01:04:49.302507 140076727973760 basic_session_run_hooks.py:260] loss = 0.012781637, step = 6340 (11.691 sec)
I0716 01:04:49.307132 140076727973760 basic_session_run_hooks.py:260] lr = 0.00015382208 (11.694 sec)
I0716 01:05:00.724810 140076727973760 basic_session_run_hooks.py:692] global_step/sec: 8.75148
I0716 01:05:00.731925 140076727973760 basic_session_run_hooks.py:260] loss = 0.014898315, step = 6440 (11.429 sec)
I0716 01:05:00.734669 140076727973760 basic_session_run_hooks.py:260] lr = 0.00015220992 (11.428 sec)
I0716 01:05:11.655709 140076727973760 basic_session_run_hooks.py:692] global_step/sec: 9.14838
I0716 01:05:11.663327 140076727973760 basic_session_run_hooks.py:260] loss = 0.013426252, step = 6540 (10.931 sec)
I0716 01:05:11.665699 140076727973760 basic_session_run_hooks.py:260] lr = 0.00015061465 (10.931 sec)
I0716 01:05:12.711960 140076727973760 basic_session_run_hooks.py:606] Saving checkpoints for 6552 into ../model/bigru_crf/model.ckpt.
I0716 01:05:13.110732 140076727973760 estimator.py:1145] Calling model_fn.
I0716 01:05:14.069994 140076727973760 estimator.py:1147] Done calling model_fn.
I0716 01:05:14.098985 140076727973760 evaluation.py:255] Starting evaluation at 2019-07-16T01:05:14Z
I0716 01:05:14.214880 140076727973760 monitored_session.py:240] Graph was finalized.
I0716 01:05:14.232043 140076727973760 saver.py:1280] Restoring parameters from ../model/bigru_crf/model.ckpt-6552
I0716 01:05:14.350061 140076727973760 session_manager.py:500] Running local_init_op.
I0716 01:05:14.376011 140076727973760 session_manager.py:502] Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
I0716 01:05:16.670392 140076727973760 evaluation.py:275] Finished evaluation at 2019-07-16-01:05:16
I0716 01:05:16.672097 140076727973760 estimator.py:2039] Saving dict for global step 6552: global_step = 6552, loss = 4.2534285
I0716 01:05:16.678471 140076727973760 estimator.py:2099] Saving 'checkpoint_path' summary for global step 6552: ../model/bigru_crf/model.ckpt-6552
I0716 01:05:16.746832 140076727973760 estimator.py:368] Loss for final step: 0.32965636.
Reading ../data/atis.test.w-intent.iob
I0716 01:05:17.371148 140076727973760 estimator.py:1145] Calling model_fn.
I0716 01:05:18.036510 140076727973760 estimator.py:1147] Done calling model_fn.
I0716 01:05:18.152105 140076727973760 monitored_session.py:240] Graph was finalized.
I0716 01:05:18.169236 140076727973760 saver.py:1280] Restoring parameters from ../model/bigru_crf/model.ckpt-6552
I0716 01:05:18.275037 140076727973760 session_manager.py:500] Running local_init_op.
I0716 01:05:18.293770 140076727973760 session_manager.py:502] Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.
  'precision', 'predicted', average, warn_for)
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1439: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples.
  'recall', 'true', average, warn_for)
I0716 01:05:21.352537 140076727973760 interactiveshell.py:2882] 
                                          precision    recall  f1-score   support

                             atis_flight      0.975     0.992     0.984       632
                            atis_airfare      0.959     0.979     0.969        48
                     atis_ground_service      0.973     1.000     0.986        36
                            atis_airline      1.000     1.000     1.000        38
                       atis_abbreviation      0.943     1.000     0.971        33
                           atis_aircraft      1.000     1.000     1.000         9
                        atis_flight_time      1.000     1.000     1.000         1
                           atis_quantity      0.375     1.000     0.545         3
                atis_flight#atis_airfare      0.750     0.250     0.375        12
                            atis_airport      1.000     0.889     0.941        18
                           atis_distance      1.000     1.000     1.000        10
                               atis_city      1.000     0.500     0.667         6
                        atis_ground_fare      1.000     0.857     0.923         7
                           atis_capacity      1.000     0.952     0.976        21
                          atis_flight_no      0.889     1.000     0.941         8
                               atis_meal      1.000     0.833     0.909         6
                        atis_restriction      0.000     0.000     0.000         0
             atis_airline#atis_flight_no      0.000     0.000     0.000         0
    atis_ground_service#atis_ground_fare      0.000     0.000     0.000         0
           atis_airfare#atis_flight_time      0.000     0.000     0.000         0
                           atis_cheapest      0.000     0.000     0.000         0
atis_aircraft#atis_flight#atis_flight_no      0.000     0.000     0.000         0

                               micro avg      0.969     0.974     0.971       888
                               macro avg      0.676     0.648     0.645       888
                            weighted avg      0.970     0.974     0.969       888

I0716 01:05:21.405068 140076727973760 interactiveshell.py:2882] 
                              precision    recall  f1-score   support

                           O      0.000     0.000     0.000       0.0
           B-toloc.city_name      0.977     0.990     0.983     716.0
         B-fromloc.city_name      0.972     0.997     0.985     704.0
           I-toloc.city_name      0.981     0.985     0.983     265.0
      B-depart_date.day_name      0.991     0.991     0.991     212.0
              B-airline_name      1.000     1.000     1.000     101.0
         I-fromloc.city_name      0.962     0.994     0.978     177.0
 B-depart_time.period_of_day      0.992     0.923     0.956     130.0
              I-airline_name      1.000     1.000     1.000      65.0
    B-depart_date.day_number      0.946     0.964     0.955      55.0
    B-depart_date.month_name      0.982     0.964     0.973      56.0
          B-depart_time.time      0.851     1.000     0.919      57.0
                B-round_trip      1.000     0.973     0.986      73.0
             B-cost_relative      1.000     0.973     0.986      37.0
                I-round_trip      1.000     1.000     1.000      71.0
                B-flight_mod      1.000     0.958     0.979      24.0
 B-depart_time.time_relative      0.969     0.954     0.961      65.0
          I-depart_time.time      0.944     0.981     0.962      52.0
         B-stoploc.city_name      0.909     1.000     0.952      20.0
                 B-city_name      0.914     0.561     0.696      57.0
                B-class_type      0.960     1.000     0.980      24.0
          B-arrive_time.time      1.000     0.912     0.954      34.0
 B-arrive_time.time_relative      1.000     0.935     0.967      31.0
                I-class_type      1.000     1.000     1.000      17.0
               B-flight_stop      1.000     1.000     1.000      21.0
          I-arrive_time.time      1.000     0.914     0.955      35.0
              B-airline_code      1.000     0.971     0.985      34.0
    I-depart_date.day_number      1.000     1.000     1.000      15.0
      I-fromloc.airport_name      0.429     1.000     0.600      15.0
      B-fromloc.airport_name      0.444     1.000     0.615      12.0
      B-arrive_date.day_name      0.786     1.000     0.880      11.0
          B-toloc.state_code      0.947     1.000     0.973      18.0
B-depart_date.today_relative      1.000     0.889     0.941       9.0
             B-flight_number      0.647     1.000     0.786      11.0
 B-depart_date.date_relative      0.895     1.000     0.944      17.0
          B-toloc.state_name      0.781     0.893     0.833      28.0
           B-fare_basis_code      0.944     1.000     0.971      17.0
               B-flight_time      1.000     1.000     1.000       1.0
                        B-or      1.000     1.000     1.000       3.0
 B-arrive_time.period_of_day      0.750     1.000     0.857       6.0
          B-meal_description      1.000     0.900     0.947      10.0
             I-cost_relative      1.000     0.667     0.800       3.0
              I-airport_name      0.917     0.379     0.537      29.0
               B-fare_amount      1.000     1.000     1.000       2.0
               I-fare_amount      1.000     1.000     1.000       2.0
                 I-city_name      1.000     0.600     0.750      30.0
        I-toloc.airport_name      1.000     1.000     1.000       3.0
            B-transport_type      1.000     1.000     1.000      10.0
    B-arrive_date.month_name      0.714     0.833     0.769       6.0
    B-arrive_date.day_number      0.714     0.833     0.769       6.0
         I-stoploc.city_name      0.909     1.000     0.952      10.0
                      B-meal      1.000     1.000     1.000      16.0
        B-fromloc.state_code      1.000     0.957     0.978      23.0
    B-depart_time.period_mod      0.833     1.000     0.909       5.0
                   B-connect      1.000     1.000     1.000       6.0
               B-flight_days      1.000     1.000     1.000      10.0
        B-toloc.airport_name      1.000     1.000     1.000       3.0
        B-fromloc.state_name      0.944     1.000     0.971      17.0
              B-airport_name      0.800     0.381     0.516      21.0
                   B-economy      1.000     1.000     1.000       6.0
               I-flight_time      1.000     1.000     1.000       1.0
             B-aircraft_code      1.000     0.818     0.900      33.0
                       B-mod      1.000     0.500     0.667       2.0
              B-airport_code      0.500     0.111     0.182       9.0
    B-depart_time.start_time      1.000     0.667     0.800       3.0
      B-depart_time.end_time      1.000     0.333     0.500       3.0
          B-depart_date.year      1.000     1.000     1.000       3.0
            I-transport_type      0.000     0.000     0.000       1.0
          B-restriction_code      0.400     1.000     0.571       4.0
    B-arrive_time.start_time      0.889     1.000     0.941       8.0
        B-toloc.airport_code      1.000     0.750     0.857       4.0
      B-arrive_time.end_time      0.889     1.000     0.941       8.0
      I-arrive_time.end_time      0.889     1.000     0.941       8.0
      I-depart_time.end_time      1.000     0.333     0.500       3.0
               I-flight_stop      0.000     0.000     0.000       0.0
      B-fromloc.airport_code      1.000     1.000     1.000       5.0
          I-restriction_code      1.000     1.000     1.000       3.0
    I-depart_time.start_time      1.000     1.000     1.000       1.0
          I-toloc.state_name      1.000     1.000     1.000       1.0
I-depart_date.today_relative      0.000     0.000     0.000       0.0
 B-arrive_date.date_relative      1.000     1.000     1.000       2.0
                I-flight_mod      1.000     0.167     0.286       6.0
                   I-economy      0.000     0.000     0.000       0.0
 B-return_date.date_relative      1.000     0.333     0.500       3.0
        I-fromloc.state_name      1.000     1.000     1.000       1.0
                B-state_code      1.000     1.000     1.000       1.0
    I-arrive_time.start_time      1.000     1.000     1.000       1.0
    I-arrive_date.day_number      0.000     0.000     0.000       0.0
                 B-meal_code      0.000     0.000     0.000       1.0
 I-depart_time.period_of_day      1.000     1.000     1.000       1.0
                  B-day_name      1.000     0.500     0.667       2.0
             B-period_of_day      1.000     0.500     0.667       4.0
        B-stoploc.state_code      0.000     0.000     0.000       0.0
    B-return_date.month_name      0.000     0.000     0.000       0.0
    B-return_date.day_number      0.000     0.000     0.000       0.0
    B-arrive_time.period_mod      0.000     0.000     0.000       0.0
                 I-meal_code      0.000     0.000     0.000       0.0
        B-toloc.country_name      1.000     1.000     1.000       1.0
                 B-days_code      1.000     1.000     1.000       1.0
 I-arrive_time.period_of_day      0.000     0.000     0.000       0.0
            I-today_relative      0.000     0.000     0.000       0.0
 B-return_time.period_of_day      0.000     0.000     0.000       0.0
                      B-time      0.000     0.000     0.000       0.0
           I-fare_basis_code      0.000     0.000     0.000       0.0
 I-arrive_time.time_relative      1.000     0.500     0.667       4.0
 I-depart_time.time_relative      0.000     0.000     0.000       1.0
            B-today_relative      0.000     0.000     0.000       0.0
                B-state_name      0.000     0.000     0.000       9.0
B-arrive_date.today_relative      0.000     0.000     0.000       0.0
    B-return_time.period_mod      0.000     0.000     0.000       0.0
                B-month_name      0.000     0.000     0.000       0.0
                B-day_number      0.000     0.000     0.000       0.0
 I-return_date.date_relative      0.750     1.000     0.857       3.0
I-return_date.today_relative      0.000     0.000     0.000       0.0
      B-stoploc.airport_name      0.000     0.000     0.000       0.0
             B-time_relative      0.000     0.000     0.000       0.0
                      I-time      0.000     0.000     0.000       0.0
    I-return_date.day_number      0.000     0.000     0.000       0.0
          I-meal_description      0.000     0.000     0.000       0.0
B-return_date.today_relative      0.000     0.000     0.000       0.0
      B-return_date.day_name      0.000     0.000     0.000       2.0

                   micro avg      0.952     0.953     0.952    3657.0
                   macro avg      0.687     0.651     0.651    3657.0
                weighted avg      0.960     0.953     0.951    3657.0

I0716 01:05:21.435657 140076727973760 interactiveshell.py:2882] Best Slot F1: 0.957
I0716 01:05:21.437177 140076727973760 estimator_training.py:186] Not using Distribute Coordinator.
I0716 01:05:21.439192 140076727973760 training.py:612] Running training and evaluation locally (non-distributed).
I0716 01:05:21.444603 140076727973760 training.py:700] Start train and evaluate loop. The evaluate will happen after every checkpoint. Checkpoint frequency is determined based on RunConfig arguments: save_checkpoints_steps 312 or save_checkpoints_secs None.
I0716 01:05:21.508128 140076727973760 estimator.py:1145] Calling model_fn.
I0716 01:05:22.700596 140076727973760 <ipython-input-6-0a85a3fd693f>:50] 
[<tf.Variable 'embedding:0' shape=(750, 300) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/gates/kernel:0' shape=(600, 600) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/gates/bias:0' shape=(600,) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/candidate/kernel:0' shape=(600, 300) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/candidate/bias:0' shape=(300,) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/gates/kernel:0' shape=(600, 600) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/gates/bias:0' shape=(600,) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/candidate/kernel:0' shape=(600, 300) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/candidate/bias:0' shape=(300,) dtype=float32_ref>,
 <tf.Variable 'dense/kernel:0' shape=(600, 23) dtype=float32_ref>,
 <tf.Variable 'dense/bias:0' shape=(23,) dtype=float32_ref>,
 <tf.Variable 'dense_1/kernel:0' shape=(600, 122) dtype=float32_ref>,
 <tf.Variable 'dense_1/bias:0' shape=(122,) dtype=float32_ref>,
 <tf.Variable 'transitions:0' shape=(122, 122) dtype=float32_ref>]
I0716 01:05:24.242048 140076727973760 estimator.py:1147] Done calling model_fn.
I0716 01:05:24.245666 140076727973760 basic_session_run_hooks.py:541] Create CheckpointSaverHook.
I0716 01:05:24.445375 140076727973760 monitored_session.py:240] Graph was finalized.
I0716 01:05:24.464034 140076727973760 saver.py:1280] Restoring parameters from ../model/bigru_crf/model.ckpt-6552
I0716 01:05:24.720573 140076727973760 session_manager.py:500] Running local_init_op.
I0716 01:05:24.756985 140076727973760 session_manager.py:502] Done running local_init_op.
I0716 01:05:26.251267 140076727973760 basic_session_run_hooks.py:606] Saving checkpoints for 6552 into ../model/bigru_crf/model.ckpt.
Reading ../data/atis.train.w-intent.iob
I0716 01:05:28.590522 140076727973760 basic_session_run_hooks.py:262] loss = 0.015382597, step = 6552
I0716 01:05:28.592610 140076727973760 basic_session_run_hooks.py:262] lr = 0.00015042433
I0716 01:05:40.191093 140076727973760 basic_session_run_hooks.py:692] global_step/sec: 8.61996
I0716 01:05:40.195444 140076727973760 basic_session_run_hooks.py:260] loss = 0.010777794, step = 6652 (11.605 sec)
I0716 01:05:40.198792 140076727973760 basic_session_run_hooks.py:260] lr = 0.00014884777 (11.606 sec)
I0716 01:05:51.054244 140076727973760 basic_session_run_hooks.py:692] global_step/sec: 9.20541
I0716 01:05:51.059835 140076727973760 basic_session_run_hooks.py:260] loss = 0.024977569, step = 6752 (10.864 sec)
I0716 01:05:51.062622 140076727973760 basic_session_run_hooks.py:260] lr = 0.00014728773 (10.864 sec)
I0716 01:06:02.207034 140076727973760 basic_session_run_hooks.py:692] global_step/sec: 8.96638
I0716 01:06:02.214545 140076727973760 basic_session_run_hooks.py:260] loss = 0.02586377, step = 6852 (11.155 sec)
I0716 01:06:02.216049 140076727973760 basic_session_run_hooks.py:260] lr = 0.00014574405 (11.153 sec)
I0716 01:06:03.474636 140076727973760 basic_session_run_hooks.py:606] Saving checkpoints for 6864 into ../model/bigru_crf/model.ckpt.
I0716 01:06:03.856474 140076727973760 estimator.py:1145] Calling model_fn.
I0716 01:06:05.033905 140076727973760 estimator.py:1147] Done calling model_fn.
I0716 01:06:05.064028 140076727973760 evaluation.py:255] Starting evaluation at 2019-07-16T01:06:05Z
I0716 01:06:05.190752 140076727973760 monitored_session.py:240] Graph was finalized.
I0716 01:06:05.209294 140076727973760 saver.py:1280] Restoring parameters from ../model/bigru_crf/model.ckpt-6864
I0716 01:06:05.326102 140076727973760 session_manager.py:500] Running local_init_op.
I0716 01:06:05.353211 140076727973760 session_manager.py:502] Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
I0716 01:06:07.668761 140076727973760 evaluation.py:275] Finished evaluation at 2019-07-16-01:06:07
I0716 01:06:07.670435 140076727973760 estimator.py:2039] Saving dict for global step 6864: global_step = 6864, loss = 3.9695408
I0716 01:06:07.679378 140076727973760 estimator.py:2099] Saving 'checkpoint_path' summary for global step 6864: ../model/bigru_crf/model.ckpt-6864
I0716 01:06:07.744103 140076727973760 estimator.py:368] Loss for final step: 0.07630234.
Reading ../data/atis.test.w-intent.iob
I0716 01:06:08.065013 140076727973760 estimator.py:1145] Calling model_fn.
I0716 01:06:08.727861 140076727973760 estimator.py:1147] Done calling model_fn.
I0716 01:06:08.843358 140076727973760 monitored_session.py:240] Graph was finalized.
I0716 01:06:08.859805 140076727973760 saver.py:1280] Restoring parameters from ../model/bigru_crf/model.ckpt-6864
I0716 01:06:08.963977 140076727973760 session_manager.py:500] Running local_init_op.
I0716 01:06:08.981795 140076727973760 session_manager.py:502] Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.
  'precision', 'predicted', average, warn_for)
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1439: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples.
  'recall', 'true', average, warn_for)
I0716 01:06:12.067632 140076727973760 interactiveshell.py:2882] 
                                          precision    recall  f1-score   support

                             atis_flight      0.977     0.992     0.984       632
                            atis_airfare      0.959     0.979     0.969        48
                     atis_ground_service      1.000     1.000     1.000        36
                            atis_airline      1.000     1.000     1.000        38
                       atis_abbreviation      0.943     1.000     0.971        33
                           atis_aircraft      0.900     1.000     0.947         9
                        atis_flight_time      1.000     1.000     1.000         1
                           atis_quantity      0.375     1.000     0.545         3
                atis_flight#atis_airfare      0.750     0.250     0.375        12
                            atis_airport      1.000     0.944     0.971        18
                           atis_distance      1.000     1.000     1.000        10
                               atis_city      1.000     0.500     0.667         6
                        atis_ground_fare      1.000     0.857     0.923         7
                           atis_capacity      1.000     0.952     0.976        21
                          atis_flight_no      0.889     1.000     0.941         8
                               atis_meal      1.000     0.833     0.909         6
                        atis_restriction      0.000     0.000     0.000         0
             atis_airline#atis_flight_no      0.000     0.000     0.000         0
    atis_ground_service#atis_ground_fare      0.000     0.000     0.000         0
           atis_airfare#atis_flight_time      0.000     0.000     0.000         0
                           atis_cheapest      0.000     0.000     0.000         0
atis_aircraft#atis_flight#atis_flight_no      0.000     0.000     0.000         0

                               micro avg      0.970     0.975     0.972       888
                               macro avg      0.672     0.650     0.644       888
                            weighted avg      0.972     0.975     0.970       888

I0716 01:06:12.118835 140076727973760 interactiveshell.py:2882] 
                              precision    recall  f1-score   support

                           O      0.000     0.000     0.000       0.0
           B-toloc.city_name      0.974     0.992     0.983     716.0
         B-fromloc.city_name      0.979     0.997     0.988     704.0
           I-toloc.city_name      0.977     0.977     0.977     265.0
      B-depart_date.day_name      0.986     0.991     0.988     212.0
              B-airline_name      1.000     1.000     1.000     101.0
         I-fromloc.city_name      0.962     0.994     0.978     177.0
 B-depart_time.period_of_day      1.000     0.915     0.956     130.0
              I-airline_name      1.000     1.000     1.000      65.0
    B-depart_date.day_number      0.981     0.964     0.972      55.0
    B-depart_date.month_name      0.982     0.964     0.973      56.0
          B-depart_time.time      0.862     0.982     0.918      57.0
                B-round_trip      1.000     0.973     0.986      73.0
             B-cost_relative      1.000     0.973     0.986      37.0
                I-round_trip      1.000     1.000     1.000      71.0
                B-flight_mod      1.000     1.000     1.000      24.0
 B-depart_time.time_relative      0.969     0.954     0.961      65.0
          I-depart_time.time      0.944     0.981     0.962      52.0
         B-stoploc.city_name      0.909     1.000     0.952      20.0
                 B-city_name      0.941     0.561     0.703      57.0
                B-class_type      0.960     1.000     0.980      24.0
          B-arrive_time.time      0.971     0.971     0.971      34.0
 B-arrive_time.time_relative      0.906     0.935     0.921      31.0
                I-class_type      1.000     1.000     1.000      17.0
               B-flight_stop      1.000     1.000     1.000      21.0
          I-arrive_time.time      1.000     0.971     0.986      35.0
              B-airline_code      1.000     0.971     0.985      34.0
    I-depart_date.day_number      1.000     1.000     1.000      15.0
      I-fromloc.airport_name      0.417     1.000     0.588      15.0
      B-fromloc.airport_name      0.462     1.000     0.632      12.0
      B-arrive_date.day_name      0.846     1.000     0.917      11.0
          B-toloc.state_code      0.947     1.000     0.973      18.0
B-depart_date.today_relative      1.000     0.889     0.941       9.0
             B-flight_number      0.579     1.000     0.733      11.0
 B-depart_date.date_relative      0.944     1.000     0.971      17.0
          B-toloc.state_name      0.781     0.893     0.833      28.0
           B-fare_basis_code      0.944     1.000     0.971      17.0
               B-flight_time      1.000     1.000     1.000       1.0
                        B-or      1.000     1.000     1.000       3.0
 B-arrive_time.period_of_day      0.857     1.000     0.923       6.0
          B-meal_description      1.000     0.900     0.947      10.0
             I-cost_relative      1.000     0.667     0.800       3.0
              I-airport_name      0.917     0.379     0.537      29.0
               B-fare_amount      1.000     1.000     1.000       2.0
               I-fare_amount      1.000     1.000     1.000       2.0
                 I-city_name      1.000     0.567     0.723      30.0
        I-toloc.airport_name      1.000     1.000     1.000       3.0
            B-transport_type      1.000     1.000     1.000      10.0
    B-arrive_date.month_name      0.714     0.833     0.769       6.0
    B-arrive_date.day_number      0.714     0.833     0.769       6.0
         I-stoploc.city_name      0.833     1.000     0.909      10.0
                      B-meal      1.000     1.000     1.000      16.0
        B-fromloc.state_code      1.000     0.957     0.978      23.0
    B-depart_time.period_mod      0.833     1.000     0.909       5.0
                   B-connect      1.000     1.000     1.000       6.0
               B-flight_days      1.000     1.000     1.000      10.0
        B-toloc.airport_name      1.000     1.000     1.000       3.0
        B-fromloc.state_name      0.944     1.000     0.971      17.0
              B-airport_name      0.800     0.381     0.516      21.0
                   B-economy      1.000     1.000     1.000       6.0
               I-flight_time      1.000     1.000     1.000       1.0
             B-aircraft_code      0.962     0.758     0.847      33.0
                       B-mod      1.000     0.500     0.667       2.0
              B-airport_code      0.500     0.111     0.182       9.0
    B-depart_time.start_time      1.000     0.667     0.800       3.0
      B-depart_time.end_time      1.000     0.333     0.500       3.0
          B-depart_date.year      1.000     1.000     1.000       3.0
            I-transport_type      0.000     0.000     0.000       1.0
          B-restriction_code      0.333     1.000     0.500       4.0
    B-arrive_time.start_time      0.889     1.000     0.941       8.0
        B-toloc.airport_code      1.000     0.750     0.857       4.0
      B-arrive_time.end_time      0.889     1.000     0.941       8.0
      I-arrive_time.end_time      0.889     1.000     0.941       8.0
      I-depart_time.end_time      1.000     0.333     0.500       3.0
               I-flight_stop      0.000     0.000     0.000       0.0
      B-fromloc.airport_code      1.000     1.000     1.000       5.0
          I-restriction_code      1.000     1.000     1.000       3.0
    I-depart_time.start_time      1.000     1.000     1.000       1.0
          I-toloc.state_name      1.000     1.000     1.000       1.0
I-depart_date.today_relative      0.000     0.000     0.000       0.0
 B-arrive_date.date_relative      1.000     1.000     1.000       2.0
                I-flight_mod      1.000     0.167     0.286       6.0
                   I-economy      0.000     0.000     0.000       0.0
 B-return_date.date_relative      1.000     0.333     0.500       3.0
        I-fromloc.state_name      1.000     1.000     1.000       1.0
                B-state_code      1.000     1.000     1.000       1.0
    I-arrive_time.start_time      1.000     1.000     1.000       1.0
    I-arrive_date.day_number      0.000     0.000     0.000       0.0
                 B-meal_code      0.000     0.000     0.000       1.0
 I-depart_time.period_of_day      1.000     1.000     1.000       1.0
                  B-day_name      1.000     0.500     0.667       2.0
             B-period_of_day      1.000     1.000     1.000       4.0
        B-stoploc.state_code      0.000     0.000     0.000       0.0
    B-return_date.month_name      0.000     0.000     0.000       0.0
    B-return_date.day_number      0.000     0.000     0.000       0.0
    B-arrive_time.period_mod      0.000     0.000     0.000       0.0
                 I-meal_code      0.000     0.000     0.000       0.0
        B-toloc.country_name      1.000     1.000     1.000       1.0
                 B-days_code      1.000     1.000     1.000       1.0
 I-arrive_time.period_of_day      0.000     0.000     0.000       0.0
            I-today_relative      0.000     0.000     0.000       0.0
 B-return_time.period_of_day      0.000     0.000     0.000       0.0
                      B-time      0.000     0.000     0.000       0.0
           I-fare_basis_code      0.000     0.000     0.000       0.0
 I-arrive_time.time_relative      0.000     0.000     0.000       4.0
 I-depart_time.time_relative      0.000     0.000     0.000       1.0
            B-today_relative      0.000     0.000     0.000       0.0
                B-state_name      0.000     0.000     0.000       9.0
B-arrive_date.today_relative      0.000     0.000     0.000       0.0
    B-return_time.period_mod      0.000     0.000     0.000       0.0
                B-month_name      0.000     0.000     0.000       0.0
                B-day_number      0.000     0.000     0.000       0.0
 I-return_date.date_relative      0.750     1.000     0.857       3.0
I-return_date.today_relative      0.000     0.000     0.000       0.0
      B-stoploc.airport_name      0.000     0.000     0.000       0.0
             B-time_relative      0.000     0.000     0.000       0.0
                      I-time      0.000     0.000     0.000       0.0
    I-return_date.day_number      0.000     0.000     0.000       0.0
          I-meal_description      0.000     0.000     0.000       0.0
B-return_date.today_relative      0.000     0.000     0.000       0.0
      B-return_date.day_name      0.000     0.000     0.000       2.0

                   micro avg      0.951     0.953     0.952    3657.0
                   macro avg      0.678     0.651     0.648    3657.0
                weighted avg      0.959     0.953     0.951    3657.0

I0716 01:06:12.148849 140076727973760 interactiveshell.py:2882] Best Slot F1: 0.957
I0716 01:06:12.150597 140076727973760 estimator_training.py:186] Not using Distribute Coordinator.
I0716 01:06:12.153935 140076727973760 training.py:612] Running training and evaluation locally (non-distributed).
I0716 01:06:12.156926 140076727973760 training.py:700] Start train and evaluate loop. The evaluate will happen after every checkpoint. Checkpoint frequency is determined based on RunConfig arguments: save_checkpoints_steps 312 or save_checkpoints_secs None.
I0716 01:06:12.222025 140076727973760 estimator.py:1145] Calling model_fn.
I0716 01:06:13.580383 140076727973760 <ipython-input-6-0a85a3fd693f>:50] 
[<tf.Variable 'embedding:0' shape=(750, 300) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/gates/kernel:0' shape=(600, 600) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/gates/bias:0' shape=(600,) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/candidate/kernel:0' shape=(600, 300) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/candidate/bias:0' shape=(300,) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/gates/kernel:0' shape=(600, 600) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/gates/bias:0' shape=(600,) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/candidate/kernel:0' shape=(600, 300) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/candidate/bias:0' shape=(300,) dtype=float32_ref>,
 <tf.Variable 'dense/kernel:0' shape=(600, 23) dtype=float32_ref>,
 <tf.Variable 'dense/bias:0' shape=(23,) dtype=float32_ref>,
 <tf.Variable 'dense_1/kernel:0' shape=(600, 122) dtype=float32_ref>,
 <tf.Variable 'dense_1/bias:0' shape=(122,) dtype=float32_ref>,
 <tf.Variable 'transitions:0' shape=(122, 122) dtype=float32_ref>]
I0716 01:06:15.088608 140076727973760 estimator.py:1147] Done calling model_fn.
I0716 01:06:15.092270 140076727973760 basic_session_run_hooks.py:541] Create CheckpointSaverHook.
I0716 01:06:15.287362 140076727973760 monitored_session.py:240] Graph was finalized.
I0716 01:06:15.305697 140076727973760 saver.py:1280] Restoring parameters from ../model/bigru_crf/model.ckpt-6864
I0716 01:06:15.573523 140076727973760 session_manager.py:500] Running local_init_op.
I0716 01:06:15.610403 140076727973760 session_manager.py:502] Done running local_init_op.
I0716 01:06:17.340473 140076727973760 basic_session_run_hooks.py:606] Saving checkpoints for 6864 into ../model/bigru_crf/model.ckpt.
Reading ../data/atis.train.w-intent.iob
I0716 01:06:19.725132 140076727973760 basic_session_run_hooks.py:262] loss = 0.014536148, step = 6864
I0716 01:06:19.726732 140076727973760 basic_session_run_hooks.py:262] lr = 0.0001455599
I0716 01:06:31.252304 140076727973760 basic_session_run_hooks.py:692] global_step/sec: 8.6747
I0716 01:06:31.259644 140076727973760 basic_session_run_hooks.py:260] loss = 0.011604054, step = 6964 (11.535 sec)
I0716 01:06:31.261172 140076727973760 basic_session_run_hooks.py:260] lr = 0.00014403433 (11.534 sec)
I0716 01:06:42.222961 140076727973760 basic_session_run_hooks.py:692] global_step/sec: 9.11524
I0716 01:06:42.231143 140076727973760 basic_session_run_hooks.py:260] loss = 0.041044094, step = 7064 (10.971 sec)
I0716 01:06:42.232728 140076727973760 basic_session_run_hooks.py:260] lr = 0.00014252475 (10.972 sec)
I0716 01:06:53.177091 140076727973760 basic_session_run_hooks.py:692] global_step/sec: 9.12898
I0716 01:06:53.180199 140076727973760 basic_session_run_hooks.py:260] loss = 0.02156842, step = 7164 (10.949 sec)
I0716 01:06:53.181736 140076727973760 basic_session_run_hooks.py:260] lr = 0.00014103096 (10.949 sec)
I0716 01:06:54.354328 140076727973760 basic_session_run_hooks.py:606] Saving checkpoints for 7176 into ../model/bigru_crf/model.ckpt.
I0716 01:06:54.759983 140076727973760 estimator.py:1145] Calling model_fn.
I0716 01:06:55.725243 140076727973760 estimator.py:1147] Done calling model_fn.
I0716 01:06:55.754721 140076727973760 evaluation.py:255] Starting evaluation at 2019-07-16T01:06:55Z
I0716 01:06:55.872021 140076727973760 monitored_session.py:240] Graph was finalized.
I0716 01:06:55.890456 140076727973760 saver.py:1280] Restoring parameters from ../model/bigru_crf/model.ckpt-7176
I0716 01:06:56.009446 140076727973760 session_manager.py:500] Running local_init_op.
I0716 01:06:56.037206 140076727973760 session_manager.py:502] Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
I0716 01:06:58.352742 140076727973760 evaluation.py:275] Finished evaluation at 2019-07-16-01:06:58
I0716 01:06:58.354333 140076727973760 estimator.py:2039] Saving dict for global step 7176: global_step = 7176, loss = 4.124317
I0716 01:06:58.361050 140076727973760 estimator.py:2099] Saving 'checkpoint_path' summary for global step 7176: ../model/bigru_crf/model.ckpt-7176
I0716 01:06:58.421768 140076727973760 estimator.py:368] Loss for final step: 0.0062238853.
Reading ../data/atis.test.w-intent.iob
I0716 01:06:58.759921 140076727973760 estimator.py:1145] Calling model_fn.
I0716 01:06:59.728709 140076727973760 estimator.py:1147] Done calling model_fn.
I0716 01:06:59.842278 140076727973760 monitored_session.py:240] Graph was finalized.
I0716 01:06:59.855162 140076727973760 saver.py:1280] Restoring parameters from ../model/bigru_crf/model.ckpt-7176
I0716 01:06:59.961207 140076727973760 session_manager.py:500] Running local_init_op.
I0716 01:06:59.983002 140076727973760 session_manager.py:502] Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.
  'precision', 'predicted', average, warn_for)
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1439: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples.
  'recall', 'true', average, warn_for)
I0716 01:07:03.139384 140076727973760 interactiveshell.py:2882] 
                                          precision    recall  f1-score   support

                             atis_flight      0.977     0.992     0.984       632
                            atis_airfare      0.959     0.979     0.969        48
                     atis_ground_service      1.000     1.000     1.000        36
                            atis_airline      1.000     1.000     1.000        38
                       atis_abbreviation      0.943     1.000     0.971        33
                           atis_aircraft      0.900     1.000     0.947         9
                        atis_flight_time      1.000     1.000     1.000         1
                           atis_quantity      0.375     1.000     0.545         3
                atis_flight#atis_airfare      0.750     0.250     0.375        12
                            atis_airport      1.000     0.944     0.971        18
                           atis_distance      1.000     1.000     1.000        10
                               atis_city      1.000     0.500     0.667         6
                        atis_ground_fare      1.000     0.857     0.923         7
                           atis_capacity      1.000     0.952     0.976        21
                          atis_flight_no      0.889     1.000     0.941         8
                               atis_meal      1.000     0.833     0.909         6
                        atis_restriction      0.000     0.000     0.000         0
             atis_airline#atis_flight_no      0.000     0.000     0.000         0
    atis_ground_service#atis_ground_fare      0.000     0.000     0.000         0
           atis_airfare#atis_flight_time      0.000     0.000     0.000         0
                           atis_cheapest      0.000     0.000     0.000         0
atis_aircraft#atis_flight#atis_flight_no      0.000     0.000     0.000         0

                               micro avg      0.970     0.975     0.972       888
                               macro avg      0.672     0.650     0.644       888
                            weighted avg      0.972     0.975     0.970       888

I0716 01:07:03.191339 140076727973760 interactiveshell.py:2882] 
                              precision    recall  f1-score   support

                           O      0.000     0.000     0.000       0.0
           B-toloc.city_name      0.973     0.993     0.983     716.0
         B-fromloc.city_name      0.980     0.997     0.989     704.0
           I-toloc.city_name      0.963     0.985     0.974     265.0
      B-depart_date.day_name      0.991     0.991     0.991     212.0
              B-airline_name      1.000     1.000     1.000     101.0
         I-fromloc.city_name      0.967     0.989     0.978     177.0
 B-depart_time.period_of_day      1.000     0.923     0.960     130.0
              I-airline_name      1.000     1.000     1.000      65.0
    B-depart_date.day_number      0.981     0.964     0.972      55.0
    B-depart_date.month_name      0.982     0.964     0.973      56.0
          B-depart_time.time      0.864     1.000     0.927      57.0
                B-round_trip      1.000     0.973     0.986      73.0
             B-cost_relative      1.000     0.973     0.986      37.0
                I-round_trip      1.000     1.000     1.000      71.0
                B-flight_mod      1.000     1.000     1.000      24.0
 B-depart_time.time_relative      0.969     0.954     0.961      65.0
          I-depart_time.time      0.944     0.981     0.962      52.0
         B-stoploc.city_name      0.952     1.000     0.976      20.0
                 B-city_name      0.909     0.526     0.667      57.0
                B-class_type      0.960     1.000     0.980      24.0
          B-arrive_time.time      0.971     0.971     0.971      34.0
 B-arrive_time.time_relative      0.906     0.935     0.921      31.0
                I-class_type      1.000     1.000     1.000      17.0
               B-flight_stop      1.000     1.000     1.000      21.0
          I-arrive_time.time      1.000     0.971     0.986      35.0
              B-airline_code      1.000     0.971     0.985      34.0
    I-depart_date.day_number      1.000     0.933     0.966      15.0
      I-fromloc.airport_name      0.441     1.000     0.612      15.0
      B-fromloc.airport_name      0.500     1.000     0.667      12.0
      B-arrive_date.day_name      0.786     1.000     0.880      11.0
          B-toloc.state_code      0.947     1.000     0.973      18.0
B-depart_date.today_relative      1.000     0.889     0.941       9.0
             B-flight_number      0.688     1.000     0.815      11.0
 B-depart_date.date_relative      0.944     1.000     0.971      17.0
          B-toloc.state_name      0.781     0.893     0.833      28.0
           B-fare_basis_code      0.944     1.000     0.971      17.0
               B-flight_time      1.000     1.000     1.000       1.0
                        B-or      1.000     1.000     1.000       3.0
 B-arrive_time.period_of_day      0.750     1.000     0.857       6.0
          B-meal_description      1.000     0.900     0.947      10.0
             I-cost_relative      1.000     0.667     0.800       3.0
              I-airport_name      0.929     0.448     0.605      29.0
               B-fare_amount      1.000     1.000     1.000       2.0
               I-fare_amount      1.000     1.000     1.000       2.0
                 I-city_name      1.000     0.500     0.667      30.0
        I-toloc.airport_name      1.000     1.000     1.000       3.0
            B-transport_type      1.000     1.000     1.000      10.0
    B-arrive_date.month_name      0.714     0.833     0.769       6.0
    B-arrive_date.day_number      0.714     0.833     0.769       6.0
         I-stoploc.city_name      0.833     1.000     0.909      10.0
                      B-meal      1.000     1.000     1.000      16.0
        B-fromloc.state_code      1.000     0.957     0.978      23.0
    B-depart_time.period_mod      1.000     1.000     1.000       5.0
                   B-connect      1.000     1.000     1.000       6.0
               B-flight_days      1.000     1.000     1.000      10.0
        B-toloc.airport_name      1.000     1.000     1.000       3.0
        B-fromloc.state_name      0.944     1.000     0.971      17.0
              B-airport_name      0.833     0.476     0.606      21.0
                   B-economy      1.000     1.000     1.000       6.0
               I-flight_time      1.000     1.000     1.000       1.0
             B-aircraft_code      0.967     0.879     0.921      33.0
                       B-mod      1.000     0.500     0.667       2.0
              B-airport_code      0.500     0.111     0.182       9.0
    B-depart_time.start_time      1.000     0.667     0.800       3.0
      B-depart_time.end_time      1.000     0.333     0.500       3.0
          B-depart_date.year      1.000     1.000     1.000       3.0
            I-transport_type      0.000     0.000     0.000       1.0
          B-restriction_code      0.444     1.000     0.615       4.0
    B-arrive_time.start_time      0.889     1.000     0.941       8.0
        B-toloc.airport_code      1.000     0.750     0.857       4.0
      B-arrive_time.end_time      0.889     1.000     0.941       8.0
      I-arrive_time.end_time      0.889     1.000     0.941       8.0
      I-depart_time.end_time      1.000     0.333     0.500       3.0
               I-flight_stop      0.000     0.000     0.000       0.0
      B-fromloc.airport_code      1.000     1.000     1.000       5.0
          I-restriction_code      1.000     1.000     1.000       3.0
    I-depart_time.start_time      1.000     1.000     1.000       1.0
          I-toloc.state_name      1.000     1.000     1.000       1.0
I-depart_date.today_relative      0.000     0.000     0.000       0.0
 B-arrive_date.date_relative      1.000     1.000     1.000       2.0
                I-flight_mod      1.000     0.167     0.286       6.0
                   I-economy      0.000     0.000     0.000       0.0
 B-return_date.date_relative      1.000     0.333     0.500       3.0
        I-fromloc.state_name      1.000     1.000     1.000       1.0
                B-state_code      1.000     1.000     1.000       1.0
    I-arrive_time.start_time      1.000     1.000     1.000       1.0
    I-arrive_date.day_number      0.000     0.000     0.000       0.0
                 B-meal_code      1.000     1.000     1.000       1.0
 I-depart_time.period_of_day      1.000     1.000     1.000       1.0
                  B-day_name      1.000     0.500     0.667       2.0
             B-period_of_day      1.000     0.750     0.857       4.0
        B-stoploc.state_code      0.000     0.000     0.000       0.0
    B-return_date.month_name      0.000     0.000     0.000       0.0
    B-return_date.day_number      0.000     0.000     0.000       0.0
    B-arrive_time.period_mod      0.000     0.000     0.000       0.0
                 I-meal_code      0.000     0.000     0.000       0.0
        B-toloc.country_name      1.000     1.000     1.000       1.0
                 B-days_code      1.000     1.000     1.000       1.0
 I-arrive_time.period_of_day      0.000     0.000     0.000       0.0
            I-today_relative      0.000     0.000     0.000       0.0
 B-return_time.period_of_day      0.000     0.000     0.000       0.0
                      B-time      0.000     0.000     0.000       0.0
           I-fare_basis_code      0.000     0.000     0.000       0.0
 I-arrive_time.time_relative      0.000     0.000     0.000       4.0
 I-depart_time.time_relative      0.000     0.000     0.000       1.0
            B-today_relative      0.000     0.000     0.000       0.0
                B-state_name      0.000     0.000     0.000       9.0
B-arrive_date.today_relative      0.000     0.000     0.000       0.0
    B-return_time.period_mod      0.000     0.000     0.000       0.0
                B-month_name      0.000     0.000     0.000       0.0
                B-day_number      0.000     0.000     0.000       0.0
 I-return_date.date_relative      0.750     1.000     0.857       3.0
I-return_date.today_relative      0.000     0.000     0.000       0.0
      B-stoploc.airport_name      0.000     0.000     0.000       0.0
             B-time_relative      0.000     0.000     0.000       0.0
                      I-time      0.000     0.000     0.000       0.0
    I-return_date.day_number      0.000     0.000     0.000       0.0
          I-meal_description      0.000     0.000     0.000       0.0
B-return_date.today_relative      0.000     0.000     0.000       0.0
      B-return_date.day_name      0.000     0.000     0.000       2.0

                   micro avg      0.953     0.955     0.954    3657.0
                   macro avg      0.689     0.659     0.658    3657.0
                weighted avg      0.960     0.955     0.952    3657.0

I0716 01:07:03.220589 140076727973760 interactiveshell.py:2882] Best Slot F1: 0.957
I0716 01:07:03.222182 140076727973760 estimator_training.py:186] Not using Distribute Coordinator.
I0716 01:07:03.224254 140076727973760 training.py:612] Running training and evaluation locally (non-distributed).
I0716 01:07:03.229186 140076727973760 training.py:700] Start train and evaluate loop. The evaluate will happen after every checkpoint. Checkpoint frequency is determined based on RunConfig arguments: save_checkpoints_steps 312 or save_checkpoints_secs None.
I0716 01:07:03.292271 140076727973760 estimator.py:1145] Calling model_fn.
I0716 01:07:04.436796 140076727973760 <ipython-input-6-0a85a3fd693f>:50] 
[<tf.Variable 'embedding:0' shape=(750, 300) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/gates/kernel:0' shape=(600, 600) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/gates/bias:0' shape=(600,) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/candidate/kernel:0' shape=(600, 300) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/candidate/bias:0' shape=(300,) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/gates/kernel:0' shape=(600, 600) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/gates/bias:0' shape=(600,) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/candidate/kernel:0' shape=(600, 300) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/candidate/bias:0' shape=(300,) dtype=float32_ref>,
 <tf.Variable 'dense/kernel:0' shape=(600, 23) dtype=float32_ref>,
 <tf.Variable 'dense/bias:0' shape=(23,) dtype=float32_ref>,
 <tf.Variable 'dense_1/kernel:0' shape=(600, 122) dtype=float32_ref>,
 <tf.Variable 'dense_1/bias:0' shape=(122,) dtype=float32_ref>,
 <tf.Variable 'transitions:0' shape=(122, 122) dtype=float32_ref>]
I0716 01:07:05.985757 140076727973760 estimator.py:1147] Done calling model_fn.
I0716 01:07:05.990235 140076727973760 basic_session_run_hooks.py:541] Create CheckpointSaverHook.
I0716 01:07:06.187187 140076727973760 monitored_session.py:240] Graph was finalized.
I0716 01:07:06.205759 140076727973760 saver.py:1280] Restoring parameters from ../model/bigru_crf/model.ckpt-7176
I0716 01:07:06.461118 140076727973760 session_manager.py:500] Running local_init_op.
I0716 01:07:06.498376 140076727973760 session_manager.py:502] Done running local_init_op.
I0716 01:07:08.033393 140076727973760 basic_session_run_hooks.py:606] Saving checkpoints for 7176 into ../model/bigru_crf/model.ckpt.
Reading ../data/atis.train.w-intent.iob
I0716 01:07:10.398428 140076727973760 basic_session_run_hooks.py:262] loss = 0.029496832, step = 7176
I0716 01:07:10.400348 140076727973760 basic_session_run_hooks.py:262] lr = 0.00014085279
I0716 01:07:22.334982 140076727973760 basic_session_run_hooks.py:692] global_step/sec: 8.37729
I0716 01:07:22.342323 140076727973760 basic_session_run_hooks.py:260] loss = 0.027537743, step = 7276 (11.943 sec)
I0716 01:07:22.343982 140076727973760 basic_session_run_hooks.py:260] lr = 0.00013937654 (11.944 sec)
I0716 01:07:33.134149 140076727973760 basic_session_run_hooks.py:692] global_step/sec: 9.25998
I0716 01:07:33.137138 140076727973760 basic_session_run_hooks.py:260] loss = 0.003356348, step = 7376 (10.796 sec)
I0716 01:07:33.140712 140076727973760 basic_session_run_hooks.py:260] lr = 0.00013791576 (10.797 sec)
I0716 01:07:44.042206 140076727973760 basic_session_run_hooks.py:692] global_step/sec: 9.16756
I0716 01:07:44.049664 140076727973760 basic_session_run_hooks.py:260] loss = 0.021856468, step = 7476 (10.913 sec)
I0716 01:07:44.051969 140076727973760 basic_session_run_hooks.py:260] lr = 0.0001364703 (10.911 sec)
I0716 01:07:45.201955 140076727973760 basic_session_run_hooks.py:606] Saving checkpoints for 7488 into ../model/bigru_crf/model.ckpt.
I0716 01:07:45.586441 140076727973760 estimator.py:1145] Calling model_fn.
I0716 01:07:46.764918 140076727973760 estimator.py:1147] Done calling model_fn.
I0716 01:07:46.793812 140076727973760 evaluation.py:255] Starting evaluation at 2019-07-16T01:07:46Z
I0716 01:07:46.919159 140076727973760 monitored_session.py:240] Graph was finalized.
I0716 01:07:46.939145 140076727973760 saver.py:1280] Restoring parameters from ../model/bigru_crf/model.ckpt-7488
I0716 01:07:47.062938 140076727973760 session_manager.py:500] Running local_init_op.
I0716 01:07:47.091711 140076727973760 session_manager.py:502] Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
I0716 01:07:49.403813 140076727973760 evaluation.py:275] Finished evaluation at 2019-07-16-01:07:49
I0716 01:07:49.405577 140076727973760 estimator.py:2039] Saving dict for global step 7488: global_step = 7488, loss = 4.285758
I0716 01:07:49.414665 140076727973760 estimator.py:2099] Saving 'checkpoint_path' summary for global step 7488: ../model/bigru_crf/model.ckpt-7488
I0716 01:07:49.480863 140076727973760 estimator.py:368] Loss for final step: 0.0068322415.
Reading ../data/atis.test.w-intent.iob
I0716 01:07:49.804447 140076727973760 estimator.py:1145] Calling model_fn.
I0716 01:07:50.461311 140076727973760 estimator.py:1147] Done calling model_fn.
I0716 01:07:50.575401 140076727973760 monitored_session.py:240] Graph was finalized.
I0716 01:07:50.591560 140076727973760 saver.py:1280] Restoring parameters from ../model/bigru_crf/model.ckpt-7488
I0716 01:07:50.693788 140076727973760 session_manager.py:500] Running local_init_op.
I0716 01:07:50.712812 140076727973760 session_manager.py:502] Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.
  'precision', 'predicted', average, warn_for)
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1439: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples.
  'recall', 'true', average, warn_for)
I0716 01:07:53.906502 140076727973760 interactiveshell.py:2882] 
                                          precision    recall  f1-score   support

                             atis_flight      0.974     0.992     0.983       632
                            atis_airfare      0.959     0.979     0.969        48
                     atis_ground_service      1.000     1.000     1.000        36
                            atis_airline      1.000     1.000     1.000        38
                       atis_abbreviation      0.943     1.000     0.971        33
                           atis_aircraft      1.000     1.000     1.000         9
                        atis_flight_time      1.000     1.000     1.000         1
                           atis_quantity      0.375     1.000     0.545         3
                atis_flight#atis_airfare      0.667     0.167     0.267        12
                            atis_airport      1.000     0.944     0.971        18
                           atis_distance      1.000     1.000     1.000        10
                               atis_city      1.000     0.500     0.667         6
                        atis_ground_fare      1.000     0.857     0.923         7
                           atis_capacity      1.000     0.952     0.976        21
                          atis_flight_no      0.889     1.000     0.941         8
                               atis_meal      1.000     0.833     0.909         6
                        atis_restriction      0.000     0.000     0.000         0
             atis_airline#atis_flight_no      0.000     0.000     0.000         0
    atis_ground_service#atis_ground_fare      0.000     0.000     0.000         0
           atis_airfare#atis_flight_time      0.000     0.000     0.000         0
                           atis_cheapest      0.000     0.000     0.000         0
atis_aircraft#atis_flight#atis_flight_no      0.000     0.000     0.000         0

                               micro avg      0.969     0.974     0.971       888
                               macro avg      0.673     0.647     0.642       888
                            weighted avg      0.969     0.974     0.968       888

I0716 01:07:53.957725 140076727973760 interactiveshell.py:2882] 
                              precision    recall  f1-score   support

                           O      0.000     0.000     0.000       0.0
           B-toloc.city_name      0.973     0.992     0.982     716.0
         B-fromloc.city_name      0.976     0.997     0.987     704.0
           I-toloc.city_name      0.966     0.977     0.972     265.0
      B-depart_date.day_name      0.991     0.991     0.991     212.0
              B-airline_name      1.000     1.000     1.000     101.0
         I-fromloc.city_name      0.962     0.989     0.975     177.0
 B-depart_time.period_of_day      0.992     0.923     0.956     130.0
              I-airline_name      1.000     1.000     1.000      65.0
    B-depart_date.day_number      0.981     0.964     0.972      55.0
    B-depart_date.month_name      0.982     0.964     0.973      56.0
          B-depart_time.time      0.848     0.982     0.911      57.0
                B-round_trip      1.000     0.973     0.986      73.0
             B-cost_relative      1.000     0.973     0.986      37.0
                I-round_trip      1.000     1.000     1.000      71.0
                B-flight_mod      1.000     1.000     1.000      24.0
 B-depart_time.time_relative      0.969     0.954     0.961      65.0
          I-depart_time.time      0.943     0.962     0.952      52.0
         B-stoploc.city_name      0.952     1.000     0.976      20.0
                 B-city_name      0.939     0.544     0.689      57.0
                B-class_type      0.960     1.000     0.980      24.0
          B-arrive_time.time      0.970     0.941     0.955      34.0
 B-arrive_time.time_relative      0.906     0.935     0.921      31.0
                I-class_type      1.000     1.000     1.000      17.0
               B-flight_stop      1.000     1.000     1.000      21.0
          I-arrive_time.time      0.971     0.971     0.971      35.0
              B-airline_code      1.000     0.971     0.985      34.0
    I-depart_date.day_number      1.000     1.000     1.000      15.0
      I-fromloc.airport_name      0.417     1.000     0.588      15.0
      B-fromloc.airport_name      0.462     1.000     0.632      12.0
      B-arrive_date.day_name      0.786     1.000     0.880      11.0
          B-toloc.state_code      0.947     1.000     0.973      18.0
B-depart_date.today_relative      1.000     0.889     0.941       9.0
             B-flight_number      0.786     1.000     0.880      11.0
 B-depart_date.date_relative      0.944     1.000     0.971      17.0
          B-toloc.state_name      0.781     0.893     0.833      28.0
           B-fare_basis_code      0.944     1.000     0.971      17.0
               B-flight_time      1.000     1.000     1.000       1.0
                        B-or      1.000     1.000     1.000       3.0
 B-arrive_time.period_of_day      0.857     1.000     0.923       6.0
          B-meal_description      1.000     0.900     0.947      10.0
             I-cost_relative      1.000     0.667     0.800       3.0
              I-airport_name      0.917     0.379     0.537      29.0
               B-fare_amount      1.000     1.000     1.000       2.0
               I-fare_amount      1.000     1.000     1.000       2.0
                 I-city_name      1.000     0.533     0.696      30.0
        I-toloc.airport_name      1.000     1.000     1.000       3.0
            B-transport_type      1.000     1.000     1.000      10.0
    B-arrive_date.month_name      0.714     0.833     0.769       6.0
    B-arrive_date.day_number      0.714     0.833     0.769       6.0
         I-stoploc.city_name      0.909     1.000     0.952      10.0
                      B-meal      1.000     1.000     1.000      16.0
        B-fromloc.state_code      1.000     0.957     0.978      23.0
    B-depart_time.period_mod      1.000     1.000     1.000       5.0
                   B-connect      1.000     1.000     1.000       6.0
               B-flight_days      1.000     1.000     1.000      10.0
        B-toloc.airport_name      1.000     1.000     1.000       3.0
        B-fromloc.state_name      0.944     1.000     0.971      17.0
              B-airport_name      0.800     0.381     0.516      21.0
                   B-economy      1.000     1.000     1.000       6.0
               I-flight_time      1.000     1.000     1.000       1.0
             B-aircraft_code      0.968     0.909     0.937      33.0
                       B-mod      1.000     0.500     0.667       2.0
              B-airport_code      0.500     0.111     0.182       9.0
    B-depart_time.start_time      1.000     1.000     1.000       3.0
      B-depart_time.end_time      1.000     0.667     0.800       3.0
          B-depart_date.year      1.000     1.000     1.000       3.0
            I-transport_type      0.000     0.000     0.000       1.0
          B-restriction_code      0.333     1.000     0.500       4.0
    B-arrive_time.start_time      1.000     1.000     1.000       8.0
        B-toloc.airport_code      1.000     0.750     0.857       4.0
      B-arrive_time.end_time      1.000     1.000     1.000       8.0
      I-arrive_time.end_time      1.000     1.000     1.000       8.0
      I-depart_time.end_time      1.000     0.667     0.800       3.0
               I-flight_stop      0.000     0.000     0.000       0.0
      B-fromloc.airport_code      1.000     1.000     1.000       5.0
          I-restriction_code      1.000     1.000     1.000       3.0
    I-depart_time.start_time      1.000     1.000     1.000       1.0
          I-toloc.state_name      1.000     1.000     1.000       1.0
I-depart_date.today_relative      0.000     0.000     0.000       0.0
 B-arrive_date.date_relative      1.000     1.000     1.000       2.0
                I-flight_mod      1.000     0.167     0.286       6.0
                   I-economy      0.000     0.000     0.000       0.0
 B-return_date.date_relative      1.000     0.333     0.500       3.0
        I-fromloc.state_name      1.000     1.000     1.000       1.0
                B-state_code      1.000     1.000     1.000       1.0
    I-arrive_time.start_time      1.000     1.000     1.000       1.0
    I-arrive_date.day_number      0.000     0.000     0.000       0.0
                 B-meal_code      0.000     0.000     0.000       1.0
 I-depart_time.period_of_day      1.000     1.000     1.000       1.0
                  B-day_name      1.000     0.500     0.667       2.0
             B-period_of_day      1.000     1.000     1.000       4.0
        B-stoploc.state_code      0.000     0.000     0.000       0.0
    B-return_date.month_name      0.000     0.000     0.000       0.0
    B-return_date.day_number      0.000     0.000     0.000       0.0
    B-arrive_time.period_mod      0.000     0.000     0.000       0.0
                 I-meal_code      0.000     0.000     0.000       0.0
        B-toloc.country_name      1.000     1.000     1.000       1.0
                 B-days_code      1.000     1.000     1.000       1.0
 I-arrive_time.period_of_day      0.000     0.000     0.000       0.0
            I-today_relative      0.000     0.000     0.000       0.0
 B-return_time.period_of_day      0.000     0.000     0.000       0.0
                      B-time      0.000     0.000     0.000       0.0
           I-fare_basis_code      0.000     0.000     0.000       0.0
 I-arrive_time.time_relative      0.000     0.000     0.000       4.0
 I-depart_time.time_relative      0.000     0.000     0.000       1.0
            B-today_relative      0.000     0.000     0.000       0.0
                B-state_name      0.000     0.000     0.000       9.0
B-arrive_date.today_relative      0.000     0.000     0.000       0.0
    B-return_time.period_mod      0.000     0.000     0.000       0.0
                B-month_name      0.000     0.000     0.000       0.0
                B-day_number      0.000     0.000     0.000       0.0
 I-return_date.date_relative      0.750     1.000     0.857       3.0
I-return_date.today_relative      0.000     0.000     0.000       0.0
      B-stoploc.airport_name      0.000     0.000     0.000       0.0
             B-time_relative      0.000     0.000     0.000       0.0
                      I-time      0.000     0.000     0.000       0.0
    I-return_date.day_number      0.000     0.000     0.000       0.0
          I-meal_description      0.000     0.000     0.000       0.0
B-return_date.today_relative      0.000     0.000     0.000       0.0
      B-return_date.day_name      0.000     0.000     0.000       2.0

                   micro avg      0.952     0.954     0.953    3657.0
                   macro avg      0.684     0.660     0.658    3657.0
                weighted avg      0.959     0.954     0.951    3657.0

I0716 01:07:53.986789 140076727973760 interactiveshell.py:2882] Best Slot F1: 0.957
I0716 01:07:53.988596 140076727973760 estimator_training.py:186] Not using Distribute Coordinator.
I0716 01:07:53.992880 140076727973760 training.py:612] Running training and evaluation locally (non-distributed).
I0716 01:07:53.995326 140076727973760 training.py:700] Start train and evaluate loop. The evaluate will happen after every checkpoint. Checkpoint frequency is determined based on RunConfig arguments: save_checkpoints_steps 312 or save_checkpoints_secs None.
I0716 01:07:54.059036 140076727973760 estimator.py:1145] Calling model_fn.
I0716 01:07:55.371352 140076727973760 <ipython-input-6-0a85a3fd693f>:50] 
[<tf.Variable 'embedding:0' shape=(750, 300) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/gates/kernel:0' shape=(600, 600) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/gates/bias:0' shape=(600,) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/candidate/kernel:0' shape=(600, 300) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/candidate/bias:0' shape=(300,) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/gates/kernel:0' shape=(600, 600) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/gates/bias:0' shape=(600,) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/candidate/kernel:0' shape=(600, 300) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/candidate/bias:0' shape=(300,) dtype=float32_ref>,
 <tf.Variable 'dense/kernel:0' shape=(600, 23) dtype=float32_ref>,
 <tf.Variable 'dense/bias:0' shape=(23,) dtype=float32_ref>,
 <tf.Variable 'dense_1/kernel:0' shape=(600, 122) dtype=float32_ref>,
 <tf.Variable 'dense_1/bias:0' shape=(122,) dtype=float32_ref>,
 <tf.Variable 'transitions:0' shape=(122, 122) dtype=float32_ref>]
I0716 01:07:56.901959 140076727973760 estimator.py:1147] Done calling model_fn.
I0716 01:07:56.905689 140076727973760 basic_session_run_hooks.py:541] Create CheckpointSaverHook.
I0716 01:07:57.101051 140076727973760 monitored_session.py:240] Graph was finalized.
I0716 01:07:57.117244 140076727973760 saver.py:1280] Restoring parameters from ../model/bigru_crf/model.ckpt-7488
I0716 01:07:57.382473 140076727973760 session_manager.py:500] Running local_init_op.
I0716 01:07:57.420250 140076727973760 session_manager.py:502] Done running local_init_op.
I0716 01:07:58.955657 140076727973760 basic_session_run_hooks.py:606] Saving checkpoints for 7488 into ../model/bigru_crf/model.ckpt.
Reading ../data/atis.train.w-intent.iob
I0716 01:08:01.308576 140076727973760 basic_session_run_hooks.py:262] loss = 0.016282866, step = 7488
I0716 01:08:01.314596 140076727973760 basic_session_run_hooks.py:262] lr = 0.00013629787
I0716 01:08:13.403026 140076727973760 basic_session_run_hooks.py:692] global_step/sec: 8.26793
I0716 01:08:13.410147 140076727973760 basic_session_run_hooks.py:260] loss = 0.010518235, step = 7588 (12.102 sec)
I0716 01:08:13.412345 140076727973760 basic_session_run_hooks.py:260] lr = 0.00013486936 (12.098 sec)
I0716 01:08:24.217446 140076727973760 basic_session_run_hooks.py:692] global_step/sec: 9.2469
I0716 01:08:24.224272 140076727973760 basic_session_run_hooks.py:260] loss = 0.011644351, step = 7688 (10.814 sec)
I0716 01:08:24.226519 140076727973760 basic_session_run_hooks.py:260] lr = 0.00013345583 (10.814 sec)
I0716 01:08:34.969874 140076727973760 basic_session_run_hooks.py:692] global_step/sec: 9.30023
I0716 01:08:34.972789 140076727973760 basic_session_run_hooks.py:260] loss = 0.0076736514, step = 7788 (10.749 sec)
I0716 01:08:34.977826 140076727973760 basic_session_run_hooks.py:260] lr = 0.00013205713 (10.751 sec)
I0716 01:08:36.136519 140076727973760 basic_session_run_hooks.py:606] Saving checkpoints for 7800 into ../model/bigru_crf/model.ckpt.
I0716 01:08:36.545670 140076727973760 estimator.py:1145] Calling model_fn.
I0716 01:08:37.729356 140076727973760 estimator.py:1147] Done calling model_fn.
I0716 01:08:37.759375 140076727973760 evaluation.py:255] Starting evaluation at 2019-07-16T01:08:37Z
I0716 01:08:37.877200 140076727973760 monitored_session.py:240] Graph was finalized.
I0716 01:08:37.894999 140076727973760 saver.py:1280] Restoring parameters from ../model/bigru_crf/model.ckpt-7800
I0716 01:08:38.016578 140076727973760 session_manager.py:500] Running local_init_op.
I0716 01:08:38.044147 140076727973760 session_manager.py:502] Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
I0716 01:08:40.338587 140076727973760 evaluation.py:275] Finished evaluation at 2019-07-16-01:08:40
I0716 01:08:40.340267 140076727973760 estimator.py:2039] Saving dict for global step 7800: global_step = 7800, loss = 4.276826
I0716 01:08:40.348180 140076727973760 estimator.py:2099] Saving 'checkpoint_path' summary for global step 7800: ../model/bigru_crf/model.ckpt-7800
I0716 01:08:40.421188 140076727973760 estimator.py:368] Loss for final step: 0.0015443178.
Reading ../data/atis.test.w-intent.iob
I0716 01:08:40.745668 140076727973760 estimator.py:1145] Calling model_fn.
I0716 01:08:41.403154 140076727973760 estimator.py:1147] Done calling model_fn.
I0716 01:08:41.516979 140076727973760 monitored_session.py:240] Graph was finalized.
I0716 01:08:41.532525 140076727973760 saver.py:1280] Restoring parameters from ../model/bigru_crf/model.ckpt-7800
I0716 01:08:41.638096 140076727973760 session_manager.py:500] Running local_init_op.
I0716 01:08:41.669364 140076727973760 session_manager.py:502] Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.
  'precision', 'predicted', average, warn_for)
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1439: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples.
  'recall', 'true', average, warn_for)
I0716 01:08:44.813206 140076727973760 interactiveshell.py:2882] 
                                          precision    recall  f1-score   support

                             atis_flight      0.975     0.992     0.984       632
                            atis_airfare      0.959     0.979     0.969        48
                     atis_ground_service      1.000     1.000     1.000        36
                            atis_airline      1.000     1.000     1.000        38
                       atis_abbreviation      0.943     1.000     0.971        33
                           atis_aircraft      1.000     1.000     1.000         9
                        atis_flight_time      1.000     1.000     1.000         1
                           atis_quantity      0.375     1.000     0.545         3
                atis_flight#atis_airfare      0.750     0.250     0.375        12
                            atis_airport      1.000     0.944     0.971        18
                           atis_distance      1.000     1.000     1.000        10
                               atis_city      1.000     0.500     0.667         6
                        atis_ground_fare      1.000     0.857     0.923         7
                           atis_capacity      1.000     0.952     0.976        21
                          atis_flight_no      0.889     1.000     0.941         8
                               atis_meal      1.000     0.833     0.909         6
                        atis_restriction      0.000     0.000     0.000         0
             atis_airline#atis_flight_no      0.000     0.000     0.000         0
    atis_ground_service#atis_ground_fare      0.000     0.000     0.000         0
           atis_airfare#atis_flight_time      0.000     0.000     0.000         0
                           atis_cheapest      0.000     0.000     0.000         0
atis_aircraft#atis_flight#atis_flight_no      0.000     0.000     0.000         0

                               micro avg      0.970     0.975     0.972       888
                               macro avg      0.677     0.650     0.647       888
                            weighted avg      0.971     0.975     0.970       888

I0716 01:08:44.863459 140076727973760 interactiveshell.py:2882] 
                              precision    recall  f1-score   support

                           O      0.000     0.000     0.000       0.0
           B-toloc.city_name      0.971     0.992     0.981     716.0
         B-fromloc.city_name      0.983     0.997     0.990     704.0
           I-toloc.city_name      0.974     0.981     0.977     265.0
      B-depart_date.day_name      0.986     0.991     0.988     212.0
              B-airline_name      1.000     1.000     1.000     101.0
         I-fromloc.city_name      0.956     0.989     0.972     177.0
 B-depart_time.period_of_day      1.000     0.923     0.960     130.0
              I-airline_name      1.000     1.000     1.000      65.0
    B-depart_date.day_number      0.981     0.964     0.972      55.0
    B-depart_date.month_name      0.982     0.964     0.973      56.0
          B-depart_time.time      0.838     1.000     0.912      57.0
                B-round_trip      1.000     0.973     0.986      73.0
             B-cost_relative      1.000     0.973     0.986      37.0
                I-round_trip      1.000     1.000     1.000      71.0
                B-flight_mod      1.000     1.000     1.000      24.0
 B-depart_time.time_relative      0.969     0.969     0.969      65.0
          I-depart_time.time      0.945     1.000     0.972      52.0
         B-stoploc.city_name      0.952     1.000     0.976      20.0
                 B-city_name      0.912     0.544     0.681      57.0
                B-class_type      0.960     1.000     0.980      24.0
          B-arrive_time.time      1.000     0.971     0.985      34.0
 B-arrive_time.time_relative      0.935     0.935     0.935      31.0
                I-class_type      1.000     1.000     1.000      17.0
               B-flight_stop      1.000     1.000     1.000      21.0
          I-arrive_time.time      1.000     0.971     0.986      35.0
              B-airline_code      1.000     0.971     0.985      34.0
    I-depart_date.day_number      1.000     1.000     1.000      15.0
      I-fromloc.airport_name      0.417     1.000     0.588      15.0
      B-fromloc.airport_name      0.462     1.000     0.632      12.0
      B-arrive_date.day_name      0.769     0.909     0.833      11.0
          B-toloc.state_code      0.947     1.000     0.973      18.0
B-depart_date.today_relative      1.000     0.889     0.941       9.0
             B-flight_number      0.579     1.000     0.733      11.0
 B-depart_date.date_relative      0.944     1.000     0.971      17.0
          B-toloc.state_name      0.781     0.893     0.833      28.0
           B-fare_basis_code      0.944     1.000     0.971      17.0
               B-flight_time      1.000     1.000     1.000       1.0
                        B-or      1.000     1.000     1.000       3.0
 B-arrive_time.period_of_day      0.857     1.000     0.923       6.0
          B-meal_description      1.000     0.900     0.947      10.0
             I-cost_relative      1.000     0.667     0.800       3.0
              I-airport_name      0.917     0.379     0.537      29.0
               B-fare_amount      1.000     1.000     1.000       2.0
               I-fare_amount      1.000     1.000     1.000       2.0
                 I-city_name      1.000     0.567     0.723      30.0
        I-toloc.airport_name      1.000     1.000     1.000       3.0
            B-transport_type      1.000     1.000     1.000      10.0
    B-arrive_date.month_name      0.714     0.833     0.769       6.0
    B-arrive_date.day_number      0.714     0.833     0.769       6.0
         I-stoploc.city_name      0.833     1.000     0.909      10.0
                      B-meal      1.000     1.000     1.000      16.0
        B-fromloc.state_code      1.000     0.957     0.978      23.0
    B-depart_time.period_mod      1.000     1.000     1.000       5.0
                   B-connect      1.000     1.000     1.000       6.0
               B-flight_days      1.000     1.000     1.000      10.0
        B-toloc.airport_name      1.000     1.000     1.000       3.0
        B-fromloc.state_name      0.944     1.000     0.971      17.0
              B-airport_name      0.800     0.381     0.516      21.0
                   B-economy      1.000     1.000     1.000       6.0
               I-flight_time      1.000     1.000     1.000       1.0
             B-aircraft_code      0.931     0.818     0.871      33.0
                       B-mod      1.000     0.500     0.667       2.0
              B-airport_code      0.500     0.111     0.182       9.0
    B-depart_time.start_time      0.667     0.667     0.667       3.0
      B-depart_time.end_time      1.000     0.333     0.500       3.0
          B-depart_date.year      1.000     1.000     1.000       3.0
            I-transport_type      0.000     0.000     0.000       1.0
          B-restriction_code      0.400     1.000     0.571       4.0
    B-arrive_time.start_time      0.889     1.000     0.941       8.0
        B-toloc.airport_code      1.000     0.750     0.857       4.0
      B-arrive_time.end_time      0.889     1.000     0.941       8.0
      I-arrive_time.end_time      0.889     1.000     0.941       8.0
      I-depart_time.end_time      1.000     0.333     0.500       3.0
               I-flight_stop      0.000     0.000     0.000       0.0
      B-fromloc.airport_code      1.000     1.000     1.000       5.0
          I-restriction_code      1.000     1.000     1.000       3.0
    I-depart_time.start_time      1.000     1.000     1.000       1.0
          I-toloc.state_name      1.000     1.000     1.000       1.0
I-depart_date.today_relative      0.000     0.000     0.000       0.0
 B-arrive_date.date_relative      1.000     1.000     1.000       2.0
                I-flight_mod      1.000     0.167     0.286       6.0
                   I-economy      0.000     0.000     0.000       0.0
 B-return_date.date_relative      1.000     0.333     0.500       3.0
        I-fromloc.state_name      1.000     1.000     1.000       1.0
                B-state_code      1.000     1.000     1.000       1.0
    I-arrive_time.start_time      1.000     1.000     1.000       1.0
    I-arrive_date.day_number      0.000     0.000     0.000       0.0
                 B-meal_code      0.000     0.000     0.000       1.0
 I-depart_time.period_of_day      1.000     1.000     1.000       1.0
                  B-day_name      1.000     0.500     0.667       2.0
             B-period_of_day      1.000     1.000     1.000       4.0
        B-stoploc.state_code      0.000     0.000     0.000       0.0
    B-return_date.month_name      0.000     0.000     0.000       0.0
    B-return_date.day_number      0.000     0.000     0.000       0.0
    B-arrive_time.period_mod      0.000     0.000     0.000       0.0
                 I-meal_code      0.000     0.000     0.000       0.0
        B-toloc.country_name      1.000     1.000     1.000       1.0
                 B-days_code      1.000     1.000     1.000       1.0
 I-arrive_time.period_of_day      0.000     0.000     0.000       0.0
            I-today_relative      0.000     0.000     0.000       0.0
 B-return_time.period_of_day      0.000     0.000     0.000       0.0
                      B-time      0.000     0.000     0.000       0.0
           I-fare_basis_code      0.000     0.000     0.000       0.0
 I-arrive_time.time_relative      0.000     0.000     0.000       4.0
 I-depart_time.time_relative      0.000     0.000     0.000       1.0
            B-today_relative      0.000     0.000     0.000       0.0
                B-state_name      0.000     0.000     0.000       9.0
B-arrive_date.today_relative      0.000     0.000     0.000       0.0
    B-return_time.period_mod      0.000     0.000     0.000       0.0
                B-month_name      0.000     0.000     0.000       0.0
                B-day_number      0.000     0.000     0.000       0.0
 I-return_date.date_relative      0.750     1.000     0.857       3.0
I-return_date.today_relative      0.000     0.000     0.000       0.0
      B-stoploc.airport_name      0.000     0.000     0.000       0.0
             B-time_relative      0.000     0.000     0.000       0.0
                      I-time      0.000     0.000     0.000       0.0
    I-return_date.day_number      0.000     0.000     0.000       0.0
          I-meal_description      0.000     0.000     0.000       0.0
B-return_date.today_relative      0.000     0.000     0.000       0.0
      B-return_date.day_name      0.000     0.000     0.000       2.0

                   micro avg      0.952     0.954     0.953    3657.0
                   macro avg      0.677     0.651     0.648    3657.0
                weighted avg      0.959     0.954     0.951    3657.0

I0716 01:08:44.895675 140076727973760 interactiveshell.py:2882] Best Slot F1: 0.957
I0716 01:08:44.897492 140076727973760 estimator_training.py:186] Not using Distribute Coordinator.
I0716 01:08:44.899365 140076727973760 training.py:612] Running training and evaluation locally (non-distributed).
I0716 01:08:44.901283 140076727973760 training.py:700] Start train and evaluate loop. The evaluate will happen after every checkpoint. Checkpoint frequency is determined based on RunConfig arguments: save_checkpoints_steps 312 or save_checkpoints_secs None.
I0716 01:08:44.967709 140076727973760 estimator.py:1145] Calling model_fn.
I0716 01:08:46.297254 140076727973760 <ipython-input-6-0a85a3fd693f>:50] 
[<tf.Variable 'embedding:0' shape=(750, 300) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/gates/kernel:0' shape=(600, 600) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/gates/bias:0' shape=(600,) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/candidate/kernel:0' shape=(600, 300) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/candidate/bias:0' shape=(300,) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/gates/kernel:0' shape=(600, 600) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/gates/bias:0' shape=(600,) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/candidate/kernel:0' shape=(600, 300) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/candidate/bias:0' shape=(300,) dtype=float32_ref>,
 <tf.Variable 'dense/kernel:0' shape=(600, 23) dtype=float32_ref>,
 <tf.Variable 'dense/bias:0' shape=(23,) dtype=float32_ref>,
 <tf.Variable 'dense_1/kernel:0' shape=(600, 122) dtype=float32_ref>,
 <tf.Variable 'dense_1/bias:0' shape=(122,) dtype=float32_ref>,
 <tf.Variable 'transitions:0' shape=(122, 122) dtype=float32_ref>]
I0716 01:08:47.864760 140076727973760 estimator.py:1147] Done calling model_fn.
I0716 01:08:47.868709 140076727973760 basic_session_run_hooks.py:541] Create CheckpointSaverHook.
I0716 01:08:48.064363 140076727973760 monitored_session.py:240] Graph was finalized.
I0716 01:08:48.082494 140076727973760 saver.py:1280] Restoring parameters from ../model/bigru_crf/model.ckpt-7800
I0716 01:08:48.337069 140076727973760 session_manager.py:500] Running local_init_op.
I0716 01:08:48.373564 140076727973760 session_manager.py:502] Done running local_init_op.
I0716 01:08:49.881341 140076727973760 basic_session_run_hooks.py:606] Saving checkpoints for 7800 into ../model/bigru_crf/model.ckpt.
Reading ../data/atis.train.w-intent.iob
I0716 01:08:52.219379 140076727973760 basic_session_run_hooks.py:262] loss = 0.01898968, step = 7800
I0716 01:08:52.221129 140076727973760 basic_session_run_hooks.py:262] lr = 0.00013189027
I0716 01:09:03.954310 140076727973760 basic_session_run_hooks.py:692] global_step/sec: 8.52122
I0716 01:09:03.962072 140076727973760 basic_session_run_hooks.py:260] loss = 0.006735981, step = 7900 (11.743 sec)
I0716 01:09:03.963712 140076727973760 basic_session_run_hooks.py:260] lr = 0.00013050795 (11.743 sec)
I0716 01:09:14.922441 140076727973760 basic_session_run_hooks.py:692] global_step/sec: 9.11729
I0716 01:09:14.929069 140076727973760 basic_session_run_hooks.py:260] loss = 0.010166981, step = 8000 (10.967 sec)
I0716 01:09:14.931562 140076727973760 basic_session_run_hooks.py:260] lr = 0.00012914014 (10.968 sec)
I0716 01:09:25.713553 140076727973760 basic_session_run_hooks.py:692] global_step/sec: 9.26687
I0716 01:09:25.721097 140076727973760 basic_session_run_hooks.py:260] loss = 0.049034335, step = 8100 (10.792 sec)
I0716 01:09:25.723180 140076727973760 basic_session_run_hooks.py:260] lr = 0.00012778665 (10.792 sec)
I0716 01:09:26.912083 140076727973760 basic_session_run_hooks.py:606] Saving checkpoints for 8112 into ../model/bigru_crf/model.ckpt.
I0716 01:09:27.307963 140076727973760 estimator.py:1145] Calling model_fn.
I0716 01:09:28.486549 140076727973760 estimator.py:1147] Done calling model_fn.
I0716 01:09:28.515740 140076727973760 evaluation.py:255] Starting evaluation at 2019-07-16T01:09:28Z
I0716 01:09:28.632753 140076727973760 monitored_session.py:240] Graph was finalized.
I0716 01:09:28.648791 140076727973760 saver.py:1280] Restoring parameters from ../model/bigru_crf/model.ckpt-8112
I0716 01:09:28.776464 140076727973760 session_manager.py:500] Running local_init_op.
I0716 01:09:28.806324 140076727973760 session_manager.py:502] Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
I0716 01:09:31.145760 140076727973760 evaluation.py:275] Finished evaluation at 2019-07-16-01:09:31
I0716 01:09:31.147464 140076727973760 estimator.py:2039] Saving dict for global step 8112: global_step = 8112, loss = 4.267621
I0716 01:09:31.154431 140076727973760 estimator.py:2099] Saving 'checkpoint_path' summary for global step 8112: ../model/bigru_crf/model.ckpt-8112
I0716 01:09:31.216267 140076727973760 estimator.py:368] Loss for final step: 0.007409137.
Reading ../data/atis.test.w-intent.iob
I0716 01:09:31.543572 140076727973760 estimator.py:1145] Calling model_fn.
I0716 01:09:32.207490 140076727973760 estimator.py:1147] Done calling model_fn.
I0716 01:09:32.320346 140076727973760 monitored_session.py:240] Graph was finalized.
I0716 01:09:32.335514 140076727973760 saver.py:1280] Restoring parameters from ../model/bigru_crf/model.ckpt-8112
I0716 01:09:32.437573 140076727973760 session_manager.py:500] Running local_init_op.
I0716 01:09:32.457037 140076727973760 session_manager.py:502] Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.
  'precision', 'predicted', average, warn_for)
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1439: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples.
  'recall', 'true', average, warn_for)
I0716 01:09:35.601731 140076727973760 interactiveshell.py:2882] 
                                          precision    recall  f1-score   support

                             atis_flight      0.975     0.992     0.984       632
                            atis_airfare      0.959     0.979     0.969        48
                     atis_ground_service      1.000     1.000     1.000        36
                            atis_airline      1.000     1.000     1.000        38
                       atis_abbreviation      0.943     1.000     0.971        33
                           atis_aircraft      1.000     1.000     1.000         9
                        atis_flight_time      1.000     1.000     1.000         1
                           atis_quantity      0.375     1.000     0.545         3
                atis_flight#atis_airfare      0.750     0.250     0.375        12
                            atis_airport      1.000     0.944     0.971        18
                           atis_distance      1.000     1.000     1.000        10
                               atis_city      1.000     0.500     0.667         6
                        atis_ground_fare      1.000     0.857     0.923         7
                           atis_capacity      1.000     0.952     0.976        21
                          atis_flight_no      0.889     1.000     0.941         8
                               atis_meal      1.000     0.833     0.909         6
                        atis_restriction      0.000     0.000     0.000         0
             atis_airline#atis_flight_no      0.000     0.000     0.000         0
    atis_ground_service#atis_ground_fare      0.000     0.000     0.000         0
           atis_airfare#atis_flight_time      0.000     0.000     0.000         0
                           atis_cheapest      0.000     0.000     0.000         0
atis_aircraft#atis_flight#atis_flight_no      0.000     0.000     0.000         0

                               micro avg      0.970     0.975     0.972       888
                               macro avg      0.677     0.650     0.647       888
                            weighted avg      0.971     0.975     0.970       888

I0716 01:09:35.655186 140076727973760 interactiveshell.py:2882] 
                              precision    recall  f1-score   support

                           O      0.000     0.000     0.000       0.0
           B-toloc.city_name      0.974     0.993     0.983     716.0
         B-fromloc.city_name      0.978     0.997     0.987     704.0
           I-toloc.city_name      0.978     0.989     0.983     265.0
      B-depart_date.day_name      0.991     0.991     0.991     212.0
              B-airline_name      1.000     1.000     1.000     101.0
         I-fromloc.city_name      0.967     0.989     0.978     177.0
 B-depart_time.period_of_day      1.000     0.923     0.960     130.0
              I-airline_name      1.000     1.000     1.000      65.0
    B-depart_date.day_number      0.981     0.964     0.972      55.0
    B-depart_date.month_name      0.982     0.964     0.973      56.0
          B-depart_time.time      0.864     1.000     0.927      57.0
                B-round_trip      1.000     0.986     0.993      73.0
             B-cost_relative      1.000     0.973     0.986      37.0
                I-round_trip      1.000     1.000     1.000      71.0
                B-flight_mod      0.828     1.000     0.906      24.0
 B-depart_time.time_relative      0.969     0.954     0.961      65.0
          I-depart_time.time      0.945     1.000     0.972      52.0
         B-stoploc.city_name      0.909     1.000     0.952      20.0
                 B-city_name      0.941     0.561     0.703      57.0
                B-class_type      0.960     1.000     0.980      24.0
          B-arrive_time.time      0.971     0.971     0.971      34.0
 B-arrive_time.time_relative      0.906     0.935     0.921      31.0
                I-class_type      1.000     1.000     1.000      17.0
               B-flight_stop      1.000     1.000     1.000      21.0
          I-arrive_time.time      1.000     0.971     0.986      35.0
              B-airline_code      1.000     0.971     0.985      34.0
    I-depart_date.day_number      1.000     0.933     0.966      15.0
      I-fromloc.airport_name      0.429     1.000     0.600      15.0
      B-fromloc.airport_name      0.462     1.000     0.632      12.0
      B-arrive_date.day_name      0.786     1.000     0.880      11.0
          B-toloc.state_code      0.947     1.000     0.973      18.0
B-depart_date.today_relative      1.000     0.889     0.941       9.0
             B-flight_number      0.688     1.000     0.815      11.0
 B-depart_date.date_relative      0.944     1.000     0.971      17.0
          B-toloc.state_name      0.781     0.893     0.833      28.0
           B-fare_basis_code      0.944     1.000     0.971      17.0
               B-flight_time      1.000     1.000     1.000       1.0
                        B-or      1.000     1.000     1.000       3.0
 B-arrive_time.period_of_day      0.857     1.000     0.923       6.0
          B-meal_description      1.000     0.900     0.947      10.0
             I-cost_relative      1.000     0.667     0.800       3.0
              I-airport_name      0.923     0.414     0.571      29.0
               B-fare_amount      1.000     1.000     1.000       2.0
               I-fare_amount      1.000     1.000     1.000       2.0
                 I-city_name      1.000     0.600     0.750      30.0
        I-toloc.airport_name      1.000     1.000     1.000       3.0
            B-transport_type      1.000     1.000     1.000      10.0
    B-arrive_date.month_name      0.714     0.833     0.769       6.0
    B-arrive_date.day_number      0.714     0.833     0.769       6.0
         I-stoploc.city_name      0.833     1.000     0.909      10.0
                      B-meal      1.000     1.000     1.000      16.0
        B-fromloc.state_code      1.000     0.957     0.978      23.0
    B-depart_time.period_mod      1.000     1.000     1.000       5.0
                   B-connect      1.000     1.000     1.000       6.0
               B-flight_days      1.000     1.000     1.000      10.0
        B-toloc.airport_name      1.000     1.000     1.000       3.0
        B-fromloc.state_name      0.944     1.000     0.971      17.0
              B-airport_name      0.818     0.429     0.562      21.0
                   B-economy      1.000     1.000     1.000       6.0
               I-flight_time      1.000     1.000     1.000       1.0
             B-aircraft_code      0.967     0.879     0.921      33.0
                       B-mod      1.000     0.500     0.667       2.0
              B-airport_code      0.500     0.111     0.182       9.0
    B-depart_time.start_time      1.000     0.667     0.800       3.0
      B-depart_time.end_time      1.000     0.333     0.500       3.0
          B-depart_date.year      1.000     1.000     1.000       3.0
            I-transport_type      0.000     0.000     0.000       1.0
          B-restriction_code      0.400     1.000     0.571       4.0
    B-arrive_time.start_time      0.889     1.000     0.941       8.0
        B-toloc.airport_code      1.000     0.750     0.857       4.0
      B-arrive_time.end_time      0.889     1.000     0.941       8.0
      I-arrive_time.end_time      0.889     1.000     0.941       8.0
      I-depart_time.end_time      1.000     0.333     0.500       3.0
               I-flight_stop      0.000     0.000     0.000       0.0
      B-fromloc.airport_code      1.000     1.000     1.000       5.0
          I-restriction_code      1.000     1.000     1.000       3.0
    I-depart_time.start_time      1.000     1.000     1.000       1.0
          I-toloc.state_name      1.000     1.000     1.000       1.0
I-depart_date.today_relative      0.000     0.000     0.000       0.0
 B-arrive_date.date_relative      1.000     1.000     1.000       2.0
                I-flight_mod      1.000     0.167     0.286       6.0
                   I-economy      0.000     0.000     0.000       0.0
 B-return_date.date_relative      1.000     0.333     0.500       3.0
        I-fromloc.state_name      1.000     1.000     1.000       1.0
                B-state_code      1.000     1.000     1.000       1.0
    I-arrive_time.start_time      1.000     1.000     1.000       1.0
    I-arrive_date.day_number      0.000     0.000     0.000       0.0
                 B-meal_code      0.000     0.000     0.000       1.0
 I-depart_time.period_of_day      1.000     1.000     1.000       1.0
                  B-day_name      1.000     0.500     0.667       2.0
             B-period_of_day      1.000     1.000     1.000       4.0
        B-stoploc.state_code      0.000     0.000     0.000       0.0
    B-return_date.month_name      0.000     0.000     0.000       0.0
    B-return_date.day_number      0.000     0.000     0.000       0.0
    B-arrive_time.period_mod      0.000     0.000     0.000       0.0
                 I-meal_code      0.000     0.000     0.000       0.0
        B-toloc.country_name      1.000     1.000     1.000       1.0
                 B-days_code      1.000     1.000     1.000       1.0
 I-arrive_time.period_of_day      0.000     0.000     0.000       0.0
            I-today_relative      0.000     0.000     0.000       0.0
 B-return_time.period_of_day      0.000     0.000     0.000       0.0
                      B-time      0.000     0.000     0.000       0.0
           I-fare_basis_code      0.000     0.000     0.000       0.0
 I-arrive_time.time_relative      0.000     0.000     0.000       4.0
 I-depart_time.time_relative      0.000     0.000     0.000       1.0
            B-today_relative      0.000     0.000     0.000       0.0
                B-state_name      0.000     0.000     0.000       9.0
B-arrive_date.today_relative      0.000     0.000     0.000       0.0
    B-return_time.period_mod      0.000     0.000     0.000       0.0
                B-month_name      0.000     0.000     0.000       0.0
                B-day_number      0.000     0.000     0.000       0.0
 I-return_date.date_relative      0.750     1.000     0.857       3.0
I-return_date.today_relative      0.000     0.000     0.000       0.0
      B-stoploc.airport_name      0.000     0.000     0.000       0.0
             B-time_relative      0.000     0.000     0.000       0.0
                      I-time      0.000     0.000     0.000       0.0
    I-return_date.day_number      0.000     0.000     0.000       0.0
          I-meal_description      0.000     0.000     0.000       0.0
B-return_date.today_relative      0.000     0.000     0.000       0.0
      B-return_date.day_name      0.000     0.000     0.000       2.0

                   micro avg      0.955     0.957     0.956    3657.0
                   macro avg      0.679     0.653     0.651    3657.0
                weighted avg      0.959     0.957     0.953    3657.0

I0716 01:09:35.683810 140076727973760 interactiveshell.py:2882] Best Slot F1: 0.957
I0716 01:09:35.685322 140076727973760 estimator_training.py:186] Not using Distribute Coordinator.
I0716 01:09:35.689028 140076727973760 training.py:612] Running training and evaluation locally (non-distributed).
I0716 01:09:35.692294 140076727973760 training.py:700] Start train and evaluate loop. The evaluate will happen after every checkpoint. Checkpoint frequency is determined based on RunConfig arguments: save_checkpoints_steps 312 or save_checkpoints_secs None.
I0716 01:09:35.756195 140076727973760 estimator.py:1145] Calling model_fn.
I0716 01:09:37.082959 140076727973760 <ipython-input-6-0a85a3fd693f>:50] 
[<tf.Variable 'embedding:0' shape=(750, 300) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/gates/kernel:0' shape=(600, 600) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/gates/bias:0' shape=(600,) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/candidate/kernel:0' shape=(600, 300) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/candidate/bias:0' shape=(300,) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/gates/kernel:0' shape=(600, 600) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/gates/bias:0' shape=(600,) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/candidate/kernel:0' shape=(600, 300) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/candidate/bias:0' shape=(300,) dtype=float32_ref>,
 <tf.Variable 'dense/kernel:0' shape=(600, 23) dtype=float32_ref>,
 <tf.Variable 'dense/bias:0' shape=(23,) dtype=float32_ref>,
 <tf.Variable 'dense_1/kernel:0' shape=(600, 122) dtype=float32_ref>,
 <tf.Variable 'dense_1/bias:0' shape=(122,) dtype=float32_ref>,
 <tf.Variable 'transitions:0' shape=(122, 122) dtype=float32_ref>]
I0716 01:09:38.608859 140076727973760 estimator.py:1147] Done calling model_fn.
I0716 01:09:38.612764 140076727973760 basic_session_run_hooks.py:541] Create CheckpointSaverHook.
I0716 01:09:38.813656 140076727973760 monitored_session.py:240] Graph was finalized.
I0716 01:09:38.834423 140076727973760 saver.py:1280] Restoring parameters from ../model/bigru_crf/model.ckpt-8112
I0716 01:09:39.098518 140076727973760 session_manager.py:500] Running local_init_op.
I0716 01:09:39.137868 140076727973760 session_manager.py:502] Done running local_init_op.
I0716 01:09:40.668308 140076727973760 basic_session_run_hooks.py:606] Saving checkpoints for 8112 into ../model/bigru_crf/model.ckpt.
Reading ../data/atis.train.w-intent.iob
I0716 01:09:43.007535 140076727973760 basic_session_run_hooks.py:262] loss = 0.21495847, step = 8112
I0716 01:09:43.009405 140076727973760 basic_session_run_hooks.py:262] lr = 0.00012762519
I0716 01:09:54.791059 140076727973760 basic_session_run_hooks.py:692] global_step/sec: 8.48608
I0716 01:09:54.798027 140076727973760 basic_session_run_hooks.py:260] loss = 0.008623324, step = 8212 (11.790 sec)
I0716 01:09:54.801039 140076727973760 basic_session_run_hooks.py:260] lr = 0.00012628757 (11.792 sec)
I0716 01:10:05.927694 140076727973760 basic_session_run_hooks.py:692] global_step/sec: 8.97926
I0716 01:10:05.933046 140076727973760 basic_session_run_hooks.py:260] loss = 0.021833928, step = 8312 (11.135 sec)
I0716 01:10:05.935793 140076727973760 basic_session_run_hooks.py:260] lr = 0.00012496399 (11.135 sec)
I0716 01:10:16.619071 140076727973760 basic_session_run_hooks.py:692] global_step/sec: 9.35334
I0716 01:10:16.623688 140076727973760 basic_session_run_hooks.py:260] loss = 0.012724008, step = 8412 (10.691 sec)
I0716 01:10:16.626594 140076727973760 basic_session_run_hooks.py:260] lr = 0.00012365427 (10.691 sec)
I0716 01:10:17.727040 140076727973760 basic_session_run_hooks.py:606] Saving checkpoints for 8424 into ../model/bigru_crf/model.ckpt.
I0716 01:10:18.120879 140076727973760 estimator.py:1145] Calling model_fn.
I0716 01:10:19.296044 140076727973760 estimator.py:1147] Done calling model_fn.
I0716 01:10:19.326250 140076727973760 evaluation.py:255] Starting evaluation at 2019-07-16T01:10:19Z
I0716 01:10:19.444738 140076727973760 monitored_session.py:240] Graph was finalized.
I0716 01:10:19.462427 140076727973760 saver.py:1280] Restoring parameters from ../model/bigru_crf/model.ckpt-8424
I0716 01:10:19.578527 140076727973760 session_manager.py:500] Running local_init_op.
I0716 01:10:19.603986 140076727973760 session_manager.py:502] Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
I0716 01:10:21.882323 140076727973760 evaluation.py:275] Finished evaluation at 2019-07-16-01:10:21
I0716 01:10:21.883863 140076727973760 estimator.py:2039] Saving dict for global step 8424: global_step = 8424, loss = 4.2223845
I0716 01:10:21.891943 140076727973760 estimator.py:2099] Saving 'checkpoint_path' summary for global step 8424: ../model/bigru_crf/model.ckpt-8424
I0716 01:10:21.953256 140076727973760 estimator.py:368] Loss for final step: 0.0011727472.
Reading ../data/atis.test.w-intent.iob
I0716 01:10:22.283310 140076727973760 estimator.py:1145] Calling model_fn.
I0716 01:10:22.955429 140076727973760 estimator.py:1147] Done calling model_fn.
I0716 01:10:23.398610 140076727973760 monitored_session.py:240] Graph was finalized.
I0716 01:10:23.416435 140076727973760 saver.py:1280] Restoring parameters from ../model/bigru_crf/model.ckpt-8424
I0716 01:10:23.523276 140076727973760 session_manager.py:500] Running local_init_op.
I0716 01:10:23.542504 140076727973760 session_manager.py:502] Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.
  'precision', 'predicted', average, warn_for)
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1439: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples.
  'recall', 'true', average, warn_for)
I0716 01:10:26.603128 140076727973760 interactiveshell.py:2882] 
                                          precision    recall  f1-score   support

                             atis_flight      0.977     0.992     0.984       632
                            atis_airfare      0.940     0.979     0.959        48
                     atis_ground_service      1.000     1.000     1.000        36
                            atis_airline      1.000     1.000     1.000        38
                       atis_abbreviation      0.943     1.000     0.971        33
                           atis_aircraft      1.000     1.000     1.000         9
                        atis_flight_time      1.000     1.000     1.000         1
                           atis_quantity      0.375     1.000     0.545         3
                atis_flight#atis_airfare      0.750     0.250     0.375        12
                            atis_airport      1.000     0.944     0.971        18
                           atis_distance      1.000     1.000     1.000        10
                               atis_city      1.000     0.500     0.667         6
                        atis_ground_fare      1.000     0.857     0.923         7
                           atis_capacity      1.000     0.952     0.976        21
                          atis_flight_no      0.889     1.000     0.941         8
                               atis_meal      1.000     0.833     0.909         6
                        atis_restriction      0.000     0.000     0.000         0
             atis_airline#atis_flight_no      0.000     0.000     0.000         0
    atis_ground_service#atis_ground_fare      0.000     0.000     0.000         0
           atis_airfare#atis_flight_time      0.000     0.000     0.000         0
                           atis_cheapest      0.000     0.000     0.000         0
atis_aircraft#atis_flight#atis_flight_no      0.000     0.000     0.000         0

                               micro avg      0.970     0.975     0.972       888
                               macro avg      0.676     0.650     0.646       888
                            weighted avg      0.972     0.975     0.970       888

I0716 01:10:26.653280 140076727973760 interactiveshell.py:2882] 
                              precision    recall  f1-score   support

                           O      0.000     0.000     0.000       0.0
           B-toloc.city_name      0.970     0.994     0.982     716.0
         B-fromloc.city_name      0.986     0.999     0.992     704.0
           I-toloc.city_name      0.967     0.989     0.978     265.0
      B-depart_date.day_name      0.986     0.991     0.988     212.0
              B-airline_name      1.000     1.000     1.000     101.0
         I-fromloc.city_name      0.978     0.989     0.983     177.0
 B-depart_time.period_of_day      1.000     0.923     0.960     130.0
              I-airline_name      1.000     1.000     1.000      65.0
    B-depart_date.day_number      0.981     0.964     0.972      55.0
    B-depart_date.month_name      0.982     0.964     0.973      56.0
          B-depart_time.time      0.864     1.000     0.927      57.0
                B-round_trip      1.000     0.973     0.986      73.0
             B-cost_relative      1.000     0.973     0.986      37.0
                I-round_trip      1.000     1.000     1.000      71.0
                B-flight_mod      1.000     1.000     1.000      24.0
 B-depart_time.time_relative      0.969     0.954     0.961      65.0
          I-depart_time.time      0.945     1.000     0.972      52.0
         B-stoploc.city_name      0.909     1.000     0.952      20.0
                 B-city_name      0.939     0.544     0.689      57.0
                B-class_type      0.960     1.000     0.980      24.0
          B-arrive_time.time      0.971     0.971     0.971      34.0
 B-arrive_time.time_relative      0.906     0.935     0.921      31.0
                I-class_type      1.000     1.000     1.000      17.0
               B-flight_stop      1.000     1.000     1.000      21.0
          I-arrive_time.time      1.000     0.971     0.986      35.0
              B-airline_code      1.000     0.912     0.954      34.0
    I-depart_date.day_number      1.000     1.000     1.000      15.0
      I-fromloc.airport_name      0.417     1.000     0.588      15.0
      B-fromloc.airport_name      0.480     1.000     0.649      12.0
      B-arrive_date.day_name      0.769     0.909     0.833      11.0
          B-toloc.state_code      0.947     1.000     0.973      18.0
B-depart_date.today_relative      1.000     0.889     0.941       9.0
             B-flight_number      0.524     1.000     0.688      11.0
 B-depart_date.date_relative      0.944     1.000     0.971      17.0
          B-toloc.state_name      0.781     0.893     0.833      28.0
           B-fare_basis_code      0.944     1.000     0.971      17.0
               B-flight_time      1.000     1.000     1.000       1.0
                        B-or      1.000     1.000     1.000       3.0
 B-arrive_time.period_of_day      0.857     1.000     0.923       6.0
          B-meal_description      1.000     0.900     0.947      10.0
             I-cost_relative      1.000     0.667     0.800       3.0
              I-airport_name      0.917     0.379     0.537      29.0
               B-fare_amount      1.000     1.000     1.000       2.0
               I-fare_amount      1.000     1.000     1.000       2.0
                 I-city_name      1.000     0.533     0.696      30.0
        I-toloc.airport_name      1.000     1.000     1.000       3.0
            B-transport_type      1.000     1.000     1.000      10.0
    B-arrive_date.month_name      0.714     0.833     0.769       6.0
    B-arrive_date.day_number      0.714     0.833     0.769       6.0
         I-stoploc.city_name      0.833     1.000     0.909      10.0
                      B-meal      1.000     1.000     1.000      16.0
        B-fromloc.state_code      1.000     0.957     0.978      23.0
    B-depart_time.period_mod      1.000     1.000     1.000       5.0
                   B-connect      1.000     1.000     1.000       6.0
               B-flight_days      1.000     1.000     1.000      10.0
        B-toloc.airport_name      1.000     1.000     1.000       3.0
        B-fromloc.state_name      0.944     1.000     0.971      17.0
              B-airport_name      0.800     0.381     0.516      21.0
                   B-economy      1.000     1.000     1.000       6.0
               I-flight_time      1.000     1.000     1.000       1.0
             B-aircraft_code      0.964     0.818     0.885      33.0
                       B-mod      1.000     0.500     0.667       2.0
              B-airport_code      0.500     0.111     0.182       9.0
    B-depart_time.start_time      0.667     0.667     0.667       3.0
      B-depart_time.end_time      1.000     0.333     0.500       3.0
          B-depart_date.year      1.000     1.000     1.000       3.0
            I-transport_type      0.000     0.000     0.000       1.0
          B-restriction_code      0.444     1.000     0.615       4.0
    B-arrive_time.start_time      0.889     1.000     0.941       8.0
        B-toloc.airport_code      1.000     0.750     0.857       4.0
      B-arrive_time.end_time      0.889     1.000     0.941       8.0
      I-arrive_time.end_time      0.889     1.000     0.941       8.0
      I-depart_time.end_time      1.000     0.333     0.500       3.0
               I-flight_stop      0.000     0.000     0.000       0.0
      B-fromloc.airport_code      1.000     1.000     1.000       5.0
          I-restriction_code      1.000     1.000     1.000       3.0
    I-depart_time.start_time      1.000     1.000     1.000       1.0
          I-toloc.state_name      1.000     1.000     1.000       1.0
I-depart_date.today_relative      0.000     0.000     0.000       0.0
 B-arrive_date.date_relative      1.000     1.000     1.000       2.0
                I-flight_mod      1.000     0.167     0.286       6.0
                   I-economy      0.000     0.000     0.000       0.0
 B-return_date.date_relative      1.000     0.333     0.500       3.0
        I-fromloc.state_name      1.000     1.000     1.000       1.0
                B-state_code      1.000     1.000     1.000       1.0
    I-arrive_time.start_time      1.000     1.000     1.000       1.0
    I-arrive_date.day_number      0.000     0.000     0.000       0.0
                 B-meal_code      1.000     1.000     1.000       1.0
 I-depart_time.period_of_day      1.000     1.000     1.000       1.0
                  B-day_name      1.000     0.500     0.667       2.0
             B-period_of_day      1.000     1.000     1.000       4.0
        B-stoploc.state_code      0.000     0.000     0.000       0.0
    B-return_date.month_name      0.000     0.000     0.000       0.0
    B-return_date.day_number      0.000     0.000     0.000       0.0
    B-arrive_time.period_mod      0.000     0.000     0.000       0.0
                 I-meal_code      0.000     0.000     0.000       0.0
        B-toloc.country_name      1.000     1.000     1.000       1.0
                 B-days_code      1.000     1.000     1.000       1.0
 I-arrive_time.period_of_day      0.000     0.000     0.000       0.0
            I-today_relative      0.000     0.000     0.000       0.0
 B-return_time.period_of_day      0.000     0.000     0.000       0.0
                      B-time      0.000     0.000     0.000       0.0
           I-fare_basis_code      0.000     0.000     0.000       0.0
 I-arrive_time.time_relative      0.000     0.000     0.000       4.0
 I-depart_time.time_relative      0.000     0.000     0.000       1.0
            B-today_relative      0.000     0.000     0.000       0.0
                B-state_name      0.000     0.000     0.000       9.0
B-arrive_date.today_relative      0.000     0.000     0.000       0.0
    B-return_time.period_mod      0.000     0.000     0.000       0.0
                B-month_name      0.000     0.000     0.000       0.0
                B-day_number      0.000     0.000     0.000       0.0
 I-return_date.date_relative      0.750     1.000     0.857       3.0
I-return_date.today_relative      0.000     0.000     0.000       0.0
      B-stoploc.airport_name      0.000     0.000     0.000       0.0
             B-time_relative      0.000     0.000     0.000       0.0
                      I-time      0.000     0.000     0.000       0.0
    I-return_date.day_number      0.000     0.000     0.000       0.0
          I-meal_description      0.000     0.000     0.000       0.0
B-return_date.today_relative      0.000     0.000     0.000       0.0
      B-return_date.day_name      0.000     0.000     0.000       2.0

                   micro avg      0.953     0.955     0.954    3657.0
                   macro avg      0.685     0.659     0.656    3657.0
                weighted avg      0.960     0.955     0.952    3657.0

I0716 01:10:26.680991 140076727973760 interactiveshell.py:2882] Best Slot F1: 0.957
I0716 01:10:26.682475 140076727973760 estimator_training.py:186] Not using Distribute Coordinator.
I0716 01:10:26.686357 140076727973760 training.py:612] Running training and evaluation locally (non-distributed).
I0716 01:10:26.689221 140076727973760 training.py:700] Start train and evaluate loop. The evaluate will happen after every checkpoint. Checkpoint frequency is determined based on RunConfig arguments: save_checkpoints_steps 312 or save_checkpoints_secs None.
I0716 01:10:26.752440 140076727973760 estimator.py:1145] Calling model_fn.
I0716 01:10:27.751741 140076727973760 <ipython-input-6-0a85a3fd693f>:50] 
[<tf.Variable 'embedding:0' shape=(750, 300) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/gates/kernel:0' shape=(600, 600) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/gates/bias:0' shape=(600,) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/candidate/kernel:0' shape=(600, 300) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/candidate/bias:0' shape=(300,) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/gates/kernel:0' shape=(600, 600) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/gates/bias:0' shape=(600,) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/candidate/kernel:0' shape=(600, 300) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/candidate/bias:0' shape=(300,) dtype=float32_ref>,
 <tf.Variable 'dense/kernel:0' shape=(600, 23) dtype=float32_ref>,
 <tf.Variable 'dense/bias:0' shape=(23,) dtype=float32_ref>,
 <tf.Variable 'dense_1/kernel:0' shape=(600, 122) dtype=float32_ref>,
 <tf.Variable 'dense_1/bias:0' shape=(122,) dtype=float32_ref>,
 <tf.Variable 'transitions:0' shape=(122, 122) dtype=float32_ref>]
I0716 01:10:29.290796 140076727973760 estimator.py:1147] Done calling model_fn.
I0716 01:10:29.299316 140076727973760 basic_session_run_hooks.py:541] Create CheckpointSaverHook.
I0716 01:10:29.495871 140076727973760 monitored_session.py:240] Graph was finalized.
I0716 01:10:29.516280 140076727973760 saver.py:1280] Restoring parameters from ../model/bigru_crf/model.ckpt-8424
I0716 01:10:29.776308 140076727973760 session_manager.py:500] Running local_init_op.
I0716 01:10:29.818665 140076727973760 session_manager.py:502] Done running local_init_op.
I0716 01:10:31.524835 140076727973760 basic_session_run_hooks.py:606] Saving checkpoints for 8424 into ../model/bigru_crf/model.ckpt.
Reading ../data/atis.train.w-intent.iob
I0716 01:10:33.876308 140076727973760 basic_session_run_hooks.py:262] loss = 0.01626001, step = 8424
I0716 01:10:33.878232 140076727973760 basic_session_run_hooks.py:262] lr = 0.00012349803
I0716 01:10:45.347473 140076727973760 basic_session_run_hooks.py:692] global_step/sec: 8.71707
I0716 01:10:45.354527 140076727973760 basic_session_run_hooks.py:260] loss = 0.093327254, step = 8524 (11.478 sec)
I0716 01:10:45.356254 140076727973760 basic_session_run_hooks.py:260] lr = 0.00012220368 (11.478 sec)
I0716 01:10:56.743962 140076727973760 basic_session_run_hooks.py:692] global_step/sec: 8.77469
I0716 01:10:56.751214 140076727973760 basic_session_run_hooks.py:260] loss = 0.014506616, step = 8624 (11.397 sec)
I0716 01:10:56.753989 140076727973760 basic_session_run_hooks.py:260] lr = 0.000120922894 (11.398 sec)
I0716 01:11:07.425936 140076727973760 basic_session_run_hooks.py:692] global_step/sec: 9.36155
I0716 01:11:07.431163 140076727973760 basic_session_run_hooks.py:260] loss = 0.018018574, step = 8724 (10.680 sec)
I0716 01:11:07.434985 140076727973760 basic_session_run_hooks.py:260] lr = 0.00011965554 (10.681 sec)
I0716 01:11:08.487125 140076727973760 basic_session_run_hooks.py:606] Saving checkpoints for 8736 into ../model/bigru_crf/model.ckpt.
I0716 01:11:08.887515 140076727973760 estimator.py:1145] Calling model_fn.
I0716 01:11:09.860071 140076727973760 estimator.py:1147] Done calling model_fn.
I0716 01:11:09.889399 140076727973760 evaluation.py:255] Starting evaluation at 2019-07-16T01:11:09Z
I0716 01:11:10.005753 140076727973760 monitored_session.py:240] Graph was finalized.
I0716 01:11:10.022321 140076727973760 saver.py:1280] Restoring parameters from ../model/bigru_crf/model.ckpt-8736
I0716 01:11:10.141157 140076727973760 session_manager.py:500] Running local_init_op.
I0716 01:11:10.167426 140076727973760 session_manager.py:502] Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
I0716 01:11:12.483723 140076727973760 evaluation.py:275] Finished evaluation at 2019-07-16-01:11:12
I0716 01:11:12.485608 140076727973760 estimator.py:2039] Saving dict for global step 8736: global_step = 8736, loss = 4.3165607
I0716 01:11:12.488287 140076727973760 estimator.py:2099] Saving 'checkpoint_path' summary for global step 8736: ../model/bigru_crf/model.ckpt-8736
I0716 01:11:12.560290 140076727973760 estimator.py:368] Loss for final step: 0.06226072.
Reading ../data/atis.test.w-intent.iob
I0716 01:11:12.890314 140076727973760 estimator.py:1145] Calling model_fn.
I0716 01:11:13.856635 140076727973760 estimator.py:1147] Done calling model_fn.
I0716 01:11:13.975959 140076727973760 monitored_session.py:240] Graph was finalized.
I0716 01:11:13.993089 140076727973760 saver.py:1280] Restoring parameters from ../model/bigru_crf/model.ckpt-8736
I0716 01:11:14.096370 140076727973760 session_manager.py:500] Running local_init_op.
I0716 01:11:14.115696 140076727973760 session_manager.py:502] Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.
  'precision', 'predicted', average, warn_for)
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1439: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples.
  'recall', 'true', average, warn_for)
I0716 01:11:17.199064 140076727973760 interactiveshell.py:2882] 
                                          precision    recall  f1-score   support

                             atis_flight      0.975     0.992     0.984       632
                            atis_airfare      0.959     0.979     0.969        48
                     atis_ground_service      1.000     1.000     1.000        36
                            atis_airline      1.000     1.000     1.000        38
                       atis_abbreviation      0.943     1.000     0.971        33
                           atis_aircraft      1.000     1.000     1.000         9
                        atis_flight_time      1.000     1.000     1.000         1
                           atis_quantity      0.375     1.000     0.545         3
                atis_flight#atis_airfare      0.750     0.250     0.375        12
                            atis_airport      1.000     0.944     0.971        18
                           atis_distance      1.000     1.000     1.000        10
                               atis_city      1.000     0.500     0.667         6
                        atis_ground_fare      1.000     0.857     0.923         7
                           atis_capacity      1.000     0.952     0.976        21
                          atis_flight_no      0.889     1.000     0.941         8
                               atis_meal      1.000     0.833     0.909         6
                        atis_restriction      0.000     0.000     0.000         0
             atis_airline#atis_flight_no      0.000     0.000     0.000         0
    atis_ground_service#atis_ground_fare      0.000     0.000     0.000         0
           atis_airfare#atis_flight_time      0.000     0.000     0.000         0
                           atis_cheapest      0.000     0.000     0.000         0
atis_aircraft#atis_flight#atis_flight_no      0.000     0.000     0.000         0

                               micro avg      0.970     0.975     0.972       888
                               macro avg      0.677     0.650     0.647       888
                            weighted avg      0.971     0.975     0.970       888

I0716 01:11:17.248256 140076727973760 interactiveshell.py:2882] 
                              precision    recall  f1-score   support

                           O      0.000     0.000     0.000       0.0
           B-toloc.city_name      0.978     0.992     0.985     716.0
         B-fromloc.city_name      0.983     0.997     0.990     704.0
           I-toloc.city_name      0.977     0.981     0.979     265.0
      B-depart_date.day_name      0.991     0.991     0.991     212.0
              B-airline_name      1.000     1.000     1.000     101.0
         I-fromloc.city_name      0.972     0.989     0.980     177.0
 B-depart_time.period_of_day      1.000     0.915     0.956     130.0
              I-airline_name      1.000     1.000     1.000      65.0
    B-depart_date.day_number      0.981     0.964     0.972      55.0
    B-depart_date.month_name      0.982     0.964     0.973      56.0
          B-depart_time.time      0.864     1.000     0.927      57.0
                B-round_trip      1.000     0.973     0.986      73.0
             B-cost_relative      1.000     0.973     0.986      37.0
                I-round_trip      1.000     1.000     1.000      71.0
                B-flight_mod      1.000     1.000     1.000      24.0
 B-depart_time.time_relative      0.969     0.954     0.961      65.0
          I-depart_time.time      0.945     1.000     0.972      52.0
         B-stoploc.city_name      0.870     1.000     0.930      20.0
                 B-city_name      0.897     0.614     0.729      57.0
                B-class_type      0.960     1.000     0.980      24.0
          B-arrive_time.time      0.971     0.971     0.971      34.0
 B-arrive_time.time_relative      0.906     0.935     0.921      31.0
                I-class_type      1.000     1.000     1.000      17.0
               B-flight_stop      1.000     1.000     1.000      21.0
          I-arrive_time.time      1.000     0.971     0.986      35.0
              B-airline_code      1.000     0.971     0.985      34.0
    I-depart_date.day_number      1.000     0.933     0.966      15.0
      I-fromloc.airport_name      0.429     1.000     0.600      15.0
      B-fromloc.airport_name      0.480     1.000     0.649      12.0
      B-arrive_date.day_name      0.786     1.000     0.880      11.0
          B-toloc.state_code      0.947     1.000     0.973      18.0
B-depart_date.today_relative      1.000     0.889     0.941       9.0
             B-flight_number      0.688     1.000     0.815      11.0
 B-depart_date.date_relative      0.944     1.000     0.971      17.0
          B-toloc.state_name      0.781     0.893     0.833      28.0
           B-fare_basis_code      0.944     1.000     0.971      17.0
               B-flight_time      1.000     1.000     1.000       1.0
                        B-or      1.000     1.000     1.000       3.0
 B-arrive_time.period_of_day      0.857     1.000     0.923       6.0
          B-meal_description      1.000     0.900     0.947      10.0
             I-cost_relative      1.000     0.667     0.800       3.0
              I-airport_name      0.923     0.414     0.571      29.0
               B-fare_amount      1.000     1.000     1.000       2.0
               I-fare_amount      1.000     1.000     1.000       2.0
                 I-city_name      0.900     0.600     0.720      30.0
        I-toloc.airport_name      1.000     1.000     1.000       3.0
            B-transport_type      1.000     1.000     1.000      10.0
    B-arrive_date.month_name      0.714     0.833     0.769       6.0
    B-arrive_date.day_number      0.714     0.833     0.769       6.0
         I-stoploc.city_name      0.833     1.000     0.909      10.0
                      B-meal      1.000     1.000     1.000      16.0
        B-fromloc.state_code      1.000     0.957     0.978      23.0
    B-depart_time.period_mod      0.833     1.000     0.909       5.0
                   B-connect      1.000     1.000     1.000       6.0
               B-flight_days      1.000     1.000     1.000      10.0
        B-toloc.airport_name      1.000     1.000     1.000       3.0
        B-fromloc.state_name      0.944     1.000     0.971      17.0
              B-airport_name      0.818     0.429     0.562      21.0
                   B-economy      1.000     1.000     1.000       6.0
               I-flight_time      1.000     1.000     1.000       1.0
             B-aircraft_code      0.967     0.879     0.921      33.0
                       B-mod      1.000     0.500     0.667       2.0
              B-airport_code      0.500     0.111     0.182       9.0
    B-depart_time.start_time      0.667     0.667     0.667       3.0
      B-depart_time.end_time      1.000     0.333     0.500       3.0
          B-depart_date.year      1.000     1.000     1.000       3.0
            I-transport_type      0.000     0.000     0.000       1.0
          B-restriction_code      0.444     1.000     0.615       4.0
    B-arrive_time.start_time      0.889     1.000     0.941       8.0
        B-toloc.airport_code      1.000     0.750     0.857       4.0
      B-arrive_time.end_time      0.889     1.000     0.941       8.0
      I-arrive_time.end_time      0.889     1.000     0.941       8.0
      I-depart_time.end_time      1.000     0.333     0.500       3.0
               I-flight_stop      0.000     0.000     0.000       0.0
      B-fromloc.airport_code      1.000     1.000     1.000       5.0
          I-restriction_code      1.000     1.000     1.000       3.0
    I-depart_time.start_time      1.000     1.000     1.000       1.0
          I-toloc.state_name      1.000     1.000     1.000       1.0
I-depart_date.today_relative      0.000     0.000     0.000       0.0
 B-arrive_date.date_relative      1.000     1.000     1.000       2.0
                I-flight_mod      1.000     0.167     0.286       6.0
                   I-economy      0.000     0.000     0.000       0.0
 B-return_date.date_relative      1.000     0.333     0.500       3.0
        I-fromloc.state_name      1.000     1.000     1.000       1.0
                B-state_code      1.000     1.000     1.000       1.0
    I-arrive_time.start_time      1.000     1.000     1.000       1.0
    I-arrive_date.day_number      0.000     0.000     0.000       0.0
                 B-meal_code      1.000     1.000     1.000       1.0
 I-depart_time.period_of_day      1.000     1.000     1.000       1.0
                  B-day_name      1.000     0.500     0.667       2.0
             B-period_of_day      1.000     1.000     1.000       4.0
        B-stoploc.state_code      0.000     0.000     0.000       0.0
    B-return_date.month_name      0.000     0.000     0.000       0.0
    B-return_date.day_number      0.000     0.000     0.000       0.0
    B-arrive_time.period_mod      0.000     0.000     0.000       0.0
                 I-meal_code      0.000     0.000     0.000       0.0
        B-toloc.country_name      1.000     1.000     1.000       1.0
                 B-days_code      1.000     1.000     1.000       1.0
 I-arrive_time.period_of_day      0.000     0.000     0.000       0.0
            I-today_relative      0.000     0.000     0.000       0.0
 B-return_time.period_of_day      0.000     0.000     0.000       0.0
                      B-time      0.000     0.000     0.000       0.0
           I-fare_basis_code      0.000     0.000     0.000       0.0
 I-arrive_time.time_relative      0.000     0.000     0.000       4.0
 I-depart_time.time_relative      0.000     0.000     0.000       1.0
            B-today_relative      0.000     0.000     0.000       0.0
                B-state_name      0.000     0.000     0.000       9.0
B-arrive_date.today_relative      0.000     0.000     0.000       0.0
    B-return_time.period_mod      0.000     0.000     0.000       0.0
                B-month_name      0.000     0.000     0.000       0.0
                B-day_number      0.000     0.000     0.000       0.0
 I-return_date.date_relative      0.750     1.000     0.857       3.0
I-return_date.today_relative      0.000     0.000     0.000       0.0
      B-stoploc.airport_name      0.000     0.000     0.000       0.0
             B-time_relative      0.000     0.000     0.000       0.0
                      I-time      0.000     0.000     0.000       0.0
    I-return_date.day_number      0.000     0.000     0.000       0.0
          I-meal_description      0.000     0.000     0.000       0.0
B-return_date.today_relative      0.000     0.000     0.000       0.0
      B-return_date.day_name      0.000     0.000     0.000       2.0

                   micro avg      0.955     0.956     0.955    3657.0
                   macro avg      0.684     0.662     0.658    3657.0
                weighted avg      0.961     0.956     0.954    3657.0

I0716 01:11:17.276987 140076727973760 interactiveshell.py:2882] Best Slot F1: 0.957
I0716 01:11:17.278667 140076727973760 estimator_training.py:186] Not using Distribute Coordinator.
I0716 01:11:17.281794 140076727973760 training.py:612] Running training and evaluation locally (non-distributed).
I0716 01:11:17.284753 140076727973760 training.py:700] Start train and evaluate loop. The evaluate will happen after every checkpoint. Checkpoint frequency is determined based on RunConfig arguments: save_checkpoints_steps 312 or save_checkpoints_secs None.
I0716 01:11:17.349402 140076727973760 estimator.py:1145] Calling model_fn.
I0716 01:11:18.550369 140076727973760 <ipython-input-6-0a85a3fd693f>:50] 
[<tf.Variable 'embedding:0' shape=(750, 300) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/gates/kernel:0' shape=(600, 600) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/gates/bias:0' shape=(600,) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/candidate/kernel:0' shape=(600, 300) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/candidate/bias:0' shape=(300,) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/gates/kernel:0' shape=(600, 600) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/gates/bias:0' shape=(600,) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/candidate/kernel:0' shape=(600, 300) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/candidate/bias:0' shape=(300,) dtype=float32_ref>,
 <tf.Variable 'dense/kernel:0' shape=(600, 23) dtype=float32_ref>,
 <tf.Variable 'dense/bias:0' shape=(23,) dtype=float32_ref>,
 <tf.Variable 'dense_1/kernel:0' shape=(600, 122) dtype=float32_ref>,
 <tf.Variable 'dense_1/bias:0' shape=(122,) dtype=float32_ref>,
 <tf.Variable 'transitions:0' shape=(122, 122) dtype=float32_ref>]
I0716 01:11:20.096776 140076727973760 estimator.py:1147] Done calling model_fn.
I0716 01:11:20.100626 140076727973760 basic_session_run_hooks.py:541] Create CheckpointSaverHook.
I0716 01:11:20.295729 140076727973760 monitored_session.py:240] Graph was finalized.
I0716 01:11:20.313691 140076727973760 saver.py:1280] Restoring parameters from ../model/bigru_crf/model.ckpt-8736
I0716 01:11:20.575412 140076727973760 session_manager.py:500] Running local_init_op.
I0716 01:11:20.612809 140076727973760 session_manager.py:502] Done running local_init_op.
I0716 01:11:22.125462 140076727973760 basic_session_run_hooks.py:606] Saving checkpoints for 8736 into ../model/bigru_crf/model.ckpt.
Reading ../data/atis.train.w-intent.iob
I0716 01:11:24.604816 140076727973760 basic_session_run_hooks.py:262] loss = 0.014761524, step = 8736
I0716 01:11:24.606609 140076727973760 basic_session_run_hooks.py:262] lr = 0.00011950435
I0716 01:11:36.316289 140076727973760 basic_session_run_hooks.py:692] global_step/sec: 8.53825
I0716 01:11:36.322754 140076727973760 basic_session_run_hooks.py:260] loss = 0.007601156, step = 8836 (11.718 sec)
I0716 01:11:36.326567 140076727973760 basic_session_run_hooks.py:260] lr = 0.00011825185 (11.720 sec)
I0716 01:11:47.213500 140076727973760 basic_session_run_hooks.py:692] global_step/sec: 9.17663
I0716 01:11:47.218210 140076727973760 basic_session_run_hooks.py:260] loss = 0.01128834, step = 8936 (10.896 sec)
I0716 01:11:47.223021 140076727973760 basic_session_run_hooks.py:260] lr = 0.000117012474 (10.896 sec)
I0716 01:11:57.817066 140076727973760 basic_session_run_hooks.py:692] global_step/sec: 9.43082
I0716 01:11:57.823532 140076727973760 basic_session_run_hooks.py:260] loss = 0.029143915, step = 9036 (10.605 sec)
I0716 01:11:57.831273 140076727973760 basic_session_run_hooks.py:260] lr = 0.00011578611 (10.608 sec)
I0716 01:11:58.999359 140076727973760 basic_session_run_hooks.py:606] Saving checkpoints for 9048 into ../model/bigru_crf/model.ckpt.
I0716 01:11:59.378431 140076727973760 estimator.py:1145] Calling model_fn.
I0716 01:12:00.546964 140076727973760 estimator.py:1147] Done calling model_fn.
I0716 01:12:00.575704 140076727973760 evaluation.py:255] Starting evaluation at 2019-07-16T01:12:00Z
I0716 01:12:00.690212 140076727973760 monitored_session.py:240] Graph was finalized.
I0716 01:12:00.705297 140076727973760 saver.py:1280] Restoring parameters from ../model/bigru_crf/model.ckpt-9048
I0716 01:12:00.830587 140076727973760 session_manager.py:500] Running local_init_op.
I0716 01:12:00.861915 140076727973760 session_manager.py:502] Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
I0716 01:12:03.183193 140076727973760 evaluation.py:275] Finished evaluation at 2019-07-16-01:12:03
I0716 01:12:03.185058 140076727973760 estimator.py:2039] Saving dict for global step 9048: global_step = 9048, loss = 4.448784
I0716 01:12:03.193594 140076727973760 estimator.py:2099] Saving 'checkpoint_path' summary for global step 9048: ../model/bigru_crf/model.ckpt-9048
I0716 01:12:03.253490 140076727973760 estimator.py:368] Loss for final step: 0.00040042185.
Reading ../data/atis.test.w-intent.iob
I0716 01:12:03.570511 140076727973760 estimator.py:1145] Calling model_fn.
I0716 01:12:04.225841 140076727973760 estimator.py:1147] Done calling model_fn.
I0716 01:12:04.338133 140076727973760 monitored_session.py:240] Graph was finalized.
I0716 01:12:04.354387 140076727973760 saver.py:1280] Restoring parameters from ../model/bigru_crf/model.ckpt-9048
I0716 01:12:04.456229 140076727973760 session_manager.py:500] Running local_init_op.
I0716 01:12:04.473731 140076727973760 session_manager.py:502] Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.
  'precision', 'predicted', average, warn_for)
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1439: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples.
  'recall', 'true', average, warn_for)
I0716 01:12:07.544022 140076727973760 interactiveshell.py:2882] 
                                          precision    recall  f1-score   support

                             atis_flight      0.975     0.992     0.984       632
                            atis_airfare      0.959     0.979     0.969        48
                     atis_ground_service      1.000     1.000     1.000        36
                            atis_airline      1.000     1.000     1.000        38
                       atis_abbreviation      0.943     1.000     0.971        33
                           atis_aircraft      1.000     1.000     1.000         9
                        atis_flight_time      1.000     1.000     1.000         1
                           atis_quantity      0.375     1.000     0.545         3
                atis_flight#atis_airfare      0.750     0.250     0.375        12
                            atis_airport      1.000     0.944     0.971        18
                           atis_distance      1.000     1.000     1.000        10
                               atis_city      1.000     0.500     0.667         6
                        atis_ground_fare      1.000     0.857     0.923         7
                           atis_capacity      1.000     0.952     0.976        21
                          atis_flight_no      0.889     1.000     0.941         8
                               atis_meal      1.000     0.833     0.909         6
                        atis_restriction      0.000     0.000     0.000         0
             atis_airline#atis_flight_no      0.000     0.000     0.000         0
    atis_ground_service#atis_ground_fare      0.000     0.000     0.000         0
           atis_airfare#atis_flight_time      0.000     0.000     0.000         0
                           atis_cheapest      0.000     0.000     0.000         0
atis_aircraft#atis_flight#atis_flight_no      0.000     0.000     0.000         0

                               micro avg      0.970     0.975     0.972       888
                               macro avg      0.677     0.650     0.647       888
                            weighted avg      0.971     0.975     0.970       888

I0716 01:12:07.593524 140076727973760 interactiveshell.py:2882] 
                              precision    recall  f1-score   support

                           O      0.000     0.000     0.000       0.0
           B-toloc.city_name      0.974     0.993     0.983     716.0
         B-fromloc.city_name      0.983     0.997     0.990     704.0
           I-toloc.city_name      0.981     0.985     0.983     265.0
      B-depart_date.day_name      0.986     0.991     0.988     212.0
              B-airline_name      1.000     1.000     1.000     101.0
         I-fromloc.city_name      0.967     0.994     0.981     177.0
 B-depart_time.period_of_day      1.000     0.923     0.960     130.0
              I-airline_name      1.000     1.000     1.000      65.0
    B-depart_date.day_number      0.981     0.964     0.972      55.0
    B-depart_date.month_name      0.982     0.964     0.973      56.0
          B-depart_time.time      0.851     1.000     0.919      57.0
                B-round_trip      1.000     0.973     0.986      73.0
             B-cost_relative      1.000     0.973     0.986      37.0
                I-round_trip      1.000     1.000     1.000      71.0
                B-flight_mod      1.000     1.000     1.000      24.0
 B-depart_time.time_relative      0.969     0.969     0.969      65.0
          I-depart_time.time      0.945     1.000     0.972      52.0
         B-stoploc.city_name      0.870     1.000     0.930      20.0
                 B-city_name      0.912     0.544     0.681      57.0
                B-class_type      0.960     1.000     0.980      24.0
          B-arrive_time.time      1.000     0.971     0.985      34.0
 B-arrive_time.time_relative      0.935     0.935     0.935      31.0
                I-class_type      1.000     1.000     1.000      17.0
               B-flight_stop      1.000     1.000     1.000      21.0
          I-arrive_time.time      1.000     0.971     0.986      35.0
              B-airline_code      1.000     0.971     0.985      34.0
    I-depart_date.day_number      1.000     1.000     1.000      15.0
      I-fromloc.airport_name      0.417     1.000     0.588      15.0
      B-fromloc.airport_name      0.462     1.000     0.632      12.0
      B-arrive_date.day_name      0.846     1.000     0.917      11.0
          B-toloc.state_code      0.947     1.000     0.973      18.0
B-depart_date.today_relative      1.000     0.889     0.941       9.0
             B-flight_number      0.550     1.000     0.710      11.0
 B-depart_date.date_relative      0.944     1.000     0.971      17.0
          B-toloc.state_name      0.781     0.893     0.833      28.0
           B-fare_basis_code      0.944     1.000     0.971      17.0
               B-flight_time      1.000     1.000     1.000       1.0
                        B-or      1.000     1.000     1.000       3.0
 B-arrive_time.period_of_day      0.857     1.000     0.923       6.0
          B-meal_description      1.000     0.900     0.947      10.0
             I-cost_relative      1.000     0.667     0.800       3.0
              I-airport_name      0.917     0.379     0.537      29.0
               B-fare_amount      1.000     1.000     1.000       2.0
               I-fare_amount      1.000     1.000     1.000       2.0
                 I-city_name      1.000     0.600     0.750      30.0
        I-toloc.airport_name      1.000     1.000     1.000       3.0
            B-transport_type      1.000     1.000     1.000      10.0
    B-arrive_date.month_name      0.714     0.833     0.769       6.0
    B-arrive_date.day_number      0.714     0.833     0.769       6.0
         I-stoploc.city_name      0.909     1.000     0.952      10.0
                      B-meal      1.000     1.000     1.000      16.0
        B-fromloc.state_code      1.000     0.957     0.978      23.0
    B-depart_time.period_mod      1.000     1.000     1.000       5.0
                   B-connect      1.000     1.000     1.000       6.0
               B-flight_days      1.000     1.000     1.000      10.0
        B-toloc.airport_name      1.000     1.000     1.000       3.0
        B-fromloc.state_name      0.944     1.000     0.971      17.0
              B-airport_name      0.800     0.381     0.516      21.0
                   B-economy      1.000     1.000     1.000       6.0
               I-flight_time      1.000     1.000     1.000       1.0
             B-aircraft_code      0.931     0.818     0.871      33.0
                       B-mod      1.000     0.500     0.667       2.0
              B-airport_code      0.500     0.111     0.182       9.0
    B-depart_time.start_time      0.667     0.667     0.667       3.0
      B-depart_time.end_time      1.000     0.333     0.500       3.0
          B-depart_date.year      1.000     1.000     1.000       3.0
            I-transport_type      0.000     0.000     0.000       1.0
          B-restriction_code      0.400     1.000     0.571       4.0
    B-arrive_time.start_time      0.889     1.000     0.941       8.0
        B-toloc.airport_code      1.000     0.750     0.857       4.0
      B-arrive_time.end_time      0.889     1.000     0.941       8.0
      I-arrive_time.end_time      0.889     1.000     0.941       8.0
      I-depart_time.end_time      1.000     0.333     0.500       3.0
               I-flight_stop      0.000     0.000     0.000       0.0
      B-fromloc.airport_code      1.000     1.000     1.000       5.0
          I-restriction_code      1.000     1.000     1.000       3.0
    I-depart_time.start_time      1.000     1.000     1.000       1.0
          I-toloc.state_name      1.000     1.000     1.000       1.0
I-depart_date.today_relative      0.000     0.000     0.000       0.0
 B-arrive_date.date_relative      1.000     1.000     1.000       2.0
                I-flight_mod      1.000     0.167     0.286       6.0
                   I-economy      0.000     0.000     0.000       0.0
 B-return_date.date_relative      1.000     0.333     0.500       3.0
        I-fromloc.state_name      1.000     1.000     1.000       1.0
                B-state_code      1.000     1.000     1.000       1.0
    I-arrive_time.start_time      1.000     1.000     1.000       1.0
    I-arrive_date.day_number      0.000     0.000     0.000       0.0
                 B-meal_code      0.000     0.000     0.000       1.0
 I-depart_time.period_of_day      1.000     1.000     1.000       1.0
                  B-day_name      1.000     0.500     0.667       2.0
             B-period_of_day      1.000     1.000     1.000       4.0
        B-stoploc.state_code      0.000     0.000     0.000       0.0
    B-return_date.month_name      0.000     0.000     0.000       0.0
    B-return_date.day_number      0.000     0.000     0.000       0.0
    B-arrive_time.period_mod      0.000     0.000     0.000       0.0
                 I-meal_code      0.000     0.000     0.000       0.0
        B-toloc.country_name      1.000     1.000     1.000       1.0
                 B-days_code      1.000     1.000     1.000       1.0
 I-arrive_time.period_of_day      0.000     0.000     0.000       0.0
            I-today_relative      0.000     0.000     0.000       0.0
 B-return_time.period_of_day      0.000     0.000     0.000       0.0
                      B-time      0.000     0.000     0.000       0.0
           I-fare_basis_code      0.000     0.000     0.000       0.0
 I-arrive_time.time_relative      0.000     0.000     0.000       4.0
 I-depart_time.time_relative      0.000     0.000     0.000       1.0
            B-today_relative      0.000     0.000     0.000       0.0
                B-state_name      0.000     0.000     0.000       9.0
B-arrive_date.today_relative      0.000     0.000     0.000       0.0
    B-return_time.period_mod      0.000     0.000     0.000       0.0
                B-month_name      0.000     0.000     0.000       0.0
                B-day_number      0.000     0.000     0.000       0.0
 I-return_date.date_relative      0.750     1.000     0.857       3.0
I-return_date.today_relative      0.000     0.000     0.000       0.0
      B-stoploc.airport_name      0.000     0.000     0.000       0.0
             B-time_relative      0.000     0.000     0.000       0.0
                      I-time      0.000     0.000     0.000       0.0
    I-return_date.day_number      0.000     0.000     0.000       0.0
          I-meal_description      0.000     0.000     0.000       0.0
B-return_date.today_relative      0.000     0.000     0.000       0.0
      B-return_date.day_name      0.000     0.000     0.000       2.0

                   micro avg      0.954     0.955     0.955    3657.0
                   macro avg      0.677     0.653     0.649    3657.0
                weighted avg      0.960     0.955     0.952    3657.0

I0716 01:12:07.635835 140076727973760 interactiveshell.py:2882] Best Slot F1: 0.957
I0716 01:12:07.641032 140076727973760 estimator_training.py:186] Not using Distribute Coordinator.
I0716 01:12:07.647711 140076727973760 training.py:612] Running training and evaluation locally (non-distributed).
I0716 01:12:07.652572 140076727973760 training.py:700] Start train and evaluate loop. The evaluate will happen after every checkpoint. Checkpoint frequency is determined based on RunConfig arguments: save_checkpoints_steps 312 or save_checkpoints_secs None.
I0716 01:12:07.734753 140076727973760 estimator.py:1145] Calling model_fn.
I0716 01:12:09.072211 140076727973760 <ipython-input-6-0a85a3fd693f>:50] 
[<tf.Variable 'embedding:0' shape=(750, 300) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/gates/kernel:0' shape=(600, 600) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/gates/bias:0' shape=(600,) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/candidate/kernel:0' shape=(600, 300) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/candidate/bias:0' shape=(300,) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/gates/kernel:0' shape=(600, 600) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/gates/bias:0' shape=(600,) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/candidate/kernel:0' shape=(600, 300) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/candidate/bias:0' shape=(300,) dtype=float32_ref>,
 <tf.Variable 'dense/kernel:0' shape=(600, 23) dtype=float32_ref>,
 <tf.Variable 'dense/bias:0' shape=(23,) dtype=float32_ref>,
 <tf.Variable 'dense_1/kernel:0' shape=(600, 122) dtype=float32_ref>,
 <tf.Variable 'dense_1/bias:0' shape=(122,) dtype=float32_ref>,
 <tf.Variable 'transitions:0' shape=(122, 122) dtype=float32_ref>]
I0716 01:12:10.614021 140076727973760 estimator.py:1147] Done calling model_fn.
I0716 01:12:10.617831 140076727973760 basic_session_run_hooks.py:541] Create CheckpointSaverHook.
I0716 01:12:10.823429 140076727973760 monitored_session.py:240] Graph was finalized.
I0716 01:12:10.843392 140076727973760 saver.py:1280] Restoring parameters from ../model/bigru_crf/model.ckpt-9048
I0716 01:12:11.109941 140076727973760 session_manager.py:500] Running local_init_op.
I0716 01:12:11.155165 140076727973760 session_manager.py:502] Done running local_init_op.
I0716 01:12:12.672407 140076727973760 basic_session_run_hooks.py:606] Saving checkpoints for 9048 into ../model/bigru_crf/model.ckpt.
Reading ../data/atis.train.w-intent.iob
I0716 01:12:15.026809 140076727973760 basic_session_run_hooks.py:262] loss = 0.007752421, step = 9048
I0716 01:12:15.028734 140076727973760 basic_session_run_hooks.py:262] lr = 0.00011563981
I0716 01:12:26.665530 140076727973760 basic_session_run_hooks.py:692] global_step/sec: 8.59147
I0716 01:12:26.672165 140076727973760 basic_session_run_hooks.py:260] loss = 0.0048512323, step = 9148 (11.645 sec)
I0716 01:12:26.674315 140076727973760 basic_session_run_hooks.py:260] lr = 0.00011442781 (11.646 sec)
I0716 01:12:37.419064 140076727973760 basic_session_run_hooks.py:692] global_step/sec: 9.29936
I0716 01:12:37.421756 140076727973760 basic_session_run_hooks.py:260] loss = 0.008699753, step = 9248 (10.750 sec)
I0716 01:12:37.427697 140076727973760 basic_session_run_hooks.py:260] lr = 0.00011322854 (10.753 sec)
I0716 01:12:48.106772 140076727973760 basic_session_run_hooks.py:692] global_step/sec: 9.35647
I0716 01:12:48.113442 140076727973760 basic_session_run_hooks.py:260] loss = 0.013349785, step = 9348 (10.692 sec)
I0716 01:12:48.115551 140076727973760 basic_session_run_hooks.py:260] lr = 0.0001120418 (10.688 sec)
I0716 01:12:49.170596 140076727973760 basic_session_run_hooks.py:606] Saving checkpoints for 9360 into ../model/bigru_crf/model.ckpt.
I0716 01:12:49.552409 140076727973760 estimator.py:1145] Calling model_fn.
I0716 01:12:50.722204 140076727973760 estimator.py:1147] Done calling model_fn.
I0716 01:12:50.750939 140076727973760 evaluation.py:255] Starting evaluation at 2019-07-16T01:12:50Z
I0716 01:12:50.867984 140076727973760 monitored_session.py:240] Graph was finalized.
I0716 01:12:50.885117 140076727973760 saver.py:1280] Restoring parameters from ../model/bigru_crf/model.ckpt-9360
I0716 01:12:51.005984 140076727973760 session_manager.py:500] Running local_init_op.
I0716 01:12:51.034503 140076727973760 session_manager.py:502] Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
I0716 01:12:53.353671 140076727973760 evaluation.py:275] Finished evaluation at 2019-07-16-01:12:53
I0716 01:12:53.355434 140076727973760 estimator.py:2039] Saving dict for global step 9360: global_step = 9360, loss = 4.462923
I0716 01:12:53.362232 140076727973760 estimator.py:2099] Saving 'checkpoint_path' summary for global step 9360: ../model/bigru_crf/model.ckpt-9360
I0716 01:12:53.423499 140076727973760 estimator.py:368] Loss for final step: 0.0055543073.
Reading ../data/atis.test.w-intent.iob
I0716 01:12:53.753203 140076727973760 estimator.py:1145] Calling model_fn.
I0716 01:12:54.408308 140076727973760 estimator.py:1147] Done calling model_fn.
I0716 01:12:54.519881 140076727973760 monitored_session.py:240] Graph was finalized.
I0716 01:12:54.537301 140076727973760 saver.py:1280] Restoring parameters from ../model/bigru_crf/model.ckpt-9360
I0716 01:12:54.640001 140076727973760 session_manager.py:500] Running local_init_op.
I0716 01:12:54.658253 140076727973760 session_manager.py:502] Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.
  'precision', 'predicted', average, warn_for)
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1439: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples.
  'recall', 'true', average, warn_for)
I0716 01:12:57.778006 140076727973760 interactiveshell.py:2882] 
                                          precision    recall  f1-score   support

                             atis_flight      0.975     0.992     0.984       632
                            atis_airfare      0.959     0.979     0.969        48
                     atis_ground_service      1.000     1.000     1.000        36
                            atis_airline      1.000     1.000     1.000        38
                       atis_abbreviation      0.943     1.000     0.971        33
                           atis_aircraft      1.000     1.000     1.000         9
                        atis_flight_time      1.000     1.000     1.000         1
                           atis_quantity      0.375     1.000     0.545         3
                atis_flight#atis_airfare      0.750     0.250     0.375        12
                            atis_airport      1.000     0.944     0.971        18
                           atis_distance      1.000     1.000     1.000        10
                               atis_city      1.000     0.500     0.667         6
                        atis_ground_fare      1.000     0.857     0.923         7
                           atis_capacity      1.000     0.952     0.976        21
                          atis_flight_no      0.889     1.000     0.941         8
                               atis_meal      1.000     0.833     0.909         6
                        atis_restriction      0.000     0.000     0.000         0
             atis_airline#atis_flight_no      0.000     0.000     0.000         0
    atis_ground_service#atis_ground_fare      0.000     0.000     0.000         0
           atis_airfare#atis_flight_time      0.000     0.000     0.000         0
                           atis_cheapest      0.000     0.000     0.000         0
atis_aircraft#atis_flight#atis_flight_no      0.000     0.000     0.000         0

                               micro avg      0.970     0.975     0.972       888
                               macro avg      0.677     0.650     0.647       888
                            weighted avg      0.971     0.975     0.970       888

I0716 01:12:57.828050 140076727973760 interactiveshell.py:2882] 
                              precision    recall  f1-score   support

                           O      0.000     0.000     0.000       0.0
           B-toloc.city_name      0.971     0.993     0.982     716.0
         B-fromloc.city_name      0.985     0.999     0.992     704.0
           I-toloc.city_name      0.960     0.985     0.972     265.0
      B-depart_date.day_name      0.986     0.991     0.988     212.0
              B-airline_name      1.000     1.000     1.000     101.0
         I-fromloc.city_name      0.972     0.994     0.983     177.0
 B-depart_time.period_of_day      0.992     0.923     0.956     130.0
              I-airline_name      1.000     1.000     1.000      65.0
    B-depart_date.day_number      0.981     0.964     0.972      55.0
    B-depart_date.month_name      0.982     0.964     0.973      56.0
          B-depart_time.time      0.864     1.000     0.927      57.0
                B-round_trip      1.000     0.973     0.986      73.0
             B-cost_relative      1.000     0.973     0.986      37.0
                I-round_trip      1.000     1.000     1.000      71.0
                B-flight_mod      1.000     1.000     1.000      24.0
 B-depart_time.time_relative      0.969     0.954     0.961      65.0
          I-depart_time.time      0.944     0.981     0.962      52.0
         B-stoploc.city_name      0.909     1.000     0.952      20.0
                 B-city_name      0.909     0.526     0.667      57.0
                B-class_type      0.960     1.000     0.980      24.0
          B-arrive_time.time      0.971     0.971     0.971      34.0
 B-arrive_time.time_relative      0.906     0.935     0.921      31.0
                I-class_type      1.000     1.000     1.000      17.0
               B-flight_stop      1.000     1.000     1.000      21.0
          I-arrive_time.time      1.000     0.971     0.986      35.0
              B-airline_code      1.000     0.971     0.985      34.0
    I-depart_date.day_number      1.000     1.000     1.000      15.0
      I-fromloc.airport_name      0.455     1.000     0.625      15.0
      B-fromloc.airport_name      0.480     1.000     0.649      12.0
      B-arrive_date.day_name      0.769     0.909     0.833      11.0
          B-toloc.state_code      0.947     1.000     0.973      18.0
B-depart_date.today_relative      1.000     0.889     0.941       9.0
             B-flight_number      0.579     1.000     0.733      11.0
 B-depart_date.date_relative      0.944     1.000     0.971      17.0
          B-toloc.state_name      0.781     0.893     0.833      28.0
           B-fare_basis_code      0.944     1.000     0.971      17.0
               B-flight_time      1.000     1.000     1.000       1.0
                        B-or      1.000     1.000     1.000       3.0
 B-arrive_time.period_of_day      0.857     1.000     0.923       6.0
          B-meal_description      1.000     0.900     0.947      10.0
             I-cost_relative      1.000     0.667     0.800       3.0
              I-airport_name      0.923     0.414     0.571      29.0
               B-fare_amount      1.000     1.000     1.000       2.0
               I-fare_amount      1.000     1.000     1.000       2.0
                 I-city_name      1.000     0.500     0.667      30.0
        I-toloc.airport_name      1.000     1.000     1.000       3.0
            B-transport_type      1.000     1.000     1.000      10.0
    B-arrive_date.month_name      0.714     0.833     0.769       6.0
    B-arrive_date.day_number      0.714     0.833     0.769       6.0
         I-stoploc.city_name      0.833     1.000     0.909      10.0
                      B-meal      1.000     1.000     1.000      16.0
        B-fromloc.state_code      1.000     0.957     0.978      23.0
    B-depart_time.period_mod      1.000     1.000     1.000       5.0
                   B-connect      1.000     1.000     1.000       6.0
               B-flight_days      1.000     1.000     1.000      10.0
        B-toloc.airport_name      1.000     1.000     1.000       3.0
        B-fromloc.state_name      0.944     1.000     0.971      17.0
              B-airport_name      0.818     0.429     0.562      21.0
                   B-economy      1.000     1.000     1.000       6.0
               I-flight_time      1.000     1.000     1.000       1.0
             B-aircraft_code      0.931     0.818     0.871      33.0
                       B-mod      1.000     0.500     0.667       2.0
              B-airport_code      0.500     0.111     0.182       9.0
    B-depart_time.start_time      1.000     0.667     0.800       3.0
      B-depart_time.end_time      1.000     0.333     0.500       3.0
          B-depart_date.year      1.000     1.000     1.000       3.0
            I-transport_type      0.000     0.000     0.000       1.0
          B-restriction_code      0.400     1.000     0.571       4.0
    B-arrive_time.start_time      0.889     1.000     0.941       8.0
        B-toloc.airport_code      1.000     0.750     0.857       4.0
      B-arrive_time.end_time      0.889     1.000     0.941       8.0
      I-arrive_time.end_time      0.889     1.000     0.941       8.0
      I-depart_time.end_time      1.000     0.333     0.500       3.0
               I-flight_stop      0.000     0.000     0.000       0.0
      B-fromloc.airport_code      1.000     1.000     1.000       5.0
          I-restriction_code      1.000     1.000     1.000       3.0
    I-depart_time.start_time      1.000     1.000     1.000       1.0
          I-toloc.state_name      1.000     1.000     1.000       1.0
I-depart_date.today_relative      0.000     0.000     0.000       0.0
 B-arrive_date.date_relative      1.000     1.000     1.000       2.0
                I-flight_mod      1.000     0.167     0.286       6.0
                   I-economy      0.000     0.000     0.000       0.0
 B-return_date.date_relative      1.000     0.333     0.500       3.0
        I-fromloc.state_name      1.000     1.000     1.000       1.0
                B-state_code      1.000     1.000     1.000       1.0
    I-arrive_time.start_time      1.000     1.000     1.000       1.0
    I-arrive_date.day_number      0.000     0.000     0.000       0.0
                 B-meal_code      0.000     0.000     0.000       1.0
 I-depart_time.period_of_day      1.000     1.000     1.000       1.0
                  B-day_name      1.000     0.500     0.667       2.0
             B-period_of_day      1.000     1.000     1.000       4.0
        B-stoploc.state_code      0.000     0.000     0.000       0.0
    B-return_date.month_name      0.000     0.000     0.000       0.0
    B-return_date.day_number      0.000     0.000     0.000       0.0
    B-arrive_time.period_mod      0.000     0.000     0.000       0.0
                 I-meal_code      0.000     0.000     0.000       0.0
        B-toloc.country_name      1.000     1.000     1.000       1.0
                 B-days_code      1.000     1.000     1.000       1.0
 I-arrive_time.period_of_day      0.000     0.000     0.000       0.0
            I-today_relative      0.000     0.000     0.000       0.0
 B-return_time.period_of_day      0.000     0.000     0.000       0.0
                      B-time      0.000     0.000     0.000       0.0
           I-fare_basis_code      0.000     0.000     0.000       0.0
 I-arrive_time.time_relative      0.000     0.000     0.000       4.0
 I-depart_time.time_relative      0.000     0.000     0.000       1.0
            B-today_relative      0.000     0.000     0.000       0.0
                B-state_name      0.000     0.000     0.000       9.0
B-arrive_date.today_relative      0.000     0.000     0.000       0.0
    B-return_time.period_mod      0.000     0.000     0.000       0.0
                B-month_name      0.000     0.000     0.000       0.0
                B-day_number      0.000     0.000     0.000       0.0
 I-return_date.date_relative      0.750     1.000     0.857       3.0
I-return_date.today_relative      0.000     0.000     0.000       0.0
      B-stoploc.airport_name      0.000     0.000     0.000       0.0
             B-time_relative      0.000     0.000     0.000       0.0
                      I-time      0.000     0.000     0.000       0.0
    I-return_date.day_number      0.000     0.000     0.000       0.0
          I-meal_description      0.000     0.000     0.000       0.0
B-return_date.today_relative      0.000     0.000     0.000       0.0
      B-return_date.day_name      0.000     0.000     0.000       2.0

                   micro avg      0.953     0.954     0.954    3657.0
                   macro avg      0.679     0.651     0.649    3657.0
                weighted avg      0.959     0.954     0.951    3657.0

I0716 01:12:57.856470 140076727973760 interactiveshell.py:2882] Best Slot F1: 0.957
I0716 01:12:57.858124 140076727973760 estimator_training.py:186] Not using Distribute Coordinator.
I0716 01:12:57.861018 140076727973760 training.py:612] Running training and evaluation locally (non-distributed).
I0716 01:12:57.864135 140076727973760 training.py:700] Start train and evaluate loop. The evaluate will happen after every checkpoint. Checkpoint frequency is determined based on RunConfig arguments: save_checkpoints_steps 312 or save_checkpoints_secs None.
I0716 01:12:57.934029 140076727973760 estimator.py:1145] Calling model_fn.
I0716 01:12:59.243638 140076727973760 <ipython-input-6-0a85a3fd693f>:50] 
[<tf.Variable 'embedding:0' shape=(750, 300) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/gates/kernel:0' shape=(600, 600) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/gates/bias:0' shape=(600,) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/candidate/kernel:0' shape=(600, 300) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/candidate/bias:0' shape=(300,) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/gates/kernel:0' shape=(600, 600) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/gates/bias:0' shape=(600,) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/candidate/kernel:0' shape=(600, 300) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/candidate/bias:0' shape=(300,) dtype=float32_ref>,
 <tf.Variable 'dense/kernel:0' shape=(600, 23) dtype=float32_ref>,
 <tf.Variable 'dense/bias:0' shape=(23,) dtype=float32_ref>,
 <tf.Variable 'dense_1/kernel:0' shape=(600, 122) dtype=float32_ref>,
 <tf.Variable 'dense_1/bias:0' shape=(122,) dtype=float32_ref>,
 <tf.Variable 'transitions:0' shape=(122, 122) dtype=float32_ref>]
I0716 01:13:00.766560 140076727973760 estimator.py:1147] Done calling model_fn.
I0716 01:13:00.770538 140076727973760 basic_session_run_hooks.py:541] Create CheckpointSaverHook.
I0716 01:13:00.972747 140076727973760 monitored_session.py:240] Graph was finalized.
I0716 01:13:00.993122 140076727973760 saver.py:1280] Restoring parameters from ../model/bigru_crf/model.ckpt-9360
I0716 01:13:01.251178 140076727973760 session_manager.py:500] Running local_init_op.
I0716 01:13:01.287374 140076727973760 session_manager.py:502] Done running local_init_op.
I0716 01:13:02.804222 140076727973760 basic_session_run_hooks.py:606] Saving checkpoints for 9360 into ../model/bigru_crf/model.ckpt.
Reading ../data/atis.train.w-intent.iob
I0716 01:13:05.201243 140076727973760 basic_session_run_hooks.py:262] loss = 0.024450902, step = 9360
I0716 01:13:05.202952 140076727973760 basic_session_run_hooks.py:262] lr = 0.00011190023
I0716 01:13:16.896420 140076727973760 basic_session_run_hooks.py:692] global_step/sec: 8.55017
I0716 01:13:16.902697 140076727973760 basic_session_run_hooks.py:260] loss = 0.009379742, step = 9460 (11.701 sec)
I0716 01:13:16.911679 140076727973760 basic_session_run_hooks.py:260] lr = 0.00011072745 (11.709 sec)
I0716 01:13:27.705688 140076727973760 basic_session_run_hooks.py:692] global_step/sec: 9.25134
I0716 01:13:27.713202 140076727973760 basic_session_run_hooks.py:260] loss = 0.0056742793, step = 9560 (10.811 sec)
I0716 01:13:27.715254 140076727973760 basic_session_run_hooks.py:260] lr = 0.00010956693 (10.804 sec)
I0716 01:13:38.515786 140076727973760 basic_session_run_hooks.py:692] global_step/sec: 9.25059
I0716 01:13:38.523210 140076727973760 basic_session_run_hooks.py:260] loss = 0.005810209, step = 9660 (10.810 sec)
I0716 01:13:38.525804 140076727973760 basic_session_run_hooks.py:260] lr = 0.00010841859 (10.810 sec)
I0716 01:13:39.725109 140076727973760 basic_session_run_hooks.py:606] Saving checkpoints for 9672 into ../model/bigru_crf/model.ckpt.
I0716 01:13:40.154939 140076727973760 estimator.py:1145] Calling model_fn.
I0716 01:13:41.334682 140076727973760 estimator.py:1147] Done calling model_fn.
I0716 01:13:41.365740 140076727973760 evaluation.py:255] Starting evaluation at 2019-07-16T01:13:41Z
I0716 01:13:41.481910 140076727973760 monitored_session.py:240] Graph was finalized.
I0716 01:13:41.499959 140076727973760 saver.py:1280] Restoring parameters from ../model/bigru_crf/model.ckpt-9672
I0716 01:13:41.627418 140076727973760 session_manager.py:500] Running local_init_op.
I0716 01:13:41.655105 140076727973760 session_manager.py:502] Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
I0716 01:13:43.981242 140076727973760 evaluation.py:275] Finished evaluation at 2019-07-16-01:13:43
I0716 01:13:43.982810 140076727973760 estimator.py:2039] Saving dict for global step 9672: global_step = 9672, loss = 4.5402493
I0716 01:13:43.989763 140076727973760 estimator.py:2099] Saving 'checkpoint_path' summary for global step 9672: ../model/bigru_crf/model.ckpt-9672
I0716 01:13:44.054733 140076727973760 estimator.py:368] Loss for final step: 0.0032354048.
Reading ../data/atis.test.w-intent.iob
I0716 01:13:44.381454 140076727973760 estimator.py:1145] Calling model_fn.
I0716 01:13:45.044774 140076727973760 estimator.py:1147] Done calling model_fn.
I0716 01:13:45.156476 140076727973760 monitored_session.py:240] Graph was finalized.
I0716 01:13:45.172793 140076727973760 saver.py:1280] Restoring parameters from ../model/bigru_crf/model.ckpt-9672
I0716 01:13:45.273963 140076727973760 session_manager.py:500] Running local_init_op.
I0716 01:13:45.290879 140076727973760 session_manager.py:502] Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.
  'precision', 'predicted', average, warn_for)
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1439: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples.
  'recall', 'true', average, warn_for)
I0716 01:13:48.763059 140076727973760 interactiveshell.py:2882] 
                                          precision    recall  f1-score   support

                             atis_flight      0.975     0.992     0.984       632
                            atis_airfare      0.979     0.979     0.979        48
                     atis_ground_service      1.000     1.000     1.000        36
                            atis_airline      1.000     1.000     1.000        38
                       atis_abbreviation      0.943     1.000     0.971        33
                           atis_aircraft      0.900     1.000     0.947         9
                        atis_flight_time      1.000     1.000     1.000         1
                           atis_quantity      0.375     1.000     0.545         3
                atis_flight#atis_airfare      0.667     0.167     0.267        12
                            atis_airport      1.000     0.944     0.971        18
                           atis_distance      1.000     1.000     1.000        10
                               atis_city      1.000     0.500     0.667         6
                        atis_ground_fare      1.000     1.000     1.000         7
                           atis_capacity      1.000     0.952     0.976        21
                          atis_flight_no      0.889     1.000     0.941         8
                               atis_meal      1.000     0.833     0.909         6
                        atis_restriction      0.000     0.000     0.000         0
             atis_airline#atis_flight_no      0.000     0.000     0.000         0
    atis_ground_service#atis_ground_fare      0.000     0.000     0.000         0
           atis_airfare#atis_flight_time      0.000     0.000     0.000         0
                           atis_cheapest      0.000     0.000     0.000         0
atis_aircraft#atis_flight#atis_flight_no      0.000     0.000     0.000         0

                               micro avg      0.970     0.975     0.972       888
                               macro avg      0.669     0.653     0.643       888
                            weighted avg      0.970     0.975     0.970       888

I0716 01:13:48.823574 140076727973760 interactiveshell.py:2882] 
                              precision    recall  f1-score   support

                           O      0.000     0.000     0.000       0.0
           B-toloc.city_name      0.978     0.992     0.985     716.0
         B-fromloc.city_name      0.983     0.997     0.990     704.0
           I-toloc.city_name      0.974     0.977     0.976     265.0
      B-depart_date.day_name      0.991     0.991     0.991     212.0
              B-airline_name      1.000     1.000     1.000     101.0
         I-fromloc.city_name      0.962     0.989     0.975     177.0
 B-depart_time.period_of_day      0.992     0.923     0.956     130.0
              I-airline_name      1.000     1.000     1.000      65.0
    B-depart_date.day_number      0.981     0.964     0.972      55.0
    B-depart_date.month_name      0.982     0.964     0.973      56.0
          B-depart_time.time      0.864     1.000     0.927      57.0
                B-round_trip      1.000     0.973     0.986      73.0
             B-cost_relative      1.000     0.973     0.986      37.0
                I-round_trip      1.000     1.000     1.000      71.0
                B-flight_mod      1.000     1.000     1.000      24.0
 B-depart_time.time_relative      0.969     0.954     0.961      65.0
          I-depart_time.time      0.944     0.981     0.962      52.0
         B-stoploc.city_name      0.909     1.000     0.952      20.0
                 B-city_name      0.897     0.614     0.729      57.0
                B-class_type      0.960     1.000     0.980      24.0
          B-arrive_time.time      0.971     0.971     0.971      34.0
 B-arrive_time.time_relative      0.906     0.935     0.921      31.0
                I-class_type      1.000     1.000     1.000      17.0
               B-flight_stop      1.000     1.000     1.000      21.0
          I-arrive_time.time      1.000     0.971     0.986      35.0
              B-airline_code      1.000     0.971     0.985      34.0
    I-depart_date.day_number      1.000     0.933     0.966      15.0
      I-fromloc.airport_name      0.441     1.000     0.612      15.0
      B-fromloc.airport_name      0.500     1.000     0.667      12.0
      B-arrive_date.day_name      0.786     1.000     0.880      11.0
          B-toloc.state_code      0.947     1.000     0.973      18.0
B-depart_date.today_relative      1.000     0.889     0.941       9.0
             B-flight_number      0.647     1.000     0.786      11.0
 B-depart_date.date_relative      0.944     1.000     0.971      17.0
          B-toloc.state_name      0.781     0.893     0.833      28.0
           B-fare_basis_code      0.944     1.000     0.971      17.0
               B-flight_time      1.000     1.000     1.000       1.0
                        B-or      1.000     1.000     1.000       3.0
 B-arrive_time.period_of_day      0.857     1.000     0.923       6.0
          B-meal_description      1.000     0.900     0.947      10.0
             I-cost_relative      1.000     0.667     0.800       3.0
              I-airport_name      0.929     0.448     0.605      29.0
               B-fare_amount      1.000     1.000     1.000       2.0
               I-fare_amount      1.000     1.000     1.000       2.0
                 I-city_name      0.900     0.600     0.720      30.0
        I-toloc.airport_name      1.000     1.000     1.000       3.0
            B-transport_type      1.000     1.000     1.000      10.0
    B-arrive_date.month_name      0.714     0.833     0.769       6.0
    B-arrive_date.day_number      0.714     0.833     0.769       6.0
         I-stoploc.city_name      0.833     1.000     0.909      10.0
                      B-meal      1.000     1.000     1.000      16.0
        B-fromloc.state_code      1.000     0.957     0.978      23.0
    B-depart_time.period_mod      1.000     1.000     1.000       5.0
                   B-connect      1.000     1.000     1.000       6.0
               B-flight_days      1.000     1.000     1.000      10.0
        B-toloc.airport_name      1.000     1.000     1.000       3.0
        B-fromloc.state_name      0.944     1.000     0.971      17.0
              B-airport_name      0.833     0.476     0.606      21.0
                   B-economy      1.000     1.000     1.000       6.0
               I-flight_time      1.000     1.000     1.000       1.0
             B-aircraft_code      0.967     0.879     0.921      33.0
                       B-mod      1.000     0.500     0.667       2.0
              B-airport_code      0.500     0.111     0.182       9.0
    B-depart_time.start_time      1.000     0.667     0.800       3.0
      B-depart_time.end_time      1.000     0.333     0.500       3.0
          B-depart_date.year      1.000     1.000     1.000       3.0
            I-transport_type      0.000     0.000     0.000       1.0
          B-restriction_code      0.400     1.000     0.571       4.0
    B-arrive_time.start_time      0.889     1.000     0.941       8.0
        B-toloc.airport_code      1.000     0.750     0.857       4.0
      B-arrive_time.end_time      0.889     1.000     0.941       8.0
      I-arrive_time.end_time      0.889     1.000     0.941       8.0
      I-depart_time.end_time      1.000     0.333     0.500       3.0
               I-flight_stop      0.000     0.000     0.000       0.0
      B-fromloc.airport_code      1.000     1.000     1.000       5.0
          I-restriction_code      1.000     1.000     1.000       3.0
    I-depart_time.start_time      1.000     1.000     1.000       1.0
          I-toloc.state_name      1.000     1.000     1.000       1.0
I-depart_date.today_relative      0.000     0.000     0.000       0.0
 B-arrive_date.date_relative      1.000     1.000     1.000       2.0
                I-flight_mod      1.000     0.167     0.286       6.0
                   I-economy      0.000     0.000     0.000       0.0
 B-return_date.date_relative      1.000     0.333     0.500       3.0
        I-fromloc.state_name      1.000     1.000     1.000       1.0
                B-state_code      1.000     1.000     1.000       1.0
    I-arrive_time.start_time      1.000     1.000     1.000       1.0
    I-arrive_date.day_number      0.000     0.000     0.000       0.0
                 B-meal_code      0.000     0.000     0.000       1.0
 I-depart_time.period_of_day      1.000     1.000     1.000       1.0
                  B-day_name      1.000     0.500     0.667       2.0
             B-period_of_day      1.000     1.000     1.000       4.0
        B-stoploc.state_code      0.000     0.000     0.000       0.0
    B-return_date.month_name      0.000     0.000     0.000       0.0
    B-return_date.day_number      0.000     0.000     0.000       0.0
    B-arrive_time.period_mod      0.000     0.000     0.000       0.0
                 I-meal_code      0.000     0.000     0.000       0.0
        B-toloc.country_name      1.000     1.000     1.000       1.0
                 B-days_code      1.000     1.000     1.000       1.0
 I-arrive_time.period_of_day      0.000     0.000     0.000       0.0
            I-today_relative      0.000     0.000     0.000       0.0
 B-return_time.period_of_day      0.000     0.000     0.000       0.0
                      B-time      0.000     0.000     0.000       0.0
           I-fare_basis_code      0.000     0.000     0.000       0.0
 I-arrive_time.time_relative      0.000     0.000     0.000       4.0
 I-depart_time.time_relative      0.000     0.000     0.000       1.0
            B-today_relative      0.000     0.000     0.000       0.0
                B-state_name      0.000     0.000     0.000       9.0
B-arrive_date.today_relative      0.000     0.000     0.000       0.0
    B-return_time.period_mod      0.000     0.000     0.000       0.0
                B-month_name      0.000     0.000     0.000       0.0
                B-day_number      0.000     0.000     0.000       0.0
 I-return_date.date_relative      0.750     1.000     0.857       3.0
I-return_date.today_relative      0.000     0.000     0.000       0.0
      B-stoploc.airport_name      0.000     0.000     0.000       0.0
             B-time_relative      0.000     0.000     0.000       0.0
                      I-time      0.000     0.000     0.000       0.0
    I-return_date.day_number      0.000     0.000     0.000       0.0
          I-meal_description      0.000     0.000     0.000       0.0
B-return_date.today_relative      0.000     0.000     0.000       0.0
      B-return_date.day_name      0.000     0.000     0.000       2.0

                   micro avg      0.955     0.956     0.955    3657.0
                   macro avg      0.680     0.654     0.652    3657.0
                weighted avg      0.960     0.956     0.954    3657.0

I0716 01:13:48.851997 140076727973760 interactiveshell.py:2882] Best Slot F1: 0.957
I0716 01:13:48.853603 140076727973760 estimator_training.py:186] Not using Distribute Coordinator.
I0716 01:13:48.858152 140076727973760 training.py:612] Running training and evaluation locally (non-distributed).
I0716 01:13:48.862947 140076727973760 training.py:700] Start train and evaluate loop. The evaluate will happen after every checkpoint. Checkpoint frequency is determined based on RunConfig arguments: save_checkpoints_steps 312 or save_checkpoints_secs None.
I0716 01:13:48.925082 140076727973760 estimator.py:1145] Calling model_fn.
I0716 01:13:49.887588 140076727973760 <ipython-input-6-0a85a3fd693f>:50] 
[<tf.Variable 'embedding:0' shape=(750, 300) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/gates/kernel:0' shape=(600, 600) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/gates/bias:0' shape=(600,) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/candidate/kernel:0' shape=(600, 300) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/candidate/bias:0' shape=(300,) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/gates/kernel:0' shape=(600, 600) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/gates/bias:0' shape=(600,) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/candidate/kernel:0' shape=(600, 300) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/candidate/bias:0' shape=(300,) dtype=float32_ref>,
 <tf.Variable 'dense/kernel:0' shape=(600, 23) dtype=float32_ref>,
 <tf.Variable 'dense/bias:0' shape=(23,) dtype=float32_ref>,
 <tf.Variable 'dense_1/kernel:0' shape=(600, 122) dtype=float32_ref>,
 <tf.Variable 'dense_1/bias:0' shape=(122,) dtype=float32_ref>,
 <tf.Variable 'transitions:0' shape=(122, 122) dtype=float32_ref>]
I0716 01:13:51.435515 140076727973760 estimator.py:1147] Done calling model_fn.
I0716 01:13:51.439323 140076727973760 basic_session_run_hooks.py:541] Create CheckpointSaverHook.
I0716 01:13:51.637001 140076727973760 monitored_session.py:240] Graph was finalized.
I0716 01:13:51.658719 140076727973760 saver.py:1280] Restoring parameters from ../model/bigru_crf/model.ckpt-9672
I0716 01:13:51.927115 140076727973760 session_manager.py:500] Running local_init_op.
I0716 01:13:51.968415 140076727973760 session_manager.py:502] Done running local_init_op.
I0716 01:13:53.466736 140076727973760 basic_session_run_hooks.py:606] Saving checkpoints for 9672 into ../model/bigru_crf/model.ckpt.
Reading ../data/atis.train.w-intent.iob
I0716 01:13:55.804607 140076727973760 basic_session_run_hooks.py:262] loss = 0.007013521, step = 9672
I0716 01:13:55.806132 140076727973760 basic_session_run_hooks.py:262] lr = 0.000108281594
I0716 01:14:07.084267 140076727973760 basic_session_run_hooks.py:692] global_step/sec: 8.86499
I0716 01:14:07.087220 140076727973760 basic_session_run_hooks.py:260] loss = 0.00772463, step = 9772 (11.283 sec)
I0716 01:14:07.092320 140076727973760 basic_session_run_hooks.py:260] lr = 0.000107146734 (11.286 sec)
I0716 01:14:17.904240 140076727973760 basic_session_run_hooks.py:692] global_step/sec: 9.24218
I0716 01:14:17.907539 140076727973760 basic_session_run_hooks.py:260] loss = 0.0033783677, step = 9872 (10.820 sec)
I0716 01:14:17.911832 140076727973760 basic_session_run_hooks.py:260] lr = 0.00010602375 (10.820 sec)
I0716 01:14:28.620991 140076727973760 basic_session_run_hooks.py:692] global_step/sec: 9.33122
I0716 01:14:28.624074 140076727973760 basic_session_run_hooks.py:260] loss = 0.0062032603, step = 9972 (10.717 sec)
I0716 01:14:28.626091 140076727973760 basic_session_run_hooks.py:260] lr = 0.00010491254 (10.714 sec)
I0716 01:14:29.815322 140076727973760 basic_session_run_hooks.py:606] Saving checkpoints for 9984 into ../model/bigru_crf/model.ckpt.
I0716 01:14:30.271186 140076727973760 estimator.py:1145] Calling model_fn.
I0716 01:14:31.453004 140076727973760 estimator.py:1147] Done calling model_fn.
I0716 01:14:31.483109 140076727973760 evaluation.py:255] Starting evaluation at 2019-07-16T01:14:31Z
I0716 01:14:31.599452 140076727973760 monitored_session.py:240] Graph was finalized.
I0716 01:14:31.614970 140076727973760 saver.py:1280] Restoring parameters from ../model/bigru_crf/model.ckpt-9984
I0716 01:14:31.734997 140076727973760 session_manager.py:500] Running local_init_op.
I0716 01:14:31.762921 140076727973760 session_manager.py:502] Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
I0716 01:14:34.102300 140076727973760 evaluation.py:275] Finished evaluation at 2019-07-16-01:14:34
I0716 01:14:34.103910 140076727973760 estimator.py:2039] Saving dict for global step 9984: global_step = 9984, loss = 4.4444613
I0716 01:14:34.112718 140076727973760 estimator.py:2099] Saving 'checkpoint_path' summary for global step 9984: ../model/bigru_crf/model.ckpt-9984
I0716 01:14:34.170863 140076727973760 estimator.py:368] Loss for final step: 0.002228734.
Reading ../data/atis.test.w-intent.iob
I0716 01:14:34.490896 140076727973760 estimator.py:1145] Calling model_fn.
I0716 01:14:35.465409 140076727973760 estimator.py:1147] Done calling model_fn.
I0716 01:14:35.578038 140076727973760 monitored_session.py:240] Graph was finalized.
I0716 01:14:35.592828 140076727973760 saver.py:1280] Restoring parameters from ../model/bigru_crf/model.ckpt-9984
I0716 01:14:35.701222 140076727973760 session_manager.py:500] Running local_init_op.
I0716 01:14:35.719745 140076727973760 session_manager.py:502] Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.
  'precision', 'predicted', average, warn_for)
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1439: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples.
  'recall', 'true', average, warn_for)
I0716 01:14:38.874283 140076727973760 interactiveshell.py:2882] 
                                          precision    recall  f1-score   support

                             atis_flight      0.975     0.992     0.984       632
                            atis_airfare      0.979     0.979     0.979        48
                     atis_ground_service      1.000     1.000     1.000        36
                            atis_airline      1.000     1.000     1.000        38
                       atis_abbreviation      0.943     1.000     0.971        33
                           atis_aircraft      1.000     1.000     1.000         9
                        atis_flight_time      1.000     1.000     1.000         1
                           atis_quantity      0.375     1.000     0.545         3
                atis_flight#atis_airfare      0.750     0.250     0.375        12
                            atis_airport      1.000     0.944     0.971        18
                           atis_distance      1.000     1.000     1.000        10
                               atis_city      1.000     0.500     0.667         6
                        atis_ground_fare      1.000     1.000     1.000         7
                           atis_capacity      1.000     0.952     0.976        21
                          atis_flight_no      0.889     1.000     0.941         8
                               atis_meal      1.000     0.833     0.909         6
                        atis_restriction      0.000     0.000     0.000         0
             atis_airline#atis_flight_no      0.000     0.000     0.000         0
    atis_ground_service#atis_ground_fare      0.000     0.000     0.000         0
           atis_airfare#atis_flight_time      0.000     0.000     0.000         0
                           atis_cheapest      0.000     0.000     0.000         0
atis_aircraft#atis_flight#atis_flight_no      0.000     0.000     0.000         0

                               micro avg      0.971     0.976     0.974       888
                               macro avg      0.678     0.657     0.651       888
                            weighted avg      0.973     0.976     0.972       888

I0716 01:14:38.926311 140076727973760 interactiveshell.py:2882] 
                              precision    recall  f1-score   support

                           O      0.000     0.000     0.000       0.0
           B-toloc.city_name      0.971     0.994     0.983     716.0
         B-fromloc.city_name      0.986     0.997     0.992     704.0
           I-toloc.city_name      0.964     0.996     0.980     265.0
      B-depart_date.day_name      0.991     0.991     0.991     212.0
              B-airline_name      1.000     1.000     1.000     101.0
         I-fromloc.city_name      0.983     0.989     0.986     177.0
 B-depart_time.period_of_day      0.992     0.923     0.956     130.0
              I-airline_name      1.000     1.000     1.000      65.0
    B-depart_date.day_number      0.981     0.964     0.972      55.0
    B-depart_date.month_name      0.982     0.964     0.973      56.0
          B-depart_time.time      0.864     1.000     0.927      57.0
                B-round_trip      1.000     0.986     0.993      73.0
             B-cost_relative      1.000     0.973     0.986      37.0
                I-round_trip      1.000     1.000     1.000      71.0
                B-flight_mod      0.960     1.000     0.980      24.0
 B-depart_time.time_relative      0.969     0.954     0.961      65.0
          I-depart_time.time      0.944     0.981     0.962      52.0
         B-stoploc.city_name      0.870     1.000     0.930      20.0
                 B-city_name      0.909     0.526     0.667      57.0
                B-class_type      0.960     1.000     0.980      24.0
          B-arrive_time.time      0.971     0.971     0.971      34.0
 B-arrive_time.time_relative      0.906     0.935     0.921      31.0
                I-class_type      1.000     1.000     1.000      17.0
               B-flight_stop      1.000     1.000     1.000      21.0
          I-arrive_time.time      1.000     0.971     0.986      35.0
              B-airline_code      1.000     0.912     0.954      34.0
    I-depart_date.day_number      1.000     0.933     0.966      15.0
      I-fromloc.airport_name      0.429     1.000     0.600      15.0
      B-fromloc.airport_name      0.480     1.000     0.649      12.0
      B-arrive_date.day_name      0.786     1.000     0.880      11.0
          B-toloc.state_code      0.947     1.000     0.973      18.0
B-depart_date.today_relative      1.000     0.889     0.941       9.0
             B-flight_number      0.579     1.000     0.733      11.0
 B-depart_date.date_relative      0.944     1.000     0.971      17.0
          B-toloc.state_name      0.781     0.893     0.833      28.0
           B-fare_basis_code      0.944     1.000     0.971      17.0
               B-flight_time      1.000     1.000     1.000       1.0
                        B-or      1.000     1.000     1.000       3.0
 B-arrive_time.period_of_day      0.857     1.000     0.923       6.0
          B-meal_description      1.000     0.900     0.947      10.0
             I-cost_relative      1.000     0.667     0.800       3.0
              I-airport_name      0.923     0.414     0.571      29.0
               B-fare_amount      1.000     1.000     1.000       2.0
               I-fare_amount      1.000     1.000     1.000       2.0
                 I-city_name      1.000     0.500     0.667      30.0
        I-toloc.airport_name      1.000     1.000     1.000       3.0
            B-transport_type      1.000     1.000     1.000      10.0
    B-arrive_date.month_name      0.714     0.833     0.769       6.0
    B-arrive_date.day_number      0.714     0.833     0.769       6.0
         I-stoploc.city_name      0.833     1.000     0.909      10.0
                      B-meal      1.000     1.000     1.000      16.0
        B-fromloc.state_code      1.000     0.957     0.978      23.0
    B-depart_time.period_mod      1.000     1.000     1.000       5.0
                   B-connect      1.000     1.000     1.000       6.0
               B-flight_days      1.000     1.000     1.000      10.0
        B-toloc.airport_name      1.000     1.000     1.000       3.0
        B-fromloc.state_name      0.944     1.000     0.971      17.0
              B-airport_name      0.818     0.429     0.562      21.0
                   B-economy      1.000     1.000     1.000       6.0
               I-flight_time      1.000     1.000     1.000       1.0
             B-aircraft_code      0.967     0.879     0.921      33.0
                       B-mod      1.000     0.500     0.667       2.0
              B-airport_code      0.500     0.111     0.182       9.0
    B-depart_time.start_time      0.667     0.667     0.667       3.0
      B-depart_time.end_time      1.000     0.333     0.500       3.0
          B-depart_date.year      1.000     1.000     1.000       3.0
            I-transport_type      0.000     0.000     0.000       1.0
          B-restriction_code      0.400     1.000     0.571       4.0
    B-arrive_time.start_time      0.889     1.000     0.941       8.0
        B-toloc.airport_code      1.000     0.750     0.857       4.0
      B-arrive_time.end_time      0.889     1.000     0.941       8.0
      I-arrive_time.end_time      0.889     1.000     0.941       8.0
      I-depart_time.end_time      1.000     0.333     0.500       3.0
               I-flight_stop      0.000     0.000     0.000       0.0
      B-fromloc.airport_code      1.000     1.000     1.000       5.0
          I-restriction_code      1.000     1.000     1.000       3.0
    I-depart_time.start_time      1.000     1.000     1.000       1.0
          I-toloc.state_name      1.000     1.000     1.000       1.0
I-depart_date.today_relative      0.000     0.000     0.000       0.0
 B-arrive_date.date_relative      1.000     1.000     1.000       2.0
                I-flight_mod      1.000     0.167     0.286       6.0
                   I-economy      0.000     0.000     0.000       0.0
 B-return_date.date_relative      1.000     0.333     0.500       3.0
        I-fromloc.state_name      1.000     1.000     1.000       1.0
                B-state_code      1.000     1.000     1.000       1.0
    I-arrive_time.start_time      1.000     1.000     1.000       1.0
    I-arrive_date.day_number      0.000     0.000     0.000       0.0
                 B-meal_code      0.000     0.000     0.000       1.0
 I-depart_time.period_of_day      1.000     1.000     1.000       1.0
                  B-day_name      1.000     0.500     0.667       2.0
             B-period_of_day      1.000     1.000     1.000       4.0
        B-stoploc.state_code      0.000     0.000     0.000       0.0
    B-return_date.month_name      0.000     0.000     0.000       0.0
    B-return_date.day_number      0.000     0.000     0.000       0.0
    B-arrive_time.period_mod      0.000     0.000     0.000       0.0
                 I-meal_code      0.000     0.000     0.000       0.0
        B-toloc.country_name      1.000     1.000     1.000       1.0
                 B-days_code      1.000     1.000     1.000       1.0
 I-arrive_time.period_of_day      0.000     0.000     0.000       0.0
            I-today_relative      0.000     0.000     0.000       0.0
 B-return_time.period_of_day      0.000     0.000     0.000       0.0
                      B-time      0.000     0.000     0.000       0.0
           I-fare_basis_code      0.000     0.000     0.000       0.0
 I-arrive_time.time_relative      0.000     0.000     0.000       4.0
 I-depart_time.time_relative      0.000     0.000     0.000       1.0
            B-today_relative      0.000     0.000     0.000       0.0
                B-state_name      0.000     0.000     0.000       9.0
B-arrive_date.today_relative      0.000     0.000     0.000       0.0
    B-return_time.period_mod      0.000     0.000     0.000       0.0
                B-month_name      0.000     0.000     0.000       0.0
                B-day_number      0.000     0.000     0.000       0.0
 I-return_date.date_relative      0.750     1.000     0.857       3.0
I-return_date.today_relative      0.000     0.000     0.000       0.0
      B-stoploc.airport_name      0.000     0.000     0.000       0.0
             B-time_relative      0.000     0.000     0.000       0.0
                      I-time      0.000     0.000     0.000       0.0
    I-return_date.day_number      0.000     0.000     0.000       0.0
          I-meal_description      0.000     0.000     0.000       0.0
B-return_date.today_relative      0.000     0.000     0.000       0.0
      B-return_date.day_name      0.000     0.000     0.000       2.0

                   micro avg      0.954     0.955     0.954    3657.0
                   macro avg      0.676     0.652     0.648    3657.0
                weighted avg      0.959     0.955     0.952    3657.0

I0716 01:14:38.957112 140076727973760 interactiveshell.py:2882] Best Slot F1: 0.957
I0716 01:14:38.958841 140076727973760 estimator_training.py:186] Not using Distribute Coordinator.
I0716 01:14:38.962038 140076727973760 training.py:612] Running training and evaluation locally (non-distributed).
I0716 01:14:38.964607 140076727973760 training.py:700] Start train and evaluate loop. The evaluate will happen after every checkpoint. Checkpoint frequency is determined based on RunConfig arguments: save_checkpoints_steps 312 or save_checkpoints_secs None.
I0716 01:14:39.030541 140076727973760 estimator.py:1145] Calling model_fn.
I0716 01:14:39.999005 140076727973760 <ipython-input-6-0a85a3fd693f>:50] 
[<tf.Variable 'embedding:0' shape=(750, 300) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/gates/kernel:0' shape=(600, 600) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/gates/bias:0' shape=(600,) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/candidate/kernel:0' shape=(600, 300) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/candidate/bias:0' shape=(300,) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/gates/kernel:0' shape=(600, 600) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/gates/bias:0' shape=(600,) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/candidate/kernel:0' shape=(600, 300) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/candidate/bias:0' shape=(300,) dtype=float32_ref>,
 <tf.Variable 'dense/kernel:0' shape=(600, 23) dtype=float32_ref>,
 <tf.Variable 'dense/bias:0' shape=(23,) dtype=float32_ref>,
 <tf.Variable 'dense_1/kernel:0' shape=(600, 122) dtype=float32_ref>,
 <tf.Variable 'dense_1/bias:0' shape=(122,) dtype=float32_ref>,
 <tf.Variable 'transitions:0' shape=(122, 122) dtype=float32_ref>]
I0716 01:14:41.556433 140076727973760 estimator.py:1147] Done calling model_fn.
I0716 01:14:41.560607 140076727973760 basic_session_run_hooks.py:541] Create CheckpointSaverHook.
I0716 01:14:41.962718 140076727973760 monitored_session.py:240] Graph was finalized.
I0716 01:14:41.985010 140076727973760 saver.py:1280] Restoring parameters from ../model/bigru_crf/model.ckpt-9984
I0716 01:14:42.245639 140076727973760 session_manager.py:500] Running local_init_op.
I0716 01:14:42.295799 140076727973760 session_manager.py:502] Done running local_init_op.
I0716 01:14:43.850384 140076727973760 basic_session_run_hooks.py:606] Saving checkpoints for 9984 into ../model/bigru_crf/model.ckpt.
Reading ../data/atis.train.w-intent.iob
I0716 01:14:46.187259 140076727973760 basic_session_run_hooks.py:262] loss = 0.006404533, step = 9984
I0716 01:14:46.189328 140076727973760 basic_session_run_hooks.py:262] lr = 0.00010477998
I0716 01:14:57.823874 140076727973760 basic_session_run_hooks.py:692] global_step/sec: 8.59322
I0716 01:14:57.832448 140076727973760 basic_session_run_hooks.py:260] loss = 0.008892389, step = 10084 (11.645 sec)
I0716 01:14:57.834204 140076727973760 basic_session_run_hooks.py:260] lr = 0.000103681814 (11.645 sec)
I0716 01:15:08.732873 140076727973760 basic_session_run_hooks.py:692] global_step/sec: 9.16677
I0716 01:15:08.741639 140076727973760 basic_session_run_hooks.py:260] loss = 0.0017661343, step = 10184 (10.909 sec)
I0716 01:15:08.743150 140076727973760 basic_session_run_hooks.py:260] lr = 0.000102595164 (10.909 sec)
I0716 01:15:19.861700 140076727973760 basic_session_run_hooks.py:692] global_step/sec: 8.98569
I0716 01:15:19.868390 140076727973760 basic_session_run_hooks.py:260] loss = 0.0033334044, step = 10284 (11.127 sec)
I0716 01:15:19.870436 140076727973760 basic_session_run_hooks.py:260] lr = 0.00010151988 (11.127 sec)
I0716 01:15:21.055633 140076727973760 basic_session_run_hooks.py:606] Saving checkpoints for 10296 into ../model/bigru_crf/model.ckpt.
I0716 01:15:21.435421 140076727973760 estimator.py:1145] Calling model_fn.
I0716 01:15:22.392880 140076727973760 estimator.py:1147] Done calling model_fn.
I0716 01:15:22.425631 140076727973760 evaluation.py:255] Starting evaluation at 2019-07-16T01:15:22Z
I0716 01:15:22.543225 140076727973760 monitored_session.py:240] Graph was finalized.
I0716 01:15:22.558738 140076727973760 saver.py:1280] Restoring parameters from ../model/bigru_crf/model.ckpt-10296
I0716 01:15:22.678631 140076727973760 session_manager.py:500] Running local_init_op.
I0716 01:15:22.705765 140076727973760 session_manager.py:502] Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
I0716 01:15:25.001873 140076727973760 evaluation.py:275] Finished evaluation at 2019-07-16-01:15:25
I0716 01:15:25.003395 140076727973760 estimator.py:2039] Saving dict for global step 10296: global_step = 10296, loss = 4.5833898
I0716 01:15:25.011453 140076727973760 estimator.py:2099] Saving 'checkpoint_path' summary for global step 10296: ../model/bigru_crf/model.ckpt-10296
I0716 01:15:25.074176 140076727973760 estimator.py:368] Loss for final step: 0.0012518166.
Reading ../data/atis.test.w-intent.iob
I0716 01:15:25.386294 140076727973760 estimator.py:1145] Calling model_fn.
I0716 01:15:26.335395 140076727973760 estimator.py:1147] Done calling model_fn.
I0716 01:15:26.449936 140076727973760 monitored_session.py:240] Graph was finalized.
I0716 01:15:26.464225 140076727973760 saver.py:1280] Restoring parameters from ../model/bigru_crf/model.ckpt-10296
I0716 01:15:26.576615 140076727973760 session_manager.py:500] Running local_init_op.
I0716 01:15:26.594811 140076727973760 session_manager.py:502] Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.
  'precision', 'predicted', average, warn_for)
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1439: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples.
  'recall', 'true', average, warn_for)
I0716 01:15:29.713815 140076727973760 interactiveshell.py:2882] 
                                          precision    recall  f1-score   support

                             atis_flight      0.975     0.992     0.984       632
                            atis_airfare      0.979     0.979     0.979        48
                     atis_ground_service      1.000     1.000     1.000        36
                            atis_airline      1.000     1.000     1.000        38
                       atis_abbreviation      0.943     1.000     0.971        33
                           atis_aircraft      1.000     1.000     1.000         9
                        atis_flight_time      1.000     1.000     1.000         1
                           atis_quantity      0.375     1.000     0.545         3
                atis_flight#atis_airfare      0.750     0.250     0.375        12
                            atis_airport      1.000     0.944     0.971        18
                           atis_distance      1.000     1.000     1.000        10
                               atis_city      1.000     0.500     0.667         6
                        atis_ground_fare      1.000     1.000     1.000         7
                           atis_capacity      1.000     0.952     0.976        21
                          atis_flight_no      0.889     1.000     0.941         8
                               atis_meal      1.000     0.833     0.909         6
                        atis_restriction      0.000     0.000     0.000         0
             atis_airline#atis_flight_no      0.000     0.000     0.000         0
    atis_ground_service#atis_ground_fare      0.000     0.000     0.000         0
           atis_airfare#atis_flight_time      0.000     0.000     0.000         0
                           atis_cheapest      0.000     0.000     0.000         0
atis_aircraft#atis_flight#atis_flight_no      0.000     0.000     0.000         0

                               micro avg      0.971     0.976     0.974       888
                               macro avg      0.678     0.657     0.651       888
                            weighted avg      0.973     0.976     0.972       888

I0716 01:15:29.764939 140076727973760 interactiveshell.py:2882] 
                              precision    recall  f1-score   support

                           O      0.000     0.000     0.000       0.0
           B-toloc.city_name      0.974     0.992     0.983     716.0
         B-fromloc.city_name      0.979     0.997     0.988     704.0
           I-toloc.city_name      0.974     0.977     0.976     265.0
      B-depart_date.day_name      0.991     0.991     0.991     212.0
              B-airline_name      1.000     1.000     1.000     101.0
         I-fromloc.city_name      0.962     0.994     0.978     177.0
 B-depart_time.period_of_day      0.992     0.923     0.956     130.0
              I-airline_name      1.000     1.000     1.000      65.0
    B-depart_date.day_number      0.981     0.964     0.972      55.0
    B-depart_date.month_name      0.982     0.964     0.973      56.0
          B-depart_time.time      0.864     1.000     0.927      57.0
                B-round_trip      1.000     0.973     0.986      73.0
             B-cost_relative      1.000     0.973     0.986      37.0
                I-round_trip      1.000     1.000     1.000      71.0
                B-flight_mod      1.000     1.000     1.000      24.0
 B-depart_time.time_relative      0.969     0.954     0.961      65.0
          I-depart_time.time      0.944     0.981     0.962      52.0
         B-stoploc.city_name      0.909     1.000     0.952      20.0
                 B-city_name      0.943     0.579     0.717      57.0
                B-class_type      0.960     1.000     0.980      24.0
          B-arrive_time.time      0.971     0.971     0.971      34.0
 B-arrive_time.time_relative      0.906     0.935     0.921      31.0
                I-class_type      1.000     1.000     1.000      17.0
               B-flight_stop      1.000     1.000     1.000      21.0
          I-arrive_time.time      1.000     0.971     0.986      35.0
              B-airline_code      1.000     0.912     0.954      34.0
    I-depart_date.day_number      1.000     0.933     0.966      15.0
      I-fromloc.airport_name      0.429     1.000     0.600      15.0
      B-fromloc.airport_name      0.480     1.000     0.649      12.0
      B-arrive_date.day_name      0.786     1.000     0.880      11.0
          B-toloc.state_code      0.947     1.000     0.973      18.0
B-depart_date.today_relative      1.000     0.889     0.941       9.0
             B-flight_number      0.550     1.000     0.710      11.0
 B-depart_date.date_relative      0.944     1.000     0.971      17.0
          B-toloc.state_name      0.781     0.893     0.833      28.0
           B-fare_basis_code      0.944     1.000     0.971      17.0
               B-flight_time      1.000     1.000     1.000       1.0
                        B-or      1.000     1.000     1.000       3.0
 B-arrive_time.period_of_day      0.857     1.000     0.923       6.0
          B-meal_description      1.000     0.900     0.947      10.0
             I-cost_relative      1.000     0.667     0.800       3.0
              I-airport_name      0.923     0.414     0.571      29.0
               B-fare_amount      1.000     1.000     1.000       2.0
               I-fare_amount      1.000     1.000     1.000       2.0
                 I-city_name      1.000     0.533     0.696      30.0
        I-toloc.airport_name      1.000     1.000     1.000       3.0
            B-transport_type      1.000     1.000     1.000      10.0
    B-arrive_date.month_name      0.714     0.833     0.769       6.0
    B-arrive_date.day_number      0.714     0.833     0.769       6.0
         I-stoploc.city_name      0.909     1.000     0.952      10.0
                      B-meal      1.000     1.000     1.000      16.0
        B-fromloc.state_code      1.000     0.957     0.978      23.0
    B-depart_time.period_mod      1.000     1.000     1.000       5.0
                   B-connect      1.000     1.000     1.000       6.0
               B-flight_days      1.000     1.000     1.000      10.0
        B-toloc.airport_name      1.000     1.000     1.000       3.0
        B-fromloc.state_name      0.944     1.000     0.971      17.0
              B-airport_name      0.818     0.429     0.562      21.0
                   B-economy      1.000     1.000     1.000       6.0
               I-flight_time      1.000     1.000     1.000       1.0
             B-aircraft_code      0.967     0.879     0.921      33.0
                       B-mod      1.000     0.500     0.667       2.0
              B-airport_code      0.500     0.111     0.182       9.0
    B-depart_time.start_time      1.000     0.667     0.800       3.0
      B-depart_time.end_time      1.000     0.333     0.500       3.0
          B-depart_date.year      1.000     1.000     1.000       3.0
            I-transport_type      0.000     0.000     0.000       1.0
          B-restriction_code      0.400     1.000     0.571       4.0
    B-arrive_time.start_time      0.889     1.000     0.941       8.0
        B-toloc.airport_code      1.000     0.750     0.857       4.0
      B-arrive_time.end_time      0.889     1.000     0.941       8.0
      I-arrive_time.end_time      0.889     1.000     0.941       8.0
      I-depart_time.end_time      1.000     0.333     0.500       3.0
               I-flight_stop      0.000     0.000     0.000       0.0
      B-fromloc.airport_code      1.000     1.000     1.000       5.0
          I-restriction_code      1.000     1.000     1.000       3.0
    I-depart_time.start_time      1.000     1.000     1.000       1.0
          I-toloc.state_name      1.000     1.000     1.000       1.0
I-depart_date.today_relative      0.000     0.000     0.000       0.0
 B-arrive_date.date_relative      1.000     1.000     1.000       2.0
                I-flight_mod      1.000     0.167     0.286       6.0
                   I-economy      0.000     0.000     0.000       0.0
 B-return_date.date_relative      1.000     0.333     0.500       3.0
        I-fromloc.state_name      1.000     1.000     1.000       1.0
                B-state_code      1.000     1.000     1.000       1.0
    I-arrive_time.start_time      1.000     1.000     1.000       1.0
    I-arrive_date.day_number      0.000     0.000     0.000       0.0
                 B-meal_code      0.000     0.000     0.000       1.0
 I-depart_time.period_of_day      1.000     1.000     1.000       1.0
                  B-day_name      1.000     0.500     0.667       2.0
             B-period_of_day      1.000     1.000     1.000       4.0
        B-stoploc.state_code      0.000     0.000     0.000       0.0
    B-return_date.month_name      0.000     0.000     0.000       0.0
    B-return_date.day_number      0.000     0.000     0.000       0.0
    B-arrive_time.period_mod      0.000     0.000     0.000       0.0
                 I-meal_code      0.000     0.000     0.000       0.0
        B-toloc.country_name      1.000     1.000     1.000       1.0
                 B-days_code      1.000     1.000     1.000       1.0
 I-arrive_time.period_of_day      0.000     0.000     0.000       0.0
            I-today_relative      0.000     0.000     0.000       0.0
 B-return_time.period_of_day      0.000     0.000     0.000       0.0
                      B-time      0.000     0.000     0.000       0.0
           I-fare_basis_code      0.000     0.000     0.000       0.0
 I-arrive_time.time_relative      0.000     0.000     0.000       4.0
 I-depart_time.time_relative      0.000     0.000     0.000       1.0
            B-today_relative      0.000     0.000     0.000       0.0
                B-state_name      0.000     0.000     0.000       9.0
B-arrive_date.today_relative      0.000     0.000     0.000       0.0
    B-return_time.period_mod      0.000     0.000     0.000       0.0
                B-month_name      0.000     0.000     0.000       0.0
                B-day_number      0.000     0.000     0.000       0.0
 I-return_date.date_relative      0.750     1.000     0.857       3.0
I-return_date.today_relative      0.000     0.000     0.000       0.0
      B-stoploc.airport_name      0.000     0.000     0.000       0.0
             B-time_relative      0.000     0.000     0.000       0.0
                      I-time      0.000     0.000     0.000       0.0
    I-return_date.day_number      0.000     0.000     0.000       0.0
          I-meal_description      0.000     0.000     0.000       0.0
B-return_date.today_relative      0.000     0.000     0.000       0.0
      B-return_date.day_name      0.000     0.000     0.000       2.0

                   micro avg      0.953     0.954     0.954    3657.0
                   macro avg      0.680     0.652     0.650    3657.0
                weighted avg      0.960     0.954     0.952    3657.0

I0716 01:15:29.793453 140076727973760 interactiveshell.py:2882] Best Slot F1: 0.957
I0716 01:15:29.795181 140076727973760 estimator_training.py:186] Not using Distribute Coordinator.
I0716 01:15:29.799246 140076727973760 training.py:612] Running training and evaluation locally (non-distributed).
I0716 01:15:29.801974 140076727973760 training.py:700] Start train and evaluate loop. The evaluate will happen after every checkpoint. Checkpoint frequency is determined based on RunConfig arguments: save_checkpoints_steps 312 or save_checkpoints_secs None.
I0716 01:15:29.866015 140076727973760 estimator.py:1145] Calling model_fn.
I0716 01:15:31.004414 140076727973760 <ipython-input-6-0a85a3fd693f>:50] 
[<tf.Variable 'embedding:0' shape=(750, 300) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/gates/kernel:0' shape=(600, 600) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/gates/bias:0' shape=(600,) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/candidate/kernel:0' shape=(600, 300) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/candidate/bias:0' shape=(300,) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/gates/kernel:0' shape=(600, 600) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/gates/bias:0' shape=(600,) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/candidate/kernel:0' shape=(600, 300) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/candidate/bias:0' shape=(300,) dtype=float32_ref>,
 <tf.Variable 'dense/kernel:0' shape=(600, 23) dtype=float32_ref>,
 <tf.Variable 'dense/bias:0' shape=(23,) dtype=float32_ref>,
 <tf.Variable 'dense_1/kernel:0' shape=(600, 122) dtype=float32_ref>,
 <tf.Variable 'dense_1/bias:0' shape=(122,) dtype=float32_ref>,
 <tf.Variable 'transitions:0' shape=(122, 122) dtype=float32_ref>]
I0716 01:15:32.526916 140076727973760 estimator.py:1147] Done calling model_fn.
I0716 01:15:32.530575 140076727973760 basic_session_run_hooks.py:541] Create CheckpointSaverHook.
I0716 01:15:32.740853 140076727973760 monitored_session.py:240] Graph was finalized.
I0716 01:15:32.759663 140076727973760 saver.py:1280] Restoring parameters from ../model/bigru_crf/model.ckpt-10296
I0716 01:15:33.023628 140076727973760 session_manager.py:500] Running local_init_op.
I0716 01:15:33.063321 140076727973760 session_manager.py:502] Done running local_init_op.
I0716 01:15:34.563801 140076727973760 basic_session_run_hooks.py:606] Saving checkpoints for 10296 into ../model/bigru_crf/model.ckpt.
Reading ../data/atis.train.w-intent.iob
I0716 01:15:36.889125 140076727973760 basic_session_run_hooks.py:262] loss = 0.008528731, step = 10296
I0716 01:15:36.891040 140076727973760 basic_session_run_hooks.py:262] lr = 0.00010139161
I0716 01:15:48.414942 140076727973760 basic_session_run_hooks.py:692] global_step/sec: 8.67586
I0716 01:15:48.420658 140076727973760 basic_session_run_hooks.py:260] loss = 0.009942888, step = 10396 (11.532 sec)
I0716 01:15:48.424746 140076727973760 basic_session_run_hooks.py:260] lr = 0.000100328936 (11.534 sec)
I0716 01:15:59.223440 140076727973760 basic_session_run_hooks.py:692] global_step/sec: 9.25198
I0716 01:15:59.231911 140076727973760 basic_session_run_hooks.py:260] loss = 0.0036947373, step = 10496 (10.811 sec)
I0716 01:15:59.233798 140076727973760 basic_session_run_hooks.py:260] lr = 9.927743e-05 (10.809 sec)
I0716 01:16:10.406613 140076727973760 basic_session_run_hooks.py:692] global_step/sec: 8.94198
I0716 01:16:10.413411 140076727973760 basic_session_run_hooks.py:260] loss = 0.0132704, step = 10596 (11.182 sec)
I0716 01:16:10.415537 140076727973760 basic_session_run_hooks.py:260] lr = 9.823692e-05 (11.182 sec)
I0716 01:16:11.627557 140076727973760 basic_session_run_hooks.py:606] Saving checkpoints for 10608 into ../model/bigru_crf/model.ckpt.
I0716 01:16:12.016364 140076727973760 estimator.py:1145] Calling model_fn.
I0716 01:16:13.197306 140076727973760 estimator.py:1147] Done calling model_fn.
I0716 01:16:13.226732 140076727973760 evaluation.py:255] Starting evaluation at 2019-07-16T01:16:13Z
I0716 01:16:13.346538 140076727973760 monitored_session.py:240] Graph was finalized.
I0716 01:16:13.362723 140076727973760 saver.py:1280] Restoring parameters from ../model/bigru_crf/model.ckpt-10608
I0716 01:16:13.479902 140076727973760 session_manager.py:500] Running local_init_op.
I0716 01:16:13.507349 140076727973760 session_manager.py:502] Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
I0716 01:16:15.792758 140076727973760 evaluation.py:275] Finished evaluation at 2019-07-16-01:16:15
I0716 01:16:15.794436 140076727973760 estimator.py:2039] Saving dict for global step 10608: global_step = 10608, loss = 4.477148
I0716 01:16:15.801826 140076727973760 estimator.py:2099] Saving 'checkpoint_path' summary for global step 10608: ../model/bigru_crf/model.ckpt-10608
I0716 01:16:15.866571 140076727973760 estimator.py:368] Loss for final step: 0.00082361675.
Reading ../data/atis.test.w-intent.iob
I0716 01:16:16.192821 140076727973760 estimator.py:1145] Calling model_fn.
I0716 01:16:16.835687 140076727973760 estimator.py:1147] Done calling model_fn.
I0716 01:16:16.954639 140076727973760 monitored_session.py:240] Graph was finalized.
I0716 01:16:16.970279 140076727973760 saver.py:1280] Restoring parameters from ../model/bigru_crf/model.ckpt-10608
I0716 01:16:17.070095 140076727973760 session_manager.py:500] Running local_init_op.
I0716 01:16:17.086987 140076727973760 session_manager.py:502] Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.
  'precision', 'predicted', average, warn_for)
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1439: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples.
  'recall', 'true', average, warn_for)
I0716 01:16:20.199586 140076727973760 interactiveshell.py:2882] 
                                          precision    recall  f1-score   support

                             atis_flight      0.978     0.992     0.985       632
                            atis_airfare      0.979     0.979     0.979        48
                     atis_ground_service      1.000     1.000     1.000        36
                            atis_airline      1.000     1.000     1.000        38
                       atis_abbreviation      0.943     1.000     0.971        33
                           atis_aircraft      0.900     1.000     0.947         9
                        atis_flight_time      1.000     1.000     1.000         1
                           atis_quantity      0.375     1.000     0.545         3
                atis_flight#atis_airfare      0.750     0.250     0.375        12
                            atis_airport      1.000     0.944     0.971        18
                           atis_distance      1.000     1.000     1.000        10
                               atis_city      1.000     0.667     0.800         6
                        atis_ground_fare      1.000     1.000     1.000         7
                           atis_capacity      1.000     0.952     0.976        21
                          atis_flight_no      0.889     1.000     0.941         8
                               atis_meal      1.000     0.833     0.909         6
                        atis_restriction      0.000     0.000     0.000         0
             atis_airline#atis_flight_no      0.000     0.000     0.000         0
    atis_ground_service#atis_ground_fare      0.000     0.000     0.000         0
           atis_airfare#atis_flight_time      0.000     0.000     0.000         0
                           atis_cheapest      0.000     0.000     0.000         0
atis_aircraft#atis_flight#atis_flight_no      0.000     0.000     0.000         0

                               micro avg      0.972     0.977     0.975       888
                               macro avg      0.673     0.664     0.655       888
                            weighted avg      0.974     0.977     0.973       888

I0716 01:16:20.251647 140076727973760 interactiveshell.py:2882] 
                              precision    recall  f1-score   support

                           O      0.000     0.000     0.000       0.0
           B-toloc.city_name      0.970     0.992     0.981     716.0
         B-fromloc.city_name      0.983     0.997     0.990     704.0
           I-toloc.city_name      0.963     0.981     0.972     265.0
      B-depart_date.day_name      0.991     0.991     0.991     212.0
              B-airline_name      1.000     1.000     1.000     101.0
         I-fromloc.city_name      0.972     0.989     0.980     177.0
 B-depart_time.period_of_day      1.000     0.923     0.960     130.0
              I-airline_name      1.000     1.000     1.000      65.0
    B-depart_date.day_number      0.981     0.964     0.972      55.0
    B-depart_date.month_name      0.982     0.964     0.973      56.0
          B-depart_time.time      0.864     1.000     0.927      57.0
                B-round_trip      1.000     0.973     0.986      73.0
             B-cost_relative      1.000     0.973     0.986      37.0
                I-round_trip      1.000     1.000     1.000      71.0
                B-flight_mod      1.000     1.000     1.000      24.0
 B-depart_time.time_relative      0.969     0.954     0.961      65.0
          I-depart_time.time      0.945     1.000     0.972      52.0
         B-stoploc.city_name      0.909     1.000     0.952      20.0
                 B-city_name      0.882     0.526     0.659      57.0
                B-class_type      0.960     1.000     0.980      24.0
          B-arrive_time.time      0.971     0.971     0.971      34.0
 B-arrive_time.time_relative      0.906     0.935     0.921      31.0
                I-class_type      1.000     1.000     1.000      17.0
               B-flight_stop      1.000     1.000     1.000      21.0
          I-arrive_time.time      1.000     0.971     0.986      35.0
              B-airline_code      1.000     0.941     0.970      34.0
    I-depart_date.day_number      1.000     0.933     0.966      15.0
      I-fromloc.airport_name      0.429     1.000     0.600      15.0
      B-fromloc.airport_name      0.480     1.000     0.649      12.0
      B-arrive_date.day_name      0.786     1.000     0.880      11.0
          B-toloc.state_code      0.947     1.000     0.973      18.0
B-depart_date.today_relative      1.000     0.889     0.941       9.0
             B-flight_number      0.611     1.000     0.759      11.0
 B-depart_date.date_relative      0.944     1.000     0.971      17.0
          B-toloc.state_name      0.781     0.893     0.833      28.0
           B-fare_basis_code      0.944     1.000     0.971      17.0
               B-flight_time      1.000     1.000     1.000       1.0
                        B-or      1.000     1.000     1.000       3.0
 B-arrive_time.period_of_day      0.857     1.000     0.923       6.0
          B-meal_description      1.000     0.900     0.947      10.0
             I-cost_relative      1.000     0.667     0.800       3.0
              I-airport_name      0.923     0.414     0.571      29.0
               B-fare_amount      1.000     1.000     1.000       2.0
               I-fare_amount      1.000     1.000     1.000       2.0
                 I-city_name      0.938     0.500     0.652      30.0
        I-toloc.airport_name      1.000     1.000     1.000       3.0
            B-transport_type      1.000     1.000     1.000      10.0
    B-arrive_date.month_name      0.714     0.833     0.769       6.0
    B-arrive_date.day_number      0.714     0.833     0.769       6.0
         I-stoploc.city_name      0.833     1.000     0.909      10.0
                      B-meal      1.000     1.000     1.000      16.0
        B-fromloc.state_code      1.000     0.957     0.978      23.0
    B-depart_time.period_mod      1.000     1.000     1.000       5.0
                   B-connect      1.000     1.000     1.000       6.0
               B-flight_days      1.000     1.000     1.000      10.0
        B-toloc.airport_name      1.000     1.000     1.000       3.0
        B-fromloc.state_name      0.944     1.000     0.971      17.0
              B-airport_name      0.818     0.429     0.562      21.0
                   B-economy      1.000     1.000     1.000       6.0
               I-flight_time      1.000     1.000     1.000       1.0
             B-aircraft_code      0.967     0.879     0.921      33.0
                       B-mod      1.000     0.500     0.667       2.0
              B-airport_code      0.500     0.111     0.182       9.0
    B-depart_time.start_time      1.000     0.667     0.800       3.0
      B-depart_time.end_time      1.000     0.333     0.500       3.0
          B-depart_date.year      1.000     1.000     1.000       3.0
            I-transport_type      0.000     0.000     0.000       1.0
          B-restriction_code      0.400     1.000     0.571       4.0
    B-arrive_time.start_time      0.889     1.000     0.941       8.0
        B-toloc.airport_code      1.000     0.750     0.857       4.0
      B-arrive_time.end_time      0.889     1.000     0.941       8.0
      I-arrive_time.end_time      0.889     1.000     0.941       8.0
      I-depart_time.end_time      1.000     0.333     0.500       3.0
               I-flight_stop      0.000     0.000     0.000       0.0
      B-fromloc.airport_code      1.000     1.000     1.000       5.0
          I-restriction_code      1.000     1.000     1.000       3.0
    I-depart_time.start_time      1.000     1.000     1.000       1.0
          I-toloc.state_name      1.000     1.000     1.000       1.0
I-depart_date.today_relative      0.000     0.000     0.000       0.0
 B-arrive_date.date_relative      1.000     1.000     1.000       2.0
                I-flight_mod      1.000     0.167     0.286       6.0
                   I-economy      0.000     0.000     0.000       0.0
 B-return_date.date_relative      1.000     0.333     0.500       3.0
        I-fromloc.state_name      1.000     1.000     1.000       1.0
                B-state_code      1.000     1.000     1.000       1.0
    I-arrive_time.start_time      1.000     1.000     1.000       1.0
    I-arrive_date.day_number      0.000     0.000     0.000       0.0
                 B-meal_code      0.000     0.000     0.000       1.0
 I-depart_time.period_of_day      1.000     1.000     1.000       1.0
                  B-day_name      1.000     0.500     0.667       2.0
             B-period_of_day      1.000     1.000     1.000       4.0
        B-stoploc.state_code      0.000     0.000     0.000       0.0
    B-return_date.month_name      0.000     0.000     0.000       0.0
    B-return_date.day_number      0.000     0.000     0.000       0.0
    B-arrive_time.period_mod      0.000     0.000     0.000       0.0
                 I-meal_code      0.000     0.000     0.000       0.0
        B-toloc.country_name      1.000     1.000     1.000       1.0
                 B-days_code      1.000     1.000     1.000       1.0
 I-arrive_time.period_of_day      0.000     0.000     0.000       0.0
            I-today_relative      0.000     0.000     0.000       0.0
 B-return_time.period_of_day      0.000     0.000     0.000       0.0
                      B-time      0.000     0.000     0.000       0.0
           I-fare_basis_code      0.000     0.000     0.000       0.0
 I-arrive_time.time_relative      0.000     0.000     0.000       4.0
 I-depart_time.time_relative      0.000     0.000     0.000       1.0
            B-today_relative      0.000     0.000     0.000       0.0
                B-state_name      0.000     0.000     0.000       9.0
B-arrive_date.today_relative      0.000     0.000     0.000       0.0
    B-return_time.period_mod      0.000     0.000     0.000       0.0
                B-month_name      0.000     0.000     0.000       0.0
                B-day_number      0.000     0.000     0.000       0.0
 I-return_date.date_relative      0.750     1.000     0.857       3.0
I-return_date.today_relative      0.000     0.000     0.000       0.0
      B-stoploc.airport_name      0.000     0.000     0.000       0.0
             B-time_relative      0.000     0.000     0.000       0.0
                      I-time      0.000     0.000     0.000       0.0
    I-return_date.day_number      0.000     0.000     0.000       0.0
          I-meal_description      0.000     0.000     0.000       0.0
B-return_date.today_relative      0.000     0.000     0.000       0.0
      B-return_date.day_name      0.000     0.000     0.000       2.0

                   micro avg      0.952     0.954     0.953    3657.0
                   macro avg      0.679     0.652     0.650    3657.0
                weighted avg      0.958     0.954     0.951    3657.0

I0716 01:16:20.279552 140076727973760 interactiveshell.py:2882] Best Slot F1: 0.957
I0716 01:16:20.281213 140076727973760 interactiveshell.py:2882] Testing Slot F1 not improved over 3 epochs, Early Stop