In [1]:
"""
We use following lines because we are running on Google Colab
If you are running notebook on a local computer, you don't need this cell
"""
from google.colab import drive
drive.mount('/content/gdrive')
import os
os.chdir('/content/gdrive/My Drive/finch/tensorflow1/spoken_language_understanding/atis/main')
Drive already mounted at /content/gdrive; to attempt to forcibly remount, call drive.mount("/content/gdrive", force_remount=True).
In [2]:
import tensorflow as tf
import tensorflow_hub as hub

import pprint
import logging
import time
import numpy as np

from sklearn.metrics import classification_report, f1_score
from pathlib import Path

print("TensorFlow Version", tf.__version__)
print('GPU Enabled:', tf.test.is_gpu_available())
TensorFlow Version 1.14.0
GPU Enabled: True
In [0]:
def get_vocab(vocab_path):
  word2idx = {}
  with open(vocab_path) as f:
    for i, line in enumerate(f):
      line = line.rstrip()
      word2idx[line] = i
  return word2idx
In [0]:
def data_generator(f_path, params):
  print('Reading', f_path)
  with open(f_path) as f:
    for line in f:
      line = line.rstrip()
      text, slot_intent = line.split('\t')
      words = text.split()[1:-1]
      slot_intent = slot_intent.split()
      slots, intent = slot_intent[1:-1], slot_intent[-1]
      assert len(words) == len(slots)
      
      yield (words, (intent, slots))
In [0]:
def dataset(is_training, params):
  _shapes = ([None], ((), [None]))
  _types = (tf.string, (tf.string, tf.string))
  _pads = ('<pad>', ('_', 'O'))
  
  if is_training:
    ds = tf.data.Dataset.from_generator(
      lambda: data_generator(params['train_path'], params),
      output_shapes = _shapes,
      output_types = _types,)
    ds = ds.shuffle(params['num_samples'])
    ds = ds.padded_batch(params['batch_size'], _shapes, _pads)
    ds = ds.prefetch(tf.data.experimental.AUTOTUNE)
  else:
    ds = tf.data.Dataset.from_generator(
      lambda: data_generator(params['test_path'], params),
      output_shapes = _shapes,
      output_types = _types,)
    ds = ds.padded_batch(1, _shapes, _pads)
    ds = ds.prefetch(tf.data.experimental.AUTOTUNE)
  
  return ds
In [0]:
def model_fn(features, labels, mode, params):
  is_training = (mode == tf.estimator.ModeKeys.TRAIN)
  
  vocab = tf.contrib.lookup.index_table_from_file(
    params['word_path'], num_oov_buckets=1)
  words = vocab.lookup(features)
  seq_len = tf.count_nonzero(words, 1, dtype=tf.int32)
  
  embedding = np.load(params['vocab_path'])
  embedding = tf.Variable(embedding, name='embedding', dtype=tf.float32)
  x = tf.nn.embedding_lookup(embedding, words)
  
  elmo = hub.Module("https://tfhub.dev/google/elmo/2", trainable=is_training)
  e = elmo(inputs={'tokens':features, 'sequence_len':seq_len}, signature='tokens', as_dict=True)['elmo']
  
  x = tf.concat((x, e), -1)
  x = tf.layers.dropout(x, params['dropout_rate'], training=is_training)
  x = tf.layers.dense(x, params['rnn_units'], tf.nn.elu)
  x = tf.layers.dropout(x, params['dropout_rate'], training=is_training)
  
  cell_fw = tf.nn.rnn_cell.GRUCell(params['rnn_units'])
  cell_bw = tf.nn.rnn_cell.GRUCell(params['rnn_units'])
  o, _ = tf.nn.bidirectional_dynamic_rnn(cell_fw, cell_bw, x, seq_len, dtype=tf.float32)
  x = tf.concat(o, -1)
  
  y_intent = tf.layers.dense(tf.reduce_max(x, 1), params['intent_size'])
  y_slots = tf.layers.dense(x, params['slot_size'])
  
  
  if labels is not None:
    intent, slots = labels
    
    vocab = tf.contrib.lookup.index_table_from_file(
      params['intent_path'], num_oov_buckets=1)
    intent = vocab.lookup(intent)
    vocab = tf.contrib.lookup.index_table_from_file(
      params['slot_path'], num_oov_buckets=1)
    slots = vocab.lookup(slots)
    
    loss_intent = tf.nn.sparse_softmax_cross_entropy_with_logits(
      labels=intent, logits=y_intent)
    loss_intent = tf.reduce_mean(loss_intent)
    
    loss_slots = tf.nn.sparse_softmax_cross_entropy_with_logits(labels=slots, logits=y_slots)
    weights = tf.cast(tf.sign(slots), tf.float32)
    padding = tf.fill(tf.shape(weights), 1e-2)
    weights = tf.where(tf.equal(weights, 0.), padding, weights)
    loss_slots = tf.reduce_mean(loss_slots * weights)
    
    loss_op = loss_intent + loss_slots

  
  if mode == tf.estimator.ModeKeys.TRAIN:
    variables = tf.trainable_variables()
    tf.logging.info('\n'+pprint.pformat(variables))
    
    grads = tf.gradients(loss_op, variables)
    grads, _ = tf.clip_by_global_norm(grads, params['clip_norm'])
    
    global_step=tf.train.get_or_create_global_step()
    decay_lr = tf.train.exponential_decay(
        params['lr'], global_step, 1000, .9)
    hook = tf.train.LoggingTensorHook({'lr': decay_lr}, every_n_iter=100)
    
    optim = tf.train.AdamOptimizer(decay_lr)
    train_op = optim.apply_gradients(
      zip(grads, variables), global_step=global_step)
    
    return tf.estimator.EstimatorSpec(mode=mode,
                                      loss=loss_op,
                                      train_op=train_op,
                                      training_hooks=[hook],)
  
  
  if mode == tf.estimator.ModeKeys.EVAL:
    return tf.estimator.EstimatorSpec(mode=mode,
                                      loss=loss_op)
  
  
  if mode == tf.estimator.ModeKeys.PREDICT:
    return tf.estimator.EstimatorSpec(mode,
      predictions={'intent': tf.argmax(y_intent, -1),
                   'slots': tf.argmax(y_slots, -1)})
In [0]:
params = {
  'model_dir': '../model/elmo_bigru',
  'log_path': '../log/elmo_bigru.txt',
  'train_path': '../data/atis.train.w-intent.iob',
  'test_path': '../data/atis.test.w-intent.iob',
  'word_path': '../vocab/word.txt',
  'vocab_path': '../vocab/word.npy',
  'intent_path': '../vocab/intent.txt',
  'slot_path': '../vocab/slot.txt',
  'batch_size': 16,
  'num_samples': 4978,
  'rnn_units': 300,
  'dropout_rate': 0.2,
  'clip_norm': 5.0,
  'lr': 3e-4,
  'num_patience': 3,
}
In [0]:
params['word2idx'] = get_vocab(params['word_path'])
params['intent2idx'] = get_vocab(params['intent_path'])
params['slot2idx'] = get_vocab(params['slot_path'])

params['word_size'] = len(params['word2idx']) + 1
params['intent_size'] = len(params['intent2idx']) + 1
params['slot_size'] = len(params['slot2idx']) + 1
In [0]:
def is_descending(history: list):
  history = history[-(params['num_patience']+1):]
  for i in range(1, len(history)):
    if history[i-1] <= history[i]:
      return False
  return True 
In [10]:
# Create directory if not exist
Path(os.path.dirname(params['log_path'])).mkdir(exist_ok=True)
Path(params['model_dir']).mkdir(exist_ok=True, parents=True)

# Logging
logger = logging.getLogger('tensorflow')
logger.setLevel(logging.INFO)
fh = logging.FileHandler(params['log_path'])
logger.addHandler(fh)

# Create an estimator
_eval_steps = params['num_samples']//params['batch_size'] + 1
config = tf.estimator.RunConfig(
  save_checkpoints_steps=_eval_steps,)

estimator = tf.estimator.Estimator(
  model_fn=model_fn,
  model_dir=params['model_dir'],
  config=config,
  params=params)

# Train on training data and Evaluate on testing data
train_spec = tf.estimator.TrainSpec(
  input_fn=lambda: dataset(is_training=True, params=params),)

eval_spec = tf.estimator.EvalSpec(
  input_fn=lambda: dataset(is_training=False, params=params),
  steps=None,
  throttle_secs=10,)

best_f1 = .0
history_f1 = []
tf.enable_eager_execution()

while True:
  tf.estimator.train_and_evaluate(estimator, train_spec, eval_spec)
  
  intent = []
  slots = []
  for w, (i, s) in dataset(is_training=False, params=params):
    intent.append(i.numpy())
    slots.append(s.numpy())
  intent = [i for batch in intent for i in batch]
  intent = [params['intent2idx'].get(str(t, 'utf-8'), len(params['intent2idx'])) for t in intent]
  slots = [j for batch in slots for i in batch for j in i]
  slots = [params['slot2idx'].get(str(s, 'utf-8'), len(params['slot2idx'])) for s in slots]

  predicted = list(estimator.predict(input_fn=lambda: dataset(is_training=False, params=params)))
  y_slots = [j for i in predicted for j in i['slots']]
  y_intent = [i['intent'] for i in predicted]
  
  logger.info('\n'+classification_report(y_true = intent,
                                         y_pred = y_intent,
                                         labels = list(params['intent2idx'].values()),
                                         target_names = list(params['intent2idx'].keys()),
                                         digits=3))

  logger.info('\n'+classification_report(y_true = slots,
                                         y_pred = y_slots,
                                         labels = list(params['slot2idx'].values()),
                                         target_names = list(params['slot2idx'].keys()),
                                         sample_weight = np.sign(slots),
                                         digits=3))
  
  f1_slots = f1_score(y_true = slots,
                      y_pred = y_slots,
                      labels = list(params['slot2idx'].values()),
                      sample_weight = np.sign(slots),
                      average='micro',)
  history_f1.append(f1_slots)

  if f1_slots > best_f1:
    best_f1 = f1_slots
  logger.info("Best Slot F1: {:.3f}".format(best_f1))
  
  if len(history_f1) > params['num_patience'] and is_descending(history_f1):
    logger.info("Testing Slot F1 not improved over {} epochs, Early Stop".format(params['num_patience']))
    break
WARNING: Logging before flag parsing goes to stderr.
I0716 02:58:30.483514 139795883440000 estimator.py:209] Using config: {'_model_dir': '../model/elmo_bigru', '_tf_random_seed': None, '_save_summary_steps': 100, '_save_checkpoints_steps': 312, '_save_checkpoints_secs': None, '_session_config': allow_soft_placement: true
graph_options {
  rewrite_options {
    meta_optimizer_iterations: ONE
  }
}
, '_keep_checkpoint_max': 5, '_keep_checkpoint_every_n_hours': 10000, '_log_step_count_steps': 100, '_train_distribute': None, '_device_fn': None, '_protocol': None, '_eval_distribute': None, '_experimental_distribute': None, '_experimental_max_worker_delay_secs': None, '_service': None, '_cluster_spec': <tensorflow.python.training.server_lib.ClusterSpec object at 0x7f2460d49240>, '_task_type': 'worker', '_task_id': 0, '_global_id_in_cluster': 0, '_master': '', '_evaluation_master': '', '_is_chief': True, '_num_ps_replicas': 0, '_num_worker_replicas': 1}
I0716 02:58:30.489848 139795883440000 estimator_training.py:186] Not using Distribute Coordinator.
I0716 02:58:30.493601 139795883440000 training.py:612] Running training and evaluation locally (non-distributed).
I0716 02:58:30.498357 139795883440000 training.py:700] Start train and evaluate loop. The evaluate will happen after every checkpoint. Checkpoint frequency is determined based on RunConfig arguments: save_checkpoints_steps 312 or save_checkpoints_secs None.
W0716 02:58:30.509913 139795883440000 deprecation.py:323] From /usr/local/lib/python3.6/dist-packages/tensorflow/python/training/training_util.py:236: Variable.initialized_value (from tensorflow.python.ops.variables) is deprecated and will be removed in a future version.
Instructions for updating:
Use Variable.read_value. Variables in 2.X are initialized automatically both in eager and graph (inside tf.defun) contexts.
W0716 02:58:30.533959 139795883440000 deprecation.py:323] From /usr/local/lib/python3.6/dist-packages/tensorflow/python/data/ops/dataset_ops.py:494: py_func (from tensorflow.python.ops.script_ops) is deprecated and will be removed in a future version.
Instructions for updating:
tf.py_func is deprecated in TF V2. Instead, there are two
    options available in V2.
    - tf.py_function takes a python function which manipulates tf eager
    tensors instead of numpy arrays. It's easy to convert a tf eager tensor to
    an ndarray (just call tensor.numpy()) but having access to eager tensors
    means `tf.py_function`s can use accelerators such as GPUs as well as
    being differentiable using a gradient tape.
    - tf.numpy_function maintains the semantics of the deprecated tf.py_func
    (it is not differentiable, and manipulates numpy arrays). It drops the
    stateful argument making all functions stateful.
    
I0716 02:58:30.582866 139795883440000 estimator.py:1145] Calling model_fn.
W0716 02:58:31.239827 139795883440000 lazy_loader.py:50] 
The TensorFlow contrib module will not be included in TensorFlow 2.0.
For more information, please see:
  * https://github.com/tensorflow/community/blob/master/rfcs/20180907-contrib-sunset.md
  * https://github.com/tensorflow/addons
  * https://github.com/tensorflow/io (for I/O related ops)
If you depend on functionality not listed there, please file an issue.

W0716 02:58:31.253417 139795883440000 deprecation.py:323] From /usr/local/lib/python3.6/dist-packages/tensorflow/python/ops/lookup_ops.py:978: add_dispatch_support.<locals>.wrapper (from tensorflow.python.ops.array_ops) is deprecated and will be removed in a future version.
Instructions for updating:
Use tf.where in 2.0, which has the same broadcast rule as np.where
W0716 02:58:31.257234 139795883440000 deprecation.py:506] From /usr/local/lib/python3.6/dist-packages/tensorflow/python/util/deprecation.py:507: calling count_nonzero (from tensorflow.python.ops.math_ops) with axis is deprecated and will be removed in a future version.
Instructions for updating:
reduction_indices is deprecated, use axis instead
I0716 02:58:32.649416 139795883440000 saver.py:1499] Saver not created because there are no variables in the graph to restore
W0716 02:58:32.737046 139795883440000 deprecation.py:323] From <ipython-input-6-d6747b40360b>:17: dropout (from tensorflow.python.layers.core) is deprecated and will be removed in a future version.
Instructions for updating:
Use keras.layers.dropout instead.
W0716 02:58:32.812015 139795883440000 deprecation.py:323] From <ipython-input-6-d6747b40360b>:18: dense (from tensorflow.python.layers.core) is deprecated and will be removed in a future version.
Instructions for updating:
Use keras.layers.dense instead.
W0716 02:58:32.817591 139795883440000 deprecation.py:506] From /usr/local/lib/python3.6/dist-packages/tensorflow/python/ops/init_ops.py:1251: calling VarianceScaling.__init__ (from tensorflow.python.ops.init_ops) with dtype is deprecated and will be removed in a future version.
Instructions for updating:
Call initializer instance with the dtype argument instead of passing it to the constructor
W0716 02:58:33.183183 139795883440000 deprecation.py:323] From <ipython-input-6-d6747b40360b>:21: GRUCell.__init__ (from tensorflow.python.ops.rnn_cell_impl) is deprecated and will be removed in a future version.
Instructions for updating:
This class is equivalent as tf.keras.layers.GRUCell, and will be replaced by that in Tensorflow 2.0.
W0716 02:58:33.185412 139795883440000 deprecation.py:323] From <ipython-input-6-d6747b40360b>:23: bidirectional_dynamic_rnn (from tensorflow.python.ops.rnn) is deprecated and will be removed in a future version.
Instructions for updating:
Please use `keras.layers.Bidirectional(keras.layers.RNN(cell))`, which is equivalent to this API
W0716 02:58:33.187494 139795883440000 deprecation.py:323] From /usr/local/lib/python3.6/dist-packages/tensorflow/python/ops/rnn.py:464: dynamic_rnn (from tensorflow.python.ops.rnn) is deprecated and will be removed in a future version.
Instructions for updating:
Please use `keras.layers.RNN(cell)`, which is equivalent to this API
W0716 02:58:33.282541 139795883440000 deprecation.py:506] From /usr/local/lib/python3.6/dist-packages/tensorflow/python/ops/rnn_cell_impl.py:564: calling Constant.__init__ (from tensorflow.python.ops.init_ops) with dtype is deprecated and will be removed in a future version.
Instructions for updating:
Call initializer instance with the dtype argument instead of passing it to the constructor
W0716 02:58:33.300598 139795883440000 deprecation.py:506] From /usr/local/lib/python3.6/dist-packages/tensorflow/python/ops/rnn_cell_impl.py:574: calling Zeros.__init__ (from tensorflow.python.ops.init_ops) with dtype is deprecated and will be removed in a future version.
Instructions for updating:
Call initializer instance with the dtype argument instead of passing it to the constructor
I0716 02:58:33.824702 139795883440000 <ipython-input-6-d6747b40360b>:55] 
[<tf.Variable 'embedding:0' shape=(750, 300) dtype=float32_ref>,
 <tf.Variable 'module/aggregation/weights:0' shape=(3,) dtype=float32>,
 <tf.Variable 'module/aggregation/scaling:0' shape=() dtype=float32>,
 <tf.Variable 'dense/kernel:0' shape=(1324, 300) dtype=float32_ref>,
 <tf.Variable 'dense/bias:0' shape=(300,) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/gates/kernel:0' shape=(600, 600) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/gates/bias:0' shape=(600,) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/candidate/kernel:0' shape=(600, 300) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/candidate/bias:0' shape=(300,) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/gates/kernel:0' shape=(600, 600) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/gates/bias:0' shape=(600,) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/candidate/kernel:0' shape=(600, 300) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/candidate/bias:0' shape=(300,) dtype=float32_ref>,
 <tf.Variable 'dense_1/kernel:0' shape=(600, 23) dtype=float32_ref>,
 <tf.Variable 'dense_1/bias:0' shape=(23,) dtype=float32_ref>,
 <tf.Variable 'dense_2/kernel:0' shape=(600, 122) dtype=float32_ref>,
 <tf.Variable 'dense_2/bias:0' shape=(122,) dtype=float32_ref>]
I0716 02:58:34.888393 139795883440000 estimator.py:1147] Done calling model_fn.
I0716 02:58:34.892489 139795883440000 basic_session_run_hooks.py:541] Create CheckpointSaverHook.
I0716 02:58:36.013862 139795883440000 monitored_session.py:240] Graph was finalized.
I0716 02:58:39.056770 139795883440000 session_manager.py:500] Running local_init_op.
I0716 02:58:39.108992 139795883440000 session_manager.py:502] Done running local_init_op.
I0716 02:58:41.689479 139795883440000 basic_session_run_hooks.py:606] Saving checkpoints for 0 into ../model/elmo_bigru/model.ckpt.
Reading ../data/atis.train.w-intent.iob
I0716 02:58:47.781142 139795883440000 basic_session_run_hooks.py:262] loss = 4.089126, step = 0
I0716 02:58:47.785583 139795883440000 basic_session_run_hooks.py:262] lr = 0.0003
I0716 02:59:17.091764 139795883440000 basic_session_run_hooks.py:692] global_step/sec: 3.41163
I0716 02:59:17.094635 139795883440000 basic_session_run_hooks.py:260] loss = 0.57814324, step = 100 (29.314 sec)
I0716 02:59:17.100646 139795883440000 basic_session_run_hooks.py:260] lr = 0.00029685578 (29.315 sec)
I0716 02:59:43.655862 139795883440000 basic_session_run_hooks.py:692] global_step/sec: 3.76447
I0716 02:59:43.659226 139795883440000 basic_session_run_hooks.py:260] loss = 0.25135267, step = 200 (26.565 sec)
I0716 02:59:43.661857 139795883440000 basic_session_run_hooks.py:260] lr = 0.00029374452 (26.561 sec)
I0716 03:00:09.468027 139795883440000 basic_session_run_hooks.py:692] global_step/sec: 3.87415
I0716 03:00:09.473734 139795883440000 basic_session_run_hooks.py:260] loss = 0.28173244, step = 300 (25.815 sec)
I0716 03:00:09.475708 139795883440000 basic_session_run_hooks.py:260] lr = 0.00029066586 (25.814 sec)
I0716 03:00:12.225202 139795883440000 basic_session_run_hooks.py:606] Saving checkpoints for 312 into ../model/elmo_bigru/model.ckpt.
I0716 03:00:14.384853 139795883440000 estimator.py:1145] Calling model_fn.
I0716 03:00:15.675234 139795883440000 saver.py:1499] Saver not created because there are no variables in the graph to restore
I0716 03:00:16.476772 139795883440000 estimator.py:1147] Done calling model_fn.
I0716 03:00:16.506482 139795883440000 evaluation.py:255] Starting evaluation at 2019-07-16T03:00:16Z
I0716 03:00:16.733509 139795883440000 monitored_session.py:240] Graph was finalized.
W0716 03:00:16.740108 139795883440000 deprecation.py:323] From /usr/local/lib/python3.6/dist-packages/tensorflow/python/training/saver.py:1276: checkpoint_exists (from tensorflow.python.training.checkpoint_management) is deprecated and will be removed in a future version.
Instructions for updating:
Use standard file APIs to check for files with this prefix.
I0716 03:00:16.754307 139795883440000 saver.py:1280] Restoring parameters from ../model/elmo_bigru/model.ckpt-312
I0716 03:00:17.626637 139795883440000 session_manager.py:500] Running local_init_op.
I0716 03:00:17.679666 139795883440000 session_manager.py:502] Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
I0716 03:01:17.274649 139795883440000 evaluation.py:275] Finished evaluation at 2019-07-16-03:01:17
I0716 03:01:17.276630 139795883440000 estimator.py:2039] Saving dict for global step 312: global_step = 312, loss = 0.6538741
I0716 03:01:17.900297 139795883440000 estimator.py:2099] Saving 'checkpoint_path' summary for global step 312: ../model/elmo_bigru/model.ckpt-312
I0716 03:01:17.977829 139795883440000 estimator.py:368] Loss for final step: 0.9662949.
Reading ../data/atis.test.w-intent.iob
I0716 03:01:18.593478 139795883440000 estimator.py:1145] Calling model_fn.
I0716 03:01:20.062015 139795883440000 saver.py:1499] Saver not created because there are no variables in the graph to restore
I0716 03:01:20.566077 139795883440000 estimator.py:1147] Done calling model_fn.
I0716 03:01:20.785360 139795883440000 monitored_session.py:240] Graph was finalized.
I0716 03:01:20.803046 139795883440000 saver.py:1280] Restoring parameters from ../model/elmo_bigru/model.ckpt-312
I0716 03:01:21.685840 139795883440000 session_manager.py:500] Running local_init_op.
I0716 03:01:21.728985 139795883440000 session_manager.py:502] Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.
  'precision', 'predicted', average, warn_for)
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1439: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples.
  'recall', 'true', average, warn_for)
I0716 03:02:19.941967 139795883440000 interactiveshell.py:2882] 
                                          precision    recall  f1-score   support

                             atis_flight      0.948     0.991     0.969       632
                            atis_airfare      0.887     0.979     0.931        48
                     atis_ground_service      0.923     1.000     0.960        36
                            atis_airline      0.925     0.974     0.949        38
                       atis_abbreviation      0.943     1.000     0.971        33
                           atis_aircraft      1.000     1.000     1.000         9
                        atis_flight_time      0.333     1.000     0.500         1
                           atis_quantity      0.429     1.000     0.600         3
                atis_flight#atis_airfare      0.000     0.000     0.000        12
                            atis_airport      1.000     0.722     0.839        18
                           atis_distance      1.000     0.900     0.947        10
                               atis_city      0.000     0.000     0.000         6
                        atis_ground_fare      1.000     0.714     0.833         7
                           atis_capacity      1.000     0.952     0.976        21
                          atis_flight_no      0.000     0.000     0.000         8
                               atis_meal      0.000     0.000     0.000         6
                        atis_restriction      0.000     0.000     0.000         0
             atis_airline#atis_flight_no      0.000     0.000     0.000         0
    atis_ground_service#atis_ground_fare      0.000     0.000     0.000         0
           atis_airfare#atis_flight_time      0.000     0.000     0.000         0
                           atis_cheapest      0.000     0.000     0.000         0
atis_aircraft#atis_flight#atis_flight_no      0.000     0.000     0.000         0

                               micro avg      0.940     0.945     0.942       888
                               macro avg      0.472     0.511     0.476       888
                            weighted avg      0.910     0.945     0.926       888

I0716 03:02:19.991601 139795883440000 interactiveshell.py:2882] 
                              precision    recall  f1-score   support

                           O      0.000     0.000     0.000       0.0
           B-toloc.city_name      0.927     0.979     0.952     716.0
         B-fromloc.city_name      0.923     0.994     0.958     704.0
           I-toloc.city_name      0.847     0.958     0.899     265.0
      B-depart_date.day_name      0.876     1.000     0.934     212.0
              B-airline_name      0.952     0.980     0.966     101.0
         I-fromloc.city_name      0.856     0.944     0.898     177.0
 B-depart_time.period_of_day      0.804     0.885     0.842     130.0
              I-airline_name      0.875     0.969     0.920      65.0
    B-depart_date.day_number      0.720     0.982     0.831      55.0
    B-depart_date.month_name      0.889     1.000     0.941      56.0
          B-depart_time.time      0.496     0.982     0.659      57.0
                B-round_trip      0.986     0.973     0.979      73.0
             B-cost_relative      0.925     1.000     0.961      37.0
                I-round_trip      0.887     1.000     0.940      71.0
                B-flight_mod      0.700     0.875     0.778      24.0
 B-depart_time.time_relative      0.735     0.938     0.824      65.0
          I-depart_time.time      0.486     1.000     0.654      52.0
         B-stoploc.city_name      0.586     0.850     0.694      20.0
                 B-city_name      0.471     0.579     0.520      57.0
                B-class_type      1.000     0.708     0.829      24.0
          B-arrive_time.time      0.714     0.294     0.417      34.0
 B-arrive_time.time_relative      0.682     0.484     0.566      31.0
                I-class_type      1.000     1.000     1.000      17.0
               B-flight_stop      1.000     1.000     1.000      21.0
          I-arrive_time.time      1.000     0.029     0.056      35.0
              B-airline_code      0.533     0.706     0.608      34.0
    I-depart_date.day_number      0.000     0.000     0.000      15.0
      I-fromloc.airport_name      0.244     0.667     0.357      15.0
      B-fromloc.airport_name      0.000     0.000     0.000      12.0
      B-arrive_date.day_name      0.000     0.000     0.000      11.0
          B-toloc.state_code      0.882     0.833     0.857      18.0
B-depart_date.today_relative      0.000     0.000     0.000       9.0
             B-flight_number      0.538     0.636     0.583      11.0
 B-depart_date.date_relative      0.857     0.706     0.774      17.0
          B-toloc.state_name      0.000     0.000     0.000      28.0
           B-fare_basis_code      0.500     1.000     0.667      17.0
               B-flight_time      0.500     1.000     0.667       1.0
                        B-or      0.000     0.000     0.000       3.0
 B-arrive_time.period_of_day      0.000     0.000     0.000       6.0
          B-meal_description      0.538     0.700     0.609      10.0
             I-cost_relative      0.000     0.000     0.000       3.0
              I-airport_name      0.000     0.000     0.000      29.0
               B-fare_amount      0.222     1.000     0.364       2.0
               I-fare_amount      0.667     1.000     0.800       2.0
                 I-city_name      0.000     0.000     0.000      30.0
        I-toloc.airport_name      0.000     0.000     0.000       3.0
            B-transport_type      1.000     0.900     0.947      10.0
    B-arrive_date.month_name      0.000     0.000     0.000       6.0
    B-arrive_date.day_number      0.000     0.000     0.000       6.0
         I-stoploc.city_name      0.000     0.000     0.000      10.0
                      B-meal      0.000     0.000     0.000      16.0
        B-fromloc.state_code      1.000     0.087     0.160      23.0
    B-depart_time.period_mod      0.000     0.000     0.000       5.0
                   B-connect      0.000     0.000     0.000       6.0
               B-flight_days      1.000     0.100     0.182      10.0
        B-toloc.airport_name      0.000     0.000     0.000       3.0
        B-fromloc.state_name      0.000     0.000     0.000      17.0
              B-airport_name      0.000     0.000     0.000      21.0
                   B-economy      0.000     0.000     0.000       6.0
               I-flight_time      0.000     0.000     0.000       1.0
             B-aircraft_code      0.000     0.000     0.000      33.0
                       B-mod      0.000     0.000     0.000       2.0
              B-airport_code      0.000     0.000     0.000       9.0
    B-depart_time.start_time      0.000     0.000     0.000       3.0
      B-depart_time.end_time      0.000     0.000     0.000       3.0
          B-depart_date.year      0.000     0.000     0.000       3.0
            I-transport_type      0.000     0.000     0.000       1.0
          B-restriction_code      0.000     0.000     0.000       4.0
    B-arrive_time.start_time      0.000     0.000     0.000       8.0
        B-toloc.airport_code      0.000     0.000     0.000       4.0
      B-arrive_time.end_time      0.000     0.000     0.000       8.0
      I-arrive_time.end_time      0.000     0.000     0.000       8.0
      I-depart_time.end_time      0.000     0.000     0.000       3.0
               I-flight_stop      0.000     0.000     0.000       0.0
      B-fromloc.airport_code      0.000     0.000     0.000       5.0
          I-restriction_code      0.000     0.000     0.000       3.0
    I-depart_time.start_time      0.000     0.000     0.000       1.0
          I-toloc.state_name      0.000     0.000     0.000       1.0
I-depart_date.today_relative      0.000     0.000     0.000       0.0
 B-arrive_date.date_relative      0.000     0.000     0.000       2.0
                I-flight_mod      0.000     0.000     0.000       6.0
                   I-economy      0.000     0.000     0.000       0.0
 B-return_date.date_relative      0.000     0.000     0.000       3.0
        I-fromloc.state_name      0.000     0.000     0.000       1.0
                B-state_code      0.000     0.000     0.000       1.0
    I-arrive_time.start_time      0.000     0.000     0.000       1.0
    I-arrive_date.day_number      0.000     0.000     0.000       0.0
                 B-meal_code      0.000     0.000     0.000       1.0
 I-depart_time.period_of_day      0.000     0.000     0.000       1.0
                  B-day_name      0.000     0.000     0.000       2.0
             B-period_of_day      0.000     0.000     0.000       4.0
        B-stoploc.state_code      0.000     0.000     0.000       0.0
    B-return_date.month_name      0.000     0.000     0.000       0.0
    B-return_date.day_number      0.000     0.000     0.000       0.0
    B-arrive_time.period_mod      0.000     0.000     0.000       0.0
                 I-meal_code      0.000     0.000     0.000       0.0
        B-toloc.country_name      0.000     0.000     0.000       1.0
                 B-days_code      0.000     0.000     0.000       1.0
 I-arrive_time.period_of_day      0.000     0.000     0.000       0.0
            I-today_relative      0.000     0.000     0.000       0.0
 B-return_time.period_of_day      0.000     0.000     0.000       0.0
                      B-time      0.000     0.000     0.000       0.0
           I-fare_basis_code      0.000     0.000     0.000       0.0
 I-arrive_time.time_relative      0.000     0.000     0.000       4.0
 I-depart_time.time_relative      0.000     0.000     0.000       1.0
            B-today_relative      0.000     0.000     0.000       0.0
                B-state_name      0.000     0.000     0.000       9.0
B-arrive_date.today_relative      0.000     0.000     0.000       0.0
    B-return_time.period_mod      0.000     0.000     0.000       0.0
                B-month_name      0.000     0.000     0.000       0.0
                B-day_number      0.000     0.000     0.000       0.0
 I-return_date.date_relative      0.000     0.000     0.000       3.0
I-return_date.today_relative      0.000     0.000     0.000       0.0
      B-stoploc.airport_name      0.000     0.000     0.000       0.0
             B-time_relative      0.000     0.000     0.000       0.0
                      I-time      0.000     0.000     0.000       0.0
    I-return_date.day_number      0.000     0.000     0.000       0.0
          I-meal_description      0.000     0.000     0.000       0.0
B-return_date.today_relative      0.000     0.000     0.000       0.0
      B-return_date.day_name      0.000     0.000     0.000       2.0

                   micro avg      0.827     0.829     0.828    3657.0
                   macro avg      0.238     0.254     0.228    3657.0
                weighted avg      0.768     0.829     0.784    3657.0

I0716 03:02:20.012179 139795883440000 interactiveshell.py:2882] Best Slot F1: 0.828
I0716 03:02:20.013774 139795883440000 estimator_training.py:186] Not using Distribute Coordinator.
I0716 03:02:20.015699 139795883440000 training.py:612] Running training and evaluation locally (non-distributed).
I0716 03:02:20.017799 139795883440000 training.py:700] Start train and evaluate loop. The evaluate will happen after every checkpoint. Checkpoint frequency is determined based on RunConfig arguments: save_checkpoints_steps 312 or save_checkpoints_secs None.
I0716 03:02:20.084496 139795883440000 estimator.py:1145] Calling model_fn.
I0716 03:02:21.517312 139795883440000 saver.py:1499] Saver not created because there are no variables in the graph to restore
I0716 03:02:22.273025 139795883440000 <ipython-input-6-d6747b40360b>:55] 
[<tf.Variable 'embedding:0' shape=(750, 300) dtype=float32_ref>,
 <tf.Variable 'module/aggregation/weights:0' shape=(3,) dtype=float32>,
 <tf.Variable 'module/aggregation/scaling:0' shape=() dtype=float32>,
 <tf.Variable 'dense/kernel:0' shape=(1324, 300) dtype=float32_ref>,
 <tf.Variable 'dense/bias:0' shape=(300,) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/gates/kernel:0' shape=(600, 600) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/gates/bias:0' shape=(600,) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/candidate/kernel:0' shape=(600, 300) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/candidate/bias:0' shape=(300,) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/gates/kernel:0' shape=(600, 600) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/gates/bias:0' shape=(600,) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/candidate/kernel:0' shape=(600, 300) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/candidate/bias:0' shape=(300,) dtype=float32_ref>,
 <tf.Variable 'dense_1/kernel:0' shape=(600, 23) dtype=float32_ref>,
 <tf.Variable 'dense_1/bias:0' shape=(23,) dtype=float32_ref>,
 <tf.Variable 'dense_2/kernel:0' shape=(600, 122) dtype=float32_ref>,
 <tf.Variable 'dense_2/bias:0' shape=(122,) dtype=float32_ref>]
I0716 03:02:23.352547 139795883440000 estimator.py:1147] Done calling model_fn.
I0716 03:02:23.356692 139795883440000 basic_session_run_hooks.py:541] Create CheckpointSaverHook.
I0716 03:02:23.677159 139795883440000 monitored_session.py:240] Graph was finalized.
I0716 03:02:23.693130 139795883440000 saver.py:1280] Restoring parameters from ../model/elmo_bigru/model.ckpt-312
W0716 03:02:24.570057 139795883440000 deprecation.py:323] From /usr/local/lib/python3.6/dist-packages/tensorflow/python/training/saver.py:1066: get_checkpoint_mtimes (from tensorflow.python.training.checkpoint_management) is deprecated and will be removed in a future version.
Instructions for updating:
Use standard file utilities to get mtimes.
I0716 03:02:24.701941 139795883440000 session_manager.py:500] Running local_init_op.
I0716 03:02:24.764321 139795883440000 session_manager.py:502] Done running local_init_op.
I0716 03:02:27.727120 139795883440000 basic_session_run_hooks.py:606] Saving checkpoints for 312 into ../model/elmo_bigru/model.ckpt.
Reading ../data/atis.train.w-intent.iob
I0716 03:02:32.666477 139795883440000 basic_session_run_hooks.py:262] loss = 0.19418588, step = 312
I0716 03:02:32.668385 139795883440000 basic_session_run_hooks.py:262] lr = 0.0002902986
I0716 03:03:00.168136 139795883440000 basic_session_run_hooks.py:692] global_step/sec: 3.63607
I0716 03:03:00.173908 139795883440000 basic_session_run_hooks.py:260] loss = 0.44005728, step = 412 (27.507 sec)
I0716 03:03:00.177161 139795883440000 basic_session_run_hooks.py:260] lr = 0.00028725603 (27.509 sec)
I0716 03:03:26.463520 139795883440000 basic_session_run_hooks.py:692] global_step/sec: 3.80294
I0716 03:03:26.471512 139795883440000 basic_session_run_hooks.py:260] loss = 0.43844122, step = 512 (26.298 sec)
I0716 03:03:26.473572 139795883440000 basic_session_run_hooks.py:260] lr = 0.00028424538 (26.296 sec)
I0716 03:03:53.220065 139795883440000 basic_session_run_hooks.py:692] global_step/sec: 3.73741
I0716 03:03:53.226559 139795883440000 basic_session_run_hooks.py:260] loss = 0.13767746, step = 612 (26.755 sec)
I0716 03:03:53.228701 139795883440000 basic_session_run_hooks.py:260] lr = 0.00028126626 (26.755 sec)
I0716 03:03:56.350152 139795883440000 basic_session_run_hooks.py:606] Saving checkpoints for 624 into ../model/elmo_bigru/model.ckpt.
I0716 03:03:58.483096 139795883440000 estimator.py:1145] Calling model_fn.
I0716 03:03:59.825540 139795883440000 saver.py:1499] Saver not created because there are no variables in the graph to restore
I0716 03:04:00.421510 139795883440000 estimator.py:1147] Done calling model_fn.
I0716 03:04:00.455102 139795883440000 evaluation.py:255] Starting evaluation at 2019-07-16T03:04:00Z
I0716 03:04:00.696269 139795883440000 monitored_session.py:240] Graph was finalized.
I0716 03:04:00.711647 139795883440000 saver.py:1280] Restoring parameters from ../model/elmo_bigru/model.ckpt-624
I0716 03:04:01.609956 139795883440000 session_manager.py:500] Running local_init_op.
I0716 03:04:01.664305 139795883440000 session_manager.py:502] Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
I0716 03:05:00.591241 139795883440000 evaluation.py:275] Finished evaluation at 2019-07-16-03:05:00
I0716 03:05:00.593144 139795883440000 estimator.py:2039] Saving dict for global step 624: global_step = 624, loss = 0.39170766
I0716 03:05:00.599527 139795883440000 estimator.py:2099] Saving 'checkpoint_path' summary for global step 624: ../model/elmo_bigru/model.ckpt-624
I0716 03:05:00.683492 139795883440000 estimator.py:368] Loss for final step: 0.19239207.
Reading ../data/atis.test.w-intent.iob
I0716 03:05:01.279512 139795883440000 estimator.py:1145] Calling model_fn.
I0716 03:05:02.865846 139795883440000 saver.py:1499] Saver not created because there are no variables in the graph to restore
I0716 03:05:03.398653 139795883440000 estimator.py:1147] Done calling model_fn.
I0716 03:05:03.780389 139795883440000 monitored_session.py:240] Graph was finalized.
I0716 03:05:03.796601 139795883440000 saver.py:1280] Restoring parameters from ../model/elmo_bigru/model.ckpt-624
I0716 03:05:04.675439 139795883440000 session_manager.py:500] Running local_init_op.
I0716 03:05:04.724084 139795883440000 session_manager.py:502] Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.
  'precision', 'predicted', average, warn_for)
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1439: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples.
  'recall', 'true', average, warn_for)
I0716 03:06:02.881382 139795883440000 interactiveshell.py:2882] 
                                          precision    recall  f1-score   support

                             atis_flight      0.981     0.987     0.984       632
                            atis_airfare      0.889     1.000     0.941        48
                     atis_ground_service      0.857     1.000     0.923        36
                            atis_airline      0.950     1.000     0.974        38
                       atis_abbreviation      0.917     1.000     0.957        33
                           atis_aircraft      1.000     1.000     1.000         9
                        atis_flight_time      0.333     1.000     0.500         1
                           atis_quantity      0.429     1.000     0.600         3
                atis_flight#atis_airfare      0.833     0.417     0.556        12
                            atis_airport      1.000     0.944     0.971        18
                           atis_distance      1.000     0.600     0.750        10
                               atis_city      1.000     0.500     0.667         6
                        atis_ground_fare      1.000     0.857     0.923         7
                           atis_capacity      1.000     0.952     0.976        21
                          atis_flight_no      1.000     1.000     1.000         8
                               atis_meal      0.000     0.000     0.000         6
                        atis_restriction      0.000     0.000     0.000         0
             atis_airline#atis_flight_no      0.000     0.000     0.000         0
    atis_ground_service#atis_ground_fare      0.000     0.000     0.000         0
           atis_airfare#atis_flight_time      0.000     0.000     0.000         0
                           atis_cheapest      0.000     0.000     0.000         0
atis_aircraft#atis_flight#atis_flight_no      0.000     0.000     0.000         0

                               micro avg      0.960     0.965     0.962       888
                               macro avg      0.600     0.603     0.578       888
                            weighted avg      0.958     0.965     0.958       888

I0716 03:06:02.919304 139795883440000 interactiveshell.py:2882] 
                              precision    recall  f1-score   support

                           O      0.000     0.000     0.000       0.0
           B-toloc.city_name      0.958     0.989     0.973     716.0
         B-fromloc.city_name      0.955     0.997     0.976     704.0
           I-toloc.city_name      0.886     0.996     0.938     265.0
      B-depart_date.day_name      0.922     1.000     0.959     212.0
              B-airline_name      0.971     0.990     0.980     101.0
         I-fromloc.city_name      0.935     0.972     0.953     177.0
 B-depart_time.period_of_day      0.949     0.862     0.903     130.0
              I-airline_name      1.000     0.969     0.984      65.0
    B-depart_date.day_number      0.902     1.000     0.948      55.0
    B-depart_date.month_name      0.902     0.982     0.940      56.0
          B-depart_time.time      0.750     0.895     0.816      57.0
                B-round_trip      0.986     0.973     0.979      73.0
             B-cost_relative      0.974     1.000     0.987      37.0
                I-round_trip      0.973     1.000     0.986      71.0
                B-flight_mod      0.786     0.917     0.846      24.0
 B-depart_time.time_relative      0.964     0.831     0.893      65.0
          I-depart_time.time      0.885     0.885     0.885      52.0
         B-stoploc.city_name      0.760     0.950     0.844      20.0
                 B-city_name      0.810     0.596     0.687      57.0
                B-class_type      0.960     1.000     0.980      24.0
          B-arrive_time.time      0.582     0.941     0.719      34.0
 B-arrive_time.time_relative      0.667     0.968     0.789      31.0
                I-class_type      1.000     1.000     1.000      17.0
               B-flight_stop      1.000     1.000     1.000      21.0
          I-arrive_time.time      0.611     0.943     0.742      35.0
              B-airline_code      0.659     0.794     0.720      34.0
    I-depart_date.day_number      0.833     1.000     0.909      15.0
      I-fromloc.airport_name      0.387     0.800     0.522      15.0
      B-fromloc.airport_name      0.158     0.250     0.194      12.0
      B-arrive_date.day_name      0.500     0.091     0.154      11.0
          B-toloc.state_code      0.900     1.000     0.947      18.0
B-depart_date.today_relative      1.000     0.556     0.714       9.0
             B-flight_number      0.833     0.909     0.870      11.0
 B-depart_date.date_relative      0.810     1.000     0.895      17.0
          B-toloc.state_name      0.545     0.214     0.308      28.0
           B-fare_basis_code      0.762     0.941     0.842      17.0
               B-flight_time      1.000     1.000     1.000       1.0
                        B-or      1.000     1.000     1.000       3.0
 B-arrive_time.period_of_day      0.429     0.500     0.462       6.0
          B-meal_description      1.000     0.800     0.889      10.0
             I-cost_relative      1.000     0.667     0.800       3.0
              I-airport_name      0.818     0.310     0.450      29.0
               B-fare_amount      1.000     1.000     1.000       2.0
               I-fare_amount      1.000     1.000     1.000       2.0
                 I-city_name      0.760     0.633     0.691      30.0
        I-toloc.airport_name      1.000     0.333     0.500       3.0
            B-transport_type      0.909     1.000     0.952      10.0
    B-arrive_date.month_name      0.000     0.000     0.000       6.0
    B-arrive_date.day_number      0.000     0.000     0.000       6.0
         I-stoploc.city_name      0.000     0.000     0.000      10.0
                      B-meal      0.941     1.000     0.970      16.0
        B-fromloc.state_code      1.000     0.913     0.955      23.0
    B-depart_time.period_mod      0.714     1.000     0.833       5.0
                   B-connect      1.000     0.667     0.800       6.0
               B-flight_days      1.000     1.000     1.000      10.0
        B-toloc.airport_name      1.000     0.333     0.500       3.0
        B-fromloc.state_name      0.000     0.000     0.000      17.0
              B-airport_name      0.500     0.286     0.364      21.0
                   B-economy      1.000     0.167     0.286       6.0
               I-flight_time      1.000     1.000     1.000       1.0
             B-aircraft_code      1.000     0.758     0.862      33.0
                       B-mod      0.000     0.000     0.000       2.0
              B-airport_code      1.000     0.222     0.364       9.0
    B-depart_time.start_time      0.667     0.667     0.667       3.0
      B-depart_time.end_time      0.000     0.000     0.000       3.0
          B-depart_date.year      0.000     0.000     0.000       3.0
            I-transport_type      0.000     0.000     0.000       1.0
          B-restriction_code      0.500     0.500     0.500       4.0
    B-arrive_time.start_time      0.000     0.000     0.000       8.0
        B-toloc.airport_code      0.000     0.000     0.000       4.0
      B-arrive_time.end_time      0.000     0.000     0.000       8.0
      I-arrive_time.end_time      0.000     0.000     0.000       8.0
      I-depart_time.end_time      0.000     0.000     0.000       3.0
               I-flight_stop      0.000     0.000     0.000       0.0
      B-fromloc.airport_code      0.000     0.000     0.000       5.0
          I-restriction_code      0.000     0.000     0.000       3.0
    I-depart_time.start_time      0.000     0.000     0.000       1.0
          I-toloc.state_name      0.000     0.000     0.000       1.0
I-depart_date.today_relative      0.000     0.000     0.000       0.0
 B-arrive_date.date_relative      0.000     0.000     0.000       2.0
                I-flight_mod      0.000     0.000     0.000       6.0
                   I-economy      0.000     0.000     0.000       0.0
 B-return_date.date_relative      0.000     0.000     0.000       3.0
        I-fromloc.state_name      0.000     0.000     0.000       1.0
                B-state_code      0.000     0.000     0.000       1.0
    I-arrive_time.start_time      0.000     0.000     0.000       1.0
    I-arrive_date.day_number      0.000     0.000     0.000       0.0
                 B-meal_code      0.000     0.000     0.000       1.0
 I-depart_time.period_of_day      0.000     0.000     0.000       1.0
                  B-day_name      0.000     0.000     0.000       2.0
             B-period_of_day      0.000     0.000     0.000       4.0
        B-stoploc.state_code      0.000     0.000     0.000       0.0
    B-return_date.month_name      0.000     0.000     0.000       0.0
    B-return_date.day_number      0.000     0.000     0.000       0.0
    B-arrive_time.period_mod      0.000     0.000     0.000       0.0
                 I-meal_code      0.000     0.000     0.000       0.0
        B-toloc.country_name      0.000     0.000     0.000       1.0
                 B-days_code      0.000     0.000     0.000       1.0
 I-arrive_time.period_of_day      0.000     0.000     0.000       0.0
            I-today_relative      0.000     0.000     0.000       0.0
 B-return_time.period_of_day      0.000     0.000     0.000       0.0
                      B-time      0.000     0.000     0.000       0.0
           I-fare_basis_code      0.000     0.000     0.000       0.0
 I-arrive_time.time_relative      0.000     0.000     0.000       4.0
 I-depart_time.time_relative      0.000     0.000     0.000       1.0
            B-today_relative      0.000     0.000     0.000       0.0
                B-state_name      0.000     0.000     0.000       9.0
B-arrive_date.today_relative      0.000     0.000     0.000       0.0
    B-return_time.period_mod      0.000     0.000     0.000       0.0
                B-month_name      0.000     0.000     0.000       0.0
                B-day_number      0.000     0.000     0.000       0.0
 I-return_date.date_relative      0.000     0.000     0.000       3.0
I-return_date.today_relative      0.000     0.000     0.000       0.0
      B-stoploc.airport_name      0.000     0.000     0.000       0.0
             B-time_relative      0.000     0.000     0.000       0.0
                      I-time      0.000     0.000     0.000       0.0
    I-return_date.day_number      0.000     0.000     0.000       0.0
          I-meal_description      0.000     0.000     0.000       0.0
B-return_date.today_relative      0.000     0.000     0.000       0.0
      B-return_date.day_name      0.000     0.000     0.000       2.0

                   micro avg      0.898     0.900     0.899    3657.0
                   macro avg      0.419     0.396     0.394    3657.0
                weighted avg      0.876     0.900     0.881    3657.0

I0716 03:06:02.940419 139795883440000 interactiveshell.py:2882] Best Slot F1: 0.899
I0716 03:06:02.942004 139795883440000 estimator_training.py:186] Not using Distribute Coordinator.
I0716 03:06:02.943432 139795883440000 training.py:612] Running training and evaluation locally (non-distributed).
I0716 03:06:02.944901 139795883440000 training.py:700] Start train and evaluate loop. The evaluate will happen after every checkpoint. Checkpoint frequency is determined based on RunConfig arguments: save_checkpoints_steps 312 or save_checkpoints_secs None.
I0716 03:06:03.011088 139795883440000 estimator.py:1145] Calling model_fn.
I0716 03:06:04.299739 139795883440000 saver.py:1499] Saver not created because there are no variables in the graph to restore
I0716 03:06:05.053959 139795883440000 <ipython-input-6-d6747b40360b>:55] 
[<tf.Variable 'embedding:0' shape=(750, 300) dtype=float32_ref>,
 <tf.Variable 'module/aggregation/weights:0' shape=(3,) dtype=float32>,
 <tf.Variable 'module/aggregation/scaling:0' shape=() dtype=float32>,
 <tf.Variable 'dense/kernel:0' shape=(1324, 300) dtype=float32_ref>,
 <tf.Variable 'dense/bias:0' shape=(300,) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/gates/kernel:0' shape=(600, 600) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/gates/bias:0' shape=(600,) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/candidate/kernel:0' shape=(600, 300) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/candidate/bias:0' shape=(300,) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/gates/kernel:0' shape=(600, 600) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/gates/bias:0' shape=(600,) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/candidate/kernel:0' shape=(600, 300) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/candidate/bias:0' shape=(300,) dtype=float32_ref>,
 <tf.Variable 'dense_1/kernel:0' shape=(600, 23) dtype=float32_ref>,
 <tf.Variable 'dense_1/bias:0' shape=(23,) dtype=float32_ref>,
 <tf.Variable 'dense_2/kernel:0' shape=(600, 122) dtype=float32_ref>,
 <tf.Variable 'dense_2/bias:0' shape=(122,) dtype=float32_ref>]
I0716 03:06:06.150071 139795883440000 estimator.py:1147] Done calling model_fn.
I0716 03:06:06.154184 139795883440000 basic_session_run_hooks.py:541] Create CheckpointSaverHook.
I0716 03:06:06.486130 139795883440000 monitored_session.py:240] Graph was finalized.
I0716 03:06:06.497994 139795883440000 saver.py:1280] Restoring parameters from ../model/elmo_bigru/model.ckpt-624
I0716 03:06:07.499193 139795883440000 session_manager.py:500] Running local_init_op.
I0716 03:06:07.571006 139795883440000 session_manager.py:502] Done running local_init_op.
I0716 03:06:10.528747 139795883440000 basic_session_run_hooks.py:606] Saving checkpoints for 624 into ../model/elmo_bigru/model.ckpt.
Reading ../data/atis.train.w-intent.iob
I0716 03:06:15.286043 139795883440000 basic_session_run_hooks.py:262] loss = 0.10650402, step = 624
I0716 03:06:15.287670 139795883440000 basic_session_run_hooks.py:262] lr = 0.00028091087
I0716 03:06:42.979780 139795883440000 basic_session_run_hooks.py:692] global_step/sec: 3.61086
I0716 03:06:42.986958 139795883440000 basic_session_run_hooks.py:260] loss = 0.10334292, step = 724 (27.701 sec)
I0716 03:06:42.989081 139795883440000 basic_session_run_hooks.py:260] lr = 0.00027796673 (27.701 sec)
I0716 03:07:09.221698 139795883440000 basic_session_run_hooks.py:692] global_step/sec: 3.81069
I0716 03:07:09.226385 139795883440000 basic_session_run_hooks.py:260] loss = 0.07500512, step = 824 (26.239 sec)
I0716 03:07:09.230600 139795883440000 basic_session_run_hooks.py:260] lr = 0.00027505343 (26.242 sec)
I0716 03:07:35.807775 139795883440000 basic_session_run_hooks.py:692] global_step/sec: 3.76137
I0716 03:07:35.813550 139795883440000 basic_session_run_hooks.py:260] loss = 0.91219175, step = 924 (26.587 sec)
I0716 03:07:35.817075 139795883440000 basic_session_run_hooks.py:260] lr = 0.00027217067 (26.586 sec)
I0716 03:07:38.272199 139795883440000 basic_session_run_hooks.py:606] Saving checkpoints for 936 into ../model/elmo_bigru/model.ckpt.
I0716 03:07:40.166394 139795883440000 estimator.py:1145] Calling model_fn.
I0716 03:07:41.555429 139795883440000 saver.py:1499] Saver not created because there are no variables in the graph to restore
I0716 03:07:42.150777 139795883440000 estimator.py:1147] Done calling model_fn.
I0716 03:07:42.180353 139795883440000 evaluation.py:255] Starting evaluation at 2019-07-16T03:07:42Z
I0716 03:07:42.412451 139795883440000 monitored_session.py:240] Graph was finalized.
I0716 03:07:42.427135 139795883440000 saver.py:1280] Restoring parameters from ../model/elmo_bigru/model.ckpt-936
I0716 03:07:43.321085 139795883440000 session_manager.py:500] Running local_init_op.
I0716 03:07:43.375441 139795883440000 session_manager.py:502] Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
I0716 03:08:42.309874 139795883440000 evaluation.py:275] Finished evaluation at 2019-07-16-03:08:42
I0716 03:08:42.311658 139795883440000 estimator.py:2039] Saving dict for global step 936: global_step = 936, loss = 0.3205106
I0716 03:08:42.318030 139795883440000 estimator.py:2099] Saving 'checkpoint_path' summary for global step 936: ../model/elmo_bigru/model.ckpt-936
I0716 03:08:42.402586 139795883440000 estimator.py:368] Loss for final step: 0.17703222.
Reading ../data/atis.test.w-intent.iob
I0716 03:08:43.001334 139795883440000 estimator.py:1145] Calling model_fn.
I0716 03:08:44.594808 139795883440000 saver.py:1499] Saver not created because there are no variables in the graph to restore
I0716 03:08:45.103219 139795883440000 estimator.py:1147] Done calling model_fn.
I0716 03:08:45.491380 139795883440000 monitored_session.py:240] Graph was finalized.
I0716 03:08:45.505645 139795883440000 saver.py:1280] Restoring parameters from ../model/elmo_bigru/model.ckpt-936
I0716 03:08:46.373692 139795883440000 session_manager.py:500] Running local_init_op.
I0716 03:08:46.417907 139795883440000 session_manager.py:502] Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.
  'precision', 'predicted', average, warn_for)
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1439: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples.
  'recall', 'true', average, warn_for)
I0716 03:09:44.563587 139795883440000 interactiveshell.py:2882] 
                                          precision    recall  f1-score   support

                             atis_flight      0.984     0.991     0.987       632
                            atis_airfare      0.960     1.000     0.980        48
                     atis_ground_service      0.947     1.000     0.973        36
                            atis_airline      1.000     1.000     1.000        38
                       atis_abbreviation      0.943     1.000     0.971        33
                           atis_aircraft      0.900     1.000     0.947         9
                        atis_flight_time      0.143     1.000     0.250         1
                           atis_quantity      0.429     1.000     0.600         3
                atis_flight#atis_airfare      0.800     0.333     0.471        12
                            atis_airport      1.000     1.000     1.000        18
                           atis_distance      1.000     0.900     0.947        10
                               atis_city      1.000     0.500     0.667         6
                        atis_ground_fare      1.000     0.857     0.923         7
                           atis_capacity      1.000     0.952     0.976        21
                          atis_flight_no      0.889     1.000     0.941         8
                               atis_meal      1.000     0.333     0.500         6
                        atis_restriction      0.000     0.000     0.000         0
             atis_airline#atis_flight_no      0.000     0.000     0.000         0
    atis_ground_service#atis_ground_fare      0.000     0.000     0.000         0
           atis_airfare#atis_flight_time      0.000     0.000     0.000         0
                           atis_cheapest      0.000     0.000     0.000         0
atis_aircraft#atis_flight#atis_flight_no      0.000     0.000     0.000         0

                               micro avg      0.968     0.973     0.970       888
                               macro avg      0.636     0.630     0.597       888
                            weighted avg      0.975     0.973     0.970       888

I0716 03:09:44.601177 139795883440000 interactiveshell.py:2882] 
                              precision    recall  f1-score   support

                           O      0.000     0.000     0.000       0.0
           B-toloc.city_name      0.970     0.993     0.981     716.0
         B-fromloc.city_name      0.966     0.999     0.982     704.0
           I-toloc.city_name      0.940     0.996     0.967     265.0
      B-depart_date.day_name      0.946     0.991     0.968     212.0
              B-airline_name      0.962     0.990     0.976     101.0
         I-fromloc.city_name      0.967     0.989     0.978     177.0
 B-depart_time.period_of_day      0.942     0.869     0.904     130.0
              I-airline_name      1.000     0.954     0.976      65.0
    B-depart_date.day_number      0.964     0.982     0.973      55.0
    B-depart_date.month_name      0.965     0.982     0.973      56.0
          B-depart_time.time      0.778     0.982     0.868      57.0
                B-round_trip      1.000     0.986     0.993      73.0
             B-cost_relative      0.973     0.973     0.973      37.0
                I-round_trip      0.986     1.000     0.993      71.0
                B-flight_mod      0.786     0.917     0.846      24.0
 B-depart_time.time_relative      0.915     1.000     0.956      65.0
          I-depart_time.time      0.812     1.000     0.897      52.0
         B-stoploc.city_name      0.900     0.900     0.900      20.0
                 B-city_name      0.872     0.596     0.708      57.0
                B-class_type      0.960     1.000     0.980      24.0
          B-arrive_time.time      0.865     0.941     0.901      34.0
 B-arrive_time.time_relative      0.879     0.935     0.906      31.0
                I-class_type      1.000     1.000     1.000      17.0
               B-flight_stop      1.000     1.000     1.000      21.0
          I-arrive_time.time      0.800     0.914     0.853      35.0
              B-airline_code      0.757     0.824     0.789      34.0
    I-depart_date.day_number      1.000     1.000     1.000      15.0
      I-fromloc.airport_name      0.394     0.867     0.542      15.0
      B-fromloc.airport_name      0.194     0.500     0.279      12.0
      B-arrive_date.day_name      0.750     0.545     0.632      11.0
          B-toloc.state_code      0.900     1.000     0.947      18.0
B-depart_date.today_relative      1.000     0.778     0.875       9.0
             B-flight_number      0.846     1.000     0.917      11.0
 B-depart_date.date_relative      0.850     1.000     0.919      17.0
          B-toloc.state_name      0.735     0.893     0.806      28.0
           B-fare_basis_code      0.882     0.882     0.882      17.0
               B-flight_time      1.000     1.000     1.000       1.0
                        B-or      1.000     1.000     1.000       3.0
 B-arrive_time.period_of_day      0.800     0.667     0.727       6.0
          B-meal_description      0.900     0.900     0.900      10.0
             I-cost_relative      1.000     0.667     0.800       3.0
              I-airport_name      0.900     0.310     0.462      29.0
               B-fare_amount      1.000     1.000     1.000       2.0
               I-fare_amount      1.000     1.000     1.000       2.0
                 I-city_name      0.800     0.533     0.640      30.0
        I-toloc.airport_name      1.000     0.333     0.500       3.0
            B-transport_type      0.909     1.000     0.952      10.0
    B-arrive_date.month_name      0.667     0.667     0.667       6.0
    B-arrive_date.day_number      0.800     0.667     0.727       6.0
         I-stoploc.city_name      0.875     0.700     0.778      10.0
                      B-meal      1.000     0.938     0.968      16.0
        B-fromloc.state_code      1.000     0.913     0.955      23.0
    B-depart_time.period_mod      0.714     1.000     0.833       5.0
                   B-connect      1.000     1.000     1.000       6.0
               B-flight_days      1.000     1.000     1.000      10.0
        B-toloc.airport_name      1.000     0.333     0.500       3.0
        B-fromloc.state_name      0.909     0.588     0.714      17.0
              B-airport_name      0.714     0.238     0.357      21.0
                   B-economy      1.000     0.333     0.500       6.0
               I-flight_time      1.000     1.000     1.000       1.0
             B-aircraft_code      1.000     0.788     0.881      33.0
                       B-mod      0.000     0.000     0.000       2.0
              B-airport_code      0.600     0.333     0.429       9.0
    B-depart_time.start_time      0.222     0.667     0.333       3.0
      B-depart_time.end_time      0.286     0.667     0.400       3.0
          B-depart_date.year      1.000     1.000     1.000       3.0
            I-transport_type      0.000     0.000     0.000       1.0
          B-restriction_code      0.667     0.500     0.571       4.0
    B-arrive_time.start_time      0.000     0.000     0.000       8.0
        B-toloc.airport_code      0.000     0.000     0.000       4.0
      B-arrive_time.end_time      0.000     0.000     0.000       8.0
      I-arrive_time.end_time      0.000     0.000     0.000       8.0
      I-depart_time.end_time      0.000     0.000     0.000       3.0
               I-flight_stop      0.000     0.000     0.000       0.0
      B-fromloc.airport_code      0.000     0.000     0.000       5.0
          I-restriction_code      1.000     1.000     1.000       3.0
    I-depart_time.start_time      0.500     1.000     0.667       1.0
          I-toloc.state_name      0.000     0.000     0.000       1.0
I-depart_date.today_relative      0.000     0.000     0.000       0.0
 B-arrive_date.date_relative      0.000     0.000     0.000       2.0
                I-flight_mod      0.000     0.000     0.000       6.0
                   I-economy      0.000     0.000     0.000       0.0
 B-return_date.date_relative      0.000     0.000     0.000       3.0
        I-fromloc.state_name      0.000     0.000     0.000       1.0
                B-state_code      1.000     1.000     1.000       1.0
    I-arrive_time.start_time      0.000     0.000     0.000       1.0
    I-arrive_date.day_number      0.000     0.000     0.000       0.0
                 B-meal_code      0.000     0.000     0.000       1.0
 I-depart_time.period_of_day      0.000     0.000     0.000       1.0
                  B-day_name      0.000     0.000     0.000       2.0
             B-period_of_day      0.000     0.000     0.000       4.0
        B-stoploc.state_code      0.000     0.000     0.000       0.0
    B-return_date.month_name      0.000     0.000     0.000       0.0
    B-return_date.day_number      0.000     0.000     0.000       0.0
    B-arrive_time.period_mod      0.000     0.000     0.000       0.0
                 I-meal_code      0.000     0.000     0.000       0.0
        B-toloc.country_name      0.000     0.000     0.000       1.0
                 B-days_code      0.000     0.000     0.000       1.0
 I-arrive_time.period_of_day      0.000     0.000     0.000       0.0
            I-today_relative      0.000     0.000     0.000       0.0
 B-return_time.period_of_day      0.000     0.000     0.000       0.0
                      B-time      0.000     0.000     0.000       0.0
           I-fare_basis_code      0.000     0.000     0.000       0.0
 I-arrive_time.time_relative      0.000     0.000     0.000       4.0
 I-depart_time.time_relative      0.000     0.000     0.000       1.0
            B-today_relative      0.000     0.000     0.000       0.0
                B-state_name      0.000     0.000     0.000       9.0
B-arrive_date.today_relative      0.000     0.000     0.000       0.0
    B-return_time.period_mod      0.000     0.000     0.000       0.0
                B-month_name      0.000     0.000     0.000       0.0
                B-day_number      0.000     0.000     0.000       0.0
 I-return_date.date_relative      0.000     0.000     0.000       3.0
I-return_date.today_relative      0.000     0.000     0.000       0.0
      B-stoploc.airport_name      0.000     0.000     0.000       0.0
             B-time_relative      0.000     0.000     0.000       0.0
                      I-time      0.000     0.000     0.000       0.0
    I-return_date.day_number      0.000     0.000     0.000       0.0
          I-meal_description      0.000     0.000     0.000       0.0
B-return_date.today_relative      0.000     0.000     0.000       0.0
      B-return_date.day_name      0.000     0.000     0.000       2.0

                   micro avg      0.923     0.925     0.924    3657.0
                   macro avg      0.494     0.479     0.474    3657.0
                weighted avg      0.913     0.925     0.914    3657.0

I0716 03:09:44.621426 139795883440000 interactiveshell.py:2882] Best Slot F1: 0.924
I0716 03:09:44.623023 139795883440000 estimator_training.py:186] Not using Distribute Coordinator.
I0716 03:09:44.624618 139795883440000 training.py:612] Running training and evaluation locally (non-distributed).
I0716 03:09:44.627876 139795883440000 training.py:700] Start train and evaluate loop. The evaluate will happen after every checkpoint. Checkpoint frequency is determined based on RunConfig arguments: save_checkpoints_steps 312 or save_checkpoints_secs None.
I0716 03:09:44.691657 139795883440000 estimator.py:1145] Calling model_fn.
I0716 03:09:45.981238 139795883440000 saver.py:1499] Saver not created because there are no variables in the graph to restore
I0716 03:09:46.733167 139795883440000 <ipython-input-6-d6747b40360b>:55] 
[<tf.Variable 'embedding:0' shape=(750, 300) dtype=float32_ref>,
 <tf.Variable 'module/aggregation/weights:0' shape=(3,) dtype=float32>,
 <tf.Variable 'module/aggregation/scaling:0' shape=() dtype=float32>,
 <tf.Variable 'dense/kernel:0' shape=(1324, 300) dtype=float32_ref>,
 <tf.Variable 'dense/bias:0' shape=(300,) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/gates/kernel:0' shape=(600, 600) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/gates/bias:0' shape=(600,) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/candidate/kernel:0' shape=(600, 300) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/candidate/bias:0' shape=(300,) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/gates/kernel:0' shape=(600, 600) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/gates/bias:0' shape=(600,) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/candidate/kernel:0' shape=(600, 300) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/candidate/bias:0' shape=(300,) dtype=float32_ref>,
 <tf.Variable 'dense_1/kernel:0' shape=(600, 23) dtype=float32_ref>,
 <tf.Variable 'dense_1/bias:0' shape=(23,) dtype=float32_ref>,
 <tf.Variable 'dense_2/kernel:0' shape=(600, 122) dtype=float32_ref>,
 <tf.Variable 'dense_2/bias:0' shape=(122,) dtype=float32_ref>]
I0716 03:09:47.810201 139795883440000 estimator.py:1147] Done calling model_fn.
I0716 03:09:47.814795 139795883440000 basic_session_run_hooks.py:541] Create CheckpointSaverHook.
I0716 03:09:48.135759 139795883440000 monitored_session.py:240] Graph was finalized.
I0716 03:09:48.153249 139795883440000 saver.py:1280] Restoring parameters from ../model/elmo_bigru/model.ckpt-936
I0716 03:09:49.177926 139795883440000 session_manager.py:500] Running local_init_op.
I0716 03:09:49.244561 139795883440000 session_manager.py:502] Done running local_init_op.
I0716 03:09:52.202533 139795883440000 basic_session_run_hooks.py:606] Saving checkpoints for 936 into ../model/elmo_bigru/model.ckpt.
Reading ../data/atis.train.w-intent.iob
I0716 03:09:57.132408 139795883440000 basic_session_run_hooks.py:262] loss = 0.09598224, step = 936
I0716 03:09:57.134508 139795883440000 basic_session_run_hooks.py:262] lr = 0.00027182678
I0716 03:10:25.028632 139795883440000 basic_session_run_hooks.py:692] global_step/sec: 3.58465
I0716 03:10:25.032232 139795883440000 basic_session_run_hooks.py:260] loss = 0.11424209, step = 1036 (27.900 sec)
I0716 03:10:25.036499 139795883440000 basic_session_run_hooks.py:260] lr = 0.00026897783 (27.902 sec)
I0716 03:10:51.357235 139795883440000 basic_session_run_hooks.py:692] global_step/sec: 3.79814
I0716 03:10:51.364395 139795883440000 basic_session_run_hooks.py:260] loss = 0.070536286, step = 1136 (26.332 sec)
I0716 03:10:51.366651 139795883440000 basic_session_run_hooks.py:260] lr = 0.00026615875 (26.330 sec)
I0716 03:11:17.504055 139795883440000 basic_session_run_hooks.py:692] global_step/sec: 3.82456
I0716 03:11:17.510384 139795883440000 basic_session_run_hooks.py:260] loss = 0.25277418, step = 1236 (26.146 sec)
I0716 03:11:17.513742 139795883440000 basic_session_run_hooks.py:260] lr = 0.0002633692 (26.147 sec)
I0716 03:11:20.337857 139795883440000 basic_session_run_hooks.py:606] Saving checkpoints for 1248 into ../model/elmo_bigru/model.ckpt.
I0716 03:11:22.240674 139795883440000 estimator.py:1145] Calling model_fn.
I0716 03:11:23.577543 139795883440000 saver.py:1499] Saver not created because there are no variables in the graph to restore
I0716 03:11:24.163589 139795883440000 estimator.py:1147] Done calling model_fn.
I0716 03:11:24.192977 139795883440000 evaluation.py:255] Starting evaluation at 2019-07-16T03:11:24Z
I0716 03:11:24.418920 139795883440000 monitored_session.py:240] Graph was finalized.
I0716 03:11:24.435197 139795883440000 saver.py:1280] Restoring parameters from ../model/elmo_bigru/model.ckpt-1248
I0716 03:11:25.325025 139795883440000 session_manager.py:500] Running local_init_op.
I0716 03:11:25.378224 139795883440000 session_manager.py:502] Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
I0716 03:12:24.341384 139795883440000 evaluation.py:275] Finished evaluation at 2019-07-16-03:12:24
I0716 03:12:24.343268 139795883440000 estimator.py:2039] Saving dict for global step 1248: global_step = 1248, loss = 0.29128006
I0716 03:12:24.351259 139795883440000 estimator.py:2099] Saving 'checkpoint_path' summary for global step 1248: ../model/elmo_bigru/model.ckpt-1248
I0716 03:12:24.436446 139795883440000 estimator.py:368] Loss for final step: 0.08681949.
Reading ../data/atis.test.w-intent.iob
I0716 03:12:25.030044 139795883440000 estimator.py:1145] Calling model_fn.
I0716 03:12:26.625313 139795883440000 saver.py:1499] Saver not created because there are no variables in the graph to restore
I0716 03:12:27.137457 139795883440000 estimator.py:1147] Done calling model_fn.
I0716 03:12:27.531526 139795883440000 monitored_session.py:240] Graph was finalized.
I0716 03:12:27.549257 139795883440000 saver.py:1280] Restoring parameters from ../model/elmo_bigru/model.ckpt-1248
I0716 03:12:28.412033 139795883440000 session_manager.py:500] Running local_init_op.
I0716 03:12:28.456053 139795883440000 session_manager.py:502] Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.
  'precision', 'predicted', average, warn_for)
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1439: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples.
  'recall', 'true', average, warn_for)
I0716 03:13:26.352146 139795883440000 interactiveshell.py:2882] 
                                          precision    recall  f1-score   support

                             atis_flight      0.987     0.989     0.988       632
                            atis_airfare      0.906     1.000     0.950        48
                     atis_ground_service      0.878     1.000     0.935        36
                            atis_airline      1.000     1.000     1.000        38
                       atis_abbreviation      0.943     1.000     0.971        33
                           atis_aircraft      1.000     1.000     1.000         9
                        atis_flight_time      0.333     1.000     0.500         1
                           atis_quantity      0.429     1.000     0.600         3
                atis_flight#atis_airfare      0.857     0.500     0.632        12
                            atis_airport      1.000     0.944     0.971        18
                           atis_distance      1.000     1.000     1.000        10
                               atis_city      1.000     0.167     0.286         6
                        atis_ground_fare      1.000     0.857     0.923         7
                           atis_capacity      1.000     0.952     0.976        21
                          atis_flight_no      0.889     1.000     0.941         8
                               atis_meal      1.000     0.667     0.800         6
                        atis_restriction      0.000     0.000     0.000         0
             atis_airline#atis_flight_no      0.000     0.000     0.000         0
    atis_ground_service#atis_ground_fare      0.000     0.000     0.000         0
           atis_airfare#atis_flight_time      0.000     0.000     0.000         0
                           atis_cheapest      0.000     0.000     0.000         0
atis_aircraft#atis_flight#atis_flight_no      0.000     0.000     0.000         0

                               micro avg      0.969     0.974     0.971       888
                               macro avg      0.646     0.640     0.612       888
                            weighted avg      0.973     0.974     0.970       888

I0716 03:13:26.388364 139795883440000 interactiveshell.py:2882] 
                              precision    recall  f1-score   support

                           O      0.000     0.000     0.000       0.0
           B-toloc.city_name      0.974     0.994     0.984     716.0
         B-fromloc.city_name      0.989     0.999     0.994     704.0
           I-toloc.city_name      0.936     0.996     0.965     265.0
      B-depart_date.day_name      0.977     0.995     0.986     212.0
              B-airline_name      0.962     0.990     0.976     101.0
         I-fromloc.city_name      0.989     0.972     0.980     177.0
 B-depart_time.period_of_day      0.975     0.885     0.927     130.0
              I-airline_name      1.000     0.954     0.976      65.0
    B-depart_date.day_number      0.982     0.982     0.982      55.0
    B-depart_date.month_name      0.982     0.964     0.973      56.0
          B-depart_time.time      0.826     1.000     0.905      57.0
                B-round_trip      1.000     0.986     0.993      73.0
             B-cost_relative      0.973     0.973     0.973      37.0
                I-round_trip      0.986     1.000     0.993      71.0
                B-flight_mod      0.786     0.917     0.846      24.0
 B-depart_time.time_relative      0.942     1.000     0.970      65.0
          I-depart_time.time      0.852     1.000     0.920      52.0
         B-stoploc.city_name      0.864     0.950     0.905      20.0
                 B-city_name      0.857     0.632     0.727      57.0
                B-class_type      0.960     1.000     0.980      24.0
          B-arrive_time.time      1.000     0.971     0.985      34.0
 B-arrive_time.time_relative      0.882     0.968     0.923      31.0
                I-class_type      1.000     1.000     1.000      17.0
               B-flight_stop      1.000     1.000     1.000      21.0
          I-arrive_time.time      0.810     0.971     0.883      35.0
              B-airline_code      0.795     0.912     0.849      34.0
    I-depart_date.day_number      1.000     1.000     1.000      15.0
      I-fromloc.airport_name      0.500     1.000     0.667      15.0
      B-fromloc.airport_name      0.355     0.917     0.512      12.0
      B-arrive_date.day_name      0.786     1.000     0.880      11.0
          B-toloc.state_code      0.900     1.000     0.947      18.0
B-depart_date.today_relative      1.000     0.778     0.875       9.0
             B-flight_number      0.846     1.000     0.917      11.0
 B-depart_date.date_relative      0.850     1.000     0.919      17.0
          B-toloc.state_name      0.806     0.893     0.847      28.0
           B-fare_basis_code      0.895     1.000     0.944      17.0
               B-flight_time      1.000     1.000     1.000       1.0
                        B-or      1.000     1.000     1.000       3.0
 B-arrive_time.period_of_day      0.667     1.000     0.800       6.0
          B-meal_description      1.000     0.900     0.947      10.0
             I-cost_relative      1.000     0.667     0.800       3.0
              I-airport_name      0.800     0.414     0.545      29.0
               B-fare_amount      1.000     1.000     1.000       2.0
               I-fare_amount      1.000     1.000     1.000       2.0
                 I-city_name      0.810     0.567     0.667      30.0
        I-toloc.airport_name      1.000     1.000     1.000       3.0
            B-transport_type      0.909     1.000     0.952      10.0
    B-arrive_date.month_name      0.714     0.833     0.769       6.0
    B-arrive_date.day_number      0.714     0.833     0.769       6.0
         I-stoploc.city_name      0.889     0.800     0.842      10.0
                      B-meal      1.000     1.000     1.000      16.0
        B-fromloc.state_code      1.000     0.913     0.955      23.0
    B-depart_time.period_mod      1.000     1.000     1.000       5.0
                   B-connect      1.000     1.000     1.000       6.0
               B-flight_days      1.000     1.000     1.000      10.0
        B-toloc.airport_name      1.000     0.333     0.500       3.0
        B-fromloc.state_name      0.941     0.941     0.941      17.0
              B-airport_name      0.556     0.238     0.333      21.0
                   B-economy      1.000     0.333     0.500       6.0
               I-flight_time      1.000     1.000     1.000       1.0
             B-aircraft_code      1.000     0.848     0.918      33.0
                       B-mod      1.000     0.500     0.667       2.0
              B-airport_code      0.750     0.333     0.462       9.0
    B-depart_time.start_time      1.000     0.333     0.500       3.0
      B-depart_time.end_time      1.000     0.333     0.500       3.0
          B-depart_date.year      1.000     1.000     1.000       3.0
            I-transport_type      0.000     0.000     0.000       1.0
          B-restriction_code      0.667     0.500     0.571       4.0
    B-arrive_time.start_time      0.889     1.000     0.941       8.0
        B-toloc.airport_code      1.000     0.750     0.857       4.0
      B-arrive_time.end_time      0.889     1.000     0.941       8.0
      I-arrive_time.end_time      0.000     0.000     0.000       8.0
      I-depart_time.end_time      0.000     0.000     0.000       3.0
               I-flight_stop      0.000     0.000     0.000       0.0
      B-fromloc.airport_code      0.800     0.800     0.800       5.0
          I-restriction_code      1.000     1.000     1.000       3.0
    I-depart_time.start_time      1.000     1.000     1.000       1.0
          I-toloc.state_name      1.000     1.000     1.000       1.0
I-depart_date.today_relative      0.000     0.000     0.000       0.0
 B-arrive_date.date_relative      1.000     0.500     0.667       2.0
                I-flight_mod      0.000     0.000     0.000       6.0
                   I-economy      0.000     0.000     0.000       0.0
 B-return_date.date_relative      0.000     0.000     0.000       3.0
        I-fromloc.state_name      0.000     0.000     0.000       1.0
                B-state_code      1.000     1.000     1.000       1.0
    I-arrive_time.start_time      1.000     1.000     1.000       1.0
    I-arrive_date.day_number      0.000     0.000     0.000       0.0
                 B-meal_code      0.000     0.000     0.000       1.0
 I-depart_time.period_of_day      0.000     0.000     0.000       1.0
                  B-day_name      0.000     0.000     0.000       2.0
             B-period_of_day      0.000     0.000     0.000       4.0
        B-stoploc.state_code      0.000     0.000     0.000       0.0
    B-return_date.month_name      0.000     0.000     0.000       0.0
    B-return_date.day_number      0.000     0.000     0.000       0.0
    B-arrive_time.period_mod      0.000     0.000     0.000       0.0
                 I-meal_code      0.000     0.000     0.000       0.0
        B-toloc.country_name      0.000     0.000     0.000       1.0
                 B-days_code      0.000     0.000     0.000       1.0
 I-arrive_time.period_of_day      0.000     0.000     0.000       0.0
            I-today_relative      0.000     0.000     0.000       0.0
 B-return_time.period_of_day      0.000     0.000     0.000       0.0
                      B-time      0.000     0.000     0.000       0.0
           I-fare_basis_code      0.000     0.000     0.000       0.0
 I-arrive_time.time_relative      0.000     0.000     0.000       4.0
 I-depart_time.time_relative      0.000     0.000     0.000       1.0
            B-today_relative      0.000     0.000     0.000       0.0
                B-state_name      0.000     0.000     0.000       9.0
B-arrive_date.today_relative      0.000     0.000     0.000       0.0
    B-return_time.period_mod      0.000     0.000     0.000       0.0
                B-month_name      0.000     0.000     0.000       0.0
                B-day_number      0.000     0.000     0.000       0.0
 I-return_date.date_relative      0.000     0.000     0.000       3.0
I-return_date.today_relative      0.000     0.000     0.000       0.0
      B-stoploc.airport_name      0.000     0.000     0.000       0.0
             B-time_relative      0.000     0.000     0.000       0.0
                      I-time      0.000     0.000     0.000       0.0
    I-return_date.day_number      0.000     0.000     0.000       0.0
          I-meal_description      0.000     0.000     0.000       0.0
B-return_date.today_relative      0.000     0.000     0.000       0.0
      B-return_date.day_name      0.000     0.000     0.000       2.0

                   micro avg      0.942     0.944     0.943    3657.0
                   macro avg      0.580     0.556     0.556    3657.0
                weighted avg      0.936     0.944     0.936    3657.0

I0716 03:13:26.409072 139795883440000 interactiveshell.py:2882] Best Slot F1: 0.943
I0716 03:13:26.410695 139795883440000 estimator_training.py:186] Not using Distribute Coordinator.
I0716 03:13:26.413439 139795883440000 training.py:612] Running training and evaluation locally (non-distributed).
I0716 03:13:26.417328 139795883440000 training.py:700] Start train and evaluate loop. The evaluate will happen after every checkpoint. Checkpoint frequency is determined based on RunConfig arguments: save_checkpoints_steps 312 or save_checkpoints_secs None.
I0716 03:13:26.484791 139795883440000 estimator.py:1145] Calling model_fn.
I0716 03:13:27.782305 139795883440000 saver.py:1499] Saver not created because there are no variables in the graph to restore
I0716 03:13:28.523794 139795883440000 <ipython-input-6-d6747b40360b>:55] 
[<tf.Variable 'embedding:0' shape=(750, 300) dtype=float32_ref>,
 <tf.Variable 'module/aggregation/weights:0' shape=(3,) dtype=float32>,
 <tf.Variable 'module/aggregation/scaling:0' shape=() dtype=float32>,
 <tf.Variable 'dense/kernel:0' shape=(1324, 300) dtype=float32_ref>,
 <tf.Variable 'dense/bias:0' shape=(300,) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/gates/kernel:0' shape=(600, 600) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/gates/bias:0' shape=(600,) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/candidate/kernel:0' shape=(600, 300) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/candidate/bias:0' shape=(300,) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/gates/kernel:0' shape=(600, 600) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/gates/bias:0' shape=(600,) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/candidate/kernel:0' shape=(600, 300) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/candidate/bias:0' shape=(300,) dtype=float32_ref>,
 <tf.Variable 'dense_1/kernel:0' shape=(600, 23) dtype=float32_ref>,
 <tf.Variable 'dense_1/bias:0' shape=(23,) dtype=float32_ref>,
 <tf.Variable 'dense_2/kernel:0' shape=(600, 122) dtype=float32_ref>,
 <tf.Variable 'dense_2/bias:0' shape=(122,) dtype=float32_ref>]
I0716 03:13:29.597792 139795883440000 estimator.py:1147] Done calling model_fn.
I0716 03:13:29.601427 139795883440000 basic_session_run_hooks.py:541] Create CheckpointSaverHook.
I0716 03:13:29.932820 139795883440000 monitored_session.py:240] Graph was finalized.
I0716 03:13:29.950454 139795883440000 saver.py:1280] Restoring parameters from ../model/elmo_bigru/model.ckpt-1248
I0716 03:13:30.966139 139795883440000 session_manager.py:500] Running local_init_op.
I0716 03:13:31.033563 139795883440000 session_manager.py:502] Done running local_init_op.
I0716 03:13:34.017776 139795883440000 basic_session_run_hooks.py:606] Saving checkpoints for 1248 into ../model/elmo_bigru/model.ckpt.
Reading ../data/atis.train.w-intent.iob
I0716 03:13:38.845421 139795883440000 basic_session_run_hooks.py:262] loss = 0.056573045, step = 1248
I0716 03:13:38.847369 139795883440000 basic_session_run_hooks.py:262] lr = 0.00026303643
I0716 03:14:06.612472 139795883440000 basic_session_run_hooks.py:692] global_step/sec: 3.60131
I0716 03:14:06.617980 139795883440000 basic_session_run_hooks.py:260] loss = 0.044386838, step = 1348 (27.773 sec)
I0716 03:14:06.620636 139795883440000 basic_session_run_hooks.py:260] lr = 0.00026027963 (27.773 sec)
I0716 03:14:33.374242 139795883440000 basic_session_run_hooks.py:692] global_step/sec: 3.73667
I0716 03:14:33.379942 139795883440000 basic_session_run_hooks.py:260] loss = 0.04956132, step = 1448 (26.762 sec)
I0716 03:14:33.382454 139795883440000 basic_session_run_hooks.py:260] lr = 0.00025755167 (26.762 sec)
I0716 03:15:00.112639 139795883440000 basic_session_run_hooks.py:692] global_step/sec: 3.73994
I0716 03:15:00.118970 139795883440000 basic_session_run_hooks.py:260] loss = 0.038445197, step = 1548 (26.739 sec)
I0716 03:15:00.121211 139795883440000 basic_session_run_hooks.py:260] lr = 0.00025485238 (26.739 sec)
I0716 03:15:02.877131 139795883440000 basic_session_run_hooks.py:606] Saving checkpoints for 1560 into ../model/elmo_bigru/model.ckpt.
W0716 03:15:04.569809 139795883440000 deprecation.py:323] From /usr/local/lib/python3.6/dist-packages/tensorflow/python/training/saver.py:960: remove_checkpoint (from tensorflow.python.training.checkpoint_management) is deprecated and will be removed in a future version.
Instructions for updating:
Use standard file APIs to delete files with this prefix.
I0716 03:15:05.077725 139795883440000 estimator.py:1145] Calling model_fn.
I0716 03:15:06.442743 139795883440000 saver.py:1499] Saver not created because there are no variables in the graph to restore
I0716 03:15:07.043466 139795883440000 estimator.py:1147] Done calling model_fn.
I0716 03:15:07.072723 139795883440000 evaluation.py:255] Starting evaluation at 2019-07-16T03:15:07Z
I0716 03:15:07.301808 139795883440000 monitored_session.py:240] Graph was finalized.
I0716 03:15:07.318938 139795883440000 saver.py:1280] Restoring parameters from ../model/elmo_bigru/model.ckpt-1560
I0716 03:15:08.226789 139795883440000 session_manager.py:500] Running local_init_op.
I0716 03:15:08.283047 139795883440000 session_manager.py:502] Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
I0716 03:16:07.308760 139795883440000 evaluation.py:275] Finished evaluation at 2019-07-16-03:16:07
I0716 03:16:07.310705 139795883440000 estimator.py:2039] Saving dict for global step 1560: global_step = 1560, loss = 0.28729033
I0716 03:16:07.316600 139795883440000 estimator.py:2099] Saving 'checkpoint_path' summary for global step 1560: ../model/elmo_bigru/model.ckpt-1560
I0716 03:16:07.401273 139795883440000 estimator.py:368] Loss for final step: 0.008345632.
Reading ../data/atis.test.w-intent.iob
I0716 03:16:07.986275 139795883440000 estimator.py:1145] Calling model_fn.
I0716 03:16:09.618869 139795883440000 saver.py:1499] Saver not created because there are no variables in the graph to restore
I0716 03:16:10.127372 139795883440000 estimator.py:1147] Done calling model_fn.
I0716 03:16:10.338745 139795883440000 monitored_session.py:240] Graph was finalized.
I0716 03:16:10.351769 139795883440000 saver.py:1280] Restoring parameters from ../model/elmo_bigru/model.ckpt-1560
I0716 03:16:11.246006 139795883440000 session_manager.py:500] Running local_init_op.
I0716 03:16:11.290265 139795883440000 session_manager.py:502] Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.
  'precision', 'predicted', average, warn_for)
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1439: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples.
  'recall', 'true', average, warn_for)
I0716 03:17:09.232142 139795883440000 interactiveshell.py:2882] 
                                          precision    recall  f1-score   support

                             atis_flight      0.984     0.992     0.988       632
                            atis_airfare      0.960     1.000     0.980        48
                     atis_ground_service      0.973     1.000     0.986        36
                            atis_airline      1.000     1.000     1.000        38
                       atis_abbreviation      0.943     1.000     0.971        33
                           atis_aircraft      1.000     0.889     0.941         9
                        atis_flight_time      0.333     1.000     0.500         1
                           atis_quantity      0.375     1.000     0.545         3
                atis_flight#atis_airfare      0.833     0.417     0.556        12
                            atis_airport      1.000     0.944     0.971        18
                           atis_distance      1.000     1.000     1.000        10
                               atis_city      0.600     0.500     0.545         6
                        atis_ground_fare      1.000     0.857     0.923         7
                           atis_capacity      1.000     0.952     0.976        21
                          atis_flight_no      0.889     1.000     0.941         8
                               atis_meal      1.000     0.667     0.800         6
                        atis_restriction      0.000     0.000     0.000         0
             atis_airline#atis_flight_no      0.000     0.000     0.000         0
    atis_ground_service#atis_ground_fare      0.000     0.000     0.000         0
           atis_airfare#atis_flight_time      0.000     0.000     0.000         0
                           atis_cheapest      0.000     0.000     0.000         0
atis_aircraft#atis_flight#atis_flight_no      0.000     0.000     0.000         0

                               micro avg      0.971     0.976     0.974       888
                               macro avg      0.631     0.646     0.619       888
                            weighted avg      0.975     0.976     0.973       888

I0716 03:17:09.269594 139795883440000 interactiveshell.py:2882] 
                              precision    recall  f1-score   support

                           O      0.000     0.000     0.000       0.0
           B-toloc.city_name      0.970     0.997     0.983     716.0
         B-fromloc.city_name      0.992     0.997     0.994     704.0
           I-toloc.city_name      0.953     1.000     0.976     265.0
      B-depart_date.day_name      0.986     0.991     0.988     212.0
              B-airline_name      0.980     0.990     0.985     101.0
         I-fromloc.city_name      0.994     0.977     0.986     177.0
 B-depart_time.period_of_day      0.983     0.877     0.927     130.0
              I-airline_name      1.000     0.969     0.984      65.0
    B-depart_date.day_number      0.981     0.964     0.972      55.0
    B-depart_date.month_name      0.982     0.964     0.973      56.0
          B-depart_time.time      0.836     0.982     0.903      57.0
                B-round_trip      1.000     0.986     0.993      73.0
             B-cost_relative      0.973     0.973     0.973      37.0
                I-round_trip      0.986     1.000     0.993      71.0
                B-flight_mod      0.800     1.000     0.889      24.0
 B-depart_time.time_relative      0.968     0.938     0.953      65.0
          I-depart_time.time      0.870     0.904     0.887      52.0
         B-stoploc.city_name      0.950     0.950     0.950      20.0
                 B-city_name      0.850     0.596     0.701      57.0
                B-class_type      0.960     1.000     0.980      24.0
          B-arrive_time.time      0.971     0.971     0.971      34.0
 B-arrive_time.time_relative      0.833     0.968     0.896      31.0
                I-class_type      1.000     1.000     1.000      17.0
               B-flight_stop      1.000     1.000     1.000      21.0
          I-arrive_time.time      0.723     0.971     0.829      35.0
              B-airline_code      0.773     1.000     0.872      34.0
    I-depart_date.day_number      1.000     0.933     0.966      15.0
      I-fromloc.airport_name      0.484     1.000     0.652      15.0
      B-fromloc.airport_name      0.429     1.000     0.600      12.0
      B-arrive_date.day_name      0.786     1.000     0.880      11.0
          B-toloc.state_code      0.900     1.000     0.947      18.0
B-depart_date.today_relative      1.000     0.778     0.875       9.0
             B-flight_number      0.786     1.000     0.880      11.0
 B-depart_date.date_relative      0.850     1.000     0.919      17.0
          B-toloc.state_name      0.889     0.857     0.873      28.0
           B-fare_basis_code      0.850     1.000     0.919      17.0
               B-flight_time      1.000     1.000     1.000       1.0
                        B-or      1.000     1.000     1.000       3.0
 B-arrive_time.period_of_day      0.600     1.000     0.750       6.0
          B-meal_description      1.000     0.900     0.947      10.0
             I-cost_relative      1.000     0.667     0.800       3.0
              I-airport_name      0.765     0.448     0.565      29.0
               B-fare_amount      1.000     1.000     1.000       2.0
               I-fare_amount      1.000     1.000     1.000       2.0
                 I-city_name      0.792     0.633     0.704      30.0
        I-toloc.airport_name      1.000     1.000     1.000       3.0
            B-transport_type      1.000     0.700     0.824      10.0
    B-arrive_date.month_name      0.714     0.833     0.769       6.0
    B-arrive_date.day_number      0.625     0.833     0.714       6.0
         I-stoploc.city_name      0.909     1.000     0.952      10.0
                      B-meal      1.000     1.000     1.000      16.0
        B-fromloc.state_code      1.000     0.913     0.955      23.0
    B-depart_time.period_mod      0.833     1.000     0.909       5.0
                   B-connect      1.000     1.000     1.000       6.0
               B-flight_days      1.000     1.000     1.000      10.0
        B-toloc.airport_name      1.000     0.333     0.500       3.0
        B-fromloc.state_name      0.941     0.941     0.941      17.0
              B-airport_name      0.615     0.381     0.471      21.0
                   B-economy      1.000     0.333     0.500       6.0
               I-flight_time      1.000     1.000     1.000       1.0
             B-aircraft_code      1.000     0.848     0.918      33.0
                       B-mod      1.000     0.500     0.667       2.0
              B-airport_code      1.000     0.111     0.200       9.0
    B-depart_time.start_time      1.000     0.667     0.800       3.0
      B-depart_time.end_time      1.000     0.333     0.500       3.0
          B-depart_date.year      1.000     1.000     1.000       3.0
            I-transport_type      0.200     1.000     0.333       1.0
          B-restriction_code      0.667     0.500     0.571       4.0
    B-arrive_time.start_time      0.889     1.000     0.941       8.0
        B-toloc.airport_code      1.000     0.750     0.857       4.0
      B-arrive_time.end_time      0.889     1.000     0.941       8.0
      I-arrive_time.end_time      1.000     0.250     0.400       8.0
      I-depart_time.end_time      1.000     0.667     0.800       3.0
               I-flight_stop      0.000     0.000     0.000       0.0
      B-fromloc.airport_code      0.800     0.800     0.800       5.0
          I-restriction_code      1.000     1.000     1.000       3.0
    I-depart_time.start_time      1.000     1.000     1.000       1.0
          I-toloc.state_name      1.000     1.000     1.000       1.0
I-depart_date.today_relative      0.000     0.000     0.000       0.0
 B-arrive_date.date_relative      1.000     0.500     0.667       2.0
                I-flight_mod      0.000     0.000     0.000       6.0
                   I-economy      0.000     0.000     0.000       0.0
 B-return_date.date_relative      0.667     0.667     0.667       3.0
        I-fromloc.state_name      0.000     0.000     0.000       1.0
                B-state_code      1.000     1.000     1.000       1.0
    I-arrive_time.start_time      0.000     0.000     0.000       1.0
    I-arrive_date.day_number      0.000     0.000     0.000       0.0
                 B-meal_code      0.000     0.000     0.000       1.0
 I-depart_time.period_of_day      0.000     0.000     0.000       1.0
                  B-day_name      0.500     0.500     0.500       2.0
             B-period_of_day      0.000     0.000     0.000       4.0
        B-stoploc.state_code      0.000     0.000     0.000       0.0
    B-return_date.month_name      0.000     0.000     0.000       0.0
    B-return_date.day_number      0.000     0.000     0.000       0.0
    B-arrive_time.period_mod      0.000     0.000     0.000       0.0
                 I-meal_code      0.000     0.000     0.000       0.0
        B-toloc.country_name      0.000     0.000     0.000       1.0
                 B-days_code      0.000     0.000     0.000       1.0
 I-arrive_time.period_of_day      0.000     0.000     0.000       0.0
            I-today_relative      0.000     0.000     0.000       0.0
 B-return_time.period_of_day      0.000     0.000     0.000       0.0
                      B-time      0.000     0.000     0.000       0.0
           I-fare_basis_code      0.000     0.000     0.000       0.0
 I-arrive_time.time_relative      0.000     0.000     0.000       4.0
 I-depart_time.time_relative      0.000     0.000     0.000       1.0
            B-today_relative      0.000     0.000     0.000       0.0
                B-state_name      0.000     0.000     0.000       9.0
B-arrive_date.today_relative      0.000     0.000     0.000       0.0
    B-return_time.period_mod      0.000     0.000     0.000       0.0
                B-month_name      0.000     0.000     0.000       0.0
                B-day_number      0.000     0.000     0.000       0.0
 I-return_date.date_relative      0.000     0.000     0.000       3.0
I-return_date.today_relative      0.000     0.000     0.000       0.0
      B-stoploc.airport_name      0.000     0.000     0.000       0.0
             B-time_relative      0.000     0.000     0.000       0.0
                      I-time      0.000     0.000     0.000       0.0
    I-return_date.day_number      0.000     0.000     0.000       0.0
          I-meal_description      0.000     0.000     0.000       0.0
B-return_date.today_relative      0.000     0.000     0.000       0.0
      B-return_date.day_name      0.000     0.000     0.000       2.0

                   micro avg      0.943     0.945     0.944    3657.0
                   macro avg      0.601     0.575     0.570    3657.0
                weighted avg      0.942     0.945     0.939    3657.0

I0716 03:17:09.291633 139795883440000 interactiveshell.py:2882] Best Slot F1: 0.944
I0716 03:17:09.293515 139795883440000 estimator_training.py:186] Not using Distribute Coordinator.
I0716 03:17:09.296332 139795883440000 training.py:612] Running training and evaluation locally (non-distributed).
I0716 03:17:09.299224 139795883440000 training.py:700] Start train and evaluate loop. The evaluate will happen after every checkpoint. Checkpoint frequency is determined based on RunConfig arguments: save_checkpoints_steps 312 or save_checkpoints_secs None.
I0716 03:17:09.362334 139795883440000 estimator.py:1145] Calling model_fn.
I0716 03:17:10.946285 139795883440000 saver.py:1499] Saver not created because there are no variables in the graph to restore
I0716 03:17:11.521585 139795883440000 <ipython-input-6-d6747b40360b>:55] 
[<tf.Variable 'embedding:0' shape=(750, 300) dtype=float32_ref>,
 <tf.Variable 'module/aggregation/weights:0' shape=(3,) dtype=float32>,
 <tf.Variable 'module/aggregation/scaling:0' shape=() dtype=float32>,
 <tf.Variable 'dense/kernel:0' shape=(1324, 300) dtype=float32_ref>,
 <tf.Variable 'dense/bias:0' shape=(300,) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/gates/kernel:0' shape=(600, 600) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/gates/bias:0' shape=(600,) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/candidate/kernel:0' shape=(600, 300) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/candidate/bias:0' shape=(300,) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/gates/kernel:0' shape=(600, 600) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/gates/bias:0' shape=(600,) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/candidate/kernel:0' shape=(600, 300) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/candidate/bias:0' shape=(300,) dtype=float32_ref>,
 <tf.Variable 'dense_1/kernel:0' shape=(600, 23) dtype=float32_ref>,
 <tf.Variable 'dense_1/bias:0' shape=(23,) dtype=float32_ref>,
 <tf.Variable 'dense_2/kernel:0' shape=(600, 122) dtype=float32_ref>,
 <tf.Variable 'dense_2/bias:0' shape=(122,) dtype=float32_ref>]
I0716 03:17:12.603037 139795883440000 estimator.py:1147] Done calling model_fn.
I0716 03:17:12.607095 139795883440000 basic_session_run_hooks.py:541] Create CheckpointSaverHook.
I0716 03:17:13.122910 139795883440000 monitored_session.py:240] Graph was finalized.
I0716 03:17:13.142551 139795883440000 saver.py:1280] Restoring parameters from ../model/elmo_bigru/model.ckpt-1560
I0716 03:17:14.162578 139795883440000 session_manager.py:500] Running local_init_op.
I0716 03:17:14.228501 139795883440000 session_manager.py:502] Done running local_init_op.
I0716 03:17:16.980484 139795883440000 basic_session_run_hooks.py:606] Saving checkpoints for 1560 into ../model/elmo_bigru/model.ckpt.
Reading ../data/atis.train.w-intent.iob
I0716 03:17:21.701616 139795883440000 basic_session_run_hooks.py:262] loss = 0.05464157, step = 1560
I0716 03:17:21.703541 139795883440000 basic_session_run_hooks.py:262] lr = 0.00025453034
I0716 03:17:49.344164 139795883440000 basic_session_run_hooks.py:692] global_step/sec: 3.61755
I0716 03:17:49.347166 139795883440000 basic_session_run_hooks.py:260] loss = 0.08211179, step = 1660 (27.646 sec)
I0716 03:17:49.349646 139795883440000 basic_session_run_hooks.py:260] lr = 0.0002518627 (27.646 sec)
I0716 03:18:15.868099 139795883440000 basic_session_run_hooks.py:692] global_step/sec: 3.77017
I0716 03:18:15.871267 139795883440000 basic_session_run_hooks.py:260] loss = 0.029694248, step = 1760 (26.524 sec)
I0716 03:18:15.873329 139795883440000 basic_session_run_hooks.py:260] lr = 0.00024922297 (26.524 sec)
I0716 03:18:41.795644 139795883440000 basic_session_run_hooks.py:692] global_step/sec: 3.8569
I0716 03:18:41.803750 139795883440000 basic_session_run_hooks.py:260] loss = 0.057554252, step = 1860 (25.932 sec)
I0716 03:18:41.805616 139795883440000 basic_session_run_hooks.py:260] lr = 0.00024661093 (25.932 sec)
I0716 03:18:44.854101 139795883440000 basic_session_run_hooks.py:606] Saving checkpoints for 1872 into ../model/elmo_bigru/model.ckpt.
I0716 03:18:46.823345 139795883440000 estimator.py:1145] Calling model_fn.
I0716 03:18:48.140162 139795883440000 saver.py:1499] Saver not created because there are no variables in the graph to restore
I0716 03:18:48.964244 139795883440000 estimator.py:1147] Done calling model_fn.
I0716 03:18:48.994191 139795883440000 evaluation.py:255] Starting evaluation at 2019-07-16T03:18:48Z
I0716 03:18:49.214614 139795883440000 monitored_session.py:240] Graph was finalized.
I0716 03:18:49.229626 139795883440000 saver.py:1280] Restoring parameters from ../model/elmo_bigru/model.ckpt-1872
I0716 03:18:50.094576 139795883440000 session_manager.py:500] Running local_init_op.
I0716 03:18:50.148860 139795883440000 session_manager.py:502] Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
I0716 03:19:49.197244 139795883440000 evaluation.py:275] Finished evaluation at 2019-07-16-03:19:49
I0716 03:19:49.198988 139795883440000 estimator.py:2039] Saving dict for global step 1872: global_step = 1872, loss = 0.27500156
I0716 03:19:49.205196 139795883440000 estimator.py:2099] Saving 'checkpoint_path' summary for global step 1872: ../model/elmo_bigru/model.ckpt-1872
I0716 03:19:49.288769 139795883440000 estimator.py:368] Loss for final step: 0.023545232.
Reading ../data/atis.test.w-intent.iob
I0716 03:19:49.877800 139795883440000 estimator.py:1145] Calling model_fn.
I0716 03:19:50.902939 139795883440000 saver.py:1499] Saver not created because there are no variables in the graph to restore
I0716 03:19:51.873017 139795883440000 estimator.py:1147] Done calling model_fn.
I0716 03:19:52.085095 139795883440000 monitored_session.py:240] Graph was finalized.
I0716 03:19:52.101049 139795883440000 saver.py:1280] Restoring parameters from ../model/elmo_bigru/model.ckpt-1872
I0716 03:19:52.988541 139795883440000 session_manager.py:500] Running local_init_op.
I0716 03:19:53.032260 139795883440000 session_manager.py:502] Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.
  'precision', 'predicted', average, warn_for)
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1439: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples.
  'recall', 'true', average, warn_for)
I0716 03:20:51.023446 139795883440000 interactiveshell.py:2882] 
                                          precision    recall  f1-score   support

                             atis_flight      0.987     0.991     0.989       632
                            atis_airfare      0.906     1.000     0.950        48
                     atis_ground_service      0.923     1.000     0.960        36
                            atis_airline      1.000     1.000     1.000        38
                       atis_abbreviation      0.943     1.000     0.971        33
                           atis_aircraft      1.000     0.889     0.941         9
                        atis_flight_time      0.500     1.000     0.667         1
                           atis_quantity      0.429     1.000     0.600         3
                atis_flight#atis_airfare      0.833     0.417     0.556        12
                            atis_airport      0.944     0.944     0.944        18
                           atis_distance      1.000     1.000     1.000        10
                               atis_city      1.000     0.500     0.667         6
                        atis_ground_fare      1.000     0.857     0.923         7
                           atis_capacity      0.952     0.952     0.952        21
                          atis_flight_no      0.889     1.000     0.941         8
                               atis_meal      1.000     0.667     0.800         6
                        atis_restriction      0.000     0.000     0.000         0
             atis_airline#atis_flight_no      0.000     0.000     0.000         0
    atis_ground_service#atis_ground_fare      0.000     0.000     0.000         0
           atis_airfare#atis_flight_time      0.000     0.000     0.000         0
                           atis_cheapest      0.000     0.000     0.000         0
atis_aircraft#atis_flight#atis_flight_no      0.000     0.000     0.000         0

                               micro avg      0.970     0.975     0.972       888
                               macro avg      0.650     0.646     0.630       888
                            weighted avg      0.973     0.975     0.971       888

I0716 03:20:51.059612 139795883440000 interactiveshell.py:2882] 
                              precision    recall  f1-score   support

                           O      0.000     0.000     0.000       0.0
           B-toloc.city_name      0.969     0.994     0.981     716.0
         B-fromloc.city_name      0.990     0.999     0.994     704.0
           I-toloc.city_name      0.960     1.000     0.980     265.0
      B-depart_date.day_name      0.986     0.991     0.988     212.0
              B-airline_name      0.981     1.000     0.990     101.0
         I-fromloc.city_name      0.989     0.994     0.992     177.0
 B-depart_time.period_of_day      0.983     0.885     0.931     130.0
              I-airline_name      1.000     0.969     0.984      65.0
    B-depart_date.day_number      0.981     0.945     0.963      55.0
    B-depart_date.month_name      0.982     0.964     0.973      56.0
          B-depart_time.time      0.792     1.000     0.884      57.0
                B-round_trip      1.000     0.986     0.993      73.0
             B-cost_relative      0.973     0.973     0.973      37.0
                I-round_trip      0.986     1.000     0.993      71.0
                B-flight_mod      0.800     1.000     0.889      24.0
 B-depart_time.time_relative      0.969     0.954     0.961      65.0
          I-depart_time.time      0.879     0.981     0.927      52.0
         B-stoploc.city_name      0.905     0.950     0.927      20.0
                 B-city_name      0.889     0.561     0.688      57.0
                B-class_type      0.960     1.000     0.980      24.0
          B-arrive_time.time      1.000     0.971     0.985      34.0
 B-arrive_time.time_relative      0.857     0.968     0.909      31.0
                I-class_type      1.000     1.000     1.000      17.0
               B-flight_stop      1.000     1.000     1.000      21.0
          I-arrive_time.time      0.944     0.971     0.958      35.0
              B-airline_code      0.825     0.971     0.892      34.0
    I-depart_date.day_number      1.000     0.933     0.966      15.0
      I-fromloc.airport_name      0.500     1.000     0.667      15.0
      B-fromloc.airport_name      0.400     1.000     0.571      12.0
      B-arrive_date.day_name      0.786     1.000     0.880      11.0
          B-toloc.state_code      1.000     1.000     1.000      18.0
B-depart_date.today_relative      1.000     0.778     0.875       9.0
             B-flight_number      0.769     0.909     0.833      11.0
 B-depart_date.date_relative      0.944     1.000     0.971      17.0
          B-toloc.state_name      0.828     0.857     0.842      28.0
           B-fare_basis_code      0.850     1.000     0.919      17.0
               B-flight_time      1.000     1.000     1.000       1.0
                        B-or      1.000     1.000     1.000       3.0
 B-arrive_time.period_of_day      0.750     1.000     0.857       6.0
          B-meal_description      1.000     0.900     0.947      10.0
             I-cost_relative      1.000     0.667     0.800       3.0
              I-airport_name      0.867     0.448     0.591      29.0
               B-fare_amount      1.000     1.000     1.000       2.0
               I-fare_amount      1.000     1.000     1.000       2.0
                 I-city_name      1.000     0.600     0.750      30.0
        I-toloc.airport_name      1.000     1.000     1.000       3.0
            B-transport_type      1.000     0.900     0.947      10.0
    B-arrive_date.month_name      0.714     0.833     0.769       6.0
    B-arrive_date.day_number      0.556     0.833     0.667       6.0
         I-stoploc.city_name      0.909     1.000     0.952      10.0
                      B-meal      1.000     1.000     1.000      16.0
        B-fromloc.state_code      1.000     1.000     1.000      23.0
    B-depart_time.period_mod      0.833     1.000     0.909       5.0
                   B-connect      1.000     1.000     1.000       6.0
               B-flight_days      1.000     1.000     1.000      10.0
        B-toloc.airport_name      1.000     0.333     0.500       3.0
        B-fromloc.state_name      0.941     0.941     0.941      17.0
              B-airport_name      0.615     0.381     0.471      21.0
                   B-economy      1.000     0.333     0.500       6.0
               I-flight_time      1.000     1.000     1.000       1.0
             B-aircraft_code      1.000     0.788     0.881      33.0
                       B-mod      1.000     0.500     0.667       2.0
              B-airport_code      1.000     0.333     0.500       9.0
    B-depart_time.start_time      1.000     1.000     1.000       3.0
      B-depart_time.end_time      1.000     0.667     0.800       3.0
          B-depart_date.year      1.000     1.000     1.000       3.0
            I-transport_type      0.167     1.000     0.286       1.0
          B-restriction_code      1.000     0.500     0.667       4.0
    B-arrive_time.start_time      1.000     1.000     1.000       8.0
        B-toloc.airport_code      1.000     0.750     0.857       4.0
      B-arrive_time.end_time      1.000     1.000     1.000       8.0
      I-arrive_time.end_time      1.000     0.875     0.933       8.0
      I-depart_time.end_time      1.000     0.667     0.800       3.0
               I-flight_stop      0.000     0.000     0.000       0.0
      B-fromloc.airport_code      0.833     1.000     0.909       5.0
          I-restriction_code      1.000     1.000     1.000       3.0
    I-depart_time.start_time      1.000     1.000     1.000       1.0
          I-toloc.state_name      1.000     1.000     1.000       1.0
I-depart_date.today_relative      0.000     0.000     0.000       0.0
 B-arrive_date.date_relative      1.000     1.000     1.000       2.0
                I-flight_mod      0.000     0.000     0.000       6.0
                   I-economy      0.000     0.000     0.000       0.0
 B-return_date.date_relative      0.500     0.667     0.571       3.0
        I-fromloc.state_name      0.000     0.000     0.000       1.0
                B-state_code      1.000     1.000     1.000       1.0
    I-arrive_time.start_time      1.000     1.000     1.000       1.0
    I-arrive_date.day_number      0.000     0.000     0.000       0.0
                 B-meal_code      0.000     0.000     0.000       1.0
 I-depart_time.period_of_day      0.000     0.000     0.000       1.0
                  B-day_name      1.000     0.500     0.667       2.0
             B-period_of_day      1.000     0.500     0.667       4.0
        B-stoploc.state_code      0.000     0.000     0.000       0.0
    B-return_date.month_name      0.000     0.000     0.000       0.0
    B-return_date.day_number      0.000     0.000     0.000       0.0
    B-arrive_time.period_mod      0.000     0.000     0.000       0.0
                 I-meal_code      0.000     0.000     0.000       0.0
        B-toloc.country_name      0.000     0.000     0.000       1.0
                 B-days_code      0.000     0.000     0.000       1.0
 I-arrive_time.period_of_day      0.000     0.000     0.000       0.0
            I-today_relative      0.000     0.000     0.000       0.0
 B-return_time.period_of_day      0.000     0.000     0.000       0.0
                      B-time      0.000     0.000     0.000       0.0
           I-fare_basis_code      0.000     0.000     0.000       0.0
 I-arrive_time.time_relative      0.000     0.000     0.000       4.0
 I-depart_time.time_relative      0.000     0.000     0.000       1.0
            B-today_relative      0.000     0.000     0.000       0.0
                B-state_name      0.000     0.000     0.000       9.0
B-arrive_date.today_relative      0.000     0.000     0.000       0.0
    B-return_time.period_mod      0.000     0.000     0.000       0.0
                B-month_name      0.000     0.000     0.000       0.0
                B-day_number      0.000     0.000     0.000       0.0
 I-return_date.date_relative      0.000     0.000     0.000       3.0
I-return_date.today_relative      0.000     0.000     0.000       0.0
      B-stoploc.airport_name      0.000     0.000     0.000       0.0
             B-time_relative      0.000     0.000     0.000       0.0
                      I-time      0.000     0.000     0.000       0.0
    I-return_date.day_number      0.000     0.000     0.000       0.0
          I-meal_description      0.000     0.000     0.000       0.0
B-return_date.today_relative      0.000     0.000     0.000       0.0
      B-return_date.day_name      0.000     0.000     0.000       2.0

                   micro avg      0.949     0.951     0.950    3657.0
                   macro avg      0.631     0.607     0.604    3657.0
                weighted avg      0.950     0.951     0.946    3657.0

I0716 03:20:51.080179 139795883440000 interactiveshell.py:2882] Best Slot F1: 0.950
I0716 03:20:51.081658 139795883440000 estimator_training.py:186] Not using Distribute Coordinator.
I0716 03:20:51.085367 139795883440000 training.py:612] Running training and evaluation locally (non-distributed).
I0716 03:20:51.088035 139795883440000 training.py:700] Start train and evaluate loop. The evaluate will happen after every checkpoint. Checkpoint frequency is determined based on RunConfig arguments: save_checkpoints_steps 312 or save_checkpoints_secs None.
I0716 03:20:51.152100 139795883440000 estimator.py:1145] Calling model_fn.
I0716 03:20:52.592101 139795883440000 saver.py:1499] Saver not created because there are no variables in the graph to restore
I0716 03:20:53.185715 139795883440000 <ipython-input-6-d6747b40360b>:55] 
[<tf.Variable 'embedding:0' shape=(750, 300) dtype=float32_ref>,
 <tf.Variable 'module/aggregation/weights:0' shape=(3,) dtype=float32>,
 <tf.Variable 'module/aggregation/scaling:0' shape=() dtype=float32>,
 <tf.Variable 'dense/kernel:0' shape=(1324, 300) dtype=float32_ref>,
 <tf.Variable 'dense/bias:0' shape=(300,) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/gates/kernel:0' shape=(600, 600) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/gates/bias:0' shape=(600,) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/candidate/kernel:0' shape=(600, 300) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/candidate/bias:0' shape=(300,) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/gates/kernel:0' shape=(600, 600) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/gates/bias:0' shape=(600,) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/candidate/kernel:0' shape=(600, 300) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/candidate/bias:0' shape=(300,) dtype=float32_ref>,
 <tf.Variable 'dense_1/kernel:0' shape=(600, 23) dtype=float32_ref>,
 <tf.Variable 'dense_1/bias:0' shape=(23,) dtype=float32_ref>,
 <tf.Variable 'dense_2/kernel:0' shape=(600, 122) dtype=float32_ref>,
 <tf.Variable 'dense_2/bias:0' shape=(122,) dtype=float32_ref>]
I0716 03:20:54.254271 139795883440000 estimator.py:1147] Done calling model_fn.
I0716 03:20:54.257858 139795883440000 basic_session_run_hooks.py:541] Create CheckpointSaverHook.
I0716 03:20:54.765297 139795883440000 monitored_session.py:240] Graph was finalized.
I0716 03:20:54.785214 139795883440000 saver.py:1280] Restoring parameters from ../model/elmo_bigru/model.ckpt-1872
I0716 03:20:55.794133 139795883440000 session_manager.py:500] Running local_init_op.
I0716 03:20:55.866584 139795883440000 session_manager.py:502] Done running local_init_op.
I0716 03:20:58.639599 139795883440000 basic_session_run_hooks.py:606] Saving checkpoints for 1872 into ../model/elmo_bigru/model.ckpt.
Reading ../data/atis.train.w-intent.iob
I0716 03:21:03.386334 139795883440000 basic_session_run_hooks.py:262] loss = 0.05346859, step = 1872
I0716 03:21:03.388282 139795883440000 basic_session_run_hooks.py:262] lr = 0.0002462993
I0716 03:21:30.505856 139795883440000 basic_session_run_hooks.py:692] global_step/sec: 3.68729
I0716 03:21:30.513989 139795883440000 basic_session_run_hooks.py:260] loss = 0.04573856, step = 1972 (27.128 sec)
I0716 03:21:30.516146 139795883440000 basic_session_run_hooks.py:260] lr = 0.00024371794 (27.128 sec)
I0716 03:21:57.266182 139795883440000 basic_session_run_hooks.py:692] global_step/sec: 3.73689
I0716 03:21:57.273312 139795883440000 basic_session_run_hooks.py:260] loss = 0.043409802, step = 2072 (26.759 sec)
I0716 03:21:57.276177 139795883440000 basic_session_run_hooks.py:260] lr = 0.00024116358 (26.760 sec)
I0716 03:22:24.066598 139795883440000 basic_session_run_hooks.py:692] global_step/sec: 3.73127
I0716 03:22:24.074424 139795883440000 basic_session_run_hooks.py:260] loss = 0.040823795, step = 2172 (26.801 sec)
I0716 03:22:24.075997 139795883440000 basic_session_run_hooks.py:260] lr = 0.00023863598 (26.800 sec)
I0716 03:22:26.920837 139795883440000 basic_session_run_hooks.py:606] Saving checkpoints for 2184 into ../model/elmo_bigru/model.ckpt.
I0716 03:22:29.112518 139795883440000 estimator.py:1145] Calling model_fn.
I0716 03:22:30.449255 139795883440000 saver.py:1499] Saver not created because there are no variables in the graph to restore
I0716 03:22:31.278379 139795883440000 estimator.py:1147] Done calling model_fn.
I0716 03:22:31.310838 139795883440000 evaluation.py:255] Starting evaluation at 2019-07-16T03:22:31Z
I0716 03:22:31.531050 139795883440000 monitored_session.py:240] Graph was finalized.
I0716 03:22:31.549881 139795883440000 saver.py:1280] Restoring parameters from ../model/elmo_bigru/model.ckpt-2184
I0716 03:22:32.448242 139795883440000 session_manager.py:500] Running local_init_op.
I0716 03:22:32.499600 139795883440000 session_manager.py:502] Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
I0716 03:23:31.168509 139795883440000 evaluation.py:275] Finished evaluation at 2019-07-16-03:23:31
I0716 03:23:31.170503 139795883440000 estimator.py:2039] Saving dict for global step 2184: global_step = 2184, loss = 0.28009278
I0716 03:23:31.177702 139795883440000 estimator.py:2099] Saving 'checkpoint_path' summary for global step 2184: ../model/elmo_bigru/model.ckpt-2184
I0716 03:23:31.260535 139795883440000 estimator.py:368] Loss for final step: 0.01047217.
Reading ../data/atis.test.w-intent.iob
I0716 03:23:31.841433 139795883440000 estimator.py:1145] Calling model_fn.
I0716 03:23:32.880039 139795883440000 saver.py:1499] Saver not created because there are no variables in the graph to restore
I0716 03:23:33.836756 139795883440000 estimator.py:1147] Done calling model_fn.
I0716 03:23:34.046449 139795883440000 monitored_session.py:240] Graph was finalized.
I0716 03:23:34.061011 139795883440000 saver.py:1280] Restoring parameters from ../model/elmo_bigru/model.ckpt-2184
I0716 03:23:34.944166 139795883440000 session_manager.py:500] Running local_init_op.
I0716 03:23:34.986050 139795883440000 session_manager.py:502] Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.
  'precision', 'predicted', average, warn_for)
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1439: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples.
  'recall', 'true', average, warn_for)
I0716 03:24:32.635773 139795883440000 interactiveshell.py:2882] 
                                          precision    recall  f1-score   support

                             atis_flight      0.986     0.992     0.989       632
                            atis_airfare      0.941     1.000     0.970        48
                     atis_ground_service      1.000     1.000     1.000        36
                            atis_airline      1.000     1.000     1.000        38
                       atis_abbreviation      0.943     1.000     0.971        33
                           atis_aircraft      1.000     1.000     1.000         9
                        atis_flight_time      0.333     1.000     0.500         1
                           atis_quantity      0.429     1.000     0.600         3
                atis_flight#atis_airfare      0.833     0.417     0.556        12
                            atis_airport      1.000     1.000     1.000        18
                           atis_distance      1.000     1.000     1.000        10
                               atis_city      0.750     0.500     0.600         6
                        atis_ground_fare      1.000     0.857     0.923         7
                           atis_capacity      1.000     0.952     0.976        21
                          atis_flight_no      0.889     1.000     0.941         8
                               atis_meal      1.000     0.833     0.909         6
                        atis_restriction      0.000     0.000     0.000         0
             atis_airline#atis_flight_no      0.000     0.000     0.000         0
    atis_ground_service#atis_ground_fare      0.000     0.000     0.000         0
           atis_airfare#atis_flight_time      0.000     0.000     0.000         0
                           atis_cheapest      0.000     0.000     0.000         0
atis_aircraft#atis_flight#atis_flight_no      0.000     0.000     0.000         0

                               micro avg      0.974     0.980     0.977       888
                               macro avg      0.641     0.661     0.633       888
                            weighted avg      0.977     0.980     0.976       888

I0716 03:24:32.672422 139795883440000 interactiveshell.py:2882] 
                              precision    recall  f1-score   support

                           O      0.000     0.000     0.000       0.0
           B-toloc.city_name      0.974     0.992     0.983     716.0
         B-fromloc.city_name      0.989     0.999     0.994     704.0
           I-toloc.city_name      0.967     0.996     0.981     265.0
      B-depart_date.day_name      0.986     0.991     0.988     212.0
              B-airline_name      0.981     1.000     0.990     101.0
         I-fromloc.city_name      0.972     0.994     0.983     177.0
 B-depart_time.period_of_day      0.983     0.915     0.948     130.0
              I-airline_name      1.000     0.969     0.984      65.0
    B-depart_date.day_number      0.964     0.964     0.964      55.0
    B-depart_date.month_name      0.982     0.964     0.973      56.0
          B-depart_time.time      0.826     1.000     0.905      57.0
                B-round_trip      1.000     0.986     0.993      73.0
             B-cost_relative      0.973     0.973     0.973      37.0
                I-round_trip      0.986     1.000     0.993      71.0
                B-flight_mod      0.800     1.000     0.889      24.0
 B-depart_time.time_relative      0.970     1.000     0.985      65.0
          I-depart_time.time      0.929     1.000     0.963      52.0
         B-stoploc.city_name      0.870     1.000     0.930      20.0
                 B-city_name      0.895     0.596     0.716      57.0
                B-class_type      0.960     1.000     0.980      24.0
          B-arrive_time.time      1.000     0.971     0.985      34.0
 B-arrive_time.time_relative      0.882     0.968     0.923      31.0
                I-class_type      1.000     1.000     1.000      17.0
               B-flight_stop      1.000     1.000     1.000      21.0
          I-arrive_time.time      1.000     0.971     0.986      35.0
              B-airline_code      0.810     1.000     0.895      34.0
    I-depart_date.day_number      1.000     0.933     0.966      15.0
      I-fromloc.airport_name      0.484     1.000     0.652      15.0
      B-fromloc.airport_name      0.400     1.000     0.571      12.0
      B-arrive_date.day_name      0.786     1.000     0.880      11.0
          B-toloc.state_code      1.000     1.000     1.000      18.0
B-depart_date.today_relative      1.000     0.778     0.875       9.0
             B-flight_number      0.786     1.000     0.880      11.0
 B-depart_date.date_relative      0.895     1.000     0.944      17.0
          B-toloc.state_name      0.833     0.893     0.862      28.0
           B-fare_basis_code      0.895     1.000     0.944      17.0
               B-flight_time      1.000     1.000     1.000       1.0
                        B-or      1.000     1.000     1.000       3.0
 B-arrive_time.period_of_day      0.750     1.000     0.857       6.0
          B-meal_description      1.000     0.900     0.947      10.0
             I-cost_relative      1.000     0.667     0.800       3.0
              I-airport_name      0.765     0.448     0.565      29.0
               B-fare_amount      1.000     1.000     1.000       2.0
               I-fare_amount      1.000     1.000     1.000       2.0
                 I-city_name      1.000     0.533     0.696      30.0
        I-toloc.airport_name      1.000     1.000     1.000       3.0
            B-transport_type      0.909     1.000     0.952      10.0
    B-arrive_date.month_name      0.714     0.833     0.769       6.0
    B-arrive_date.day_number      0.625     0.833     0.714       6.0
         I-stoploc.city_name      0.909     1.000     0.952      10.0
                      B-meal      1.000     1.000     1.000      16.0
        B-fromloc.state_code      1.000     1.000     1.000      23.0
    B-depart_time.period_mod      1.000     1.000     1.000       5.0
                   B-connect      1.000     1.000     1.000       6.0
               B-flight_days      1.000     1.000     1.000      10.0
        B-toloc.airport_name      1.000     1.000     1.000       3.0
        B-fromloc.state_name      0.941     0.941     0.941      17.0
              B-airport_name      0.636     0.333     0.437      21.0
                   B-economy      1.000     0.333     0.500       6.0
               I-flight_time      1.000     1.000     1.000       1.0
             B-aircraft_code      1.000     0.758     0.862      33.0
                       B-mod      1.000     0.500     0.667       2.0
              B-airport_code      1.000     0.444     0.615       9.0
    B-depart_time.start_time      1.000     0.667     0.800       3.0
      B-depart_time.end_time      1.000     0.333     0.500       3.0
          B-depart_date.year      1.000     1.000     1.000       3.0
            I-transport_type      0.000     0.000     0.000       1.0
          B-restriction_code      1.000     0.500     0.667       4.0
    B-arrive_time.start_time      0.889     1.000     0.941       8.0
        B-toloc.airport_code      1.000     0.750     0.857       4.0
      B-arrive_time.end_time      0.889     1.000     0.941       8.0
      I-arrive_time.end_time      1.000     1.000     1.000       8.0
      I-depart_time.end_time      1.000     0.667     0.800       3.0
               I-flight_stop      0.000     0.000     0.000       0.0
      B-fromloc.airport_code      0.714     1.000     0.833       5.0
          I-restriction_code      1.000     1.000     1.000       3.0
    I-depart_time.start_time      1.000     1.000     1.000       1.0
          I-toloc.state_name      1.000     1.000     1.000       1.0
I-depart_date.today_relative      0.000     0.000     0.000       0.0
 B-arrive_date.date_relative      1.000     1.000     1.000       2.0
                I-flight_mod      0.000     0.000     0.000       6.0
                   I-economy      0.000     0.000     0.000       0.0
 B-return_date.date_relative      0.667     0.667     0.667       3.0
        I-fromloc.state_name      0.000     0.000     0.000       1.0
                B-state_code      1.000     1.000     1.000       1.0
    I-arrive_time.start_time      1.000     1.000     1.000       1.0
    I-arrive_date.day_number      0.000     0.000     0.000       0.0
                 B-meal_code      0.000     0.000     0.000       1.0
 I-depart_time.period_of_day      0.000     0.000     0.000       1.0
                  B-day_name      1.000     0.500     0.667       2.0
             B-period_of_day      1.000     0.500     0.667       4.0
        B-stoploc.state_code      0.000     0.000     0.000       0.0
    B-return_date.month_name      0.000     0.000     0.000       0.0
    B-return_date.day_number      0.000     0.000     0.000       0.0
    B-arrive_time.period_mod      0.000     0.000     0.000       0.0
                 I-meal_code      0.000     0.000     0.000       0.0
        B-toloc.country_name      0.000     0.000     0.000       1.0
                 B-days_code      0.000     0.000     0.000       1.0
 I-arrive_time.period_of_day      0.000     0.000     0.000       0.0
            I-today_relative      0.000     0.000     0.000       0.0
 B-return_time.period_of_day      0.000     0.000     0.000       0.0
                      B-time      0.000     0.000     0.000       0.0
           I-fare_basis_code      0.000     0.000     0.000       0.0
 I-arrive_time.time_relative      0.000     0.000     0.000       4.0
 I-depart_time.time_relative      0.000     0.000     0.000       1.0
            B-today_relative      0.000     0.000     0.000       0.0
                B-state_name      0.000     0.000     0.000       9.0
B-arrive_date.today_relative      0.000     0.000     0.000       0.0
    B-return_time.period_mod      0.000     0.000     0.000       0.0
                B-month_name      0.000     0.000     0.000       0.0
                B-day_number      0.000     0.000     0.000       0.0
 I-return_date.date_relative      0.000     0.000     0.000       3.0
I-return_date.today_relative      0.000     0.000     0.000       0.0
      B-stoploc.airport_name      0.000     0.000     0.000       0.0
             B-time_relative      0.000     0.000     0.000       0.0
                      I-time      0.000     0.000     0.000       0.0
    I-return_date.day_number      0.000     0.000     0.000       0.0
          I-meal_description      0.000     0.000     0.000       0.0
B-return_date.today_relative      0.000     0.000     0.000       0.0
      B-return_date.day_name      0.000     0.000     0.000       2.0

                   micro avg      0.952     0.953     0.952    3657.0
                   macro avg      0.630     0.603     0.604    3657.0
                weighted avg      0.951     0.953     0.948    3657.0

I0716 03:24:32.695803 139795883440000 interactiveshell.py:2882] Best Slot F1: 0.952
I0716 03:24:32.697161 139795883440000 estimator_training.py:186] Not using Distribute Coordinator.
I0716 03:24:32.700752 139795883440000 training.py:612] Running training and evaluation locally (non-distributed).
I0716 03:24:32.704385 139795883440000 training.py:700] Start train and evaluate loop. The evaluate will happen after every checkpoint. Checkpoint frequency is determined based on RunConfig arguments: save_checkpoints_steps 312 or save_checkpoints_secs None.
I0716 03:24:32.768478 139795883440000 estimator.py:1145] Calling model_fn.
I0716 03:24:34.197245 139795883440000 saver.py:1499] Saver not created because there are no variables in the graph to restore
I0716 03:24:34.793243 139795883440000 <ipython-input-6-d6747b40360b>:55] 
[<tf.Variable 'embedding:0' shape=(750, 300) dtype=float32_ref>,
 <tf.Variable 'module/aggregation/weights:0' shape=(3,) dtype=float32>,
 <tf.Variable 'module/aggregation/scaling:0' shape=() dtype=float32>,
 <tf.Variable 'dense/kernel:0' shape=(1324, 300) dtype=float32_ref>,
 <tf.Variable 'dense/bias:0' shape=(300,) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/gates/kernel:0' shape=(600, 600) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/gates/bias:0' shape=(600,) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/candidate/kernel:0' shape=(600, 300) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/candidate/bias:0' shape=(300,) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/gates/kernel:0' shape=(600, 600) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/gates/bias:0' shape=(600,) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/candidate/kernel:0' shape=(600, 300) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/candidate/bias:0' shape=(300,) dtype=float32_ref>,
 <tf.Variable 'dense_1/kernel:0' shape=(600, 23) dtype=float32_ref>,
 <tf.Variable 'dense_1/bias:0' shape=(23,) dtype=float32_ref>,
 <tf.Variable 'dense_2/kernel:0' shape=(600, 122) dtype=float32_ref>,
 <tf.Variable 'dense_2/bias:0' shape=(122,) dtype=float32_ref>]
I0716 03:24:35.869684 139795883440000 estimator.py:1147] Done calling model_fn.
I0716 03:24:35.874225 139795883440000 basic_session_run_hooks.py:541] Create CheckpointSaverHook.
I0716 03:24:36.372488 139795883440000 monitored_session.py:240] Graph was finalized.
I0716 03:24:36.389989 139795883440000 saver.py:1280] Restoring parameters from ../model/elmo_bigru/model.ckpt-2184
I0716 03:24:37.422055 139795883440000 session_manager.py:500] Running local_init_op.
I0716 03:24:37.489724 139795883440000 session_manager.py:502] Done running local_init_op.
I0716 03:24:40.400031 139795883440000 basic_session_run_hooks.py:606] Saving checkpoints for 2184 into ../model/elmo_bigru/model.ckpt.
Reading ../data/atis.train.w-intent.iob
I0716 03:24:45.578757 139795883440000 basic_session_run_hooks.py:262] loss = 0.063943386, step = 2184
I0716 03:24:45.580540 139795883440000 basic_session_run_hooks.py:262] lr = 0.00023833448
I0716 03:25:12.153598 139795883440000 basic_session_run_hooks.py:692] global_step/sec: 3.76288
I0716 03:25:12.158720 139795883440000 basic_session_run_hooks.py:260] loss = 0.03020731, step = 2284 (26.580 sec)
I0716 03:25:12.161730 139795883440000 basic_session_run_hooks.py:260] lr = 0.00023583656 (26.581 sec)
I0716 03:25:39.568844 139795883440000 basic_session_run_hooks.py:692] global_step/sec: 3.6476
I0716 03:25:39.576084 139795883440000 basic_session_run_hooks.py:260] loss = 0.03212006, step = 2384 (27.417 sec)
I0716 03:25:39.578353 139795883440000 basic_session_run_hooks.py:260] lr = 0.00023336482 (27.417 sec)
I0716 03:26:05.588598 139795883440000 basic_session_run_hooks.py:692] global_step/sec: 3.84323
I0716 03:26:05.595238 139795883440000 basic_session_run_hooks.py:260] loss = 0.14932747, step = 2484 (26.019 sec)
I0716 03:26:05.597378 139795883440000 basic_session_run_hooks.py:260] lr = 0.00023091899 (26.019 sec)
I0716 03:26:08.265371 139795883440000 basic_session_run_hooks.py:606] Saving checkpoints for 2496 into ../model/elmo_bigru/model.ckpt.
I0716 03:26:10.106337 139795883440000 estimator.py:1145] Calling model_fn.
I0716 03:26:11.422580 139795883440000 saver.py:1499] Saver not created because there are no variables in the graph to restore
I0716 03:26:12.238691 139795883440000 estimator.py:1147] Done calling model_fn.
I0716 03:26:12.267982 139795883440000 evaluation.py:255] Starting evaluation at 2019-07-16T03:26:12Z
I0716 03:26:12.498234 139795883440000 monitored_session.py:240] Graph was finalized.
I0716 03:26:12.514398 139795883440000 saver.py:1280] Restoring parameters from ../model/elmo_bigru/model.ckpt-2496
I0716 03:26:13.408679 139795883440000 session_manager.py:500] Running local_init_op.
I0716 03:26:13.464711 139795883440000 session_manager.py:502] Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
I0716 03:27:11.979592 139795883440000 evaluation.py:275] Finished evaluation at 2019-07-16-03:27:11
I0716 03:27:11.981393 139795883440000 estimator.py:2039] Saving dict for global step 2496: global_step = 2496, loss = 0.280597
I0716 03:27:11.989431 139795883440000 estimator.py:2099] Saving 'checkpoint_path' summary for global step 2496: ../model/elmo_bigru/model.ckpt-2496
I0716 03:27:12.073182 139795883440000 estimator.py:368] Loss for final step: 0.07099749.
Reading ../data/atis.test.w-intent.iob
I0716 03:27:12.656421 139795883440000 estimator.py:1145] Calling model_fn.
I0716 03:27:13.679173 139795883440000 saver.py:1499] Saver not created because there are no variables in the graph to restore
I0716 03:27:14.652549 139795883440000 estimator.py:1147] Done calling model_fn.
I0716 03:27:14.870477 139795883440000 monitored_session.py:240] Graph was finalized.
I0716 03:27:14.887186 139795883440000 saver.py:1280] Restoring parameters from ../model/elmo_bigru/model.ckpt-2496
I0716 03:27:15.759628 139795883440000 session_manager.py:500] Running local_init_op.
I0716 03:27:15.808227 139795883440000 session_manager.py:502] Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.
  'precision', 'predicted', average, warn_for)
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1439: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples.
  'recall', 'true', average, warn_for)
I0716 03:28:13.653455 139795883440000 interactiveshell.py:2882] 
                                          precision    recall  f1-score   support

                             atis_flight      0.987     0.992     0.990       632
                            atis_airfare      0.941     1.000     0.970        48
                     atis_ground_service      0.973     1.000     0.986        36
                            atis_airline      1.000     1.000     1.000        38
                       atis_abbreviation      0.943     1.000     0.971        33
                           atis_aircraft      1.000     0.889     0.941         9
                        atis_flight_time      0.333     1.000     0.500         1
                           atis_quantity      0.429     1.000     0.600         3
                atis_flight#atis_airfare      0.857     0.500     0.632        12
                            atis_airport      1.000     0.944     0.971        18
                           atis_distance      1.000     1.000     1.000        10
                               atis_city      0.600     0.500     0.545         6
                        atis_ground_fare      1.000     0.857     0.923         7
                           atis_capacity      0.952     0.952     0.952        21
                          atis_flight_no      0.889     1.000     0.941         8
                               atis_meal      1.000     0.667     0.800         6
                        atis_restriction      0.000     0.000     0.000         0
             atis_airline#atis_flight_no      0.000     0.000     0.000         0
    atis_ground_service#atis_ground_fare      0.000     0.000     0.000         0
           atis_airfare#atis_flight_time      0.000     0.000     0.000         0
                           atis_cheapest      0.000     0.000     0.000         0
atis_aircraft#atis_flight#atis_flight_no      0.000     0.000     0.000         0

                               micro avg      0.972     0.977     0.975       888
                               macro avg      0.632     0.650     0.624       888
                            weighted avg      0.975     0.977     0.975       888

I0716 03:28:13.689820 139795883440000 interactiveshell.py:2882] 
                              precision    recall  f1-score   support

                           O      0.000     0.000     0.000       0.0
           B-toloc.city_name      0.974     0.993     0.983     716.0
         B-fromloc.city_name      0.989     0.999     0.994     704.0
           I-toloc.city_name      0.964     1.000     0.981     265.0
      B-depart_date.day_name      0.990     0.972     0.981     212.0
              B-airline_name      0.990     1.000     0.995     101.0
         I-fromloc.city_name      0.994     0.994     0.994     177.0
 B-depart_time.period_of_day      0.983     0.915     0.948     130.0
              I-airline_name      1.000     0.985     0.992      65.0
    B-depart_date.day_number      0.981     0.945     0.963      55.0
    B-depart_date.month_name      0.981     0.946     0.964      56.0
          B-depart_time.time      0.851     1.000     0.919      57.0
                B-round_trip      1.000     1.000     1.000      73.0
             B-cost_relative      0.973     0.973     0.973      37.0
                I-round_trip      1.000     1.000     1.000      71.0
                B-flight_mod      0.800     1.000     0.889      24.0
 B-depart_time.time_relative      0.969     0.954     0.961      65.0
          I-depart_time.time      0.945     1.000     0.972      52.0
         B-stoploc.city_name      0.909     1.000     0.952      20.0
                 B-city_name      0.900     0.632     0.742      57.0
                B-class_type      0.960     1.000     0.980      24.0
          B-arrive_time.time      0.971     0.971     0.971      34.0
 B-arrive_time.time_relative      0.833     0.968     0.896      31.0
                I-class_type      1.000     1.000     1.000      17.0
               B-flight_stop      1.000     1.000     1.000      21.0
          I-arrive_time.time      1.000     0.971     0.986      35.0
              B-airline_code      0.850     1.000     0.919      34.0
    I-depart_date.day_number      1.000     0.933     0.966      15.0
      I-fromloc.airport_name      0.405     1.000     0.577      15.0
      B-fromloc.airport_name      0.414     1.000     0.585      12.0
      B-arrive_date.day_name      0.611     1.000     0.759      11.0
          B-toloc.state_code      1.000     1.000     1.000      18.0
B-depart_date.today_relative      1.000     0.889     0.941       9.0
             B-flight_number      0.786     1.000     0.880      11.0
 B-depart_date.date_relative      0.944     1.000     0.971      17.0
          B-toloc.state_name      0.833     0.893     0.862      28.0
           B-fare_basis_code      0.850     1.000     0.919      17.0
               B-flight_time      1.000     1.000     1.000       1.0
                        B-or      1.000     1.000     1.000       3.0
 B-arrive_time.period_of_day      0.750     1.000     0.857       6.0
          B-meal_description      1.000     0.900     0.947      10.0
             I-cost_relative      1.000     0.667     0.800       3.0
              I-airport_name      0.833     0.345     0.488      29.0
               B-fare_amount      1.000     1.000     1.000       2.0
               I-fare_amount      1.000     1.000     1.000       2.0
                 I-city_name      1.000     0.567     0.723      30.0
        I-toloc.airport_name      1.000     1.000     1.000       3.0
            B-transport_type      1.000     1.000     1.000      10.0
    B-arrive_date.month_name      0.625     0.833     0.714       6.0
    B-arrive_date.day_number      0.556     0.833     0.667       6.0
         I-stoploc.city_name      0.909     1.000     0.952      10.0
                      B-meal      1.000     1.000     1.000      16.0
        B-fromloc.state_code      1.000     1.000     1.000      23.0
    B-depart_time.period_mod      0.833     1.000     0.909       5.0
                   B-connect      1.000     1.000     1.000       6.0
               B-flight_days      1.000     1.000     1.000      10.0
        B-toloc.airport_name      1.000     1.000     1.000       3.0
        B-fromloc.state_name      0.941     0.941     0.941      17.0
              B-airport_name      0.636     0.333     0.437      21.0
                   B-economy      1.000     0.333     0.500       6.0
               I-flight_time      1.000     1.000     1.000       1.0
             B-aircraft_code      1.000     0.848     0.918      33.0
                       B-mod      1.000     0.500     0.667       2.0
              B-airport_code      1.000     0.444     0.615       9.0
    B-depart_time.start_time      1.000     0.667     0.800       3.0
      B-depart_time.end_time      1.000     0.333     0.500       3.0
          B-depart_date.year      1.000     1.000     1.000       3.0
            I-transport_type      1.000     1.000     1.000       1.0
          B-restriction_code      1.000     0.500     0.667       4.0
    B-arrive_time.start_time      0.889     1.000     0.941       8.0
        B-toloc.airport_code      1.000     0.750     0.857       4.0
      B-arrive_time.end_time      0.889     1.000     0.941       8.0
      I-arrive_time.end_time      0.889     1.000     0.941       8.0
      I-depart_time.end_time      1.000     0.333     0.500       3.0
               I-flight_stop      0.000     0.000     0.000       0.0
      B-fromloc.airport_code      0.625     1.000     0.769       5.0
          I-restriction_code      1.000     1.000     1.000       3.0
    I-depart_time.start_time      1.000     1.000     1.000       1.0
          I-toloc.state_name      1.000     1.000     1.000       1.0
I-depart_date.today_relative      0.000     0.000     0.000       0.0
 B-arrive_date.date_relative      1.000     1.000     1.000       2.0
                I-flight_mod      0.000     0.000     0.000       6.0
                   I-economy      0.000     0.000     0.000       0.0
 B-return_date.date_relative      0.500     0.667     0.571       3.0
        I-fromloc.state_name      0.000     0.000     0.000       1.0
                B-state_code      1.000     1.000     1.000       1.0
    I-arrive_time.start_time      1.000     1.000     1.000       1.0
    I-arrive_date.day_number      0.000     0.000     0.000       0.0
                 B-meal_code      0.000     0.000     0.000       1.0
 I-depart_time.period_of_day      0.000     0.000     0.000       1.0
                  B-day_name      1.000     0.500     0.667       2.0
             B-period_of_day      1.000     0.750     0.857       4.0
        B-stoploc.state_code      0.000     0.000     0.000       0.0
    B-return_date.month_name      0.000     0.000     0.000       0.0
    B-return_date.day_number      0.000     0.000     0.000       0.0
    B-arrive_time.period_mod      0.000     0.000     0.000       0.0
                 I-meal_code      0.000     0.000     0.000       0.0
        B-toloc.country_name      1.000     1.000     1.000       1.0
                 B-days_code      0.000     0.000     0.000       1.0
 I-arrive_time.period_of_day      0.000     0.000     0.000       0.0
            I-today_relative      0.000     0.000     0.000       0.0
 B-return_time.period_of_day      0.000     0.000     0.000       0.0
                      B-time      0.000     0.000     0.000       0.0
           I-fare_basis_code      0.000     0.000     0.000       0.0
 I-arrive_time.time_relative      0.000     0.000     0.000       4.0
 I-depart_time.time_relative      0.000     0.000     0.000       1.0
            B-today_relative      0.000     0.000     0.000       0.0
                B-state_name      0.000     0.000     0.000       9.0
B-arrive_date.today_relative      0.000     0.000     0.000       0.0
    B-return_time.period_mod      0.000     0.000     0.000       0.0
                B-month_name      0.000     0.000     0.000       0.0
                B-day_number      0.000     0.000     0.000       0.0
 I-return_date.date_relative      0.000     0.000     0.000       3.0
I-return_date.today_relative      0.000     0.000     0.000       0.0
      B-stoploc.airport_name      0.000     0.000     0.000       0.0
             B-time_relative      0.000     0.000     0.000       0.0
                      I-time      0.000     0.000     0.000       0.0
    I-return_date.day_number      0.000     0.000     0.000       0.0
          I-meal_description      0.000     0.000     0.000       0.0
B-return_date.today_relative      0.000     0.000     0.000       0.0
      B-return_date.day_name      0.000     0.000     0.000       2.0

                   micro avg      0.952     0.954     0.953    3657.0
                   macro avg      0.641     0.620     0.616    3657.0
                weighted avg      0.953     0.954     0.949    3657.0

I0716 03:28:13.711204 139795883440000 interactiveshell.py:2882] Best Slot F1: 0.953
I0716 03:28:13.712770 139795883440000 estimator_training.py:186] Not using Distribute Coordinator.
I0716 03:28:13.714238 139795883440000 training.py:612] Running training and evaluation locally (non-distributed).
I0716 03:28:13.717311 139795883440000 training.py:700] Start train and evaluate loop. The evaluate will happen after every checkpoint. Checkpoint frequency is determined based on RunConfig arguments: save_checkpoints_steps 312 or save_checkpoints_secs None.
I0716 03:28:13.781014 139795883440000 estimator.py:1145] Calling model_fn.
I0716 03:28:15.236994 139795883440000 saver.py:1499] Saver not created because there are no variables in the graph to restore
I0716 03:28:15.821968 139795883440000 <ipython-input-6-d6747b40360b>:55] 
[<tf.Variable 'embedding:0' shape=(750, 300) dtype=float32_ref>,
 <tf.Variable 'module/aggregation/weights:0' shape=(3,) dtype=float32>,
 <tf.Variable 'module/aggregation/scaling:0' shape=() dtype=float32>,
 <tf.Variable 'dense/kernel:0' shape=(1324, 300) dtype=float32_ref>,
 <tf.Variable 'dense/bias:0' shape=(300,) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/gates/kernel:0' shape=(600, 600) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/gates/bias:0' shape=(600,) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/candidate/kernel:0' shape=(600, 300) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/candidate/bias:0' shape=(300,) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/gates/kernel:0' shape=(600, 600) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/gates/bias:0' shape=(600,) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/candidate/kernel:0' shape=(600, 300) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/candidate/bias:0' shape=(300,) dtype=float32_ref>,
 <tf.Variable 'dense_1/kernel:0' shape=(600, 23) dtype=float32_ref>,
 <tf.Variable 'dense_1/bias:0' shape=(23,) dtype=float32_ref>,
 <tf.Variable 'dense_2/kernel:0' shape=(600, 122) dtype=float32_ref>,
 <tf.Variable 'dense_2/bias:0' shape=(122,) dtype=float32_ref>]
I0716 03:28:17.068133 139795883440000 estimator.py:1147] Done calling model_fn.
I0716 03:28:17.072277 139795883440000 basic_session_run_hooks.py:541] Create CheckpointSaverHook.
I0716 03:28:17.395323 139795883440000 monitored_session.py:240] Graph was finalized.
I0716 03:28:17.413097 139795883440000 saver.py:1280] Restoring parameters from ../model/elmo_bigru/model.ckpt-2496
I0716 03:28:18.424999 139795883440000 session_manager.py:500] Running local_init_op.
I0716 03:28:18.496368 139795883440000 session_manager.py:502] Done running local_init_op.
I0716 03:28:21.246337 139795883440000 basic_session_run_hooks.py:606] Saving checkpoints for 2496 into ../model/elmo_bigru/model.ckpt.
Reading ../data/atis.train.w-intent.iob
I0716 03:28:25.928198 139795883440000 basic_session_run_hooks.py:262] loss = 0.040984906, step = 2496
I0716 03:28:25.930085 139795883440000 basic_session_run_hooks.py:262] lr = 0.0002306272
I0716 03:28:52.934669 139795883440000 basic_session_run_hooks.py:692] global_step/sec: 3.70273
I0716 03:28:52.941412 139795883440000 basic_session_run_hooks.py:260] loss = 0.06381842, step = 2596 (27.013 sec)
I0716 03:28:52.943911 139795883440000 basic_session_run_hooks.py:260] lr = 0.00022821005 (27.014 sec)
I0716 03:29:19.840715 139795883440000 basic_session_run_hooks.py:692] global_step/sec: 3.71663
I0716 03:29:19.844252 139795883440000 basic_session_run_hooks.py:260] loss = 0.023345564, step = 2696 (26.903 sec)
I0716 03:29:19.846118 139795883440000 basic_session_run_hooks.py:260] lr = 0.00022581825 (26.902 sec)
I0716 03:29:46.032870 139795883440000 basic_session_run_hooks.py:692] global_step/sec: 3.81795
I0716 03:29:46.036398 139795883440000 basic_session_run_hooks.py:260] loss = 0.038965724, step = 2796 (26.192 sec)
I0716 03:29:46.041105 139795883440000 basic_session_run_hooks.py:260] lr = 0.00022345151 (26.195 sec)
I0716 03:29:48.910070 139795883440000 basic_session_run_hooks.py:606] Saving checkpoints for 2808 into ../model/elmo_bigru/model.ckpt.
I0716 03:29:51.028875 139795883440000 estimator.py:1145] Calling model_fn.
I0716 03:29:52.357199 139795883440000 saver.py:1499] Saver not created because there are no variables in the graph to restore
I0716 03:29:53.193941 139795883440000 estimator.py:1147] Done calling model_fn.
I0716 03:29:53.224582 139795883440000 evaluation.py:255] Starting evaluation at 2019-07-16T03:29:53Z
I0716 03:29:53.445019 139795883440000 monitored_session.py:240] Graph was finalized.
I0716 03:29:53.458856 139795883440000 saver.py:1280] Restoring parameters from ../model/elmo_bigru/model.ckpt-2808
I0716 03:29:54.366625 139795883440000 session_manager.py:500] Running local_init_op.
I0716 03:29:54.422200 139795883440000 session_manager.py:502] Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
I0716 03:30:53.035401 139795883440000 evaluation.py:275] Finished evaluation at 2019-07-16-03:30:53
I0716 03:30:53.037463 139795883440000 estimator.py:2039] Saving dict for global step 2808: global_step = 2808, loss = 0.2947929
I0716 03:30:53.044780 139795883440000 estimator.py:2099] Saving 'checkpoint_path' summary for global step 2808: ../model/elmo_bigru/model.ckpt-2808
I0716 03:30:53.122354 139795883440000 estimator.py:368] Loss for final step: 0.00338955.
Reading ../data/atis.test.w-intent.iob
I0716 03:30:53.703723 139795883440000 estimator.py:1145] Calling model_fn.
I0716 03:30:54.741008 139795883440000 saver.py:1499] Saver not created because there are no variables in the graph to restore
I0716 03:30:55.715564 139795883440000 estimator.py:1147] Done calling model_fn.
I0716 03:30:55.930930 139795883440000 monitored_session.py:240] Graph was finalized.
I0716 03:30:55.946676 139795883440000 saver.py:1280] Restoring parameters from ../model/elmo_bigru/model.ckpt-2808
I0716 03:30:56.829934 139795883440000 session_manager.py:500] Running local_init_op.
I0716 03:30:56.878928 139795883440000 session_manager.py:502] Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.
  'precision', 'predicted', average, warn_for)
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1439: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples.
  'recall', 'true', average, warn_for)
I0716 03:31:54.581436 139795883440000 interactiveshell.py:2882] 
                                          precision    recall  f1-score   support

                             atis_flight      0.984     0.992     0.988       632
                            atis_airfare      0.941     1.000     0.970        48
                     atis_ground_service      0.973     1.000     0.986        36
                            atis_airline      1.000     1.000     1.000        38
                       atis_abbreviation      0.943     1.000     0.971        33
                           atis_aircraft      1.000     0.889     0.941         9
                        atis_flight_time      0.333     1.000     0.500         1
                           atis_quantity      0.429     1.000     0.600         3
                atis_flight#atis_airfare      0.833     0.417     0.556        12
                            atis_airport      1.000     0.944     0.971        18
                           atis_distance      1.000     1.000     1.000        10
                               atis_city      1.000     0.500     0.667         6
                        atis_ground_fare      1.000     0.857     0.923         7
                           atis_capacity      0.952     0.952     0.952        21
                          atis_flight_no      0.889     1.000     0.941         8
                               atis_meal      1.000     0.833     0.909         6
                        atis_restriction      0.000     0.000     0.000         0
             atis_airline#atis_flight_no      0.000     0.000     0.000         0
    atis_ground_service#atis_ground_fare      0.000     0.000     0.000         0
           atis_airfare#atis_flight_time      0.000     0.000     0.000         0
                           atis_cheapest      0.000     0.000     0.000         0
atis_aircraft#atis_flight#atis_flight_no      0.000     0.000     0.000         0

                               micro avg      0.972     0.977     0.975       888
                               macro avg      0.649     0.654     0.631       888
                            weighted avg      0.975     0.977     0.974       888

I0716 03:31:54.617678 139795883440000 interactiveshell.py:2882] 
                              precision    recall  f1-score   support

                           O      0.000     0.000     0.000       0.0
           B-toloc.city_name      0.974     0.993     0.983     716.0
         B-fromloc.city_name      0.992     1.000     0.996     704.0
           I-toloc.city_name      0.967     0.992     0.980     265.0
      B-depart_date.day_name      0.991     0.991     0.991     212.0
              B-airline_name      1.000     1.000     1.000     101.0
         I-fromloc.city_name      0.967     0.994     0.981     177.0
 B-depart_time.period_of_day      0.983     0.908     0.944     130.0
              I-airline_name      0.985     1.000     0.992      65.0
    B-depart_date.day_number      0.981     0.964     0.972      55.0
    B-depart_date.month_name      0.981     0.946     0.964      56.0
          B-depart_time.time      0.851     1.000     0.919      57.0
                B-round_trip      1.000     1.000     1.000      73.0
             B-cost_relative      0.973     0.973     0.973      37.0
                I-round_trip      1.000     1.000     1.000      71.0
                B-flight_mod      0.828     1.000     0.906      24.0
 B-depart_time.time_relative      0.970     1.000     0.985      65.0
          I-depart_time.time      0.912     1.000     0.954      52.0
         B-stoploc.city_name      0.870     1.000     0.930      20.0
                 B-city_name      0.917     0.579     0.710      57.0
                B-class_type      0.960     1.000     0.980      24.0
          B-arrive_time.time      1.000     0.971     0.985      34.0
 B-arrive_time.time_relative      0.882     0.968     0.923      31.0
                I-class_type      1.000     1.000     1.000      17.0
               B-flight_stop      1.000     1.000     1.000      21.0
          I-arrive_time.time      1.000     0.971     0.986      35.0
              B-airline_code      0.872     1.000     0.932      34.0
    I-depart_date.day_number      1.000     0.933     0.966      15.0
      I-fromloc.airport_name      0.441     1.000     0.612      15.0
      B-fromloc.airport_name      0.414     1.000     0.585      12.0
      B-arrive_date.day_name      0.786     1.000     0.880      11.0
          B-toloc.state_code      1.000     1.000     1.000      18.0
B-depart_date.today_relative      1.000     0.889     0.941       9.0
             B-flight_number      0.786     1.000     0.880      11.0
 B-depart_date.date_relative      0.944     1.000     0.971      17.0
          B-toloc.state_name      0.833     0.893     0.862      28.0
           B-fare_basis_code      0.895     1.000     0.944      17.0
               B-flight_time      1.000     1.000     1.000       1.0
                        B-or      1.000     1.000     1.000       3.0
 B-arrive_time.period_of_day      0.750     1.000     0.857       6.0
          B-meal_description      1.000     0.900     0.947      10.0
             I-cost_relative      1.000     0.667     0.800       3.0
              I-airport_name      0.846     0.379     0.524      29.0
               B-fare_amount      1.000     1.000     1.000       2.0
               I-fare_amount      1.000     1.000     1.000       2.0
                 I-city_name      1.000     0.567     0.723      30.0
        I-toloc.airport_name      1.000     1.000     1.000       3.0
            B-transport_type      1.000     1.000     1.000      10.0
    B-arrive_date.month_name      0.625     0.833     0.714       6.0
    B-arrive_date.day_number      0.625     0.833     0.714       6.0
         I-stoploc.city_name      0.909     1.000     0.952      10.0
                      B-meal      1.000     1.000     1.000      16.0
        B-fromloc.state_code      1.000     1.000     1.000      23.0
    B-depart_time.period_mod      1.000     1.000     1.000       5.0
                   B-connect      1.000     1.000     1.000       6.0
               B-flight_days      1.000     1.000     1.000      10.0
        B-toloc.airport_name      1.000     1.000     1.000       3.0
        B-fromloc.state_name      0.895     1.000     0.944      17.0
              B-airport_name      0.636     0.333     0.437      21.0
                   B-economy      1.000     0.333     0.500       6.0
               I-flight_time      1.000     1.000     1.000       1.0
             B-aircraft_code      1.000     0.848     0.918      33.0
                       B-mod      1.000     0.500     0.667       2.0
              B-airport_code      1.000     0.444     0.615       9.0
    B-depart_time.start_time      1.000     1.000     1.000       3.0
      B-depart_time.end_time      1.000     0.667     0.800       3.0
          B-depart_date.year      1.000     1.000     1.000       3.0
            I-transport_type      0.000     0.000     0.000       1.0
          B-restriction_code      0.667     0.500     0.571       4.0
    B-arrive_time.start_time      1.000     1.000     1.000       8.0
        B-toloc.airport_code      1.000     0.750     0.857       4.0
      B-arrive_time.end_time      1.000     1.000     1.000       8.0
      I-arrive_time.end_time      1.000     1.000     1.000       8.0
      I-depart_time.end_time      1.000     0.667     0.800       3.0
               I-flight_stop      0.000     0.000     0.000       0.0
      B-fromloc.airport_code      0.625     1.000     0.769       5.0
          I-restriction_code      1.000     1.000     1.000       3.0
    I-depart_time.start_time      1.000     1.000     1.000       1.0
          I-toloc.state_name      1.000     1.000     1.000       1.0
I-depart_date.today_relative      0.000     0.000     0.000       0.0
 B-arrive_date.date_relative      1.000     1.000     1.000       2.0
                I-flight_mod      1.000     0.167     0.286       6.0
                   I-economy      0.000     0.000     0.000       0.0
 B-return_date.date_relative      0.667     0.667     0.667       3.0
        I-fromloc.state_name      0.000     0.000     0.000       1.0
                B-state_code      1.000     1.000     1.000       1.0
    I-arrive_time.start_time      1.000     1.000     1.000       1.0
    I-arrive_date.day_number      0.000     0.000     0.000       0.0
                 B-meal_code      0.000     0.000     0.000       1.0
 I-depart_time.period_of_day      0.000     0.000     0.000       1.0
                  B-day_name      1.000     0.500     0.667       2.0
             B-period_of_day      1.000     0.750     0.857       4.0
        B-stoploc.state_code      0.000     0.000     0.000       0.0
    B-return_date.month_name      0.000     0.000     0.000       0.0
    B-return_date.day_number      0.000     0.000     0.000       0.0
    B-arrive_time.period_mod      0.000     0.000     0.000       0.0
                 I-meal_code      0.000     0.000     0.000       0.0
        B-toloc.country_name      0.000     0.000     0.000       1.0
                 B-days_code      0.000     0.000     0.000       1.0
 I-arrive_time.period_of_day      0.000     0.000     0.000       0.0
            I-today_relative      0.000     0.000     0.000       0.0
 B-return_time.period_of_day      0.000     0.000     0.000       0.0
                      B-time      0.000     0.000     0.000       0.0
           I-fare_basis_code      0.000     0.000     0.000       0.0
 I-arrive_time.time_relative      0.000     0.000     0.000       4.0
 I-depart_time.time_relative      0.000     0.000     0.000       1.0
            B-today_relative      0.000     0.000     0.000       0.0
                B-state_name      0.000     0.000     0.000       9.0
B-arrive_date.today_relative      0.000     0.000     0.000       0.0
    B-return_time.period_mod      0.000     0.000     0.000       0.0
                B-month_name      0.000     0.000     0.000       0.0
                B-day_number      0.000     0.000     0.000       0.0
 I-return_date.date_relative      1.000     0.333     0.500       3.0
I-return_date.today_relative      0.000     0.000     0.000       0.0
      B-stoploc.airport_name      0.000     0.000     0.000       0.0
             B-time_relative      0.000     0.000     0.000       0.0
                      I-time      0.000     0.000     0.000       0.0
    I-return_date.day_number      0.000     0.000     0.000       0.0
          I-meal_description      0.000     0.000     0.000       0.0
B-return_date.today_relative      0.000     0.000     0.000       0.0
      B-return_date.day_name      0.000     0.000     0.000       2.0

                   micro avg      0.954     0.956     0.955    3657.0
                   macro avg      0.646     0.617     0.617    3657.0
                weighted avg      0.957     0.956     0.951    3657.0

I0716 03:31:54.637969 139795883440000 interactiveshell.py:2882] Best Slot F1: 0.955
I0716 03:31:54.639525 139795883440000 estimator_training.py:186] Not using Distribute Coordinator.
I0716 03:31:54.640842 139795883440000 training.py:612] Running training and evaluation locally (non-distributed).
I0716 03:31:54.644369 139795883440000 training.py:700] Start train and evaluate loop. The evaluate will happen after every checkpoint. Checkpoint frequency is determined based on RunConfig arguments: save_checkpoints_steps 312 or save_checkpoints_secs None.
I0716 03:31:54.707897 139795883440000 estimator.py:1145] Calling model_fn.
I0716 03:31:56.134528 139795883440000 saver.py:1499] Saver not created because there are no variables in the graph to restore
I0716 03:31:56.710400 139795883440000 <ipython-input-6-d6747b40360b>:55] 
[<tf.Variable 'embedding:0' shape=(750, 300) dtype=float32_ref>,
 <tf.Variable 'module/aggregation/weights:0' shape=(3,) dtype=float32>,
 <tf.Variable 'module/aggregation/scaling:0' shape=() dtype=float32>,
 <tf.Variable 'dense/kernel:0' shape=(1324, 300) dtype=float32_ref>,
 <tf.Variable 'dense/bias:0' shape=(300,) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/gates/kernel:0' shape=(600, 600) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/gates/bias:0' shape=(600,) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/candidate/kernel:0' shape=(600, 300) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/candidate/bias:0' shape=(300,) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/gates/kernel:0' shape=(600, 600) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/gates/bias:0' shape=(600,) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/candidate/kernel:0' shape=(600, 300) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/candidate/bias:0' shape=(300,) dtype=float32_ref>,
 <tf.Variable 'dense_1/kernel:0' shape=(600, 23) dtype=float32_ref>,
 <tf.Variable 'dense_1/bias:0' shape=(23,) dtype=float32_ref>,
 <tf.Variable 'dense_2/kernel:0' shape=(600, 122) dtype=float32_ref>,
 <tf.Variable 'dense_2/bias:0' shape=(122,) dtype=float32_ref>]
I0716 03:31:57.955630 139795883440000 estimator.py:1147] Done calling model_fn.
I0716 03:31:57.959849 139795883440000 basic_session_run_hooks.py:541] Create CheckpointSaverHook.
I0716 03:31:58.293367 139795883440000 monitored_session.py:240] Graph was finalized.
I0716 03:31:58.313203 139795883440000 saver.py:1280] Restoring parameters from ../model/elmo_bigru/model.ckpt-2808
I0716 03:31:59.349800 139795883440000 session_manager.py:500] Running local_init_op.
I0716 03:31:59.419791 139795883440000 session_manager.py:502] Done running local_init_op.
I0716 03:32:02.178851 139795883440000 basic_session_run_hooks.py:606] Saving checkpoints for 2808 into ../model/elmo_bigru/model.ckpt.
Reading ../data/atis.train.w-intent.iob
I0716 03:32:07.011035 139795883440000 basic_session_run_hooks.py:262] loss = 0.0189911, step = 2808
I0716 03:32:07.013246 139795883440000 basic_session_run_hooks.py:262] lr = 0.00022316918
I0716 03:32:34.563869 139795883440000 basic_session_run_hooks.py:692] global_step/sec: 3.62933
I0716 03:32:34.570090 139795883440000 basic_session_run_hooks.py:260] loss = 0.03583881, step = 2908 (27.559 sec)
I0716 03:32:34.573180 139795883440000 basic_session_run_hooks.py:260] lr = 0.00022083019 (27.560 sec)
I0716 03:33:01.101560 139795883440000 basic_session_run_hooks.py:692] global_step/sec: 3.76822
I0716 03:33:01.107414 139795883440000 basic_session_run_hooks.py:260] loss = 0.032352217, step = 3008 (26.537 sec)
I0716 03:33:01.111151 139795883440000 basic_session_run_hooks.py:260] lr = 0.00021851574 (26.538 sec)
I0716 03:33:27.693025 139795883440000 basic_session_run_hooks.py:692] global_step/sec: 3.76061
I0716 03:33:27.695716 139795883440000 basic_session_run_hooks.py:260] loss = 0.026988506, step = 3108 (26.588 sec)
I0716 03:33:27.700540 139795883440000 basic_session_run_hooks.py:260] lr = 0.00021622553 (26.589 sec)
I0716 03:33:30.370072 139795883440000 basic_session_run_hooks.py:606] Saving checkpoints for 3120 into ../model/elmo_bigru/model.ckpt.
I0716 03:33:32.249582 139795883440000 estimator.py:1145] Calling model_fn.
I0716 03:33:33.591853 139795883440000 saver.py:1499] Saver not created because there are no variables in the graph to restore
I0716 03:33:34.398364 139795883440000 estimator.py:1147] Done calling model_fn.
I0716 03:33:34.436570 139795883440000 evaluation.py:255] Starting evaluation at 2019-07-16T03:33:34Z
I0716 03:33:34.662846 139795883440000 monitored_session.py:240] Graph was finalized.
I0716 03:33:34.680534 139795883440000 saver.py:1280] Restoring parameters from ../model/elmo_bigru/model.ckpt-3120
I0716 03:33:35.565032 139795883440000 session_manager.py:500] Running local_init_op.
I0716 03:33:35.620205 139795883440000 session_manager.py:502] Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
I0716 03:34:34.403399 139795883440000 evaluation.py:275] Finished evaluation at 2019-07-16-03:34:34
I0716 03:34:34.405219 139795883440000 estimator.py:2039] Saving dict for global step 3120: global_step = 3120, loss = 0.29238617
I0716 03:34:34.413099 139795883440000 estimator.py:2099] Saving 'checkpoint_path' summary for global step 3120: ../model/elmo_bigru/model.ckpt-3120
I0716 03:34:34.496270 139795883440000 estimator.py:368] Loss for final step: 0.018303238.
Reading ../data/atis.test.w-intent.iob
I0716 03:34:35.101234 139795883440000 estimator.py:1145] Calling model_fn.
I0716 03:34:36.580038 139795883440000 saver.py:1499] Saver not created because there are no variables in the graph to restore
I0716 03:34:37.104504 139795883440000 estimator.py:1147] Done calling model_fn.
I0716 03:34:37.313228 139795883440000 monitored_session.py:240] Graph was finalized.
I0716 03:34:37.329845 139795883440000 saver.py:1280] Restoring parameters from ../model/elmo_bigru/model.ckpt-3120
I0716 03:34:38.225905 139795883440000 session_manager.py:500] Running local_init_op.
I0716 03:34:38.282472 139795883440000 session_manager.py:502] Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.
  'precision', 'predicted', average, warn_for)
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1439: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples.
  'recall', 'true', average, warn_for)
I0716 03:35:36.634260 139795883440000 interactiveshell.py:2882] 
                                          precision    recall  f1-score   support

                             atis_flight      0.984     0.992     0.988       632
                            atis_airfare      0.941     1.000     0.970        48
                     atis_ground_service      0.973     1.000     0.986        36
                            atis_airline      1.000     1.000     1.000        38
                       atis_abbreviation      0.943     1.000     0.971        33
                           atis_aircraft      1.000     0.889     0.941         9
                        atis_flight_time      0.333     1.000     0.500         1
                           atis_quantity      0.375     1.000     0.545         3
                atis_flight#atis_airfare      0.833     0.417     0.556        12
                            atis_airport      1.000     0.944     0.971        18
                           atis_distance      1.000     1.000     1.000        10
                               atis_city      1.000     0.500     0.667         6
                        atis_ground_fare      1.000     0.857     0.923         7
                           atis_capacity      1.000     0.952     0.976        21
                          atis_flight_no      0.889     1.000     0.941         8
                               atis_meal      1.000     0.833     0.909         6
                        atis_restriction      0.000     0.000     0.000         0
             atis_airline#atis_flight_no      0.000     0.000     0.000         0
    atis_ground_service#atis_ground_fare      0.000     0.000     0.000         0
           atis_airfare#atis_flight_time      0.000     0.000     0.000         0
                           atis_cheapest      0.000     0.000     0.000         0
atis_aircraft#atis_flight#atis_flight_no      0.000     0.000     0.000         0

                               micro avg      0.972     0.977     0.975       888
                               macro avg      0.649     0.654     0.629       888
                            weighted avg      0.976     0.977     0.974       888

I0716 03:35:36.670582 139795883440000 interactiveshell.py:2882] 
                              precision    recall  f1-score   support

                           O      0.000     0.000     0.000       0.0
           B-toloc.city_name      0.974     0.993     0.983     716.0
         B-fromloc.city_name      0.992     0.999     0.995     704.0
           I-toloc.city_name      0.974     0.989     0.981     265.0
      B-depart_date.day_name      0.991     0.991     0.991     212.0
              B-airline_name      1.000     1.000     1.000     101.0
         I-fromloc.city_name      0.978     0.994     0.986     177.0
 B-depart_time.period_of_day      0.984     0.923     0.952     130.0
              I-airline_name      1.000     1.000     1.000      65.0
    B-depart_date.day_number      0.981     0.964     0.972      55.0
    B-depart_date.month_name      0.981     0.946     0.964      56.0
          B-depart_time.time      0.864     1.000     0.927      57.0
                B-round_trip      1.000     1.000     1.000      73.0
             B-cost_relative      0.973     0.973     0.973      37.0
                I-round_trip      1.000     1.000     1.000      71.0
                B-flight_mod      0.828     1.000     0.906      24.0
 B-depart_time.time_relative      0.970     0.985     0.977      65.0
          I-depart_time.time      0.945     1.000     0.972      52.0
         B-stoploc.city_name      0.870     1.000     0.930      20.0
                 B-city_name      0.900     0.632     0.742      57.0
                B-class_type      0.960     1.000     0.980      24.0
          B-arrive_time.time      0.971     0.971     0.971      34.0
 B-arrive_time.time_relative      0.857     0.968     0.909      31.0
                I-class_type      1.000     1.000     1.000      17.0
               B-flight_stop      1.000     1.000     1.000      21.0
          I-arrive_time.time      1.000     0.971     0.986      35.0
              B-airline_code      0.895     1.000     0.944      34.0
    I-depart_date.day_number      1.000     0.933     0.966      15.0
      I-fromloc.airport_name      0.429     1.000     0.600      15.0
      B-fromloc.airport_name      0.400     1.000     0.571      12.0
      B-arrive_date.day_name      0.786     1.000     0.880      11.0
          B-toloc.state_code      1.000     1.000     1.000      18.0
B-depart_date.today_relative      1.000     0.889     0.941       9.0
             B-flight_number      0.786     1.000     0.880      11.0
 B-depart_date.date_relative      0.944     1.000     0.971      17.0
          B-toloc.state_name      0.833     0.893     0.862      28.0
           B-fare_basis_code      0.895     1.000     0.944      17.0
               B-flight_time      1.000     1.000     1.000       1.0
                        B-or      1.000     1.000     1.000       3.0
 B-arrive_time.period_of_day      0.750     1.000     0.857       6.0
          B-meal_description      1.000     0.900     0.947      10.0
             I-cost_relative      1.000     0.667     0.800       3.0
              I-airport_name      0.846     0.379     0.524      29.0
               B-fare_amount      1.000     1.000     1.000       2.0
               I-fare_amount      1.000     1.000     1.000       2.0
                 I-city_name      1.000     0.633     0.776      30.0
        I-toloc.airport_name      1.000     1.000     1.000       3.0
            B-transport_type      1.000     1.000     1.000      10.0
    B-arrive_date.month_name      0.625     0.833     0.714       6.0
    B-arrive_date.day_number      0.625     0.833     0.714       6.0
         I-stoploc.city_name      0.909     1.000     0.952      10.0
                      B-meal      1.000     1.000     1.000      16.0
        B-fromloc.state_code      1.000     1.000     1.000      23.0
    B-depart_time.period_mod      1.000     1.000     1.000       5.0
                   B-connect      1.000     1.000     1.000       6.0
               B-flight_days      1.000     1.000     1.000      10.0
        B-toloc.airport_name      1.000     1.000     1.000       3.0
        B-fromloc.state_name      1.000     1.000     1.000      17.0
              B-airport_name      0.636     0.333     0.437      21.0
                   B-economy      1.000     0.667     0.800       6.0
               I-flight_time      1.000     1.000     1.000       1.0
             B-aircraft_code      1.000     0.848     0.918      33.0
                       B-mod      1.000     0.500     0.667       2.0
              B-airport_code      1.000     0.444     0.615       9.0
    B-depart_time.start_time      1.000     0.667     0.800       3.0
      B-depart_time.end_time      1.000     0.333     0.500       3.0
          B-depart_date.year      1.000     1.000     1.000       3.0
            I-transport_type      1.000     1.000     1.000       1.0
          B-restriction_code      0.750     0.750     0.750       4.0
    B-arrive_time.start_time      0.889     1.000     0.941       8.0
        B-toloc.airport_code      1.000     0.750     0.857       4.0
      B-arrive_time.end_time      0.889     1.000     0.941       8.0
      I-arrive_time.end_time      0.889     1.000     0.941       8.0
      I-depart_time.end_time      1.000     0.333     0.500       3.0
               I-flight_stop      0.000     0.000     0.000       0.0
      B-fromloc.airport_code      0.625     1.000     0.769       5.0
          I-restriction_code      1.000     1.000     1.000       3.0
    I-depart_time.start_time      1.000     1.000     1.000       1.0
          I-toloc.state_name      1.000     1.000     1.000       1.0
I-depart_date.today_relative      0.000     0.000     0.000       0.0
 B-arrive_date.date_relative      1.000     1.000     1.000       2.0
                I-flight_mod      1.000     0.167     0.286       6.0
                   I-economy      0.000     0.000     0.000       0.0
 B-return_date.date_relative      0.500     0.333     0.400       3.0
        I-fromloc.state_name      1.000     1.000     1.000       1.0
                B-state_code      1.000     1.000     1.000       1.0
    I-arrive_time.start_time      1.000     1.000     1.000       1.0
    I-arrive_date.day_number      0.000     0.000     0.000       0.0
                 B-meal_code      0.000     0.000     0.000       1.0
 I-depart_time.period_of_day      0.000     0.000     0.000       1.0
                  B-day_name      1.000     0.500     0.667       2.0
             B-period_of_day      1.000     0.750     0.857       4.0
        B-stoploc.state_code      0.000     0.000     0.000       0.0
    B-return_date.month_name      0.000     0.000     0.000       0.0
    B-return_date.day_number      0.000     0.000     0.000       0.0
    B-arrive_time.period_mod      0.000     0.000     0.000       0.0
                 I-meal_code      0.000     0.000     0.000       0.0
        B-toloc.country_name      1.000     1.000     1.000       1.0
                 B-days_code      0.000     0.000     0.000       1.0
 I-arrive_time.period_of_day      0.000     0.000     0.000       0.0
            I-today_relative      0.000     0.000     0.000       0.0
 B-return_time.period_of_day      0.000     0.000     0.000       0.0
                      B-time      0.000     0.000     0.000       0.0
           I-fare_basis_code      0.000     0.000     0.000       0.0
 I-arrive_time.time_relative      0.000     0.000     0.000       4.0
 I-depart_time.time_relative      0.000     0.000     0.000       1.0
            B-today_relative      0.000     0.000     0.000       0.0
                B-state_name      0.000     0.000     0.000       9.0
B-arrive_date.today_relative      0.000     0.000     0.000       0.0
    B-return_time.period_mod      0.000     0.000     0.000       0.0
                B-month_name      0.000     0.000     0.000       0.0
                B-day_number      0.000     0.000     0.000       0.0
 I-return_date.date_relative      1.000     0.333     0.500       3.0
I-return_date.today_relative      0.000     0.000     0.000       0.0
      B-stoploc.airport_name      0.000     0.000     0.000       0.0
             B-time_relative      0.000     0.000     0.000       0.0
                      I-time      0.000     0.000     0.000       0.0
    I-return_date.day_number      0.000     0.000     0.000       0.0
          I-meal_description      0.000     0.000     0.000       0.0
B-return_date.today_relative      0.000     0.000     0.000       0.0
      B-return_date.day_name      0.000     0.000     0.000       2.0

                   micro avg      0.956     0.958     0.957    3657.0
                   macro avg      0.669     0.636     0.637    3657.0
                weighted avg      0.959     0.958     0.954    3657.0

I0716 03:35:36.690505 139795883440000 interactiveshell.py:2882] Best Slot F1: 0.957
I0716 03:35:36.692061 139795883440000 estimator_training.py:186] Not using Distribute Coordinator.
I0716 03:35:36.693465 139795883440000 training.py:612] Running training and evaluation locally (non-distributed).
I0716 03:35:36.696232 139795883440000 training.py:700] Start train and evaluate loop. The evaluate will happen after every checkpoint. Checkpoint frequency is determined based on RunConfig arguments: save_checkpoints_steps 312 or save_checkpoints_secs None.
I0716 03:35:36.762451 139795883440000 estimator.py:1145] Calling model_fn.
I0716 03:35:38.223496 139795883440000 saver.py:1499] Saver not created because there are no variables in the graph to restore
I0716 03:35:38.814246 139795883440000 <ipython-input-6-d6747b40360b>:55] 
[<tf.Variable 'embedding:0' shape=(750, 300) dtype=float32_ref>,
 <tf.Variable 'module/aggregation/weights:0' shape=(3,) dtype=float32>,
 <tf.Variable 'module/aggregation/scaling:0' shape=() dtype=float32>,
 <tf.Variable 'dense/kernel:0' shape=(1324, 300) dtype=float32_ref>,
 <tf.Variable 'dense/bias:0' shape=(300,) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/gates/kernel:0' shape=(600, 600) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/gates/bias:0' shape=(600,) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/candidate/kernel:0' shape=(600, 300) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/candidate/bias:0' shape=(300,) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/gates/kernel:0' shape=(600, 600) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/gates/bias:0' shape=(600,) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/candidate/kernel:0' shape=(600, 300) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/candidate/bias:0' shape=(300,) dtype=float32_ref>,
 <tf.Variable 'dense_1/kernel:0' shape=(600, 23) dtype=float32_ref>,
 <tf.Variable 'dense_1/bias:0' shape=(23,) dtype=float32_ref>,
 <tf.Variable 'dense_2/kernel:0' shape=(600, 122) dtype=float32_ref>,
 <tf.Variable 'dense_2/bias:0' shape=(122,) dtype=float32_ref>]
I0716 03:35:40.079383 139795883440000 estimator.py:1147] Done calling model_fn.
I0716 03:35:40.083529 139795883440000 basic_session_run_hooks.py:541] Create CheckpointSaverHook.
I0716 03:35:40.399303 139795883440000 monitored_session.py:240] Graph was finalized.
I0716 03:35:40.417211 139795883440000 saver.py:1280] Restoring parameters from ../model/elmo_bigru/model.ckpt-3120
I0716 03:35:41.444351 139795883440000 session_manager.py:500] Running local_init_op.
I0716 03:35:41.522269 139795883440000 session_manager.py:502] Done running local_init_op.
I0716 03:35:44.345134 139795883440000 basic_session_run_hooks.py:606] Saving checkpoints for 3120 into ../model/elmo_bigru/model.ckpt.
Reading ../data/atis.train.w-intent.iob
I0716 03:35:49.160026 139795883440000 basic_session_run_hooks.py:262] loss = 0.024522787, step = 3120
I0716 03:35:49.161925 139795883440000 basic_session_run_hooks.py:262] lr = 0.00021595233
I0716 03:36:17.451791 139795883440000 basic_session_run_hooks.py:692] global_step/sec: 3.53452
I0716 03:36:17.455758 139795883440000 basic_session_run_hooks.py:260] loss = 0.018387439, step = 3220 (28.296 sec)
I0716 03:36:17.459416 139795883440000 basic_session_run_hooks.py:260] lr = 0.00021368897 (28.297 sec)
I0716 03:36:43.690851 139795883440000 basic_session_run_hooks.py:692] global_step/sec: 3.81111
I0716 03:36:43.696416 139795883440000 basic_session_run_hooks.py:260] loss = 0.028793802, step = 3320 (26.241 sec)
I0716 03:36:43.700000 139795883440000 basic_session_run_hooks.py:260] lr = 0.00021144934 (26.241 sec)
I0716 03:37:10.121157 139795883440000 basic_session_run_hooks.py:692] global_step/sec: 3.78353
I0716 03:37:10.128551 139795883440000 basic_session_run_hooks.py:260] loss = 0.023584345, step = 3420 (26.432 sec)
I0716 03:37:10.131360 139795883440000 basic_session_run_hooks.py:260] lr = 0.0002092332 (26.431 sec)
I0716 03:37:13.255368 139795883440000 basic_session_run_hooks.py:606] Saving checkpoints for 3432 into ../model/elmo_bigru/model.ckpt.
I0716 03:37:15.275616 139795883440000 estimator.py:1145] Calling model_fn.
I0716 03:37:16.615622 139795883440000 saver.py:1499] Saver not created because there are no variables in the graph to restore
I0716 03:37:17.416712 139795883440000 estimator.py:1147] Done calling model_fn.
I0716 03:37:17.444679 139795883440000 evaluation.py:255] Starting evaluation at 2019-07-16T03:37:17Z
I0716 03:37:17.674989 139795883440000 monitored_session.py:240] Graph was finalized.
I0716 03:37:17.688753 139795883440000 saver.py:1280] Restoring parameters from ../model/elmo_bigru/model.ckpt-3432
I0716 03:37:18.592427 139795883440000 session_manager.py:500] Running local_init_op.
I0716 03:37:18.653311 139795883440000 session_manager.py:502] Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
I0716 03:38:17.490103 139795883440000 evaluation.py:275] Finished evaluation at 2019-07-16-03:38:17
I0716 03:38:17.492123 139795883440000 estimator.py:2039] Saving dict for global step 3432: global_step = 3432, loss = 0.28893816
I0716 03:38:17.498728 139795883440000 estimator.py:2099] Saving 'checkpoint_path' summary for global step 3432: ../model/elmo_bigru/model.ckpt-3432
I0716 03:38:17.585021 139795883440000 estimator.py:368] Loss for final step: 0.008152063.
Reading ../data/atis.test.w-intent.iob
I0716 03:38:18.181552 139795883440000 estimator.py:1145] Calling model_fn.
I0716 03:38:19.680203 139795883440000 saver.py:1499] Saver not created because there are no variables in the graph to restore
I0716 03:38:20.192303 139795883440000 estimator.py:1147] Done calling model_fn.
I0716 03:38:20.402776 139795883440000 monitored_session.py:240] Graph was finalized.
I0716 03:38:20.419113 139795883440000 saver.py:1280] Restoring parameters from ../model/elmo_bigru/model.ckpt-3432
I0716 03:38:21.312413 139795883440000 session_manager.py:500] Running local_init_op.
I0716 03:38:21.359920 139795883440000 session_manager.py:502] Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.
  'precision', 'predicted', average, warn_for)
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1439: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples.
  'recall', 'true', average, warn_for)
I0716 03:39:19.167973 139795883440000 interactiveshell.py:2882] 
                                          precision    recall  f1-score   support

                             atis_flight      0.986     0.992     0.989       632
                            atis_airfare      0.941     1.000     0.970        48
                     atis_ground_service      0.973     1.000     0.986        36
                            atis_airline      1.000     1.000     1.000        38
                       atis_abbreviation      0.943     1.000     0.971        33
                           atis_aircraft      1.000     0.889     0.941         9
                        atis_flight_time      0.333     1.000     0.500         1
                           atis_quantity      0.429     1.000     0.600         3
                atis_flight#atis_airfare      0.833     0.417     0.556        12
                            atis_airport      1.000     0.944     0.971        18
                           atis_distance      1.000     1.000     1.000        10
                               atis_city      0.750     0.500     0.600         6
                        atis_ground_fare      1.000     0.857     0.923         7
                           atis_capacity      0.952     0.952     0.952        21
                          atis_flight_no      0.889     1.000     0.941         8
                               atis_meal      1.000     0.833     0.909         6
                        atis_restriction      0.000     0.000     0.000         0
             atis_airline#atis_flight_no      0.000     0.000     0.000         0
    atis_ground_service#atis_ground_fare      0.000     0.000     0.000         0
           atis_airfare#atis_flight_time      0.000     0.000     0.000         0
                           atis_cheapest      0.000     0.000     0.000         0
atis_aircraft#atis_flight#atis_flight_no      0.000     0.000     0.000         0

                               micro avg      0.972     0.977     0.975       888
                               macro avg      0.638     0.654     0.628       888
                            weighted avg      0.975     0.977     0.974       888

I0716 03:39:19.204613 139795883440000 interactiveshell.py:2882] 
                              precision    recall  f1-score   support

                           O      0.000     0.000     0.000       0.0
           B-toloc.city_name      0.977     0.993     0.985     716.0
         B-fromloc.city_name      0.993     0.999     0.996     704.0
           I-toloc.city_name      0.985     0.989     0.987     265.0
      B-depart_date.day_name      0.991     0.991     0.991     212.0
              B-airline_name      0.990     1.000     0.995     101.0
         I-fromloc.city_name      0.967     0.994     0.981     177.0
 B-depart_time.period_of_day      0.984     0.923     0.952     130.0
              I-airline_name      1.000     0.985     0.992      65.0
    B-depart_date.day_number      0.964     0.964     0.964      55.0
    B-depart_date.month_name      0.982     0.964     0.973      56.0
          B-depart_time.time      0.864     1.000     0.927      57.0
                B-round_trip      1.000     1.000     1.000      73.0
             B-cost_relative      0.973     0.973     0.973      37.0
                I-round_trip      1.000     1.000     1.000      71.0
                B-flight_mod      0.828     1.000     0.906      24.0
 B-depart_time.time_relative      0.970     0.985     0.977      65.0
          I-depart_time.time      0.945     1.000     0.972      52.0
         B-stoploc.city_name      0.870     1.000     0.930      20.0
                 B-city_name      0.886     0.684     0.772      57.0
                B-class_type      0.960     1.000     0.980      24.0
          B-arrive_time.time      0.971     0.971     0.971      34.0
 B-arrive_time.time_relative      0.857     0.968     0.909      31.0
                I-class_type      1.000     1.000     1.000      17.0
               B-flight_stop      1.000     1.000     1.000      21.0
          I-arrive_time.time      1.000     0.971     0.986      35.0
              B-airline_code      0.850     1.000     0.919      34.0
    I-depart_date.day_number      1.000     1.000     1.000      15.0
      I-fromloc.airport_name      0.469     1.000     0.638      15.0
      B-fromloc.airport_name      0.429     1.000     0.600      12.0
      B-arrive_date.day_name      0.786     1.000     0.880      11.0
          B-toloc.state_code      0.947     1.000     0.973      18.0
B-depart_date.today_relative      1.000     0.889     0.941       9.0
             B-flight_number      0.846     1.000     0.917      11.0
 B-depart_date.date_relative      0.944     1.000     0.971      17.0
          B-toloc.state_name      0.774     0.857     0.814      28.0
           B-fare_basis_code      0.850     1.000     0.919      17.0
               B-flight_time      1.000     1.000     1.000       1.0
                        B-or      1.000     1.000     1.000       3.0
 B-arrive_time.period_of_day      0.750     1.000     0.857       6.0
          B-meal_description      1.000     0.900     0.947      10.0
             I-cost_relative      1.000     0.667     0.800       3.0
              I-airport_name      0.867     0.448     0.591      29.0
               B-fare_amount      1.000     1.000     1.000       2.0
               I-fare_amount      1.000     1.000     1.000       2.0
                 I-city_name      1.000     0.667     0.800      30.0
        I-toloc.airport_name      1.000     1.000     1.000       3.0
            B-transport_type      1.000     1.000     1.000      10.0
    B-arrive_date.month_name      0.714     0.833     0.769       6.0
    B-arrive_date.day_number      0.714     0.833     0.769       6.0
         I-stoploc.city_name      0.909     1.000     0.952      10.0
                      B-meal      1.000     1.000     1.000      16.0
        B-fromloc.state_code      1.000     0.957     0.978      23.0
    B-depart_time.period_mod      1.000     1.000     1.000       5.0
                   B-connect      1.000     1.000     1.000       6.0
               B-flight_days      1.000     1.000     1.000      10.0
        B-toloc.airport_name      1.000     1.000     1.000       3.0
        B-fromloc.state_name      1.000     1.000     1.000      17.0
              B-airport_name      0.800     0.381     0.516      21.0
                   B-economy      1.000     1.000     1.000       6.0
               I-flight_time      1.000     1.000     1.000       1.0
             B-aircraft_code      1.000     0.848     0.918      33.0
                       B-mod      1.000     0.500     0.667       2.0
              B-airport_code      1.000     0.444     0.615       9.0
    B-depart_time.start_time      1.000     1.000     1.000       3.0
      B-depart_time.end_time      1.000     0.667     0.800       3.0
          B-depart_date.year      1.000     1.000     1.000       3.0
            I-transport_type      1.000     1.000     1.000       1.0
          B-restriction_code      1.000     0.500     0.667       4.0
    B-arrive_time.start_time      1.000     1.000     1.000       8.0
        B-toloc.airport_code      1.000     0.750     0.857       4.0
      B-arrive_time.end_time      1.000     1.000     1.000       8.0
      I-arrive_time.end_time      0.889     1.000     0.941       8.0
      I-depart_time.end_time      1.000     0.333     0.500       3.0
               I-flight_stop      0.000     0.000     0.000       0.0
      B-fromloc.airport_code      0.625     1.000     0.769       5.0
          I-restriction_code      1.000     1.000     1.000       3.0
    I-depart_time.start_time      1.000     1.000     1.000       1.0
          I-toloc.state_name      1.000     1.000     1.000       1.0
I-depart_date.today_relative      0.000     0.000     0.000       0.0
 B-arrive_date.date_relative      1.000     1.000     1.000       2.0
                I-flight_mod      1.000     0.167     0.286       6.0
                   I-economy      0.000     0.000     0.000       0.0
 B-return_date.date_relative      0.500     0.333     0.400       3.0
        I-fromloc.state_name      1.000     1.000     1.000       1.0
                B-state_code      1.000     1.000     1.000       1.0
    I-arrive_time.start_time      1.000     1.000     1.000       1.0
    I-arrive_date.day_number      0.000     0.000     0.000       0.0
                 B-meal_code      0.000     0.000     0.000       1.0
 I-depart_time.period_of_day      0.000     0.000     0.000       1.0
                  B-day_name      1.000     0.500     0.667       2.0
             B-period_of_day      1.000     0.750     0.857       4.0
        B-stoploc.state_code      0.000     0.000     0.000       0.0
    B-return_date.month_name      0.000     0.000     0.000       0.0
    B-return_date.day_number      0.000     0.000     0.000       0.0
    B-arrive_time.period_mod      0.000     0.000     0.000       0.0
                 I-meal_code      0.000     0.000     0.000       0.0
        B-toloc.country_name      1.000     1.000     1.000       1.0
                 B-days_code      0.000     0.000     0.000       1.0
 I-arrive_time.period_of_day      0.000     0.000     0.000       0.0
            I-today_relative      0.000     0.000     0.000       0.0
 B-return_time.period_of_day      0.000     0.000     0.000       0.0
                      B-time      0.000     0.000     0.000       0.0
           I-fare_basis_code      0.000     0.000     0.000       0.0
 I-arrive_time.time_relative      0.000     0.000     0.000       4.0
 I-depart_time.time_relative      0.000     0.000     0.000       1.0
            B-today_relative      0.000     0.000     0.000       0.0
                B-state_name      0.000     0.000     0.000       9.0
B-arrive_date.today_relative      0.000     0.000     0.000       0.0
    B-return_time.period_mod      0.000     0.000     0.000       0.0
                B-month_name      0.000     0.000     0.000       0.0
                B-day_number      0.000     0.000     0.000       0.0
 I-return_date.date_relative      1.000     0.333     0.500       3.0
I-return_date.today_relative      0.000     0.000     0.000       0.0
      B-stoploc.airport_name      0.000     0.000     0.000       0.0
             B-time_relative      0.000     0.000     0.000       0.0
                      I-time      0.000     0.000     0.000       0.0
    I-return_date.day_number      0.000     0.000     0.000       0.0
          I-meal_description      0.000     0.000     0.000       0.0
B-return_date.today_relative      0.000     0.000     0.000       0.0
      B-return_date.day_name      0.000     0.000     0.000       2.0

                   micro avg      0.959     0.960     0.959    3657.0
                   macro avg      0.675     0.644     0.646    3657.0
                weighted avg      0.960     0.960     0.956    3657.0

I0716 03:39:19.230432 139795883440000 interactiveshell.py:2882] Best Slot F1: 0.959
I0716 03:39:19.231839 139795883440000 estimator_training.py:186] Not using Distribute Coordinator.
I0716 03:39:19.233781 139795883440000 training.py:612] Running training and evaluation locally (non-distributed).
I0716 03:39:19.237845 139795883440000 training.py:700] Start train and evaluate loop. The evaluate will happen after every checkpoint. Checkpoint frequency is determined based on RunConfig arguments: save_checkpoints_steps 312 or save_checkpoints_secs None.
I0716 03:39:19.309082 139795883440000 estimator.py:1145] Calling model_fn.
I0716 03:39:20.755266 139795883440000 saver.py:1499] Saver not created because there are no variables in the graph to restore
I0716 03:39:21.346002 139795883440000 <ipython-input-6-d6747b40360b>:55] 
[<tf.Variable 'embedding:0' shape=(750, 300) dtype=float32_ref>,
 <tf.Variable 'module/aggregation/weights:0' shape=(3,) dtype=float32>,
 <tf.Variable 'module/aggregation/scaling:0' shape=() dtype=float32>,
 <tf.Variable 'dense/kernel:0' shape=(1324, 300) dtype=float32_ref>,
 <tf.Variable 'dense/bias:0' shape=(300,) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/gates/kernel:0' shape=(600, 600) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/gates/bias:0' shape=(600,) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/candidate/kernel:0' shape=(600, 300) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/candidate/bias:0' shape=(300,) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/gates/kernel:0' shape=(600, 600) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/gates/bias:0' shape=(600,) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/candidate/kernel:0' shape=(600, 300) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/candidate/bias:0' shape=(300,) dtype=float32_ref>,
 <tf.Variable 'dense_1/kernel:0' shape=(600, 23) dtype=float32_ref>,
 <tf.Variable 'dense_1/bias:0' shape=(23,) dtype=float32_ref>,
 <tf.Variable 'dense_2/kernel:0' shape=(600, 122) dtype=float32_ref>,
 <tf.Variable 'dense_2/bias:0' shape=(122,) dtype=float32_ref>]
I0716 03:39:22.593348 139795883440000 estimator.py:1147] Done calling model_fn.
I0716 03:39:22.597057 139795883440000 basic_session_run_hooks.py:541] Create CheckpointSaverHook.
I0716 03:39:22.914428 139795883440000 monitored_session.py:240] Graph was finalized.
I0716 03:39:22.932276 139795883440000 saver.py:1280] Restoring parameters from ../model/elmo_bigru/model.ckpt-3432
I0716 03:39:23.970994 139795883440000 session_manager.py:500] Running local_init_op.
I0716 03:39:24.039549 139795883440000 session_manager.py:502] Done running local_init_op.
I0716 03:39:26.760696 139795883440000 basic_session_run_hooks.py:606] Saving checkpoints for 3432 into ../model/elmo_bigru/model.ckpt.
Reading ../data/atis.train.w-intent.iob
I0716 03:39:31.528347 139795883440000 basic_session_run_hooks.py:262] loss = 0.02635604, step = 3432
I0716 03:39:31.530147 139795883440000 basic_session_run_hooks.py:262] lr = 0.00020896882
I0716 03:39:58.348940 139795883440000 basic_session_run_hooks.py:692] global_step/sec: 3.7284
I0716 03:39:58.355597 139795883440000 basic_session_run_hooks.py:260] loss = 0.016621402, step = 3532 (26.827 sec)
I0716 03:39:58.357238 139795883440000 basic_session_run_hooks.py:260] lr = 0.00020677868 (26.827 sec)
I0716 03:40:25.918095 139795883440000 basic_session_run_hooks.py:692] global_step/sec: 3.62724
I0716 03:40:25.924374 139795883440000 basic_session_run_hooks.py:260] loss = 0.022599978, step = 3632 (27.569 sec)
I0716 03:40:25.927362 139795883440000 basic_session_run_hooks.py:260] lr = 0.00020461148 (27.570 sec)
I0716 03:40:52.109565 139795883440000 basic_session_run_hooks.py:692] global_step/sec: 3.81804
I0716 03:40:52.116390 139795883440000 basic_session_run_hooks.py:260] loss = 0.032441977, step = 3732 (26.192 sec)
I0716 03:40:52.118647 139795883440000 basic_session_run_hooks.py:260] lr = 0.00020246702 (26.191 sec)
I0716 03:40:54.840354 139795883440000 basic_session_run_hooks.py:606] Saving checkpoints for 3744 into ../model/elmo_bigru/model.ckpt.
I0716 03:40:56.758404 139795883440000 estimator.py:1145] Calling model_fn.
I0716 03:40:58.077687 139795883440000 saver.py:1499] Saver not created because there are no variables in the graph to restore
I0716 03:40:58.906549 139795883440000 estimator.py:1147] Done calling model_fn.
I0716 03:40:58.940602 139795883440000 evaluation.py:255] Starting evaluation at 2019-07-16T03:40:58Z
I0716 03:40:59.164282 139795883440000 monitored_session.py:240] Graph was finalized.
I0716 03:40:59.180455 139795883440000 saver.py:1280] Restoring parameters from ../model/elmo_bigru/model.ckpt-3744
I0716 03:41:00.080443 139795883440000 session_manager.py:500] Running local_init_op.
I0716 03:41:00.139611 139795883440000 session_manager.py:502] Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
I0716 03:41:58.817108 139795883440000 evaluation.py:275] Finished evaluation at 2019-07-16-03:41:58
I0716 03:41:58.819010 139795883440000 estimator.py:2039] Saving dict for global step 3744: global_step = 3744, loss = 0.29848006
I0716 03:41:58.826589 139795883440000 estimator.py:2099] Saving 'checkpoint_path' summary for global step 3744: ../model/elmo_bigru/model.ckpt-3744
I0716 03:41:58.910535 139795883440000 estimator.py:368] Loss for final step: 0.009342727.
Reading ../data/atis.test.w-intent.iob
I0716 03:41:59.536495 139795883440000 estimator.py:1145] Calling model_fn.
I0716 03:42:01.035902 139795883440000 saver.py:1499] Saver not created because there are no variables in the graph to restore
I0716 03:42:01.564124 139795883440000 estimator.py:1147] Done calling model_fn.
I0716 03:42:01.776388 139795883440000 monitored_session.py:240] Graph was finalized.
I0716 03:42:01.793310 139795883440000 saver.py:1280] Restoring parameters from ../model/elmo_bigru/model.ckpt-3744
I0716 03:42:02.692504 139795883440000 session_manager.py:500] Running local_init_op.
I0716 03:42:02.742432 139795883440000 session_manager.py:502] Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.
  'precision', 'predicted', average, warn_for)
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1439: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples.
  'recall', 'true', average, warn_for)
I0716 03:43:00.717076 139795883440000 interactiveshell.py:2882] 
                                          precision    recall  f1-score   support

                             atis_flight      0.984     0.991     0.987       632
                            atis_airfare      0.941     1.000     0.970        48
                     atis_ground_service      0.973     1.000     0.986        36
                            atis_airline      1.000     1.000     1.000        38
                       atis_abbreviation      0.943     1.000     0.971        33
                           atis_aircraft      1.000     0.889     0.941         9
                        atis_flight_time      0.500     1.000     0.667         1
                           atis_quantity      0.375     1.000     0.545         3
                atis_flight#atis_airfare      0.833     0.417     0.556        12
                            atis_airport      0.944     0.944     0.944        18
                           atis_distance      1.000     1.000     1.000        10
                               atis_city      0.750     0.500     0.600         6
                        atis_ground_fare      1.000     0.857     0.923         7
                           atis_capacity      1.000     0.952     0.976        21
                          atis_flight_no      0.889     1.000     0.941         8
                               atis_meal      1.000     0.833     0.909         6
                        atis_restriction      0.000     0.000     0.000         0
             atis_airline#atis_flight_no      0.000     0.000     0.000         0
    atis_ground_service#atis_ground_fare      0.000     0.000     0.000         0
           atis_airfare#atis_flight_time      0.000     0.000     0.000         0
                           atis_cheapest      0.000     0.000     0.000         0
atis_aircraft#atis_flight#atis_flight_no      0.000     0.000     0.000         0

                               micro avg      0.971     0.976     0.974       888
                               macro avg      0.642     0.654     0.633       888
                            weighted avg      0.974     0.976     0.973       888

I0716 03:43:00.753411 139795883440000 interactiveshell.py:2882] 
                              precision    recall  f1-score   support

                           O      0.000     0.000     0.000       0.0
           B-toloc.city_name      0.975     0.993     0.984     716.0
         B-fromloc.city_name      0.990     1.000     0.995     704.0
           I-toloc.city_name      0.981     0.985     0.983     265.0
      B-depart_date.day_name      0.991     0.991     0.991     212.0
              B-airline_name      0.990     1.000     0.995     101.0
         I-fromloc.city_name      0.957     1.000     0.978     177.0
 B-depart_time.period_of_day      0.984     0.931     0.957     130.0
              I-airline_name      1.000     0.985     0.992      65.0
    B-depart_date.day_number      0.981     0.964     0.972      55.0
    B-depart_date.month_name      0.982     0.964     0.973      56.0
          B-depart_time.time      0.851     1.000     0.919      57.0
                B-round_trip      1.000     1.000     1.000      73.0
             B-cost_relative      0.973     0.973     0.973      37.0
                I-round_trip      1.000     1.000     1.000      71.0
                B-flight_mod      0.828     1.000     0.906      24.0
 B-depart_time.time_relative      0.955     0.985     0.970      65.0
          I-depart_time.time      0.945     1.000     0.972      52.0
         B-stoploc.city_name      0.870     1.000     0.930      20.0
                 B-city_name      0.902     0.649     0.755      57.0
                B-class_type      0.960     1.000     0.980      24.0
          B-arrive_time.time      1.000     0.971     0.985      34.0
 B-arrive_time.time_relative      1.000     0.935     0.967      31.0
                I-class_type      1.000     1.000     1.000      17.0
               B-flight_stop      1.000     1.000     1.000      21.0
          I-arrive_time.time      1.000     0.971     0.986      35.0
              B-airline_code      0.872     1.000     0.932      34.0
    I-depart_date.day_number      1.000     1.000     1.000      15.0
      I-fromloc.airport_name      0.455     1.000     0.625      15.0
      B-fromloc.airport_name      0.444     1.000     0.615      12.0
      B-arrive_date.day_name      0.786     1.000     0.880      11.0
          B-toloc.state_code      1.000     1.000     1.000      18.0
B-depart_date.today_relative      1.000     0.889     0.941       9.0
             B-flight_number      0.733     1.000     0.846      11.0
 B-depart_date.date_relative      0.944     1.000     0.971      17.0
          B-toloc.state_name      0.781     0.893     0.833      28.0
           B-fare_basis_code      0.850     1.000     0.919      17.0
               B-flight_time      1.000     1.000     1.000       1.0
                        B-or      1.000     1.000     1.000       3.0
 B-arrive_time.period_of_day      0.857     1.000     0.923       6.0
          B-meal_description      1.000     0.900     0.947      10.0
             I-cost_relative      1.000     0.667     0.800       3.0
              I-airport_name      0.833     0.345     0.488      29.0
               B-fare_amount      1.000     1.000     1.000       2.0
               I-fare_amount      1.000     1.000     1.000       2.0
                 I-city_name      0.944     0.567     0.708      30.0
        I-toloc.airport_name      1.000     1.000     1.000       3.0
            B-transport_type      1.000     1.000     1.000      10.0
    B-arrive_date.month_name      0.714     0.833     0.769       6.0
    B-arrive_date.day_number      0.714     0.833     0.769       6.0
         I-stoploc.city_name      0.909     1.000     0.952      10.0
                      B-meal      1.000     1.000     1.000      16.0
        B-fromloc.state_code      1.000     1.000     1.000      23.0
    B-depart_time.period_mod      0.833     1.000     0.909       5.0
                   B-connect      1.000     1.000     1.000       6.0
               B-flight_days      1.000     1.000     1.000      10.0
        B-toloc.airport_name      1.000     1.000     1.000       3.0
        B-fromloc.state_name      0.944     1.000     0.971      17.0
              B-airport_name      0.750     0.286     0.414      21.0
                   B-economy      1.000     1.000     1.000       6.0
               I-flight_time      1.000     1.000     1.000       1.0
             B-aircraft_code      1.000     0.818     0.900      33.0
                       B-mod      1.000     0.500     0.667       2.0
              B-airport_code      1.000     0.556     0.714       9.0
    B-depart_time.start_time      0.750     1.000     0.857       3.0
      B-depart_time.end_time      0.667     0.667     0.667       3.0
          B-depart_date.year      1.000     1.000     1.000       3.0
            I-transport_type      1.000     1.000     1.000       1.0
          B-restriction_code      0.667     0.500     0.571       4.0
    B-arrive_time.start_time      1.000     0.875     0.933       8.0
        B-toloc.airport_code      1.000     0.750     0.857       4.0
      B-arrive_time.end_time      1.000     0.875     0.933       8.0
      I-arrive_time.end_time      1.000     0.875     0.933       8.0
      I-depart_time.end_time      0.667     0.667     0.667       3.0
               I-flight_stop      0.000     0.000     0.000       0.0
      B-fromloc.airport_code      0.714     1.000     0.833       5.0
          I-restriction_code      1.000     1.000     1.000       3.0
    I-depart_time.start_time      1.000     1.000     1.000       1.0
          I-toloc.state_name      1.000     1.000     1.000       1.0
I-depart_date.today_relative      0.000     0.000     0.000       0.0
 B-arrive_date.date_relative      1.000     1.000     1.000       2.0
                I-flight_mod      1.000     0.167     0.286       6.0
                   I-economy      0.000     0.000     0.000       0.0
 B-return_date.date_relative      1.000     0.667     0.800       3.0
        I-fromloc.state_name      1.000     1.000     1.000       1.0
                B-state_code      1.000     1.000     1.000       1.0
    I-arrive_time.start_time      1.000     1.000     1.000       1.0
    I-arrive_date.day_number      0.000     0.000     0.000       0.0
                 B-meal_code      0.000     0.000     0.000       1.0
 I-depart_time.period_of_day      0.000     0.000     0.000       1.0
                  B-day_name      1.000     0.500     0.667       2.0
             B-period_of_day      1.000     0.750     0.857       4.0
        B-stoploc.state_code      0.000     0.000     0.000       0.0
    B-return_date.month_name      0.000     0.000     0.000       0.0
    B-return_date.day_number      0.000     0.000     0.000       0.0
    B-arrive_time.period_mod      0.000     0.000     0.000       0.0
                 I-meal_code      0.000     0.000     0.000       0.0
        B-toloc.country_name      1.000     1.000     1.000       1.0
                 B-days_code      0.000     0.000     0.000       1.0
 I-arrive_time.period_of_day      0.000     0.000     0.000       0.0
            I-today_relative      0.000     0.000     0.000       0.0
 B-return_time.period_of_day      0.000     0.000     0.000       0.0
                      B-time      0.000     0.000     0.000       0.0
           I-fare_basis_code      0.000     0.000     0.000       0.0
 I-arrive_time.time_relative      1.000     1.000     1.000       4.0
 I-depart_time.time_relative      0.000     0.000     0.000       1.0
            B-today_relative      0.000     0.000     0.000       0.0
                B-state_name      0.000     0.000     0.000       9.0
B-arrive_date.today_relative      0.000     0.000     0.000       0.0
    B-return_time.period_mod      0.000     0.000     0.000       0.0
                B-month_name      0.000     0.000     0.000       0.0
                B-day_number      0.000     0.000     0.000       0.0
 I-return_date.date_relative      1.000     0.667     0.800       3.0
I-return_date.today_relative      0.000     0.000     0.000       0.0
      B-stoploc.airport_name      0.000     0.000     0.000       0.0
             B-time_relative      0.000     0.000     0.000       0.0
                      I-time      0.000     0.000     0.000       0.0
    I-return_date.day_number      0.000     0.000     0.000       0.0
          I-meal_description      0.000     0.000     0.000       0.0
B-return_date.today_relative      0.000     0.000     0.000       0.0
      B-return_date.day_name      0.000     0.000     0.000       2.0

                   micro avg      0.958     0.959     0.958    3657.0
                   macro avg      0.677     0.656     0.656    3657.0
                weighted avg      0.960     0.959     0.955    3657.0

I0716 03:43:00.773989 139795883440000 interactiveshell.py:2882] Best Slot F1: 0.959
I0716 03:43:00.775548 139795883440000 estimator_training.py:186] Not using Distribute Coordinator.
I0716 03:43:00.779031 139795883440000 training.py:612] Running training and evaluation locally (non-distributed).
I0716 03:43:00.781644 139795883440000 training.py:700] Start train and evaluate loop. The evaluate will happen after every checkpoint. Checkpoint frequency is determined based on RunConfig arguments: save_checkpoints_steps 312 or save_checkpoints_secs None.
I0716 03:43:00.846108 139795883440000 estimator.py:1145] Calling model_fn.
I0716 03:43:02.279944 139795883440000 saver.py:1499] Saver not created because there are no variables in the graph to restore
I0716 03:43:02.867092 139795883440000 <ipython-input-6-d6747b40360b>:55] 
[<tf.Variable 'embedding:0' shape=(750, 300) dtype=float32_ref>,
 <tf.Variable 'module/aggregation/weights:0' shape=(3,) dtype=float32>,
 <tf.Variable 'module/aggregation/scaling:0' shape=() dtype=float32>,
 <tf.Variable 'dense/kernel:0' shape=(1324, 300) dtype=float32_ref>,
 <tf.Variable 'dense/bias:0' shape=(300,) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/gates/kernel:0' shape=(600, 600) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/gates/bias:0' shape=(600,) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/candidate/kernel:0' shape=(600, 300) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/candidate/bias:0' shape=(300,) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/gates/kernel:0' shape=(600, 600) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/gates/bias:0' shape=(600,) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/candidate/kernel:0' shape=(600, 300) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/candidate/bias:0' shape=(300,) dtype=float32_ref>,
 <tf.Variable 'dense_1/kernel:0' shape=(600, 23) dtype=float32_ref>,
 <tf.Variable 'dense_1/bias:0' shape=(23,) dtype=float32_ref>,
 <tf.Variable 'dense_2/kernel:0' shape=(600, 122) dtype=float32_ref>,
 <tf.Variable 'dense_2/bias:0' shape=(122,) dtype=float32_ref>]
I0716 03:43:04.119152 139795883440000 estimator.py:1147] Done calling model_fn.
I0716 03:43:04.123580 139795883440000 basic_session_run_hooks.py:541] Create CheckpointSaverHook.
I0716 03:43:04.439007 139795883440000 monitored_session.py:240] Graph was finalized.
I0716 03:43:04.455222 139795883440000 saver.py:1280] Restoring parameters from ../model/elmo_bigru/model.ckpt-3744
I0716 03:43:05.489947 139795883440000 session_manager.py:500] Running local_init_op.
I0716 03:43:05.554122 139795883440000 session_manager.py:502] Done running local_init_op.
I0716 03:43:08.292708 139795883440000 basic_session_run_hooks.py:606] Saving checkpoints for 3744 into ../model/elmo_bigru/model.ckpt.
Reading ../data/atis.train.w-intent.iob
I0716 03:43:12.975174 139795883440000 basic_session_run_hooks.py:262] loss = 0.021001698, step = 3744
I0716 03:43:12.977192 139795883440000 basic_session_run_hooks.py:262] lr = 0.00020221119
I0716 03:43:39.822550 139795883440000 basic_session_run_hooks.py:692] global_step/sec: 3.72469
I0716 03:43:39.827977 139795883440000 basic_session_run_hooks.py:260] loss = 0.02767645, step = 3844 (26.853 sec)
I0716 03:43:39.830882 139795883440000 basic_session_run_hooks.py:260] lr = 0.00020009186 (26.854 sec)
I0716 03:44:06.863698 139795883440000 basic_session_run_hooks.py:692] global_step/sec: 3.69806
I0716 03:44:06.869125 139795883440000 basic_session_run_hooks.py:260] loss = 0.018864132, step = 3944 (27.041 sec)
I0716 03:44:06.872111 139795883440000 basic_session_run_hooks.py:260] lr = 0.00019799476 (27.041 sec)
I0716 03:44:33.213929 139795883440000 basic_session_run_hooks.py:692] global_step/sec: 3.79504
I0716 03:44:33.216701 139795883440000 basic_session_run_hooks.py:260] loss = 0.027750138, step = 4044 (26.348 sec)
I0716 03:44:33.222939 139795883440000 basic_session_run_hooks.py:260] lr = 0.00019591961 (26.351 sec)
I0716 03:44:36.275014 139795883440000 basic_session_run_hooks.py:606] Saving checkpoints for 4056 into ../model/elmo_bigru/model.ckpt.
I0716 03:44:38.252676 139795883440000 estimator.py:1145] Calling model_fn.
I0716 03:44:39.814150 139795883440000 saver.py:1499] Saver not created because there are no variables in the graph to restore
I0716 03:44:40.388385 139795883440000 estimator.py:1147] Done calling model_fn.
I0716 03:44:40.418080 139795883440000 evaluation.py:255] Starting evaluation at 2019-07-16T03:44:40Z
I0716 03:44:40.637779 139795883440000 monitored_session.py:240] Graph was finalized.
I0716 03:44:40.655734 139795883440000 saver.py:1280] Restoring parameters from ../model/elmo_bigru/model.ckpt-4056
I0716 03:44:41.561204 139795883440000 session_manager.py:500] Running local_init_op.
I0716 03:44:41.618479 139795883440000 session_manager.py:502] Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
I0716 03:45:40.799308 139795883440000 evaluation.py:275] Finished evaluation at 2019-07-16-03:45:40
I0716 03:45:40.801134 139795883440000 estimator.py:2039] Saving dict for global step 4056: global_step = 4056, loss = 0.313336
I0716 03:45:40.808373 139795883440000 estimator.py:2099] Saving 'checkpoint_path' summary for global step 4056: ../model/elmo_bigru/model.ckpt-4056
I0716 03:45:40.894330 139795883440000 estimator.py:368] Loss for final step: 0.010575725.
Reading ../data/atis.test.w-intent.iob
I0716 03:45:41.482816 139795883440000 estimator.py:1145] Calling model_fn.
I0716 03:45:42.989626 139795883440000 saver.py:1499] Saver not created because there are no variables in the graph to restore
I0716 03:45:43.509806 139795883440000 estimator.py:1147] Done calling model_fn.
I0716 03:45:43.724382 139795883440000 monitored_session.py:240] Graph was finalized.
I0716 03:45:43.739015 139795883440000 saver.py:1280] Restoring parameters from ../model/elmo_bigru/model.ckpt-4056
I0716 03:45:44.637495 139795883440000 session_manager.py:500] Running local_init_op.
I0716 03:45:44.689667 139795883440000 session_manager.py:502] Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.
  'precision', 'predicted', average, warn_for)
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1439: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples.
  'recall', 'true', average, warn_for)
I0716 03:46:42.461421 139795883440000 interactiveshell.py:2882] 
                                          precision    recall  f1-score   support

                             atis_flight      0.986     0.991     0.988       632
                            atis_airfare      0.941     1.000     0.970        48
                     atis_ground_service      1.000     1.000     1.000        36
                            atis_airline      1.000     1.000     1.000        38
                       atis_abbreviation      0.943     1.000     0.971        33
                           atis_aircraft      1.000     0.889     0.941         9
                        atis_flight_time      0.333     1.000     0.500         1
                           atis_quantity      0.375     1.000     0.545         3
                atis_flight#atis_airfare      0.833     0.417     0.556        12
                            atis_airport      0.947     1.000     0.973        18
                           atis_distance      1.000     1.000     1.000        10
                               atis_city      0.750     0.500     0.600         6
                        atis_ground_fare      1.000     0.857     0.923         7
                           atis_capacity      1.000     0.952     0.976        21
                          atis_flight_no      0.889     1.000     0.941         8
                               atis_meal      1.000     0.833     0.909         6
                        atis_restriction      0.000     0.000     0.000         0
             atis_airline#atis_flight_no      0.000     0.000     0.000         0
    atis_ground_service#atis_ground_fare      0.000     0.000     0.000         0
           atis_airfare#atis_flight_time      0.000     0.000     0.000         0
                           atis_cheapest      0.000     0.000     0.000         0
atis_aircraft#atis_flight#atis_flight_no      0.000     0.000     0.000         0

                               micro avg      0.972     0.977     0.975       888
                               macro avg      0.636     0.656     0.627       888
                            weighted avg      0.976     0.977     0.975       888

I0716 03:46:42.499803 139795883440000 interactiveshell.py:2882] 
                              precision    recall  f1-score   support

                           O      0.000     0.000     0.000       0.0
           B-toloc.city_name      0.974     0.993     0.983     716.0
         B-fromloc.city_name      0.989     0.999     0.994     704.0
           I-toloc.city_name      0.978     0.992     0.985     265.0
      B-depart_date.day_name      0.991     0.991     0.991     212.0
              B-airline_name      1.000     1.000     1.000     101.0
         I-fromloc.city_name      0.978     0.994     0.986     177.0
 B-depart_time.period_of_day      0.983     0.915     0.948     130.0
              I-airline_name      1.000     1.000     1.000      65.0
    B-depart_date.day_number      0.981     0.964     0.972      55.0
    B-depart_date.month_name      0.981     0.946     0.964      56.0
          B-depart_time.time      0.864     1.000     0.927      57.0
                B-round_trip      1.000     1.000     1.000      73.0
             B-cost_relative      0.973     0.973     0.973      37.0
                I-round_trip      1.000     1.000     1.000      71.0
                B-flight_mod      0.828     1.000     0.906      24.0
 B-depart_time.time_relative      0.970     0.985     0.977      65.0
          I-depart_time.time      0.929     1.000     0.963      52.0
         B-stoploc.city_name      0.952     1.000     0.976      20.0
                 B-city_name      0.923     0.632     0.750      57.0
                B-class_type      0.960     1.000     0.980      24.0
          B-arrive_time.time      0.971     0.971     0.971      34.0
 B-arrive_time.time_relative      0.857     0.968     0.909      31.0
                I-class_type      1.000     1.000     1.000      17.0
               B-flight_stop      1.000     1.000     1.000      21.0
          I-arrive_time.time      1.000     0.971     0.986      35.0
              B-airline_code      0.872     1.000     0.932      34.0
    I-depart_date.day_number      1.000     0.933     0.966      15.0
      I-fromloc.airport_name      0.417     1.000     0.588      15.0
      B-fromloc.airport_name      0.400     1.000     0.571      12.0
      B-arrive_date.day_name      0.786     1.000     0.880      11.0
          B-toloc.state_code      1.000     1.000     1.000      18.0
B-depart_date.today_relative      1.000     0.889     0.941       9.0
             B-flight_number      0.733     1.000     0.846      11.0
 B-depart_date.date_relative      0.944     1.000     0.971      17.0
          B-toloc.state_name      0.781     0.893     0.833      28.0
           B-fare_basis_code      0.850     1.000     0.919      17.0
               B-flight_time      1.000     1.000     1.000       1.0
                        B-or      1.000     1.000     1.000       3.0
 B-arrive_time.period_of_day      0.750     1.000     0.857       6.0
          B-meal_description      1.000     0.900     0.947      10.0
             I-cost_relative      1.000     0.667     0.800       3.0
              I-airport_name      0.833     0.345     0.488      29.0
               B-fare_amount      1.000     1.000     1.000       2.0
               I-fare_amount      1.000     1.000     1.000       2.0
                 I-city_name      1.000     0.567     0.723      30.0
        I-toloc.airport_name      1.000     1.000     1.000       3.0
            B-transport_type      1.000     1.000     1.000      10.0
    B-arrive_date.month_name      0.625     0.833     0.714       6.0
    B-arrive_date.day_number      0.625     0.833     0.714       6.0
         I-stoploc.city_name      0.909     1.000     0.952      10.0
                      B-meal      1.000     1.000     1.000      16.0
        B-fromloc.state_code      1.000     1.000     1.000      23.0
    B-depart_time.period_mod      1.000     1.000     1.000       5.0
                   B-connect      1.000     1.000     1.000       6.0
               B-flight_days      1.000     1.000     1.000      10.0
        B-toloc.airport_name      1.000     1.000     1.000       3.0
        B-fromloc.state_name      0.944     1.000     0.971      17.0
              B-airport_name      0.778     0.333     0.467      21.0
                   B-economy      1.000     1.000     1.000       6.0
               I-flight_time      1.000     1.000     1.000       1.0
             B-aircraft_code      1.000     0.818     0.900      33.0
                       B-mod      1.000     0.500     0.667       2.0
              B-airport_code      1.000     0.444     0.615       9.0
    B-depart_time.start_time      1.000     0.667     0.800       3.0
      B-depart_time.end_time      1.000     0.333     0.500       3.0
          B-depart_date.year      1.000     1.000     1.000       3.0
            I-transport_type      1.000     1.000     1.000       1.0
          B-restriction_code      0.667     0.500     0.571       4.0
    B-arrive_time.start_time      0.889     1.000     0.941       8.0
        B-toloc.airport_code      1.000     0.750     0.857       4.0
      B-arrive_time.end_time      0.889     1.000     0.941       8.0
      I-arrive_time.end_time      0.889     1.000     0.941       8.0
      I-depart_time.end_time      1.000     0.333     0.500       3.0
               I-flight_stop      0.000     0.000     0.000       0.0
      B-fromloc.airport_code      0.625     1.000     0.769       5.0
          I-restriction_code      1.000     1.000     1.000       3.0
    I-depart_time.start_time      1.000     1.000     1.000       1.0
          I-toloc.state_name      1.000     1.000     1.000       1.0
I-depart_date.today_relative      0.000     0.000     0.000       0.0
 B-arrive_date.date_relative      1.000     1.000     1.000       2.0
                I-flight_mod      1.000     0.167     0.286       6.0
                   I-economy      0.000     0.000     0.000       0.0
 B-return_date.date_relative      1.000     0.667     0.800       3.0
        I-fromloc.state_name      1.000     1.000     1.000       1.0
                B-state_code      1.000     1.000     1.000       1.0
    I-arrive_time.start_time      1.000     1.000     1.000       1.0
    I-arrive_date.day_number      0.000     0.000     0.000       0.0
                 B-meal_code      0.000     0.000     0.000       1.0
 I-depart_time.period_of_day      0.000     0.000     0.000       1.0
                  B-day_name      1.000     0.500     0.667       2.0
             B-period_of_day      1.000     0.750     0.857       4.0
        B-stoploc.state_code      0.000     0.000     0.000       0.0
    B-return_date.month_name      0.000     0.000     0.000       0.0
    B-return_date.day_number      0.000     0.000     0.000       0.0
    B-arrive_time.period_mod      0.000     0.000     0.000       0.0
                 I-meal_code      0.000     0.000     0.000       0.0
        B-toloc.country_name      1.000     1.000     1.000       1.0
                 B-days_code      0.000     0.000     0.000       1.0
 I-arrive_time.period_of_day      0.000     0.000     0.000       0.0
            I-today_relative      0.000     0.000     0.000       0.0
 B-return_time.period_of_day      0.000     0.000     0.000       0.0
                      B-time      0.000     0.000     0.000       0.0
           I-fare_basis_code      0.000     0.000     0.000       0.0
 I-arrive_time.time_relative      0.000     0.000     0.000       4.0
 I-depart_time.time_relative      0.000     0.000     0.000       1.0
            B-today_relative      0.000     0.000     0.000       0.0
                B-state_name      0.000     0.000     0.000       9.0
B-arrive_date.today_relative      0.000     0.000     0.000       0.0
    B-return_time.period_mod      0.000     0.000     0.000       0.0
                B-month_name      0.000     0.000     0.000       0.0
                B-day_number      0.000     0.000     0.000       0.0
 I-return_date.date_relative      1.000     0.667     0.800       3.0
I-return_date.today_relative      0.000     0.000     0.000       0.0
      B-stoploc.airport_name      0.000     0.000     0.000       0.0
             B-time_relative      0.000     0.000     0.000       0.0
                      I-time      0.000     0.000     0.000       0.0
    I-return_date.day_number      0.000     0.000     0.000       0.0
          I-meal_description      0.000     0.000     0.000       0.0
B-return_date.today_relative      0.000     0.000     0.000       0.0
      B-return_date.day_name      0.000     0.000     0.000       2.0

                   micro avg      0.956     0.957     0.957    3657.0
                   macro avg      0.672     0.641     0.642    3657.0
                weighted avg      0.959     0.957     0.953    3657.0

I0716 03:46:42.520813 139795883440000 interactiveshell.py:2882] Best Slot F1: 0.959
I0716 03:46:42.522331 139795883440000 estimator_training.py:186] Not using Distribute Coordinator.
I0716 03:46:42.524306 139795883440000 training.py:612] Running training and evaluation locally (non-distributed).
I0716 03:46:42.525812 139795883440000 training.py:700] Start train and evaluate loop. The evaluate will happen after every checkpoint. Checkpoint frequency is determined based on RunConfig arguments: save_checkpoints_steps 312 or save_checkpoints_secs None.
I0716 03:46:42.591641 139795883440000 estimator.py:1145] Calling model_fn.
I0716 03:46:44.033873 139795883440000 saver.py:1499] Saver not created because there are no variables in the graph to restore
I0716 03:46:44.784013 139795883440000 <ipython-input-6-d6747b40360b>:55] 
[<tf.Variable 'embedding:0' shape=(750, 300) dtype=float32_ref>,
 <tf.Variable 'module/aggregation/weights:0' shape=(3,) dtype=float32>,
 <tf.Variable 'module/aggregation/scaling:0' shape=() dtype=float32>,
 <tf.Variable 'dense/kernel:0' shape=(1324, 300) dtype=float32_ref>,
 <tf.Variable 'dense/bias:0' shape=(300,) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/gates/kernel:0' shape=(600, 600) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/gates/bias:0' shape=(600,) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/candidate/kernel:0' shape=(600, 300) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/candidate/bias:0' shape=(300,) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/gates/kernel:0' shape=(600, 600) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/gates/bias:0' shape=(600,) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/candidate/kernel:0' shape=(600, 300) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/candidate/bias:0' shape=(300,) dtype=float32_ref>,
 <tf.Variable 'dense_1/kernel:0' shape=(600, 23) dtype=float32_ref>,
 <tf.Variable 'dense_1/bias:0' shape=(23,) dtype=float32_ref>,
 <tf.Variable 'dense_2/kernel:0' shape=(600, 122) dtype=float32_ref>,
 <tf.Variable 'dense_2/bias:0' shape=(122,) dtype=float32_ref>]
I0716 03:46:45.850750 139795883440000 estimator.py:1147] Done calling model_fn.
I0716 03:46:45.854489 139795883440000 basic_session_run_hooks.py:541] Create CheckpointSaverHook.
I0716 03:46:46.181741 139795883440000 monitored_session.py:240] Graph was finalized.
I0716 03:46:46.198333 139795883440000 saver.py:1280] Restoring parameters from ../model/elmo_bigru/model.ckpt-4056
I0716 03:46:47.237629 139795883440000 session_manager.py:500] Running local_init_op.
I0716 03:46:47.309618 139795883440000 session_manager.py:502] Done running local_init_op.
I0716 03:46:50.114432 139795883440000 basic_session_run_hooks.py:606] Saving checkpoints for 4056 into ../model/elmo_bigru/model.ckpt.
Reading ../data/atis.train.w-intent.iob
I0716 03:46:54.964871 139795883440000 basic_session_run_hooks.py:262] loss = 0.015439781, step = 4056
I0716 03:46:54.966714 139795883440000 basic_session_run_hooks.py:262] lr = 0.00019567205
I0716 03:47:22.732358 139795883440000 basic_session_run_hooks.py:692] global_step/sec: 3.60126
I0716 03:47:22.735428 139795883440000 basic_session_run_hooks.py:260] loss = 0.022223681, step = 4156 (27.771 sec)
I0716 03:47:22.740602 139795883440000 basic_session_run_hooks.py:260] lr = 0.0001936213 (27.774 sec)
I0716 03:47:49.557179 139795883440000 basic_session_run_hooks.py:692] global_step/sec: 3.72788
I0716 03:47:49.562354 139795883440000 basic_session_run_hooks.py:260] loss = 0.024592336, step = 4256 (26.827 sec)
I0716 03:47:49.565015 139795883440000 basic_session_run_hooks.py:260] lr = 0.00019159199 (26.824 sec)
I0716 03:48:15.762630 139795883440000 basic_session_run_hooks.py:692] global_step/sec: 3.816
I0716 03:48:15.770267 139795883440000 basic_session_run_hooks.py:260] loss = 0.019611156, step = 4356 (26.208 sec)
I0716 03:48:15.772212 139795883440000 basic_session_run_hooks.py:260] lr = 0.00018958394 (26.207 sec)
I0716 03:48:18.542946 139795883440000 basic_session_run_hooks.py:606] Saving checkpoints for 4368 into ../model/elmo_bigru/model.ckpt.
I0716 03:48:20.516006 139795883440000 estimator.py:1145] Calling model_fn.
I0716 03:48:21.880723 139795883440000 saver.py:1499] Saver not created because there are no variables in the graph to restore
I0716 03:48:22.436344 139795883440000 estimator.py:1147] Done calling model_fn.
I0716 03:48:22.465405 139795883440000 evaluation.py:255] Starting evaluation at 2019-07-16T03:48:22Z
I0716 03:48:22.684628 139795883440000 monitored_session.py:240] Graph was finalized.
I0716 03:48:22.697028 139795883440000 saver.py:1280] Restoring parameters from ../model/elmo_bigru/model.ckpt-4368
I0716 03:48:23.581039 139795883440000 session_manager.py:500] Running local_init_op.
I0716 03:48:23.639722 139795883440000 session_manager.py:502] Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
I0716 03:49:22.386981 139795883440000 evaluation.py:275] Finished evaluation at 2019-07-16-03:49:22
I0716 03:49:22.388689 139795883440000 estimator.py:2039] Saving dict for global step 4368: global_step = 4368, loss = 0.3263626
I0716 03:49:22.396470 139795883440000 estimator.py:2099] Saving 'checkpoint_path' summary for global step 4368: ../model/elmo_bigru/model.ckpt-4368
I0716 03:49:22.478740 139795883440000 estimator.py:368] Loss for final step: 0.012535343.
Reading ../data/atis.test.w-intent.iob
I0716 03:49:23.071287 139795883440000 estimator.py:1145] Calling model_fn.
I0716 03:49:24.734490 139795883440000 saver.py:1499] Saver not created because there are no variables in the graph to restore
I0716 03:49:25.242751 139795883440000 estimator.py:1147] Done calling model_fn.
I0716 03:49:25.466029 139795883440000 monitored_session.py:240] Graph was finalized.
I0716 03:49:25.482319 139795883440000 saver.py:1280] Restoring parameters from ../model/elmo_bigru/model.ckpt-4368
I0716 03:49:26.366806 139795883440000 session_manager.py:500] Running local_init_op.
I0716 03:49:26.414819 139795883440000 session_manager.py:502] Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.
  'precision', 'predicted', average, warn_for)
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1439: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples.
  'recall', 'true', average, warn_for)
I0716 03:50:24.146742 139795883440000 interactiveshell.py:2882] 
                                          precision    recall  f1-score   support

                             atis_flight      0.978     0.992     0.985       632
                            atis_airfare      0.941     1.000     0.970        48
                     atis_ground_service      1.000     1.000     1.000        36
                            atis_airline      1.000     1.000     1.000        38
                       atis_abbreviation      0.943     1.000     0.971        33
                           atis_aircraft      1.000     0.889     0.941         9
                        atis_flight_time      0.500     1.000     0.667         1
                           atis_quantity      0.375     1.000     0.545         3
                atis_flight#atis_airfare      0.833     0.417     0.556        12
                            atis_airport      1.000     0.944     0.971        18
                           atis_distance      1.000     1.000     1.000        10
                               atis_city      1.000     0.167     0.286         6
                        atis_ground_fare      1.000     0.857     0.923         7
                           atis_capacity      1.000     0.952     0.976        21
                          atis_flight_no      0.889     1.000     0.941         8
                               atis_meal      1.000     0.833     0.909         6
                        atis_restriction      0.000     0.000     0.000         0
             atis_airline#atis_flight_no      0.000     0.000     0.000         0
    atis_ground_service#atis_ground_fare      0.000     0.000     0.000         0
           atis_airfare#atis_flight_time      0.000     0.000     0.000         0
                           atis_cheapest      0.000     0.000     0.000         0
atis_aircraft#atis_flight#atis_flight_no      0.000     0.000     0.000         0

                               micro avg      0.970     0.975     0.972       888
                               macro avg      0.657     0.639     0.620       888
                            weighted avg      0.973     0.975     0.970       888

I0716 03:50:24.183134 139795883440000 interactiveshell.py:2882] 
                              precision    recall  f1-score   support

                           O      0.000     0.000     0.000       0.0
           B-toloc.city_name      0.973     0.994     0.983     716.0
         B-fromloc.city_name      0.989     1.000     0.994     704.0
           I-toloc.city_name      0.978     0.989     0.983     265.0
      B-depart_date.day_name      0.991     0.991     0.991     212.0
              B-airline_name      1.000     1.000     1.000     101.0
         I-fromloc.city_name      0.967     1.000     0.983     177.0
 B-depart_time.period_of_day      0.984     0.923     0.952     130.0
              I-airline_name      1.000     1.000     1.000      65.0
    B-depart_date.day_number      0.981     0.964     0.972      55.0
    B-depart_date.month_name      0.982     0.964     0.973      56.0
          B-depart_time.time      0.864     1.000     0.927      57.0
                B-round_trip      1.000     1.000     1.000      73.0
             B-cost_relative      0.973     0.973     0.973      37.0
                I-round_trip      1.000     1.000     1.000      71.0
                B-flight_mod      0.828     1.000     0.906      24.0
 B-depart_time.time_relative      0.955     0.985     0.970      65.0
          I-depart_time.time      0.945     1.000     0.972      52.0
         B-stoploc.city_name      0.952     1.000     0.976      20.0
                 B-city_name      0.897     0.614     0.729      57.0
                B-class_type      0.960     1.000     0.980      24.0
          B-arrive_time.time      0.971     0.971     0.971      34.0
 B-arrive_time.time_relative      0.853     0.935     0.892      31.0
                I-class_type      1.000     1.000     1.000      17.0
               B-flight_stop      1.000     1.000     1.000      21.0
          I-arrive_time.time      1.000     0.971     0.986      35.0
              B-airline_code      0.810     1.000     0.895      34.0
    I-depart_date.day_number      1.000     1.000     1.000      15.0
      I-fromloc.airport_name      0.455     1.000     0.625      15.0
      B-fromloc.airport_name      0.429     1.000     0.600      12.0
      B-arrive_date.day_name      0.786     1.000     0.880      11.0
          B-toloc.state_code      1.000     1.000     1.000      18.0
B-depart_date.today_relative      1.000     0.889     0.941       9.0
             B-flight_number      0.786     1.000     0.880      11.0
 B-depart_date.date_relative      0.944     1.000     0.971      17.0
          B-toloc.state_name      0.781     0.893     0.833      28.0
           B-fare_basis_code      0.850     1.000     0.919      17.0
               B-flight_time      1.000     1.000     1.000       1.0
                        B-or      1.000     1.000     1.000       3.0
 B-arrive_time.period_of_day      0.750     1.000     0.857       6.0
          B-meal_description      1.000     0.900     0.947      10.0
             I-cost_relative      1.000     0.667     0.800       3.0
              I-airport_name      0.917     0.379     0.537      29.0
               B-fare_amount      1.000     1.000     1.000       2.0
               I-fare_amount      1.000     1.000     1.000       2.0
                 I-city_name      1.000     0.533     0.696      30.0
        I-toloc.airport_name      0.750     1.000     0.857       3.0
            B-transport_type      1.000     1.000     1.000      10.0
    B-arrive_date.month_name      0.714     0.833     0.769       6.0
    B-arrive_date.day_number      0.714     0.833     0.769       6.0
         I-stoploc.city_name      1.000     1.000     1.000      10.0
                      B-meal      1.000     1.000     1.000      16.0
        B-fromloc.state_code      1.000     1.000     1.000      23.0
    B-depart_time.period_mod      1.000     1.000     1.000       5.0
                   B-connect      1.000     1.000     1.000       6.0
               B-flight_days      1.000     1.000     1.000      10.0
        B-toloc.airport_name      1.000     1.000     1.000       3.0
        B-fromloc.state_name      0.944     1.000     0.971      17.0
              B-airport_name      0.750     0.286     0.414      21.0
                   B-economy      1.000     1.000     1.000       6.0
               I-flight_time      1.000     1.000     1.000       1.0
             B-aircraft_code      1.000     0.848     0.918      33.0
                       B-mod      1.000     0.500     0.667       2.0
              B-airport_code      1.000     0.222     0.364       9.0
    B-depart_time.start_time      1.000     1.000     1.000       3.0
      B-depart_time.end_time      1.000     0.667     0.800       3.0
          B-depart_date.year      1.000     1.000     1.000       3.0
            I-transport_type      0.000     0.000     0.000       1.0
          B-restriction_code      1.000     0.500     0.667       4.0
    B-arrive_time.start_time      1.000     1.000     1.000       8.0
        B-toloc.airport_code      1.000     0.750     0.857       4.0
      B-arrive_time.end_time      1.000     1.000     1.000       8.0
      I-arrive_time.end_time      1.000     1.000     1.000       8.0
      I-depart_time.end_time      1.000     0.667     0.800       3.0
               I-flight_stop      0.000     0.000     0.000       0.0
      B-fromloc.airport_code      0.625     1.000     0.769       5.0
          I-restriction_code      1.000     1.000     1.000       3.0
    I-depart_time.start_time      1.000     1.000     1.000       1.0
          I-toloc.state_name      1.000     1.000     1.000       1.0
I-depart_date.today_relative      0.000     0.000     0.000       0.0
 B-arrive_date.date_relative      1.000     1.000     1.000       2.0
                I-flight_mod      1.000     0.167     0.286       6.0
                   I-economy      0.000     0.000     0.000       0.0
 B-return_date.date_relative      1.000     0.333     0.500       3.0
        I-fromloc.state_name      1.000     1.000     1.000       1.0
                B-state_code      1.000     1.000     1.000       1.0
    I-arrive_time.start_time      1.000     1.000     1.000       1.0
    I-arrive_date.day_number      0.000     0.000     0.000       0.0
                 B-meal_code      0.000     0.000     0.000       1.0
 I-depart_time.period_of_day      0.000     0.000     0.000       1.0
                  B-day_name      1.000     0.500     0.667       2.0
             B-period_of_day      1.000     0.750     0.857       4.0
        B-stoploc.state_code      0.000     0.000     0.000       0.0
    B-return_date.month_name      0.000     0.000     0.000       0.0
    B-return_date.day_number      0.000     0.000     0.000       0.0
    B-arrive_time.period_mod      0.000     0.000     0.000       0.0
                 I-meal_code      0.000     0.000     0.000       0.0
        B-toloc.country_name      1.000     1.000     1.000       1.0
                 B-days_code      0.000     0.000     0.000       1.0
 I-arrive_time.period_of_day      0.000     0.000     0.000       0.0
            I-today_relative      0.000     0.000     0.000       0.0
 B-return_time.period_of_day      0.000     0.000     0.000       0.0
                      B-time      0.000     0.000     0.000       0.0
           I-fare_basis_code      0.000     0.000     0.000       0.0
 I-arrive_time.time_relative      0.000     0.000     0.000       4.0
 I-depart_time.time_relative      0.000     0.000     0.000       1.0
            B-today_relative      0.000     0.000     0.000       0.0
                B-state_name      0.000     0.000     0.000       9.0
B-arrive_date.today_relative      0.000     0.000     0.000       0.0
    B-return_time.period_mod      0.000     0.000     0.000       0.0
                B-month_name      0.000     0.000     0.000       0.0
                B-day_number      0.000     0.000     0.000       0.0
 I-return_date.date_relative      1.000     0.667     0.800       3.0
I-return_date.today_relative      0.000     0.000     0.000       0.0
      B-stoploc.airport_name      0.000     0.000     0.000       0.0
             B-time_relative      0.000     0.000     0.000       0.0
                      I-time      0.000     0.000     0.000       0.0
    I-return_date.day_number      0.000     0.000     0.000       0.0
          I-meal_description      0.000     0.000     0.000       0.0
B-return_date.today_relative      0.000     0.000     0.000       0.0
      B-return_date.day_name      0.000     0.000     0.000       2.0

                   micro avg      0.956     0.958     0.957    3657.0
                   macro avg      0.670     0.637     0.638    3657.0
                weighted avg      0.959     0.958     0.953    3657.0

I0716 03:50:24.203364 139795883440000 interactiveshell.py:2882] Best Slot F1: 0.959
I0716 03:50:24.204905 139795883440000 estimator_training.py:186] Not using Distribute Coordinator.
I0716 03:50:24.207797 139795883440000 training.py:612] Running training and evaluation locally (non-distributed).
I0716 03:50:24.211139 139795883440000 training.py:700] Start train and evaluate loop. The evaluate will happen after every checkpoint. Checkpoint frequency is determined based on RunConfig arguments: save_checkpoints_steps 312 or save_checkpoints_secs None.
I0716 03:50:24.276541 139795883440000 estimator.py:1145] Calling model_fn.
I0716 03:50:25.731236 139795883440000 saver.py:1499] Saver not created because there are no variables in the graph to restore
I0716 03:50:26.477844 139795883440000 <ipython-input-6-d6747b40360b>:55] 
[<tf.Variable 'embedding:0' shape=(750, 300) dtype=float32_ref>,
 <tf.Variable 'module/aggregation/weights:0' shape=(3,) dtype=float32>,
 <tf.Variable 'module/aggregation/scaling:0' shape=() dtype=float32>,
 <tf.Variable 'dense/kernel:0' shape=(1324, 300) dtype=float32_ref>,
 <tf.Variable 'dense/bias:0' shape=(300,) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/gates/kernel:0' shape=(600, 600) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/gates/bias:0' shape=(600,) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/candidate/kernel:0' shape=(600, 300) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/candidate/bias:0' shape=(300,) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/gates/kernel:0' shape=(600, 600) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/gates/bias:0' shape=(600,) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/candidate/kernel:0' shape=(600, 300) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/candidate/bias:0' shape=(300,) dtype=float32_ref>,
 <tf.Variable 'dense_1/kernel:0' shape=(600, 23) dtype=float32_ref>,
 <tf.Variable 'dense_1/bias:0' shape=(23,) dtype=float32_ref>,
 <tf.Variable 'dense_2/kernel:0' shape=(600, 122) dtype=float32_ref>,
 <tf.Variable 'dense_2/bias:0' shape=(122,) dtype=float32_ref>]
I0716 03:50:27.547219 139795883440000 estimator.py:1147] Done calling model_fn.
I0716 03:50:27.551077 139795883440000 basic_session_run_hooks.py:541] Create CheckpointSaverHook.
I0716 03:50:27.879000 139795883440000 monitored_session.py:240] Graph was finalized.
I0716 03:50:27.895506 139795883440000 saver.py:1280] Restoring parameters from ../model/elmo_bigru/model.ckpt-4368
I0716 03:50:28.932056 139795883440000 session_manager.py:500] Running local_init_op.
I0716 03:50:29.001270 139795883440000 session_manager.py:502] Done running local_init_op.
I0716 03:50:32.065008 139795883440000 basic_session_run_hooks.py:606] Saving checkpoints for 4368 into ../model/elmo_bigru/model.ckpt.
Reading ../data/atis.train.w-intent.iob
I0716 03:50:37.141797 139795883440000 basic_session_run_hooks.py:262] loss = 0.018032221, step = 4368
I0716 03:50:37.143822 139795883440000 basic_session_run_hooks.py:262] lr = 0.00018934443
I0716 03:51:04.711982 139795883440000 basic_session_run_hooks.py:692] global_step/sec: 3.627
I0716 03:51:04.719229 139795883440000 basic_session_run_hooks.py:260] loss = 0.026767438, step = 4468 (27.577 sec)
I0716 03:51:04.721602 139795883440000 basic_session_run_hooks.py:260] lr = 0.00018735994 (27.578 sec)
I0716 03:51:30.516970 139795883440000 basic_session_run_hooks.py:692] global_step/sec: 3.87521
I0716 03:51:30.519589 139795883440000 basic_session_run_hooks.py:260] loss = 0.021363582, step = 4568 (25.800 sec)
I0716 03:51:30.523059 139795883440000 basic_session_run_hooks.py:260] lr = 0.00018539626 (25.801 sec)
I0716 03:51:57.386001 139795883440000 basic_session_run_hooks.py:692] global_step/sec: 3.72176
I0716 03:51:57.389219 139795883440000 basic_session_run_hooks.py:260] loss = 0.029927757, step = 4668 (26.870 sec)
I0716 03:51:57.393661 139795883440000 basic_session_run_hooks.py:260] lr = 0.00018345319 (26.871 sec)
I0716 03:52:00.255527 139795883440000 basic_session_run_hooks.py:606] Saving checkpoints for 4680 into ../model/elmo_bigru/model.ckpt.
I0716 03:52:02.224086 139795883440000 estimator.py:1145] Calling model_fn.
I0716 03:52:03.564811 139795883440000 saver.py:1499] Saver not created because there are no variables in the graph to restore
I0716 03:52:04.140324 139795883440000 estimator.py:1147] Done calling model_fn.
I0716 03:52:04.169637 139795883440000 evaluation.py:255] Starting evaluation at 2019-07-16T03:52:04Z
I0716 03:52:04.391069 139795883440000 monitored_session.py:240] Graph was finalized.
I0716 03:52:04.406590 139795883440000 saver.py:1280] Restoring parameters from ../model/elmo_bigru/model.ckpt-4680
I0716 03:52:05.334580 139795883440000 session_manager.py:500] Running local_init_op.
I0716 03:52:05.390577 139795883440000 session_manager.py:502] Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
I0716 03:53:04.278062 139795883440000 evaluation.py:275] Finished evaluation at 2019-07-16-03:53:04
I0716 03:53:04.279991 139795883440000 estimator.py:2039] Saving dict for global step 4680: global_step = 4680, loss = 0.28495464
I0716 03:53:04.287950 139795883440000 estimator.py:2099] Saving 'checkpoint_path' summary for global step 4680: ../model/elmo_bigru/model.ckpt-4680
I0716 03:53:04.370682 139795883440000 estimator.py:368] Loss for final step: 0.12527332.
Reading ../data/atis.test.w-intent.iob
I0716 03:53:04.963579 139795883440000 estimator.py:1145] Calling model_fn.
I0716 03:53:06.610333 139795883440000 saver.py:1499] Saver not created because there are no variables in the graph to restore
I0716 03:53:07.134688 139795883440000 estimator.py:1147] Done calling model_fn.
I0716 03:53:07.528035 139795883440000 monitored_session.py:240] Graph was finalized.
I0716 03:53:07.542665 139795883440000 saver.py:1280] Restoring parameters from ../model/elmo_bigru/model.ckpt-4680
I0716 03:53:08.440604 139795883440000 session_manager.py:500] Running local_init_op.
I0716 03:53:08.489479 139795883440000 session_manager.py:502] Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.
  'precision', 'predicted', average, warn_for)
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1439: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples.
  'recall', 'true', average, warn_for)
I0716 03:54:06.450646 139795883440000 interactiveshell.py:2882] 
                                          precision    recall  f1-score   support

                             atis_flight      0.991     0.992     0.991       632
                            atis_airfare      0.906     1.000     0.950        48
                     atis_ground_service      1.000     1.000     1.000        36
                            atis_airline      0.974     1.000     0.987        38
                       atis_abbreviation      0.943     1.000     0.971        33
                           atis_aircraft      1.000     0.889     0.941         9
                        atis_flight_time      0.500     1.000     0.667         1
                           atis_quantity      0.429     1.000     0.600         3
                atis_flight#atis_airfare      0.857     0.500     0.632        12
                            atis_airport      1.000     0.944     0.971        18
                           atis_distance      1.000     1.000     1.000        10
                               atis_city      0.667     0.667     0.667         6
                        atis_ground_fare      1.000     0.857     0.923         7
                           atis_capacity      0.952     0.952     0.952        21
                          atis_flight_no      0.889     1.000     0.941         8
                               atis_meal      1.000     0.667     0.800         6
                        atis_restriction      0.000     0.000     0.000         0
             atis_airline#atis_flight_no      0.000     0.000     0.000         0
    atis_ground_service#atis_ground_fare      0.000     0.000     0.000         0
           atis_airfare#atis_flight_time      0.000     0.000     0.000         0
                           atis_cheapest      0.000     0.000     0.000         0
atis_aircraft#atis_flight#atis_flight_no      0.000     0.000     0.000         0

                               micro avg      0.973     0.979     0.976       888
                               macro avg      0.641     0.658     0.636       888
                            weighted avg      0.976     0.979     0.976       888

I0716 03:54:06.487351 139795883440000 interactiveshell.py:2882] 
                              precision    recall  f1-score   support

                           O      0.000     0.000     0.000       0.0
           B-toloc.city_name      0.977     0.993     0.985     716.0
         B-fromloc.city_name      0.990     0.999     0.994     704.0
           I-toloc.city_name      0.985     0.992     0.989     265.0
      B-depart_date.day_name      0.991     0.991     0.991     212.0
              B-airline_name      1.000     1.000     1.000     101.0
         I-fromloc.city_name      0.983     0.994     0.989     177.0
 B-depart_time.period_of_day      0.984     0.923     0.952     130.0
              I-airline_name      0.985     1.000     0.992      65.0
    B-depart_date.day_number      0.981     0.964     0.972      55.0
    B-depart_date.month_name      0.981     0.946     0.964      56.0
          B-depart_time.time      0.864     1.000     0.927      57.0
                B-round_trip      1.000     1.000     1.000      73.0
             B-cost_relative      0.973     0.973     0.973      37.0
                I-round_trip      1.000     1.000     1.000      71.0
                B-flight_mod      0.828     1.000     0.906      24.0
 B-depart_time.time_relative      0.970     0.985     0.977      65.0
          I-depart_time.time      0.945     1.000     0.972      52.0
         B-stoploc.city_name      0.909     1.000     0.952      20.0
                 B-city_name      0.844     0.667     0.745      57.0
                B-class_type      0.960     1.000     0.980      24.0
          B-arrive_time.time      0.971     0.971     0.971      34.0
 B-arrive_time.time_relative      0.909     0.968     0.937      31.0
                I-class_type      1.000     1.000     1.000      17.0
               B-flight_stop      1.000     1.000     1.000      21.0
          I-arrive_time.time      1.000     0.971     0.986      35.0
              B-airline_code      0.895     1.000     0.944      34.0
    I-depart_date.day_number      1.000     0.933     0.966      15.0
      I-fromloc.airport_name      0.429     1.000     0.600      15.0
      B-fromloc.airport_name      0.400     1.000     0.571      12.0
      B-arrive_date.day_name      0.786     1.000     0.880      11.0
          B-toloc.state_code      1.000     1.000     1.000      18.0
B-depart_date.today_relative      1.000     0.889     0.941       9.0
             B-flight_number      0.733     1.000     0.846      11.0
 B-depart_date.date_relative      0.944     1.000     0.971      17.0
          B-toloc.state_name      0.926     0.893     0.909      28.0
           B-fare_basis_code      0.895     1.000     0.944      17.0
               B-flight_time      1.000     1.000     1.000       1.0
                        B-or      1.000     1.000     1.000       3.0
 B-arrive_time.period_of_day      0.750     1.000     0.857       6.0
          B-meal_description      1.000     0.900     0.947      10.0
             I-cost_relative      1.000     0.667     0.800       3.0
              I-airport_name      0.846     0.379     0.524      29.0
               B-fare_amount      1.000     1.000     1.000       2.0
               I-fare_amount      1.000     1.000     1.000       2.0
                 I-city_name      1.000     0.667     0.800      30.0
        I-toloc.airport_name      1.000     1.000     1.000       3.0
            B-transport_type      1.000     1.000     1.000      10.0
    B-arrive_date.month_name      0.625     0.833     0.714       6.0
    B-arrive_date.day_number      0.625     0.833     0.714       6.0
         I-stoploc.city_name      0.909     1.000     0.952      10.0
                      B-meal      0.941     1.000     0.970      16.0
        B-fromloc.state_code      1.000     1.000     1.000      23.0
    B-depart_time.period_mod      1.000     1.000     1.000       5.0
                   B-connect      1.000     1.000     1.000       6.0
               B-flight_days      1.000     1.000     1.000      10.0
        B-toloc.airport_name      1.000     1.000     1.000       3.0
        B-fromloc.state_name      1.000     0.941     0.970      17.0
              B-airport_name      0.583     0.333     0.424      21.0
                   B-economy      1.000     1.000     1.000       6.0
               I-flight_time      1.000     1.000     1.000       1.0
             B-aircraft_code      1.000     0.818     0.900      33.0
                       B-mod      1.000     0.500     0.667       2.0
              B-airport_code      1.000     0.444     0.615       9.0
    B-depart_time.start_time      1.000     0.667     0.800       3.0
      B-depart_time.end_time      1.000     0.333     0.500       3.0
          B-depart_date.year      1.000     1.000     1.000       3.0
            I-transport_type      1.000     1.000     1.000       1.0
          B-restriction_code      0.800     1.000     0.889       4.0
    B-arrive_time.start_time      0.889     1.000     0.941       8.0
        B-toloc.airport_code      1.000     0.750     0.857       4.0
      B-arrive_time.end_time      0.889     1.000     0.941       8.0
      I-arrive_time.end_time      0.889     1.000     0.941       8.0
      I-depart_time.end_time      1.000     0.333     0.500       3.0
               I-flight_stop      0.000     0.000     0.000       0.0
      B-fromloc.airport_code      0.714     1.000     0.833       5.0
          I-restriction_code      1.000     1.000     1.000       3.0
    I-depart_time.start_time      1.000     1.000     1.000       1.0
          I-toloc.state_name      1.000     1.000     1.000       1.0
I-depart_date.today_relative      0.000     0.000     0.000       0.0
 B-arrive_date.date_relative      1.000     1.000     1.000       2.0
                I-flight_mod      1.000     0.167     0.286       6.0
                   I-economy      0.000     0.000     0.000       0.0
 B-return_date.date_relative      1.000     0.333     0.500       3.0
        I-fromloc.state_name      1.000     1.000     1.000       1.0
                B-state_code      1.000     1.000     1.000       1.0
    I-arrive_time.start_time      1.000     1.000     1.000       1.0
    I-arrive_date.day_number      0.000     0.000     0.000       0.0
                 B-meal_code      0.000     0.000     0.000       1.0
 I-depart_time.period_of_day      0.000     0.000     0.000       1.0
                  B-day_name      1.000     0.500     0.667       2.0
             B-period_of_day      1.000     0.750     0.857       4.0
        B-stoploc.state_code      0.000     0.000     0.000       0.0
    B-return_date.month_name      0.000     0.000     0.000       0.0
    B-return_date.day_number      0.000     0.000     0.000       0.0
    B-arrive_time.period_mod      0.000     0.000     0.000       0.0
                 I-meal_code      0.000     0.000     0.000       0.0
        B-toloc.country_name      0.000     0.000     0.000       1.0
                 B-days_code      0.000     0.000     0.000       1.0
 I-arrive_time.period_of_day      0.000     0.000     0.000       0.0
            I-today_relative      0.000     0.000     0.000       0.0
 B-return_time.period_of_day      0.000     0.000     0.000       0.0
                      B-time      0.000     0.000     0.000       0.0
           I-fare_basis_code      0.000     0.000     0.000       0.0
 I-arrive_time.time_relative      1.000     0.500     0.667       4.0
 I-depart_time.time_relative      0.000     0.000     0.000       1.0
            B-today_relative      0.000     0.000     0.000       0.0
                B-state_name      0.000     0.000     0.000       9.0
B-arrive_date.today_relative      0.000     0.000     0.000       0.0
    B-return_time.period_mod      0.000     0.000     0.000       0.0
                B-month_name      0.000     0.000     0.000       0.0
                B-day_number      0.000     0.000     0.000       0.0
 I-return_date.date_relative      1.000     0.667     0.800       3.0
I-return_date.today_relative      0.000     0.000     0.000       0.0
      B-stoploc.airport_name      0.000     0.000     0.000       0.0
             B-time_relative      0.000     0.000     0.000       0.0
                      I-time      0.000     0.000     0.000       0.0
    I-return_date.day_number      0.000     0.000     0.000       0.0
          I-meal_description      0.000     0.000     0.000       0.0
B-return_date.today_relative      0.000     0.000     0.000       0.0
      B-return_date.day_name      0.000     0.000     0.000       2.0

                   micro avg      0.958     0.960     0.959    3657.0
                   macro avg      0.674     0.639     0.641    3657.0
                weighted avg      0.961     0.960     0.956    3657.0

I0716 03:54:06.507676 139795883440000 interactiveshell.py:2882] Best Slot F1: 0.959
I0716 03:54:06.509477 139795883440000 estimator_training.py:186] Not using Distribute Coordinator.
I0716 03:54:06.514797 139795883440000 training.py:612] Running training and evaluation locally (non-distributed).
I0716 03:54:06.517106 139795883440000 training.py:700] Start train and evaluate loop. The evaluate will happen after every checkpoint. Checkpoint frequency is determined based on RunConfig arguments: save_checkpoints_steps 312 or save_checkpoints_secs None.
I0716 03:54:06.581527 139795883440000 estimator.py:1145] Calling model_fn.
I0716 03:54:07.902871 139795883440000 saver.py:1499] Saver not created because there are no variables in the graph to restore
I0716 03:54:08.654480 139795883440000 <ipython-input-6-d6747b40360b>:55] 
[<tf.Variable 'embedding:0' shape=(750, 300) dtype=float32_ref>,
 <tf.Variable 'module/aggregation/weights:0' shape=(3,) dtype=float32>,
 <tf.Variable 'module/aggregation/scaling:0' shape=() dtype=float32>,
 <tf.Variable 'dense/kernel:0' shape=(1324, 300) dtype=float32_ref>,
 <tf.Variable 'dense/bias:0' shape=(300,) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/gates/kernel:0' shape=(600, 600) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/gates/bias:0' shape=(600,) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/candidate/kernel:0' shape=(600, 300) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/candidate/bias:0' shape=(300,) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/gates/kernel:0' shape=(600, 600) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/gates/bias:0' shape=(600,) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/candidate/kernel:0' shape=(600, 300) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/candidate/bias:0' shape=(300,) dtype=float32_ref>,
 <tf.Variable 'dense_1/kernel:0' shape=(600, 23) dtype=float32_ref>,
 <tf.Variable 'dense_1/bias:0' shape=(23,) dtype=float32_ref>,
 <tf.Variable 'dense_2/kernel:0' shape=(600, 122) dtype=float32_ref>,
 <tf.Variable 'dense_2/bias:0' shape=(122,) dtype=float32_ref>]
I0716 03:54:09.724922 139795883440000 estimator.py:1147] Done calling model_fn.
I0716 03:54:09.729112 139795883440000 basic_session_run_hooks.py:541] Create CheckpointSaverHook.
I0716 03:54:10.054003 139795883440000 monitored_session.py:240] Graph was finalized.
I0716 03:54:10.073008 139795883440000 saver.py:1280] Restoring parameters from ../model/elmo_bigru/model.ckpt-4680
I0716 03:54:11.115986 139795883440000 session_manager.py:500] Running local_init_op.
I0716 03:54:11.183035 139795883440000 session_manager.py:502] Done running local_init_op.
I0716 03:54:14.195451 139795883440000 basic_session_run_hooks.py:606] Saving checkpoints for 4680 into ../model/elmo_bigru/model.ckpt.
Reading ../data/atis.train.w-intent.iob
I0716 03:54:18.893395 139795883440000 basic_session_run_hooks.py:262] loss = 0.023944816, step = 4680
I0716 03:54:18.895148 139795883440000 basic_session_run_hooks.py:262] lr = 0.00018322139
I0716 03:54:47.028027 139795883440000 basic_session_run_hooks.py:692] global_step/sec: 3.55429
I0716 03:54:47.030983 139795883440000 basic_session_run_hooks.py:260] loss = 0.018975822, step = 4780 (28.138 sec)
I0716 03:54:47.036970 139795883440000 basic_session_run_hooks.py:260] lr = 0.00018130106 (28.142 sec)
I0716 03:55:12.942830 139795883440000 basic_session_run_hooks.py:692] global_step/sec: 3.85878
I0716 03:55:12.945505 139795883440000 basic_session_run_hooks.py:260] loss = 0.015009847, step = 4880 (25.915 sec)
I0716 03:55:12.950030 139795883440000 basic_session_run_hooks.py:260] lr = 0.00017940091 (25.913 sec)
I0716 03:55:39.014466 139795883440000 basic_session_run_hooks.py:692] global_step/sec: 3.83559
I0716 03:55:39.021158 139795883440000 basic_session_run_hooks.py:260] loss = 0.01981283, step = 4980 (26.076 sec)
I0716 03:55:39.023919 139795883440000 basic_session_run_hooks.py:260] lr = 0.00017752068 (26.074 sec)
I0716 03:55:41.939601 139795883440000 basic_session_run_hooks.py:606] Saving checkpoints for 4992 into ../model/elmo_bigru/model.ckpt.
I0716 03:55:44.422773 139795883440000 estimator.py:1145] Calling model_fn.
I0716 03:55:45.824679 139795883440000 saver.py:1499] Saver not created because there are no variables in the graph to restore
I0716 03:55:46.418750 139795883440000 estimator.py:1147] Done calling model_fn.
I0716 03:55:46.448998 139795883440000 evaluation.py:255] Starting evaluation at 2019-07-16T03:55:46Z
I0716 03:55:46.681682 139795883440000 monitored_session.py:240] Graph was finalized.
I0716 03:55:46.696691 139795883440000 saver.py:1280] Restoring parameters from ../model/elmo_bigru/model.ckpt-4992
I0716 03:55:47.641804 139795883440000 session_manager.py:500] Running local_init_op.
I0716 03:55:47.704630 139795883440000 session_manager.py:502] Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
I0716 03:56:46.573673 139795883440000 evaluation.py:275] Finished evaluation at 2019-07-16-03:56:46
I0716 03:56:46.575764 139795883440000 estimator.py:2039] Saving dict for global step 4992: global_step = 4992, loss = 0.31630886
I0716 03:56:46.581027 139795883440000 estimator.py:2099] Saving 'checkpoint_path' summary for global step 4992: ../model/elmo_bigru/model.ckpt-4992
I0716 03:56:46.659699 139795883440000 estimator.py:368] Loss for final step: 0.006171497.
Reading ../data/atis.test.w-intent.iob
I0716 03:56:47.247567 139795883440000 estimator.py:1145] Calling model_fn.
I0716 03:56:48.900026 139795883440000 saver.py:1499] Saver not created because there are no variables in the graph to restore
I0716 03:56:49.408294 139795883440000 estimator.py:1147] Done calling model_fn.
I0716 03:56:49.634571 139795883440000 monitored_session.py:240] Graph was finalized.
I0716 03:56:49.648227 139795883440000 saver.py:1280] Restoring parameters from ../model/elmo_bigru/model.ckpt-4992
I0716 03:56:50.536725 139795883440000 session_manager.py:500] Running local_init_op.
I0716 03:56:50.585864 139795883440000 session_manager.py:502] Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.
  'precision', 'predicted', average, warn_for)
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1439: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples.
  'recall', 'true', average, warn_for)
I0716 03:57:48.300082 139795883440000 interactiveshell.py:2882] 
                                          precision    recall  f1-score   support

                             atis_flight      0.984     0.992     0.988       632
                            atis_airfare      0.941     1.000     0.970        48
                     atis_ground_service      1.000     1.000     1.000        36
                            atis_airline      0.974     1.000     0.987        38
                       atis_abbreviation      0.943     1.000     0.971        33
                           atis_aircraft      1.000     0.889     0.941         9
                        atis_flight_time      0.333     1.000     0.500         1
                           atis_quantity      0.375     1.000     0.545         3
                atis_flight#atis_airfare      0.833     0.417     0.556        12
                            atis_airport      1.000     0.944     0.971        18
                           atis_distance      1.000     1.000     1.000        10
                               atis_city      1.000     0.500     0.667         6
                        atis_ground_fare      1.000     0.857     0.923         7
                           atis_capacity      1.000     0.952     0.976        21
                          atis_flight_no      0.889     1.000     0.941         8
                               atis_meal      1.000     0.833     0.909         6
                        atis_restriction      0.000     0.000     0.000         0
             atis_airline#atis_flight_no      0.000     0.000     0.000         0
    atis_ground_service#atis_ground_fare      0.000     0.000     0.000         0
           atis_airfare#atis_flight_time      0.000     0.000     0.000         0
                           atis_cheapest      0.000     0.000     0.000         0
atis_aircraft#atis_flight#atis_flight_no      0.000     0.000     0.000         0

                               micro avg      0.972     0.977     0.975       888
                               macro avg      0.649     0.654     0.629       888
                            weighted avg      0.976     0.977     0.974       888

I0716 03:57:48.337573 139795883440000 interactiveshell.py:2882] 
                              precision    recall  f1-score   support

                           O      0.000     0.000     0.000       0.0
           B-toloc.city_name      0.974     0.994     0.984     716.0
         B-fromloc.city_name      0.990     0.999     0.994     704.0
           I-toloc.city_name      0.967     0.992     0.980     265.0
      B-depart_date.day_name      0.991     0.991     0.991     212.0
              B-airline_name      1.000     1.000     1.000     101.0
         I-fromloc.city_name      0.978     0.994     0.986     177.0
 B-depart_time.period_of_day      0.983     0.915     0.948     130.0
              I-airline_name      1.000     1.000     1.000      65.0
    B-depart_date.day_number      0.981     0.964     0.972      55.0
    B-depart_date.month_name      0.981     0.946     0.964      56.0
          B-depart_time.time      0.864     1.000     0.927      57.0
                B-round_trip      1.000     1.000     1.000      73.0
             B-cost_relative      0.973     0.973     0.973      37.0
                I-round_trip      1.000     1.000     1.000      71.0
                B-flight_mod      0.828     1.000     0.906      24.0
 B-depart_time.time_relative      0.970     0.985     0.977      65.0
          I-depart_time.time      0.929     1.000     0.963      52.0
         B-stoploc.city_name      0.909     1.000     0.952      20.0
                 B-city_name      0.860     0.649     0.740      57.0
                B-class_type      0.960     1.000     0.980      24.0
          B-arrive_time.time      0.971     0.971     0.971      34.0
 B-arrive_time.time_relative      0.857     0.968     0.909      31.0
                I-class_type      1.000     1.000     1.000      17.0
               B-flight_stop      1.000     1.000     1.000      21.0
          I-arrive_time.time      1.000     0.971     0.986      35.0
              B-airline_code      0.971     1.000     0.986      34.0
    I-depart_date.day_number      1.000     0.933     0.966      15.0
      I-fromloc.airport_name      0.417     1.000     0.588      15.0
      B-fromloc.airport_name      0.429     1.000     0.600      12.0
      B-arrive_date.day_name      0.786     1.000     0.880      11.0
          B-toloc.state_code      1.000     1.000     1.000      18.0
B-depart_date.today_relative      1.000     0.889     0.941       9.0
             B-flight_number      0.733     1.000     0.846      11.0
 B-depart_date.date_relative      0.944     1.000     0.971      17.0
          B-toloc.state_name      0.857     0.857     0.857      28.0
           B-fare_basis_code      0.850     1.000     0.919      17.0
               B-flight_time      1.000     1.000     1.000       1.0
                        B-or      1.000     1.000     1.000       3.0
 B-arrive_time.period_of_day      0.750     1.000     0.857       6.0
          B-meal_description      1.000     0.900     0.947      10.0
             I-cost_relative      1.000     0.667     0.800       3.0
              I-airport_name      0.833     0.345     0.488      29.0
               B-fare_amount      1.000     1.000     1.000       2.0
               I-fare_amount      1.000     1.000     1.000       2.0
                 I-city_name      1.000     0.567     0.723      30.0
        I-toloc.airport_name      1.000     1.000     1.000       3.0
            B-transport_type      1.000     1.000     1.000      10.0
    B-arrive_date.month_name      0.625     0.833     0.714       6.0
    B-arrive_date.day_number      0.714     0.833     0.769       6.0
         I-stoploc.city_name      0.909     1.000     0.952      10.0
                      B-meal      1.000     1.000     1.000      16.0
        B-fromloc.state_code      1.000     1.000     1.000      23.0
    B-depart_time.period_mod      1.000     1.000     1.000       5.0
                   B-connect      1.000     1.000     1.000       6.0
               B-flight_days      1.000     1.000     1.000      10.0
        B-toloc.airport_name      1.000     1.000     1.000       3.0
        B-fromloc.state_name      1.000     1.000     1.000      17.0
              B-airport_name      0.636     0.333     0.437      21.0
                   B-economy      1.000     0.333     0.500       6.0
               I-flight_time      1.000     1.000     1.000       1.0
             B-aircraft_code      1.000     0.818     0.900      33.0
                       B-mod      1.000     0.500     0.667       2.0
              B-airport_code      0.667     0.444     0.533       9.0
    B-depart_time.start_time      1.000     0.667     0.800       3.0
      B-depart_time.end_time      1.000     0.333     0.500       3.0
          B-depart_date.year      1.000     1.000     1.000       3.0
            I-transport_type      0.000     0.000     0.000       1.0
          B-restriction_code      0.800     1.000     0.889       4.0
    B-arrive_time.start_time      0.889     1.000     0.941       8.0
        B-toloc.airport_code      1.000     0.750     0.857       4.0
      B-arrive_time.end_time      0.889     1.000     0.941       8.0
      I-arrive_time.end_time      0.889     1.000     0.941       8.0
      I-depart_time.end_time      1.000     0.333     0.500       3.0
               I-flight_stop      0.000     0.000     0.000       0.0
      B-fromloc.airport_code      0.625     1.000     0.769       5.0
          I-restriction_code      1.000     1.000     1.000       3.0
    I-depart_time.start_time      1.000     1.000     1.000       1.0
          I-toloc.state_name      1.000     1.000     1.000       1.0
I-depart_date.today_relative      0.000     0.000     0.000       0.0
 B-arrive_date.date_relative      1.000     1.000     1.000       2.0
                I-flight_mod      1.000     0.167     0.286       6.0
                   I-economy      0.000     0.000     0.000       0.0
 B-return_date.date_relative      1.000     0.667     0.800       3.0
        I-fromloc.state_name      1.000     1.000     1.000       1.0
                B-state_code      1.000     1.000     1.000       1.0
    I-arrive_time.start_time      1.000     1.000     1.000       1.0
    I-arrive_date.day_number      0.000     0.000     0.000       0.0
                 B-meal_code      0.000     0.000     0.000       1.0
 I-depart_time.period_of_day      0.000     0.000     0.000       1.0
                  B-day_name      1.000     0.500     0.667       2.0
             B-period_of_day      1.000     0.750     0.857       4.0
        B-stoploc.state_code      0.000     0.000     0.000       0.0
    B-return_date.month_name      0.000     0.000     0.000       0.0
    B-return_date.day_number      0.000     0.000     0.000       0.0
    B-arrive_time.period_mod      0.000     0.000     0.000       0.0
                 I-meal_code      0.000     0.000     0.000       0.0
        B-toloc.country_name      1.000     1.000     1.000       1.0
                 B-days_code      0.000     0.000     0.000       1.0
 I-arrive_time.period_of_day      0.000     0.000     0.000       0.0
            I-today_relative      0.000     0.000     0.000       0.0
 B-return_time.period_of_day      0.000     0.000     0.000       0.0
                      B-time      0.000     0.000     0.000       0.0
           I-fare_basis_code      0.000     0.000     0.000       0.0
 I-arrive_time.time_relative      0.000     0.000     0.000       4.0
 I-depart_time.time_relative      0.000     0.000     0.000       1.0
            B-today_relative      0.000     0.000     0.000       0.0
                B-state_name      0.000     0.000     0.000       9.0
B-arrive_date.today_relative      0.000     0.000     0.000       0.0
    B-return_time.period_mod      0.000     0.000     0.000       0.0
                B-month_name      0.000     0.000     0.000       0.0
                B-day_number      0.000     0.000     0.000       0.0
 I-return_date.date_relative      1.000     0.667     0.800       3.0
I-return_date.today_relative      0.000     0.000     0.000       0.0
      B-stoploc.airport_name      0.000     0.000     0.000       0.0
             B-time_relative      0.000     0.000     0.000       0.0
                      I-time      0.000     0.000     0.000       0.0
    I-return_date.day_number      0.000     0.000     0.000       0.0
          I-meal_description      0.000     0.000     0.000       0.0
B-return_date.today_relative      0.000     0.000     0.000       0.0
      B-return_date.day_name      0.000     0.000     0.000       2.0

                   micro avg      0.955     0.957     0.956    3657.0
                   macro avg      0.663     0.631     0.632    3657.0
                weighted avg      0.957     0.957     0.952    3657.0

I0716 03:57:48.358943 139795883440000 interactiveshell.py:2882] Best Slot F1: 0.959
I0716 03:57:48.360575 139795883440000 estimator_training.py:186] Not using Distribute Coordinator.
I0716 03:57:48.362465 139795883440000 training.py:612] Running training and evaluation locally (non-distributed).
I0716 03:57:48.364193 139795883440000 training.py:700] Start train and evaluate loop. The evaluate will happen after every checkpoint. Checkpoint frequency is determined based on RunConfig arguments: save_checkpoints_steps 312 or save_checkpoints_secs None.
I0716 03:57:48.430262 139795883440000 estimator.py:1145] Calling model_fn.
I0716 03:57:50.047309 139795883440000 saver.py:1499] Saver not created because there are no variables in the graph to restore
I0716 03:57:50.645043 139795883440000 <ipython-input-6-d6747b40360b>:55] 
[<tf.Variable 'embedding:0' shape=(750, 300) dtype=float32_ref>,
 <tf.Variable 'module/aggregation/weights:0' shape=(3,) dtype=float32>,
 <tf.Variable 'module/aggregation/scaling:0' shape=() dtype=float32>,
 <tf.Variable 'dense/kernel:0' shape=(1324, 300) dtype=float32_ref>,
 <tf.Variable 'dense/bias:0' shape=(300,) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/gates/kernel:0' shape=(600, 600) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/gates/bias:0' shape=(600,) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/candidate/kernel:0' shape=(600, 300) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/candidate/bias:0' shape=(300,) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/gates/kernel:0' shape=(600, 600) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/gates/bias:0' shape=(600,) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/candidate/kernel:0' shape=(600, 300) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/candidate/bias:0' shape=(300,) dtype=float32_ref>,
 <tf.Variable 'dense_1/kernel:0' shape=(600, 23) dtype=float32_ref>,
 <tf.Variable 'dense_1/bias:0' shape=(23,) dtype=float32_ref>,
 <tf.Variable 'dense_2/kernel:0' shape=(600, 122) dtype=float32_ref>,
 <tf.Variable 'dense_2/bias:0' shape=(122,) dtype=float32_ref>]
I0716 03:57:51.722529 139795883440000 estimator.py:1147] Done calling model_fn.
I0716 03:57:51.725912 139795883440000 basic_session_run_hooks.py:541] Create CheckpointSaverHook.
I0716 03:57:52.252693 139795883440000 monitored_session.py:240] Graph was finalized.
I0716 03:57:52.268751 139795883440000 saver.py:1280] Restoring parameters from ../model/elmo_bigru/model.ckpt-4992
I0716 03:57:53.320537 139795883440000 session_manager.py:500] Running local_init_op.
I0716 03:57:53.384846 139795883440000 session_manager.py:502] Done running local_init_op.
I0716 03:57:56.176370 139795883440000 basic_session_run_hooks.py:606] Saving checkpoints for 4992 into ../model/elmo_bigru/model.ckpt.
Reading ../data/atis.train.w-intent.iob
I0716 03:58:00.755765 139795883440000 basic_session_run_hooks.py:262] loss = 0.018470936, step = 4992
I0716 03:58:00.757390 139795883440000 basic_session_run_hooks.py:262] lr = 0.00017729634
I0716 03:58:28.238779 139795883440000 basic_session_run_hooks.py:692] global_step/sec: 3.63854
I0716 03:58:28.243356 139795883440000 basic_session_run_hooks.py:260] loss = 0.011454404, step = 5092 (27.488 sec)
I0716 03:58:28.247864 139795883440000 basic_session_run_hooks.py:260] lr = 0.00017543815 (27.490 sec)
I0716 03:58:54.253058 139795883440000 basic_session_run_hooks.py:692] global_step/sec: 3.84406
I0716 03:58:54.259320 139795883440000 basic_session_run_hooks.py:260] loss = 0.016113179, step = 5192 (26.016 sec)
I0716 03:58:54.261050 139795883440000 basic_session_run_hooks.py:260] lr = 0.00017359943 (26.013 sec)
I0716 03:59:20.595207 139795883440000 basic_session_run_hooks.py:692] global_step/sec: 3.79619
I0716 03:59:20.600870 139795883440000 basic_session_run_hooks.py:260] loss = 0.014878462, step = 5292 (26.342 sec)
I0716 03:59:20.602448 139795883440000 basic_session_run_hooks.py:260] lr = 0.00017177999 (26.341 sec)
I0716 03:59:23.645135 139795883440000 basic_session_run_hooks.py:606] Saving checkpoints for 5304 into ../model/elmo_bigru/model.ckpt.
I0716 03:59:25.506482 139795883440000 estimator.py:1145] Calling model_fn.
I0716 03:59:26.849341 139795883440000 saver.py:1499] Saver not created because there are no variables in the graph to restore
I0716 03:59:27.427505 139795883440000 estimator.py:1147] Done calling model_fn.
I0716 03:59:27.457143 139795883440000 evaluation.py:255] Starting evaluation at 2019-07-16T03:59:27Z
I0716 03:59:27.937038 139795883440000 monitored_session.py:240] Graph was finalized.
I0716 03:59:27.952298 139795883440000 saver.py:1280] Restoring parameters from ../model/elmo_bigru/model.ckpt-5304
I0716 03:59:28.839442 139795883440000 session_manager.py:500] Running local_init_op.
I0716 03:59:28.898953 139795883440000 session_manager.py:502] Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
I0716 04:00:27.577882 139795883440000 evaluation.py:275] Finished evaluation at 2019-07-16-04:00:27
I0716 04:00:27.579757 139795883440000 estimator.py:2039] Saving dict for global step 5304: global_step = 5304, loss = 0.3132215
I0716 04:00:27.585969 139795883440000 estimator.py:2099] Saving 'checkpoint_path' summary for global step 5304: ../model/elmo_bigru/model.ckpt-5304
I0716 04:00:27.668098 139795883440000 estimator.py:368] Loss for final step: 0.009373315.
Reading ../data/atis.test.w-intent.iob
I0716 04:00:28.264626 139795883440000 estimator.py:1145] Calling model_fn.
I0716 04:00:29.294847 139795883440000 saver.py:1499] Saver not created because there are no variables in the graph to restore
I0716 04:00:29.809007 139795883440000 estimator.py:1147] Done calling model_fn.
I0716 04:00:30.021951 139795883440000 monitored_session.py:240] Graph was finalized.
I0716 04:00:30.037668 139795883440000 saver.py:1280] Restoring parameters from ../model/elmo_bigru/model.ckpt-5304
I0716 04:00:30.928122 139795883440000 session_manager.py:500] Running local_init_op.
I0716 04:00:30.974180 139795883440000 session_manager.py:502] Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.
  'precision', 'predicted', average, warn_for)
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1439: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples.
  'recall', 'true', average, warn_for)
I0716 04:01:29.005527 139795883440000 interactiveshell.py:2882] 
                                          precision    recall  f1-score   support

                             atis_flight      0.986     0.992     0.989       632
                            atis_airfare      0.941     1.000     0.970        48
                     atis_ground_service      1.000     1.000     1.000        36
                            atis_airline      0.974     1.000     0.987        38
                       atis_abbreviation      0.943     1.000     0.971        33
                           atis_aircraft      1.000     0.889     0.941         9
                        atis_flight_time      0.500     1.000     0.667         1
                           atis_quantity      0.429     1.000     0.600         3
                atis_flight#atis_airfare      0.833     0.417     0.556        12
                            atis_airport      1.000     0.944     0.971        18
                           atis_distance      1.000     1.000     1.000        10
                               atis_city      0.667     0.667     0.667         6
                        atis_ground_fare      1.000     0.857     0.923         7
                           atis_capacity      0.952     0.952     0.952        21
                          atis_flight_no      0.889     1.000     0.941         8
                               atis_meal      1.000     0.667     0.800         6
                        atis_restriction      0.000     0.000     0.000         0
             atis_airline#atis_flight_no      0.000     0.000     0.000         0
    atis_ground_service#atis_ground_fare      0.000     0.000     0.000         0
           atis_airfare#atis_flight_time      0.000     0.000     0.000         0
                           atis_cheapest      0.000     0.000     0.000         0
atis_aircraft#atis_flight#atis_flight_no      0.000     0.000     0.000         0

                               micro avg      0.972     0.977     0.975       888
                               macro avg      0.642     0.654     0.633       888
                            weighted avg      0.974     0.977     0.974       888

I0716 04:01:29.041616 139795883440000 interactiveshell.py:2882] 
                              precision    recall  f1-score   support

                           O      0.000     0.000     0.000       0.0
           B-toloc.city_name      0.977     0.993     0.985     716.0
         B-fromloc.city_name      0.992     0.999     0.995     704.0
           I-toloc.city_name      0.981     0.992     0.987     265.0
      B-depart_date.day_name      0.991     0.991     0.991     212.0
              B-airline_name      1.000     1.000     1.000     101.0
         I-fromloc.city_name      0.983     1.000     0.992     177.0
 B-depart_time.period_of_day      0.983     0.908     0.944     130.0
              I-airline_name      1.000     1.000     1.000      65.0
    B-depart_date.day_number      0.981     0.964     0.972      55.0
    B-depart_date.month_name      0.981     0.946     0.964      56.0
          B-depart_time.time      0.864     1.000     0.927      57.0
                B-round_trip      1.000     1.000     1.000      73.0
             B-cost_relative      0.973     0.973     0.973      37.0
                I-round_trip      1.000     1.000     1.000      71.0
                B-flight_mod      0.828     1.000     0.906      24.0
 B-depart_time.time_relative      0.970     0.985     0.977      65.0
          I-depart_time.time      0.912     1.000     0.954      52.0
         B-stoploc.city_name      0.833     1.000     0.909      20.0
                 B-city_name      0.884     0.667     0.760      57.0
                B-class_type      0.960     1.000     0.980      24.0
          B-arrive_time.time      0.971     0.971     0.971      34.0
 B-arrive_time.time_relative      0.882     0.968     0.923      31.0
                I-class_type      1.000     1.000     1.000      17.0
               B-flight_stop      1.000     1.000     1.000      21.0
          I-arrive_time.time      1.000     0.971     0.986      35.0
              B-airline_code      0.971     1.000     0.986      34.0
    I-depart_date.day_number      1.000     0.933     0.966      15.0
      I-fromloc.airport_name      0.441     1.000     0.612      15.0
      B-fromloc.airport_name      0.444     1.000     0.615      12.0
      B-arrive_date.day_name      0.786     1.000     0.880      11.0
          B-toloc.state_code      1.000     1.000     1.000      18.0
B-depart_date.today_relative      1.000     0.889     0.941       9.0
             B-flight_number      0.733     1.000     0.846      11.0
 B-depart_date.date_relative      0.944     1.000     0.971      17.0
          B-toloc.state_name      0.833     0.893     0.862      28.0
           B-fare_basis_code      0.773     1.000     0.872      17.0
               B-flight_time      1.000     1.000     1.000       1.0
                        B-or      1.000     1.000     1.000       3.0
 B-arrive_time.period_of_day      0.750     1.000     0.857       6.0
          B-meal_description      1.000     0.900     0.947      10.0
             I-cost_relative      1.000     0.667     0.800       3.0
              I-airport_name      0.846     0.379     0.524      29.0
               B-fare_amount      1.000     1.000     1.000       2.0
               I-fare_amount      1.000     1.000     1.000       2.0
                 I-city_name      1.000     0.633     0.776      30.0
        I-toloc.airport_name      1.000     1.000     1.000       3.0
            B-transport_type      1.000     1.000     1.000      10.0
    B-arrive_date.month_name      0.625     0.833     0.714       6.0
    B-arrive_date.day_number      0.714     0.833     0.769       6.0
         I-stoploc.city_name      0.909     1.000     0.952      10.0
                      B-meal      1.000     1.000     1.000      16.0
        B-fromloc.state_code      1.000     1.000     1.000      23.0
    B-depart_time.period_mod      1.000     1.000     1.000       5.0
                   B-connect      1.000     1.000     1.000       6.0
               B-flight_days      1.000     1.000     1.000      10.0
        B-toloc.airport_name      1.000     1.000     1.000       3.0
        B-fromloc.state_name      1.000     1.000     1.000      17.0
              B-airport_name      0.636     0.333     0.437      21.0
                   B-economy      1.000     1.000     1.000       6.0
               I-flight_time      1.000     1.000     1.000       1.0
             B-aircraft_code      1.000     0.818     0.900      33.0
                       B-mod      1.000     0.500     0.667       2.0
              B-airport_code      1.000     0.444     0.615       9.0
    B-depart_time.start_time      1.000     0.667     0.800       3.0
      B-depart_time.end_time      1.000     0.333     0.500       3.0
          B-depart_date.year      1.000     1.000     1.000       3.0
            I-transport_type      1.000     1.000     1.000       1.0
          B-restriction_code      0.800     1.000     0.889       4.0
    B-arrive_time.start_time      0.889     1.000     0.941       8.0
        B-toloc.airport_code      1.000     0.750     0.857       4.0
      B-arrive_time.end_time      0.889     1.000     0.941       8.0
      I-arrive_time.end_time      0.889     1.000     0.941       8.0
      I-depart_time.end_time      1.000     0.333     0.500       3.0
               I-flight_stop      0.000     0.000     0.000       0.0
      B-fromloc.airport_code      0.556     1.000     0.714       5.0
          I-restriction_code      1.000     1.000     1.000       3.0
    I-depart_time.start_time      1.000     1.000     1.000       1.0
          I-toloc.state_name      1.000     1.000     1.000       1.0
I-depart_date.today_relative      0.000     0.000     0.000       0.0
 B-arrive_date.date_relative      1.000     1.000     1.000       2.0
                I-flight_mod      1.000     0.167     0.286       6.0
                   I-economy      0.000     0.000     0.000       0.0
 B-return_date.date_relative      1.000     0.667     0.800       3.0
        I-fromloc.state_name      1.000     1.000     1.000       1.0
                B-state_code      1.000     1.000     1.000       1.0
    I-arrive_time.start_time      1.000     1.000     1.000       1.0
    I-arrive_date.day_number      0.000     0.000     0.000       0.0
                 B-meal_code      0.000     0.000     0.000       1.0
 I-depart_time.period_of_day      0.000     0.000     0.000       1.0
                  B-day_name      1.000     0.500     0.667       2.0
             B-period_of_day      1.000     0.750     0.857       4.0
        B-stoploc.state_code      0.000     0.000     0.000       0.0
    B-return_date.month_name      0.000     0.000     0.000       0.0
    B-return_date.day_number      0.000     0.000     0.000       0.0
    B-arrive_time.period_mod      0.000     0.000     0.000       0.0
                 I-meal_code      0.000     0.000     0.000       0.0
        B-toloc.country_name      1.000     1.000     1.000       1.0
                 B-days_code      0.000     0.000     0.000       1.0
 I-arrive_time.period_of_day      0.000     0.000     0.000       0.0
            I-today_relative      0.000     0.000     0.000       0.0
 B-return_time.period_of_day      0.000     0.000     0.000       0.0
                      B-time      0.000     0.000     0.000       0.0
           I-fare_basis_code      0.000     0.000     0.000       0.0
 I-arrive_time.time_relative      1.000     0.250     0.400       4.0
 I-depart_time.time_relative      0.000     0.000     0.000       1.0
            B-today_relative      0.000     0.000     0.000       0.0
                B-state_name      0.000     0.000     0.000       9.0
B-arrive_date.today_relative      0.000     0.000     0.000       0.0
    B-return_time.period_mod      0.000     0.000     0.000       0.0
                B-month_name      0.000     0.000     0.000       0.0
                B-day_number      0.000     0.000     0.000       0.0
 I-return_date.date_relative      1.000     0.667     0.800       3.0
I-return_date.today_relative      0.000     0.000     0.000       0.0
      B-stoploc.airport_name      0.000     0.000     0.000       0.0
             B-time_relative      0.000     0.000     0.000       0.0
                      I-time      0.000     0.000     0.000       0.0
    I-return_date.day_number      0.000     0.000     0.000       0.0
          I-meal_description      0.000     0.000     0.000       0.0
B-return_date.today_relative      0.000     0.000     0.000       0.0
      B-return_date.day_name      0.000     0.000     0.000       2.0

                   micro avg      0.958     0.960     0.959    3657.0
                   macro avg      0.681     0.648     0.649    3657.0
                weighted avg      0.961     0.960     0.956    3657.0

I0716 04:01:29.061470 139795883440000 interactiveshell.py:2882] Best Slot F1: 0.959
I0716 04:01:29.063330 139795883440000 estimator_training.py:186] Not using Distribute Coordinator.
I0716 04:01:29.065515 139795883440000 training.py:612] Running training and evaluation locally (non-distributed).
I0716 04:01:29.068432 139795883440000 training.py:700] Start train and evaluate loop. The evaluate will happen after every checkpoint. Checkpoint frequency is determined based on RunConfig arguments: save_checkpoints_steps 312 or save_checkpoints_secs None.
I0716 04:01:29.132260 139795883440000 estimator.py:1145] Calling model_fn.
I0716 04:01:31.079405 139795883440000 saver.py:1499] Saver not created because there are no variables in the graph to restore
I0716 04:01:31.658530 139795883440000 <ipython-input-6-d6747b40360b>:55] 
[<tf.Variable 'embedding:0' shape=(750, 300) dtype=float32_ref>,
 <tf.Variable 'module/aggregation/weights:0' shape=(3,) dtype=float32>,
 <tf.Variable 'module/aggregation/scaling:0' shape=() dtype=float32>,
 <tf.Variable 'dense/kernel:0' shape=(1324, 300) dtype=float32_ref>,
 <tf.Variable 'dense/bias:0' shape=(300,) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/gates/kernel:0' shape=(600, 600) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/gates/bias:0' shape=(600,) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/candidate/kernel:0' shape=(600, 300) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/candidate/bias:0' shape=(300,) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/gates/kernel:0' shape=(600, 600) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/gates/bias:0' shape=(600,) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/candidate/kernel:0' shape=(600, 300) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/candidate/bias:0' shape=(300,) dtype=float32_ref>,
 <tf.Variable 'dense_1/kernel:0' shape=(600, 23) dtype=float32_ref>,
 <tf.Variable 'dense_1/bias:0' shape=(23,) dtype=float32_ref>,
 <tf.Variable 'dense_2/kernel:0' shape=(600, 122) dtype=float32_ref>,
 <tf.Variable 'dense_2/bias:0' shape=(122,) dtype=float32_ref>]
I0716 04:01:32.732068 139795883440000 estimator.py:1147] Done calling model_fn.
I0716 04:01:32.735795 139795883440000 basic_session_run_hooks.py:541] Create CheckpointSaverHook.
I0716 04:01:33.267443 139795883440000 monitored_session.py:240] Graph was finalized.
I0716 04:01:33.287615 139795883440000 saver.py:1280] Restoring parameters from ../model/elmo_bigru/model.ckpt-5304
I0716 04:01:34.324961 139795883440000 session_manager.py:500] Running local_init_op.
I0716 04:01:34.390909 139795883440000 session_manager.py:502] Done running local_init_op.
I0716 04:01:37.174355 139795883440000 basic_session_run_hooks.py:606] Saving checkpoints for 5304 into ../model/elmo_bigru/model.ckpt.
Reading ../data/atis.train.w-intent.iob
I0716 04:01:41.832364 139795883440000 basic_session_run_hooks.py:262] loss = 0.01692254, step = 5304
I0716 04:01:41.834603 139795883440000 basic_session_run_hooks.py:262] lr = 0.00017156293
I0716 04:02:08.931746 139795883440000 basic_session_run_hooks.py:692] global_step/sec: 3.69005
I0716 04:02:08.937292 139795883440000 basic_session_run_hooks.py:260] loss = 0.016833542, step = 5404 (27.105 sec)
I0716 04:02:08.938875 139795883440000 basic_session_run_hooks.py:260] lr = 0.00016976481 (27.104 sec)
I0716 04:02:35.486347 139795883440000 basic_session_run_hooks.py:692] global_step/sec: 3.76582
I0716 04:02:35.491731 139795883440000 basic_session_run_hooks.py:260] loss = 0.017936949, step = 5504 (26.554 sec)
I0716 04:02:35.493553 139795883440000 basic_session_run_hooks.py:260] lr = 0.00016798556 (26.555 sec)
I0716 04:03:02.122225 139795883440000 basic_session_run_hooks.py:692] global_step/sec: 3.75434
I0716 04:03:02.127030 139795883440000 basic_session_run_hooks.py:260] loss = 0.013849929, step = 5604 (26.635 sec)
I0716 04:03:02.130866 139795883440000 basic_session_run_hooks.py:260] lr = 0.00016622494 (26.637 sec)
I0716 04:03:04.851873 139795883440000 basic_session_run_hooks.py:606] Saving checkpoints for 5616 into ../model/elmo_bigru/model.ckpt.
I0716 04:03:06.795298 139795883440000 estimator.py:1145] Calling model_fn.
I0716 04:03:08.144297 139795883440000 saver.py:1499] Saver not created because there are no variables in the graph to restore
I0716 04:03:08.724595 139795883440000 estimator.py:1147] Done calling model_fn.
I0716 04:03:08.752614 139795883440000 evaluation.py:255] Starting evaluation at 2019-07-16T04:03:08Z
I0716 04:03:09.239857 139795883440000 monitored_session.py:240] Graph was finalized.
I0716 04:03:09.260785 139795883440000 saver.py:1280] Restoring parameters from ../model/elmo_bigru/model.ckpt-5616
I0716 04:03:10.142755 139795883440000 session_manager.py:500] Running local_init_op.
I0716 04:03:10.198426 139795883440000 session_manager.py:502] Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
I0716 04:04:08.997198 139795883440000 evaluation.py:275] Finished evaluation at 2019-07-16-04:04:08
I0716 04:04:08.999469 139795883440000 estimator.py:2039] Saving dict for global step 5616: global_step = 5616, loss = 0.32418722
I0716 04:04:09.004559 139795883440000 estimator.py:2099] Saving 'checkpoint_path' summary for global step 5616: ../model/elmo_bigru/model.ckpt-5616
I0716 04:04:09.090413 139795883440000 estimator.py:368] Loss for final step: 0.009361349.
Reading ../data/atis.test.w-intent.iob
I0716 04:04:09.697658 139795883440000 estimator.py:1145] Calling model_fn.
I0716 04:04:10.716241 139795883440000 saver.py:1499] Saver not created because there are no variables in the graph to restore
I0716 04:04:11.224436 139795883440000 estimator.py:1147] Done calling model_fn.
I0716 04:04:11.439194 139795883440000 monitored_session.py:240] Graph was finalized.
I0716 04:04:11.453939 139795883440000 saver.py:1280] Restoring parameters from ../model/elmo_bigru/model.ckpt-5616
I0716 04:04:12.335368 139795883440000 session_manager.py:500] Running local_init_op.
I0716 04:04:12.383247 139795883440000 session_manager.py:502] Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.
  'precision', 'predicted', average, warn_for)
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1439: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples.
  'recall', 'true', average, warn_for)
I0716 04:05:10.217454 139795883440000 interactiveshell.py:2882] 
                                          precision    recall  f1-score   support

                             atis_flight      0.984     0.992     0.988       632
                            atis_airfare      0.941     1.000     0.970        48
                     atis_ground_service      1.000     1.000     1.000        36
                            atis_airline      0.974     1.000     0.987        38
                       atis_abbreviation      0.943     1.000     0.971        33
                           atis_aircraft      1.000     0.889     0.941         9
                        atis_flight_time      0.500     1.000     0.667         1
                           atis_quantity      0.375     1.000     0.545         3
                atis_flight#atis_airfare      0.833     0.417     0.556        12
                            atis_airport      1.000     0.944     0.971        18
                           atis_distance      1.000     1.000     1.000        10
                               atis_city      0.600     0.500     0.545         6
                        atis_ground_fare      1.000     0.857     0.923         7
                           atis_capacity      1.000     0.952     0.976        21
                          atis_flight_no      0.889     1.000     0.941         8
                               atis_meal      1.000     0.667     0.800         6
                        atis_restriction      0.000     0.000     0.000         0
             atis_airline#atis_flight_no      0.000     0.000     0.000         0
    atis_ground_service#atis_ground_fare      0.000     0.000     0.000         0
           atis_airfare#atis_flight_time      0.000     0.000     0.000         0
                           atis_cheapest      0.000     0.000     0.000         0
atis_aircraft#atis_flight#atis_flight_no      0.000     0.000     0.000         0

                               micro avg      0.971     0.976     0.974       888
                               macro avg      0.638     0.646     0.626       888
                            weighted avg      0.974     0.976     0.973       888

I0716 04:05:10.252486 139795883440000 interactiveshell.py:2882] 
                              precision    recall  f1-score   support

                           O      0.000     0.000     0.000       0.0
           B-toloc.city_name      0.978     0.990     0.984     716.0
         B-fromloc.city_name      0.983     0.999     0.991     704.0
           I-toloc.city_name      0.985     0.977     0.981     265.0
      B-depart_date.day_name      0.991     0.991     0.991     212.0
              B-airline_name      1.000     1.000     1.000     101.0
         I-fromloc.city_name      0.957     0.994     0.975     177.0
 B-depart_time.period_of_day      0.983     0.908     0.944     130.0
              I-airline_name      1.000     1.000     1.000      65.0
    B-depart_date.day_number      0.981     0.964     0.972      55.0
    B-depart_date.month_name      0.982     0.964     0.973      56.0
          B-depart_time.time      0.864     1.000     0.927      57.0
                B-round_trip      1.000     1.000     1.000      73.0
             B-cost_relative      0.973     0.973     0.973      37.0
                I-round_trip      1.000     1.000     1.000      71.0
                B-flight_mod      0.828     1.000     0.906      24.0
 B-depart_time.time_relative      0.955     0.985     0.970      65.0
          I-depart_time.time      0.912     1.000     0.954      52.0
         B-stoploc.city_name      0.952     1.000     0.976      20.0
                 B-city_name      0.927     0.667     0.776      57.0
                B-class_type      0.960     1.000     0.980      24.0
          B-arrive_time.time      0.971     0.971     0.971      34.0
 B-arrive_time.time_relative      0.879     0.935     0.906      31.0
                I-class_type      1.000     1.000     1.000      17.0
               B-flight_stop      1.000     1.000     1.000      21.0
          I-arrive_time.time      1.000     0.971     0.986      35.0
              B-airline_code      0.919     1.000     0.958      34.0
    I-depart_date.day_number      1.000     0.933     0.966      15.0
      I-fromloc.airport_name      0.417     1.000     0.588      15.0
      B-fromloc.airport_name      0.414     1.000     0.585      12.0
      B-arrive_date.day_name      0.786     1.000     0.880      11.0
          B-toloc.state_code      1.000     1.000     1.000      18.0
B-depart_date.today_relative      1.000     0.889     0.941       9.0
             B-flight_number      0.733     1.000     0.846      11.0
 B-depart_date.date_relative      0.944     1.000     0.971      17.0
          B-toloc.state_name      0.833     0.893     0.862      28.0
           B-fare_basis_code      0.895     1.000     0.944      17.0
               B-flight_time      1.000     1.000     1.000       1.0
                        B-or      1.000     1.000     1.000       3.0
 B-arrive_time.period_of_day      0.750     1.000     0.857       6.0
          B-meal_description      1.000     0.900     0.947      10.0
             I-cost_relative      1.000     0.667     0.800       3.0
              I-airport_name      0.769     0.345     0.476      29.0
               B-fare_amount      1.000     1.000     1.000       2.0
               I-fare_amount      1.000     1.000     1.000       2.0
                 I-city_name      1.000     0.567     0.723      30.0
        I-toloc.airport_name      1.000     1.000     1.000       3.0
            B-transport_type      1.000     1.000     1.000      10.0
    B-arrive_date.month_name      0.714     0.833     0.769       6.0
    B-arrive_date.day_number      0.714     0.833     0.769       6.0
         I-stoploc.city_name      0.909     1.000     0.952      10.0
                      B-meal      1.000     1.000     1.000      16.0
        B-fromloc.state_code      1.000     1.000     1.000      23.0
    B-depart_time.period_mod      1.000     1.000     1.000       5.0
                   B-connect      1.000     1.000     1.000       6.0
               B-flight_days      1.000     1.000     1.000      10.0
        B-toloc.airport_name      1.000     1.000     1.000       3.0
        B-fromloc.state_name      0.944     1.000     0.971      17.0
              B-airport_name      0.636     0.333     0.437      21.0
                   B-economy      1.000     1.000     1.000       6.0
               I-flight_time      1.000     1.000     1.000       1.0
             B-aircraft_code      1.000     0.818     0.900      33.0
                       B-mod      1.000     0.500     0.667       2.0
              B-airport_code      1.000     0.444     0.615       9.0
    B-depart_time.start_time      1.000     1.000     1.000       3.0
      B-depart_time.end_time      1.000     0.333     0.500       3.0
          B-depart_date.year      1.000     1.000     1.000       3.0
            I-transport_type      1.000     1.000     1.000       1.0
          B-restriction_code      0.800     1.000     0.889       4.0
    B-arrive_time.start_time      1.000     1.000     1.000       8.0
        B-toloc.airport_code      1.000     0.750     0.857       4.0
      B-arrive_time.end_time      0.889     1.000     0.941       8.0
      I-arrive_time.end_time      0.889     1.000     0.941       8.0
      I-depart_time.end_time      1.000     0.333     0.500       3.0
               I-flight_stop      0.000     0.000     0.000       0.0
      B-fromloc.airport_code      0.625     1.000     0.769       5.0
          I-restriction_code      1.000     1.000     1.000       3.0
    I-depart_time.start_time      1.000     1.000     1.000       1.0
          I-toloc.state_name      1.000     1.000     1.000       1.0
I-depart_date.today_relative      0.000     0.000     0.000       0.0
 B-arrive_date.date_relative      1.000     1.000     1.000       2.0
                I-flight_mod      1.000     0.167     0.286       6.0
                   I-economy      0.000     0.000     0.000       0.0
 B-return_date.date_relative      1.000     0.667     0.800       3.0
        I-fromloc.state_name      1.000     1.000     1.000       1.0
                B-state_code      1.000     1.000     1.000       1.0
    I-arrive_time.start_time      1.000     1.000     1.000       1.0
    I-arrive_date.day_number      0.000     0.000     0.000       0.0
                 B-meal_code      0.000     0.000     0.000       1.0
 I-depart_time.period_of_day      0.000     0.000     0.000       1.0
                  B-day_name      1.000     0.500     0.667       2.0
             B-period_of_day      1.000     0.750     0.857       4.0
        B-stoploc.state_code      0.000     0.000     0.000       0.0
    B-return_date.month_name      0.000     0.000     0.000       0.0
    B-return_date.day_number      0.000     0.000     0.000       0.0
    B-arrive_time.period_mod      0.000     0.000     0.000       0.0
                 I-meal_code      0.000     0.000     0.000       0.0
        B-toloc.country_name      1.000     1.000     1.000       1.0
                 B-days_code      0.000     0.000     0.000       1.0
 I-arrive_time.period_of_day      0.000     0.000     0.000       0.0
            I-today_relative      0.000     0.000     0.000       0.0
 B-return_time.period_of_day      0.000     0.000     0.000       0.0
                      B-time      0.000     0.000     0.000       0.0
           I-fare_basis_code      0.000     0.000     0.000       0.0
 I-arrive_time.time_relative      1.000     0.250     0.400       4.0
 I-depart_time.time_relative      0.000     0.000     0.000       1.0
            B-today_relative      0.000     0.000     0.000       0.0
                B-state_name      0.000     0.000     0.000       9.0
B-arrive_date.today_relative      0.000     0.000     0.000       0.0
    B-return_time.period_mod      0.000     0.000     0.000       0.0
                B-month_name      0.000     0.000     0.000       0.0
                B-day_number      0.000     0.000     0.000       0.0
 I-return_date.date_relative      1.000     0.667     0.800       3.0
I-return_date.today_relative      0.000     0.000     0.000       0.0
      B-stoploc.airport_name      0.000     0.000     0.000       0.0
             B-time_relative      0.000     0.000     0.000       0.0
                      I-time      0.000     0.000     0.000       0.0
    I-return_date.day_number      0.000     0.000     0.000       0.0
          I-meal_description      0.000     0.000     0.000       0.0
B-return_date.today_relative      0.000     0.000     0.000       0.0
      B-return_date.day_name      0.000     0.000     0.000       2.0

                   micro avg      0.956     0.957     0.956    3657.0
                   macro avg      0.683     0.650     0.651    3657.0
                weighted avg      0.959     0.957     0.953    3657.0

I0716 04:05:10.271334 139795883440000 interactiveshell.py:2882] Best Slot F1: 0.959
I0716 04:05:10.273087 139795883440000 estimator_training.py:186] Not using Distribute Coordinator.
I0716 04:05:10.274667 139795883440000 training.py:612] Running training and evaluation locally (non-distributed).
I0716 04:05:10.277130 139795883440000 training.py:700] Start train and evaluate loop. The evaluate will happen after every checkpoint. Checkpoint frequency is determined based on RunConfig arguments: save_checkpoints_steps 312 or save_checkpoints_secs None.
I0716 04:05:10.341723 139795883440000 estimator.py:1145] Calling model_fn.
I0716 04:05:12.143309 139795883440000 saver.py:1499] Saver not created because there are no variables in the graph to restore
I0716 04:05:12.895039 139795883440000 <ipython-input-6-d6747b40360b>:55] 
[<tf.Variable 'embedding:0' shape=(750, 300) dtype=float32_ref>,
 <tf.Variable 'module/aggregation/weights:0' shape=(3,) dtype=float32>,
 <tf.Variable 'module/aggregation/scaling:0' shape=() dtype=float32>,
 <tf.Variable 'dense/kernel:0' shape=(1324, 300) dtype=float32_ref>,
 <tf.Variable 'dense/bias:0' shape=(300,) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/gates/kernel:0' shape=(600, 600) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/gates/bias:0' shape=(600,) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/candidate/kernel:0' shape=(600, 300) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/candidate/bias:0' shape=(300,) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/gates/kernel:0' shape=(600, 600) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/gates/bias:0' shape=(600,) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/candidate/kernel:0' shape=(600, 300) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/candidate/bias:0' shape=(300,) dtype=float32_ref>,
 <tf.Variable 'dense_1/kernel:0' shape=(600, 23) dtype=float32_ref>,
 <tf.Variable 'dense_1/bias:0' shape=(23,) dtype=float32_ref>,
 <tf.Variable 'dense_2/kernel:0' shape=(600, 122) dtype=float32_ref>,
 <tf.Variable 'dense_2/bias:0' shape=(122,) dtype=float32_ref>]
I0716 04:05:13.963755 139795883440000 estimator.py:1147] Done calling model_fn.
I0716 04:05:13.967592 139795883440000 basic_session_run_hooks.py:541] Create CheckpointSaverHook.
I0716 04:05:14.292666 139795883440000 monitored_session.py:240] Graph was finalized.
I0716 04:05:14.311049 139795883440000 saver.py:1280] Restoring parameters from ../model/elmo_bigru/model.ckpt-5616
I0716 04:05:15.342847 139795883440000 session_manager.py:500] Running local_init_op.
I0716 04:05:15.411236 139795883440000 session_manager.py:502] Done running local_init_op.
I0716 04:05:18.447129 139795883440000 basic_session_run_hooks.py:606] Saving checkpoints for 5616 into ../model/elmo_bigru/model.ckpt.
Reading ../data/atis.train.w-intent.iob
I0716 04:05:23.309352 139795883440000 basic_session_run_hooks.py:262] loss = 0.024021134, step = 5616
I0716 04:05:23.311173 139795883440000 basic_session_run_hooks.py:262] lr = 0.00016601493
I0716 04:05:50.599553 139795883440000 basic_session_run_hooks.py:692] global_step/sec: 3.66423
I0716 04:05:50.605078 139795883440000 basic_session_run_hooks.py:260] loss = 0.021718727, step = 5716 (27.296 sec)
I0716 04:05:50.606785 139795883440000 basic_session_run_hooks.py:260] lr = 0.00016427496 (27.296 sec)
I0716 04:06:17.753937 139795883440000 basic_session_run_hooks.py:692] global_step/sec: 3.68265
I0716 04:06:17.759037 139795883440000 basic_session_run_hooks.py:260] loss = 0.018358195, step = 5816 (27.154 sec)
I0716 04:06:17.761049 139795883440000 basic_session_run_hooks.py:260] lr = 0.00016255325 (27.154 sec)
I0716 04:06:43.939122 139795883440000 basic_session_run_hooks.py:692] global_step/sec: 3.81895
I0716 04:06:43.942589 139795883440000 basic_session_run_hooks.py:260] loss = 0.014227215, step = 5916 (26.184 sec)
I0716 04:06:43.945022 139795883440000 basic_session_run_hooks.py:260] lr = 0.00016084957 (26.184 sec)
I0716 04:06:46.659544 139795883440000 basic_session_run_hooks.py:606] Saving checkpoints for 5928 into ../model/elmo_bigru/model.ckpt.
I0716 04:06:48.588439 139795883440000 estimator.py:1145] Calling model_fn.
I0716 04:06:49.920546 139795883440000 saver.py:1499] Saver not created because there are no variables in the graph to restore
I0716 04:06:50.490733 139795883440000 estimator.py:1147] Done calling model_fn.
I0716 04:06:50.519657 139795883440000 evaluation.py:255] Starting evaluation at 2019-07-16T04:06:50Z
I0716 04:06:50.741883 139795883440000 monitored_session.py:240] Graph was finalized.
I0716 04:06:50.756609 139795883440000 saver.py:1280] Restoring parameters from ../model/elmo_bigru/model.ckpt-5928
I0716 04:06:51.675721 139795883440000 session_manager.py:500] Running local_init_op.
I0716 04:06:51.730380 139795883440000 session_manager.py:502] Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
I0716 04:07:50.468441 139795883440000 evaluation.py:275] Finished evaluation at 2019-07-16-04:07:50
I0716 04:07:50.470516 139795883440000 estimator.py:2039] Saving dict for global step 5928: global_step = 5928, loss = 0.31755933
I0716 04:07:50.476090 139795883440000 estimator.py:2099] Saving 'checkpoint_path' summary for global step 5928: ../model/elmo_bigru/model.ckpt-5928
I0716 04:07:50.557745 139795883440000 estimator.py:368] Loss for final step: 0.0038370441.
Reading ../data/atis.test.w-intent.iob
I0716 04:07:51.160962 139795883440000 estimator.py:1145] Calling model_fn.
I0716 04:07:52.780646 139795883440000 saver.py:1499] Saver not created because there are no variables in the graph to restore
I0716 04:07:53.467393 139795883440000 estimator.py:1147] Done calling model_fn.
I0716 04:07:53.679161 139795883440000 monitored_session.py:240] Graph was finalized.
I0716 04:07:53.694185 139795883440000 saver.py:1280] Restoring parameters from ../model/elmo_bigru/model.ckpt-5928
I0716 04:07:54.575456 139795883440000 session_manager.py:500] Running local_init_op.
I0716 04:07:54.622498 139795883440000 session_manager.py:502] Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.
  'precision', 'predicted', average, warn_for)
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1439: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples.
  'recall', 'true', average, warn_for)
I0716 04:08:52.490920 139795883440000 interactiveshell.py:2882] 
                                          precision    recall  f1-score   support

                             atis_flight      0.986     0.992     0.989       632
                            atis_airfare      0.941     1.000     0.970        48
                     atis_ground_service      0.973     1.000     0.986        36
                            atis_airline      1.000     1.000     1.000        38
                       atis_abbreviation      0.943     1.000     0.971        33
                           atis_aircraft      1.000     0.889     0.941         9
                        atis_flight_time      0.500     1.000     0.667         1
                           atis_quantity      0.429     1.000     0.600         3
                atis_flight#atis_airfare      0.833     0.417     0.556        12
                            atis_airport      1.000     0.944     0.971        18
                           atis_distance      1.000     1.000     1.000        10
                               atis_city      0.667     0.667     0.667         6
                        atis_ground_fare      1.000     0.857     0.923         7
                           atis_capacity      0.952     0.952     0.952        21
                          atis_flight_no      0.889     1.000     0.941         8
                               atis_meal      1.000     0.667     0.800         6
                        atis_restriction      0.000     0.000     0.000         0
             atis_airline#atis_flight_no      0.000     0.000     0.000         0
    atis_ground_service#atis_ground_fare      0.000     0.000     0.000         0
           atis_airfare#atis_flight_time      0.000     0.000     0.000         0
                           atis_cheapest      0.000     0.000     0.000         0
atis_aircraft#atis_flight#atis_flight_no      0.000     0.000     0.000         0

                               micro avg      0.972     0.977     0.975       888
                               macro avg      0.641     0.654     0.633       888
                            weighted avg      0.974     0.977     0.974       888

I0716 04:08:52.525501 139795883440000 interactiveshell.py:2882] 
                              precision    recall  f1-score   support

                           O      0.000     0.000     0.000       0.0
           B-toloc.city_name      0.977     0.993     0.985     716.0
         B-fromloc.city_name      0.989     0.999     0.994     704.0
           I-toloc.city_name      0.985     0.992     0.989     265.0
      B-depart_date.day_name      0.991     0.991     0.991     212.0
              B-airline_name      1.000     1.000     1.000     101.0
         I-fromloc.city_name      0.983     0.994     0.989     177.0
 B-depart_time.period_of_day      0.992     0.900     0.944     130.0
              I-airline_name      1.000     1.000     1.000      65.0
    B-depart_date.day_number      0.981     0.945     0.963      55.0
    B-depart_date.month_name      0.981     0.946     0.964      56.0
          B-depart_time.time      0.862     0.982     0.918      57.0
                B-round_trip      1.000     1.000     1.000      73.0
             B-cost_relative      0.973     0.973     0.973      37.0
                I-round_trip      1.000     1.000     1.000      71.0
                B-flight_mod      0.828     1.000     0.906      24.0
 B-depart_time.time_relative      0.984     0.954     0.969      65.0
          I-depart_time.time      0.912     1.000     0.954      52.0
         B-stoploc.city_name      0.909     1.000     0.952      20.0
                 B-city_name      0.905     0.667     0.768      57.0
                B-class_type      0.960     1.000     0.980      24.0
          B-arrive_time.time      0.943     0.971     0.957      34.0
 B-arrive_time.time_relative      0.833     0.968     0.896      31.0
                I-class_type      1.000     1.000     1.000      17.0
               B-flight_stop      1.000     1.000     1.000      21.0
          I-arrive_time.time      1.000     0.971     0.986      35.0
              B-airline_code      0.971     1.000     0.986      34.0
    I-depart_date.day_number      1.000     0.933     0.966      15.0
      I-fromloc.airport_name      0.429     1.000     0.600      15.0
      B-fromloc.airport_name      0.429     1.000     0.600      12.0
      B-arrive_date.day_name      0.786     1.000     0.880      11.0
          B-toloc.state_code      1.000     1.000     1.000      18.0
B-depart_date.today_relative      1.000     0.889     0.941       9.0
             B-flight_number      0.733     1.000     0.846      11.0
 B-depart_date.date_relative      0.944     1.000     0.971      17.0
          B-toloc.state_name      0.833     0.893     0.862      28.0
           B-fare_basis_code      0.850     1.000     0.919      17.0
               B-flight_time      1.000     1.000     1.000       1.0
                        B-or      1.000     1.000     1.000       3.0
 B-arrive_time.period_of_day      0.667     1.000     0.800       6.0
          B-meal_description      1.000     0.900     0.947      10.0
             I-cost_relative      1.000     0.667     0.800       3.0
              I-airport_name      0.846     0.379     0.524      29.0
               B-fare_amount      1.000     1.000     1.000       2.0
               I-fare_amount      1.000     1.000     1.000       2.0
                 I-city_name      1.000     0.667     0.800      30.0
        I-toloc.airport_name      1.000     1.000     1.000       3.0
            B-transport_type      1.000     1.000     1.000      10.0
    B-arrive_date.month_name      0.625     0.833     0.714       6.0
    B-arrive_date.day_number      0.625     0.833     0.714       6.0
         I-stoploc.city_name      0.909     1.000     0.952      10.0
                      B-meal      1.000     1.000     1.000      16.0
        B-fromloc.state_code      1.000     1.000     1.000      23.0
    B-depart_time.period_mod      0.833     1.000     0.909       5.0
                   B-connect      1.000     1.000     1.000       6.0
               B-flight_days      1.000     1.000     1.000      10.0
        B-toloc.airport_name      1.000     1.000     1.000       3.0
        B-fromloc.state_name      1.000     1.000     1.000      17.0
              B-airport_name      0.636     0.333     0.437      21.0
                   B-economy      1.000     1.000     1.000       6.0
               I-flight_time      1.000     1.000     1.000       1.0
             B-aircraft_code      1.000     0.879     0.935      33.0
                       B-mod      1.000     0.500     0.667       2.0
              B-airport_code      1.000     0.556     0.714       9.0
    B-depart_time.start_time      1.000     0.667     0.800       3.0
      B-depart_time.end_time      1.000     0.333     0.500       3.0
          B-depart_date.year      1.000     1.000     1.000       3.0
            I-transport_type      1.000     1.000     1.000       1.0
          B-restriction_code      0.800     1.000     0.889       4.0
    B-arrive_time.start_time      0.889     1.000     0.941       8.0
        B-toloc.airport_code      1.000     0.750     0.857       4.0
      B-arrive_time.end_time      0.889     1.000     0.941       8.0
      I-arrive_time.end_time      0.889     1.000     0.941       8.0
      I-depart_time.end_time      1.000     0.333     0.500       3.0
               I-flight_stop      0.000     0.000     0.000       0.0
      B-fromloc.airport_code      0.714     1.000     0.833       5.0
          I-restriction_code      1.000     1.000     1.000       3.0
    I-depart_time.start_time      1.000     1.000     1.000       1.0
          I-toloc.state_name      1.000     1.000     1.000       1.0
I-depart_date.today_relative      0.000     0.000     0.000       0.0
 B-arrive_date.date_relative      1.000     1.000     1.000       2.0
                I-flight_mod      1.000     0.167     0.286       6.0
                   I-economy      0.000     0.000     0.000       0.0
 B-return_date.date_relative      1.000     0.333     0.500       3.0
        I-fromloc.state_name      1.000     1.000     1.000       1.0
                B-state_code      1.000     1.000     1.000       1.0
    I-arrive_time.start_time      1.000     1.000     1.000       1.0
    I-arrive_date.day_number      0.000     0.000     0.000       0.0
                 B-meal_code      0.000     0.000     0.000       1.0
 I-depart_time.period_of_day      1.000     1.000     1.000       1.0
                  B-day_name      1.000     0.500     0.667       2.0
             B-period_of_day      1.000     0.750     0.857       4.0
        B-stoploc.state_code      0.000     0.000     0.000       0.0
    B-return_date.month_name      0.000     0.000     0.000       0.0
    B-return_date.day_number      0.000     0.000     0.000       0.0
    B-arrive_time.period_mod      0.000     0.000     0.000       0.0
                 I-meal_code      0.000     0.000     0.000       0.0
        B-toloc.country_name      1.000     1.000     1.000       1.0
                 B-days_code      0.000     0.000     0.000       1.0
 I-arrive_time.period_of_day      0.000     0.000     0.000       0.0
            I-today_relative      0.000     0.000     0.000       0.0
 B-return_time.period_of_day      0.000     0.000     0.000       0.0
                      B-time      0.000     0.000     0.000       0.0
           I-fare_basis_code      0.000     0.000     0.000       0.0
 I-arrive_time.time_relative      0.000     0.000     0.000       4.0
 I-depart_time.time_relative      0.000     0.000     0.000       1.0
            B-today_relative      0.000     0.000     0.000       0.0
                B-state_name      0.000     0.000     0.000       9.0
B-arrive_date.today_relative      0.000     0.000     0.000       0.0
    B-return_time.period_mod      0.000     0.000     0.000       0.0
                B-month_name      0.000     0.000     0.000       0.0
                B-day_number      0.000     0.000     0.000       0.0
 I-return_date.date_relative      0.667     0.667     0.667       3.0
I-return_date.today_relative      0.000     0.000     0.000       0.0
      B-stoploc.airport_name      0.000     0.000     0.000       0.0
             B-time_relative      0.000     0.000     0.000       0.0
                      I-time      0.000     0.000     0.000       0.0
    I-return_date.day_number      0.000     0.000     0.000       0.0
          I-meal_description      0.000     0.000     0.000       0.0
B-return_date.today_relative      0.000     0.000     0.000       0.0
      B-return_date.day_name      0.000     0.000     0.000       2.0

                   micro avg      0.957     0.959     0.958    3657.0
                   macro avg      0.677     0.653     0.651    3657.0
                weighted avg      0.960     0.959     0.955    3657.0

I0716 04:08:52.544500 139795883440000 interactiveshell.py:2882] Best Slot F1: 0.959
I0716 04:08:52.546095 139795883440000 estimator_training.py:186] Not using Distribute Coordinator.
I0716 04:08:52.551183 139795883440000 training.py:612] Running training and evaluation locally (non-distributed).
I0716 04:08:52.552832 139795883440000 training.py:700] Start train and evaluate loop. The evaluate will happen after every checkpoint. Checkpoint frequency is determined based on RunConfig arguments: save_checkpoints_steps 312 or save_checkpoints_secs None.
I0716 04:08:52.613224 139795883440000 estimator.py:1145] Calling model_fn.
I0716 04:08:54.098511 139795883440000 saver.py:1499] Saver not created because there are no variables in the graph to restore
I0716 04:08:54.677569 139795883440000 <ipython-input-6-d6747b40360b>:55] 
[<tf.Variable 'embedding:0' shape=(750, 300) dtype=float32_ref>,
 <tf.Variable 'module/aggregation/weights:0' shape=(3,) dtype=float32>,
 <tf.Variable 'module/aggregation/scaling:0' shape=() dtype=float32>,
 <tf.Variable 'dense/kernel:0' shape=(1324, 300) dtype=float32_ref>,
 <tf.Variable 'dense/bias:0' shape=(300,) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/gates/kernel:0' shape=(600, 600) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/gates/bias:0' shape=(600,) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/candidate/kernel:0' shape=(600, 300) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/candidate/bias:0' shape=(300,) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/gates/kernel:0' shape=(600, 600) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/gates/bias:0' shape=(600,) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/candidate/kernel:0' shape=(600, 300) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/candidate/bias:0' shape=(300,) dtype=float32_ref>,
 <tf.Variable 'dense_1/kernel:0' shape=(600, 23) dtype=float32_ref>,
 <tf.Variable 'dense_1/bias:0' shape=(23,) dtype=float32_ref>,
 <tf.Variable 'dense_2/kernel:0' shape=(600, 122) dtype=float32_ref>,
 <tf.Variable 'dense_2/bias:0' shape=(122,) dtype=float32_ref>]
I0716 04:08:55.731200 139795883440000 estimator.py:1147] Done calling model_fn.
I0716 04:08:55.735250 139795883440000 basic_session_run_hooks.py:541] Create CheckpointSaverHook.
I0716 04:08:56.253782 139795883440000 monitored_session.py:240] Graph was finalized.
I0716 04:08:56.271320 139795883440000 saver.py:1280] Restoring parameters from ../model/elmo_bigru/model.ckpt-5928
I0716 04:08:57.309556 139795883440000 session_manager.py:500] Running local_init_op.
I0716 04:08:57.385339 139795883440000 session_manager.py:502] Done running local_init_op.
I0716 04:09:00.167442 139795883440000 basic_session_run_hooks.py:606] Saving checkpoints for 5928 into ../model/elmo_bigru/model.ckpt.
Reading ../data/atis.train.w-intent.iob
I0716 04:09:04.799297 139795883440000 basic_session_run_hooks.py:262] loss = 0.013556904, step = 5928
I0716 04:09:04.801827 139795883440000 basic_session_run_hooks.py:262] lr = 0.0001606463
I0716 04:09:32.558158 139795883440000 basic_session_run_hooks.py:692] global_step/sec: 3.60234
I0716 04:09:32.561326 139795883440000 basic_session_run_hooks.py:260] loss = 0.013189382, step = 6028 (27.762 sec)
I0716 04:09:32.565477 139795883440000 basic_session_run_hooks.py:260] lr = 0.00015896263 (27.764 sec)
I0716 04:09:59.291026 139795883440000 basic_session_run_hooks.py:692] global_step/sec: 3.74072
I0716 04:09:59.293781 139795883440000 basic_session_run_hooks.py:260] loss = 0.024098143, step = 6128 (26.732 sec)
I0716 04:09:59.297652 139795883440000 basic_session_run_hooks.py:260] lr = 0.00015729658 (26.732 sec)
I0716 04:10:25.366129 139795883440000 basic_session_run_hooks.py:692] global_step/sec: 3.83507
I0716 04:10:25.371350 139795883440000 basic_session_run_hooks.py:260] loss = 0.016176166, step = 6228 (26.078 sec)
I0716 04:10:25.373055 139795883440000 basic_session_run_hooks.py:260] lr = 0.00015564801 (26.075 sec)
I0716 04:10:28.079471 139795883440000 basic_session_run_hooks.py:606] Saving checkpoints for 6240 into ../model/elmo_bigru/model.ckpt.
I0716 04:10:30.156975 139795883440000 estimator.py:1145] Calling model_fn.
I0716 04:10:31.521165 139795883440000 saver.py:1499] Saver not created because there are no variables in the graph to restore
I0716 04:10:32.103361 139795883440000 estimator.py:1147] Done calling model_fn.
I0716 04:10:32.132641 139795883440000 evaluation.py:255] Starting evaluation at 2019-07-16T04:10:32Z
I0716 04:10:32.350379 139795883440000 monitored_session.py:240] Graph was finalized.
I0716 04:10:32.365226 139795883440000 saver.py:1280] Restoring parameters from ../model/elmo_bigru/model.ckpt-6240
I0716 04:10:33.253172 139795883440000 session_manager.py:500] Running local_init_op.
I0716 04:10:33.307003 139795883440000 session_manager.py:502] Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
I0716 04:11:32.190853 139795883440000 evaluation.py:275] Finished evaluation at 2019-07-16-04:11:32
I0716 04:11:32.192710 139795883440000 estimator.py:2039] Saving dict for global step 6240: global_step = 6240, loss = 0.32681456
I0716 04:11:32.198311 139795883440000 estimator.py:2099] Saving 'checkpoint_path' summary for global step 6240: ../model/elmo_bigru/model.ckpt-6240
I0716 04:11:32.275698 139795883440000 estimator.py:368] Loss for final step: 0.009180053.
Reading ../data/atis.test.w-intent.iob
I0716 04:11:32.887591 139795883440000 estimator.py:1145] Calling model_fn.
I0716 04:11:34.520857 139795883440000 saver.py:1499] Saver not created because there are no variables in the graph to restore
I0716 04:11:35.211205 139795883440000 estimator.py:1147] Done calling model_fn.
I0716 04:11:35.421146 139795883440000 monitored_session.py:240] Graph was finalized.
I0716 04:11:35.437683 139795883440000 saver.py:1280] Restoring parameters from ../model/elmo_bigru/model.ckpt-6240
I0716 04:11:36.325077 139795883440000 session_manager.py:500] Running local_init_op.
I0716 04:11:36.370754 139795883440000 session_manager.py:502] Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.
  'precision', 'predicted', average, warn_for)
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1439: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples.
  'recall', 'true', average, warn_for)
I0716 04:12:33.999751 139795883440000 interactiveshell.py:2882] 
                                          precision    recall  f1-score   support

                             atis_flight      0.983     0.992     0.987       632
                            atis_airfare      0.941     1.000     0.970        48
                     atis_ground_service      1.000     1.000     1.000        36
                            atis_airline      1.000     1.000     1.000        38
                       atis_abbreviation      0.943     1.000     0.971        33
                           atis_aircraft      1.000     0.889     0.941         9
                        atis_flight_time      0.500     1.000     0.667         1
                           atis_quantity      0.429     1.000     0.600         3
                atis_flight#atis_airfare      0.833     0.417     0.556        12
                            atis_airport      1.000     1.000     1.000        18
                           atis_distance      1.000     1.000     1.000        10
                               atis_city      1.000     0.500     0.667         6
                        atis_ground_fare      1.000     0.857     0.923         7
                           atis_capacity      0.952     0.952     0.952        21
                          atis_flight_no      0.889     1.000     0.941         8
                               atis_meal      1.000     0.833     0.909         6
                        atis_restriction      0.000     0.000     0.000         0
             atis_airline#atis_flight_no      0.000     0.000     0.000         0
    atis_ground_service#atis_ground_fare      0.000     0.000     0.000         0
           atis_airfare#atis_flight_time      0.000     0.000     0.000         0
                           atis_cheapest      0.000     0.000     0.000         0
atis_aircraft#atis_flight#atis_flight_no      0.000     0.000     0.000         0

                               micro avg      0.973     0.979     0.976       888
                               macro avg      0.658     0.656     0.640       888
                            weighted avg      0.976     0.979     0.975       888

I0716 04:12:34.034637 139795883440000 interactiveshell.py:2882] 
                              precision    recall  f1-score   support

                           O      0.000     0.000     0.000       0.0
           B-toloc.city_name      0.974     0.993     0.983     716.0
         B-fromloc.city_name      0.987     1.000     0.994     704.0
           I-toloc.city_name      0.981     0.985     0.983     265.0
      B-depart_date.day_name      0.991     0.991     0.991     212.0
              B-airline_name      1.000     1.000     1.000     101.0
         I-fromloc.city_name      0.973     1.000     0.986     177.0
 B-depart_time.period_of_day      0.984     0.923     0.952     130.0
              I-airline_name      1.000     1.000     1.000      65.0
    B-depart_date.day_number      0.981     0.964     0.972      55.0
    B-depart_date.month_name      0.982     0.964     0.973      56.0
          B-depart_time.time      0.864     1.000     0.927      57.0
                B-round_trip      1.000     1.000     1.000      73.0
             B-cost_relative      0.973     0.973     0.973      37.0
                I-round_trip      1.000     1.000     1.000      71.0
                B-flight_mod      0.828     1.000     0.906      24.0
 B-depart_time.time_relative      0.955     0.985     0.970      65.0
          I-depart_time.time      0.945     1.000     0.972      52.0
         B-stoploc.city_name      0.952     1.000     0.976      20.0
                 B-city_name      0.925     0.649     0.763      57.0
                B-class_type      0.960     1.000     0.980      24.0
          B-arrive_time.time      0.971     0.971     0.971      34.0
 B-arrive_time.time_relative      0.967     0.935     0.951      31.0
                I-class_type      1.000     1.000     1.000      17.0
               B-flight_stop      1.000     1.000     1.000      21.0
          I-arrive_time.time      1.000     0.971     0.986      35.0
              B-airline_code      0.919     1.000     0.958      34.0
    I-depart_date.day_number      1.000     0.933     0.966      15.0
      I-fromloc.airport_name      0.429     1.000     0.600      15.0
      B-fromloc.airport_name      0.429     1.000     0.600      12.0
      B-arrive_date.day_name      0.786     1.000     0.880      11.0
          B-toloc.state_code      1.000     1.000     1.000      18.0
B-depart_date.today_relative      1.000     0.889     0.941       9.0
             B-flight_number      0.733     1.000     0.846      11.0
 B-depart_date.date_relative      0.944     1.000     0.971      17.0
          B-toloc.state_name      0.828     0.857     0.842      28.0
           B-fare_basis_code      0.850     1.000     0.919      17.0
               B-flight_time      1.000     1.000     1.000       1.0
                        B-or      1.000     1.000     1.000       3.0
 B-arrive_time.period_of_day      0.750     1.000     0.857       6.0
          B-meal_description      1.000     0.900     0.947      10.0
             I-cost_relative      1.000     0.667     0.800       3.0
              I-airport_name      0.818     0.310     0.450      29.0
               B-fare_amount      1.000     1.000     1.000       2.0
               I-fare_amount      1.000     1.000     1.000       2.0
                 I-city_name      0.947     0.600     0.735      30.0
        I-toloc.airport_name      1.000     1.000     1.000       3.0
            B-transport_type      1.000     1.000     1.000      10.0
    B-arrive_date.month_name      0.714     0.833     0.769       6.0
    B-arrive_date.day_number      0.714     0.833     0.769       6.0
         I-stoploc.city_name      0.909     1.000     0.952      10.0
                      B-meal      1.000     1.000     1.000      16.0
        B-fromloc.state_code      1.000     1.000     1.000      23.0
    B-depart_time.period_mod      1.000     1.000     1.000       5.0
                   B-connect      1.000     1.000     1.000       6.0
               B-flight_days      1.000     1.000     1.000      10.0
        B-toloc.airport_name      1.000     1.000     1.000       3.0
        B-fromloc.state_name      0.944     1.000     0.971      17.0
              B-airport_name      0.636     0.333     0.437      21.0
                   B-economy      1.000     1.000     1.000       6.0
               I-flight_time      1.000     1.000     1.000       1.0
             B-aircraft_code      1.000     0.818     0.900      33.0
                       B-mod      1.000     0.500     0.667       2.0
              B-airport_code      1.000     0.444     0.615       9.0
    B-depart_time.start_time      1.000     0.667     0.800       3.0
      B-depart_time.end_time      1.000     0.333     0.500       3.0
          B-depart_date.year      1.000     1.000     1.000       3.0
            I-transport_type      0.000     0.000     0.000       1.0
          B-restriction_code      0.800     1.000     0.889       4.0
    B-arrive_time.start_time      0.889     1.000     0.941       8.0
        B-toloc.airport_code      1.000     0.750     0.857       4.0
      B-arrive_time.end_time      0.889     1.000     0.941       8.0
      I-arrive_time.end_time      0.889     1.000     0.941       8.0
      I-depart_time.end_time      1.000     0.333     0.500       3.0
               I-flight_stop      0.000     0.000     0.000       0.0
      B-fromloc.airport_code      0.625     1.000     0.769       5.0
          I-restriction_code      1.000     1.000     1.000       3.0
    I-depart_time.start_time      1.000     1.000     1.000       1.0
          I-toloc.state_name      1.000     1.000     1.000       1.0
I-depart_date.today_relative      0.000     0.000     0.000       0.0
 B-arrive_date.date_relative      1.000     1.000     1.000       2.0
                I-flight_mod      1.000     0.167     0.286       6.0
                   I-economy      0.000     0.000     0.000       0.0
 B-return_date.date_relative      1.000     0.333     0.500       3.0
        I-fromloc.state_name      1.000     1.000     1.000       1.0
                B-state_code      1.000     1.000     1.000       1.0
    I-arrive_time.start_time      1.000     1.000     1.000       1.0
    I-arrive_date.day_number      0.000     0.000     0.000       0.0
                 B-meal_code      0.000     0.000     0.000       1.0
 I-depart_time.period_of_day      0.000     0.000     0.000       1.0
                  B-day_name      1.000     0.500     0.667       2.0
             B-period_of_day      1.000     0.750     0.857       4.0
        B-stoploc.state_code      0.000     0.000     0.000       0.0
    B-return_date.month_name      0.000     0.000     0.000       0.0
    B-return_date.day_number      0.000     0.000     0.000       0.0
    B-arrive_time.period_mod      0.000     0.000     0.000       0.0
                 I-meal_code      0.000     0.000     0.000       0.0
        B-toloc.country_name      1.000     1.000     1.000       1.0
                 B-days_code      0.000     0.000     0.000       1.0
 I-arrive_time.period_of_day      0.000     0.000     0.000       0.0
            I-today_relative      0.000     0.000     0.000       0.0
 B-return_time.period_of_day      0.000     0.000     0.000       0.0
                      B-time      0.000     0.000     0.000       0.0
           I-fare_basis_code      0.000     0.000     0.000       0.0
 I-arrive_time.time_relative      1.000     1.000     1.000       4.0
 I-depart_time.time_relative      0.000     0.000     0.000       1.0
            B-today_relative      0.000     0.000     0.000       0.0
                B-state_name      0.000     0.000     0.000       9.0
B-arrive_date.today_relative      0.000     0.000     0.000       0.0
    B-return_time.period_mod      0.000     0.000     0.000       0.0
                B-month_name      0.000     0.000     0.000       0.0
                B-day_number      0.000     0.000     0.000       0.0
 I-return_date.date_relative      0.667     0.667     0.667       3.0
I-return_date.today_relative      0.000     0.000     0.000       0.0
      B-stoploc.airport_name      0.000     0.000     0.000       0.0
             B-time_relative      0.000     0.000     0.000       0.0
                      I-time      0.000     0.000     0.000       0.0
    I-return_date.day_number      0.000     0.000     0.000       0.0
          I-meal_description      0.000     0.000     0.000       0.0
B-return_date.today_relative      0.000     0.000     0.000       0.0
      B-return_date.day_name      0.000     0.000     0.000       2.0

                   micro avg      0.957     0.959     0.958    3657.0
                   macro avg      0.672     0.642     0.642    3657.0
                weighted avg      0.960     0.959     0.955    3657.0

I0716 04:12:34.054486 139795883440000 interactiveshell.py:2882] Best Slot F1: 0.959
I0716 04:12:34.056142 139795883440000 estimator_training.py:186] Not using Distribute Coordinator.
I0716 04:12:34.058927 139795883440000 training.py:612] Running training and evaluation locally (non-distributed).
I0716 04:12:34.061445 139795883440000 training.py:700] Start train and evaluate loop. The evaluate will happen after every checkpoint. Checkpoint frequency is determined based on RunConfig arguments: save_checkpoints_steps 312 or save_checkpoints_secs None.
I0716 04:12:34.123326 139795883440000 estimator.py:1145] Calling model_fn.
I0716 04:12:35.612440 139795883440000 saver.py:1499] Saver not created because there are no variables in the graph to restore
I0716 04:12:36.184969 139795883440000 <ipython-input-6-d6747b40360b>:55] 
[<tf.Variable 'embedding:0' shape=(750, 300) dtype=float32_ref>,
 <tf.Variable 'module/aggregation/weights:0' shape=(3,) dtype=float32>,
 <tf.Variable 'module/aggregation/scaling:0' shape=() dtype=float32>,
 <tf.Variable 'dense/kernel:0' shape=(1324, 300) dtype=float32_ref>,
 <tf.Variable 'dense/bias:0' shape=(300,) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/gates/kernel:0' shape=(600, 600) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/gates/bias:0' shape=(600,) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/candidate/kernel:0' shape=(600, 300) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/candidate/bias:0' shape=(300,) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/gates/kernel:0' shape=(600, 600) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/gates/bias:0' shape=(600,) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/candidate/kernel:0' shape=(600, 300) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/candidate/bias:0' shape=(300,) dtype=float32_ref>,
 <tf.Variable 'dense_1/kernel:0' shape=(600, 23) dtype=float32_ref>,
 <tf.Variable 'dense_1/bias:0' shape=(23,) dtype=float32_ref>,
 <tf.Variable 'dense_2/kernel:0' shape=(600, 122) dtype=float32_ref>,
 <tf.Variable 'dense_2/bias:0' shape=(122,) dtype=float32_ref>]
I0716 04:12:37.262094 139795883440000 estimator.py:1147] Done calling model_fn.
I0716 04:12:37.266188 139795883440000 basic_session_run_hooks.py:541] Create CheckpointSaverHook.
I0716 04:12:37.794167 139795883440000 monitored_session.py:240] Graph was finalized.
I0716 04:12:37.813221 139795883440000 saver.py:1280] Restoring parameters from ../model/elmo_bigru/model.ckpt-6240
I0716 04:12:38.844015 139795883440000 session_manager.py:500] Running local_init_op.
I0716 04:12:38.915020 139795883440000 session_manager.py:502] Done running local_init_op.
I0716 04:12:41.675868 139795883440000 basic_session_run_hooks.py:606] Saving checkpoints for 6240 into ../model/elmo_bigru/model.ckpt.
Reading ../data/atis.train.w-intent.iob
I0716 04:12:46.442596 139795883440000 basic_session_run_hooks.py:262] loss = 0.015512767, step = 6240
I0716 04:12:46.444594 139795883440000 basic_session_run_hooks.py:262] lr = 0.00015545134
I0716 04:13:13.395112 139795883440000 basic_session_run_hooks.py:692] global_step/sec: 3.71013
I0716 04:13:13.400246 139795883440000 basic_session_run_hooks.py:260] loss = 0.022132397, step = 6340 (26.958 sec)
I0716 04:13:13.401929 139795883440000 basic_session_run_hooks.py:260] lr = 0.00015382208 (26.957 sec)
I0716 04:13:39.564567 139795883440000 basic_session_run_hooks.py:692] global_step/sec: 3.82124
I0716 04:13:39.569588 139795883440000 basic_session_run_hooks.py:260] loss = 0.016259994, step = 6440 (26.169 sec)
I0716 04:13:39.574711 139795883440000 basic_session_run_hooks.py:260] lr = 0.00015220992 (26.173 sec)
I0716 04:14:06.450009 139795883440000 basic_session_run_hooks.py:692] global_step/sec: 3.71949
I0716 04:14:06.456068 139795883440000 basic_session_run_hooks.py:260] loss = 0.019697318, step = 6540 (26.886 sec)
I0716 04:14:06.457782 139795883440000 basic_session_run_hooks.py:260] lr = 0.00015061465 (26.883 sec)
I0716 04:14:09.685927 139795883440000 basic_session_run_hooks.py:606] Saving checkpoints for 6552 into ../model/elmo_bigru/model.ckpt.
I0716 04:14:11.658162 139795883440000 estimator.py:1145] Calling model_fn.
I0716 04:14:13.006481 139795883440000 saver.py:1499] Saver not created because there are no variables in the graph to restore
I0716 04:14:13.593723 139795883440000 estimator.py:1147] Done calling model_fn.
I0716 04:14:13.623715 139795883440000 evaluation.py:255] Starting evaluation at 2019-07-16T04:14:13Z
I0716 04:14:14.108688 139795883440000 monitored_session.py:240] Graph was finalized.
I0716 04:14:14.124477 139795883440000 saver.py:1280] Restoring parameters from ../model/elmo_bigru/model.ckpt-6552
I0716 04:14:15.020569 139795883440000 session_manager.py:500] Running local_init_op.
I0716 04:14:15.077372 139795883440000 session_manager.py:502] Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
I0716 04:15:13.894021 139795883440000 evaluation.py:275] Finished evaluation at 2019-07-16-04:15:13
I0716 04:15:13.895933 139795883440000 estimator.py:2039] Saving dict for global step 6552: global_step = 6552, loss = 0.3396054
I0716 04:15:13.901288 139795883440000 estimator.py:2099] Saving 'checkpoint_path' summary for global step 6552: ../model/elmo_bigru/model.ckpt-6552
I0716 04:15:13.987282 139795883440000 estimator.py:368] Loss for final step: 0.005030209.
Reading ../data/atis.test.w-intent.iob
I0716 04:15:14.588431 139795883440000 estimator.py:1145] Calling model_fn.
I0716 04:15:15.618718 139795883440000 saver.py:1499] Saver not created because there are no variables in the graph to restore
I0716 04:15:16.139860 139795883440000 estimator.py:1147] Done calling model_fn.
I0716 04:15:16.355150 139795883440000 monitored_session.py:240] Graph was finalized.
I0716 04:15:16.373098 139795883440000 saver.py:1280] Restoring parameters from ../model/elmo_bigru/model.ckpt-6552
I0716 04:15:17.265295 139795883440000 session_manager.py:500] Running local_init_op.
I0716 04:15:17.321921 139795883440000 session_manager.py:502] Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.
  'precision', 'predicted', average, warn_for)
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1439: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples.
  'recall', 'true', average, warn_for)
I0716 04:16:15.382930 139795883440000 interactiveshell.py:2882] 
                                          precision    recall  f1-score   support

                             atis_flight      0.983     0.992     0.987       632
                            atis_airfare      0.941     1.000     0.970        48
                     atis_ground_service      1.000     1.000     1.000        36
                            atis_airline      0.974     1.000     0.987        38
                       atis_abbreviation      0.943     1.000     0.971        33
                           atis_aircraft      1.000     0.889     0.941         9
                        atis_flight_time      0.500     1.000     0.667         1
                           atis_quantity      0.375     1.000     0.545         3
                atis_flight#atis_airfare      0.833     0.417     0.556        12
                            atis_airport      1.000     0.944     0.971        18
                           atis_distance      1.000     1.000     1.000        10
                               atis_city      1.000     0.500     0.667         6
                        atis_ground_fare      1.000     0.857     0.923         7
                           atis_capacity      1.000     0.952     0.976        21
                          atis_flight_no      0.889     1.000     0.941         8
                               atis_meal      1.000     0.833     0.909         6
                        atis_restriction      0.000     0.000     0.000         0
             atis_airline#atis_flight_no      0.000     0.000     0.000         0
    atis_ground_service#atis_ground_fare      0.000     0.000     0.000         0
           atis_airfare#atis_flight_time      0.000     0.000     0.000         0
                           atis_cheapest      0.000     0.000     0.000         0
atis_aircraft#atis_flight#atis_flight_no      0.000     0.000     0.000         0

                               micro avg      0.972     0.977     0.975       888
                               macro avg      0.656     0.654     0.637       888
                            weighted avg      0.975     0.977     0.974       888

I0716 04:16:15.417814 139795883440000 interactiveshell.py:2882] 
                              precision    recall  f1-score   support

                           O      0.000     0.000     0.000       0.0
           B-toloc.city_name      0.975     0.992     0.983     716.0
         B-fromloc.city_name      0.989     0.999     0.994     704.0
           I-toloc.city_name      0.978     0.992     0.985     265.0
      B-depart_date.day_name      0.991     0.991     0.991     212.0
              B-airline_name      1.000     1.000     1.000     101.0
         I-fromloc.city_name      0.978     1.000     0.989     177.0
 B-depart_time.period_of_day      0.992     0.923     0.956     130.0
              I-airline_name      1.000     1.000     1.000      65.0
    B-depart_date.day_number      0.981     0.964     0.972      55.0
    B-depart_date.month_name      0.981     0.946     0.964      56.0
          B-depart_time.time      0.864     1.000     0.927      57.0
                B-round_trip      1.000     1.000     1.000      73.0
             B-cost_relative      0.973     0.973     0.973      37.0
                I-round_trip      1.000     1.000     1.000      71.0
                B-flight_mod      0.828     1.000     0.906      24.0
 B-depart_time.time_relative      0.970     0.985     0.977      65.0
          I-depart_time.time      0.945     1.000     0.972      52.0
         B-stoploc.city_name      0.870     1.000     0.930      20.0
                 B-city_name      0.902     0.649     0.755      57.0
                B-class_type      0.960     1.000     0.980      24.0
          B-arrive_time.time      0.971     0.971     0.971      34.0
 B-arrive_time.time_relative      0.906     0.935     0.921      31.0
                I-class_type      1.000     1.000     1.000      17.0
               B-flight_stop      1.000     1.000     1.000      21.0
          I-arrive_time.time      1.000     0.971     0.986      35.0
              B-airline_code      0.850     1.000     0.919      34.0
    I-depart_date.day_number      1.000     0.933     0.966      15.0
      I-fromloc.airport_name      0.441     1.000     0.612      15.0
      B-fromloc.airport_name      0.444     1.000     0.615      12.0
      B-arrive_date.day_name      0.786     1.000     0.880      11.0
          B-toloc.state_code      1.000     1.000     1.000      18.0
B-depart_date.today_relative      1.000     0.889     0.941       9.0
             B-flight_number      0.733     1.000     0.846      11.0
 B-depart_date.date_relative      0.944     1.000     0.971      17.0
          B-toloc.state_name      0.833     0.893     0.862      28.0
           B-fare_basis_code      0.895     1.000     0.944      17.0
               B-flight_time      1.000     1.000     1.000       1.0
                        B-or      1.000     1.000     1.000       3.0
 B-arrive_time.period_of_day      0.750     1.000     0.857       6.0
          B-meal_description      1.000     0.900     0.947      10.0
             I-cost_relative      1.000     0.667     0.800       3.0
              I-airport_name      0.833     0.345     0.488      29.0
               B-fare_amount      1.000     1.000     1.000       2.0
               I-fare_amount      1.000     1.000     1.000       2.0
                 I-city_name      0.947     0.600     0.735      30.0
        I-toloc.airport_name      1.000     1.000     1.000       3.0
            B-transport_type      1.000     1.000     1.000      10.0
    B-arrive_date.month_name      0.625     0.833     0.714       6.0
    B-arrive_date.day_number      0.714     0.833     0.769       6.0
         I-stoploc.city_name      0.909     1.000     0.952      10.0
                      B-meal      1.000     1.000     1.000      16.0
        B-fromloc.state_code      1.000     1.000     1.000      23.0
    B-depart_time.period_mod      1.000     1.000     1.000       5.0
                   B-connect      1.000     1.000     1.000       6.0
               B-flight_days      1.000     1.000     1.000      10.0
        B-toloc.airport_name      1.000     1.000     1.000       3.0
        B-fromloc.state_name      0.944     1.000     0.971      17.0
              B-airport_name      0.636     0.333     0.437      21.0
                   B-economy      1.000     1.000     1.000       6.0
               I-flight_time      1.000     1.000     1.000       1.0
             B-aircraft_code      1.000     0.818     0.900      33.0
                       B-mod      1.000     0.500     0.667       2.0
              B-airport_code      1.000     0.444     0.615       9.0
    B-depart_time.start_time      1.000     0.667     0.800       3.0
      B-depart_time.end_time      1.000     0.333     0.500       3.0
          B-depart_date.year      1.000     1.000     1.000       3.0
            I-transport_type      1.000     1.000     1.000       1.0
          B-restriction_code      1.000     0.500     0.667       4.0
    B-arrive_time.start_time      0.889     1.000     0.941       8.0
        B-toloc.airport_code      1.000     0.750     0.857       4.0
      B-arrive_time.end_time      0.889     1.000     0.941       8.0
      I-arrive_time.end_time      0.889     1.000     0.941       8.0
      I-depart_time.end_time      1.000     0.333     0.500       3.0
               I-flight_stop      0.000     0.000     0.000       0.0
      B-fromloc.airport_code      0.625     1.000     0.769       5.0
          I-restriction_code      1.000     1.000     1.000       3.0
    I-depart_time.start_time      1.000     1.000     1.000       1.0
          I-toloc.state_name      1.000     1.000     1.000       1.0
I-depart_date.today_relative      0.000     0.000     0.000       0.0
 B-arrive_date.date_relative      1.000     1.000     1.000       2.0
                I-flight_mod      1.000     0.167     0.286       6.0
                   I-economy      0.000     0.000     0.000       0.0
 B-return_date.date_relative      1.000     0.333     0.500       3.0
        I-fromloc.state_name      1.000     1.000     1.000       1.0
                B-state_code      1.000     1.000     1.000       1.0
    I-arrive_time.start_time      1.000     1.000     1.000       1.0
    I-arrive_date.day_number      0.000     0.000     0.000       0.0
                 B-meal_code      0.000     0.000     0.000       1.0
 I-depart_time.period_of_day      1.000     1.000     1.000       1.0
                  B-day_name      1.000     0.500     0.667       2.0
             B-period_of_day      1.000     0.750     0.857       4.0
        B-stoploc.state_code      0.000     0.000     0.000       0.0
    B-return_date.month_name      0.000     0.000     0.000       0.0
    B-return_date.day_number      0.000     0.000     0.000       0.0
    B-arrive_time.period_mod      0.000     0.000     0.000       0.0
                 I-meal_code      0.000     0.000     0.000       0.0
        B-toloc.country_name      1.000     1.000     1.000       1.0
                 B-days_code      1.000     1.000     1.000       1.0
 I-arrive_time.period_of_day      0.000     0.000     0.000       0.0
            I-today_relative      0.000     0.000     0.000       0.0
 B-return_time.period_of_day      0.000     0.000     0.000       0.0
                      B-time      0.000     0.000     0.000       0.0
           I-fare_basis_code      0.000     0.000     0.000       0.0
 I-arrive_time.time_relative      1.000     0.500     0.667       4.0
 I-depart_time.time_relative      0.000     0.000     0.000       1.0
            B-today_relative      0.000     0.000     0.000       0.0
                B-state_name      0.000     0.000     0.000       9.0
B-arrive_date.today_relative      0.000     0.000     0.000       0.0
    B-return_time.period_mod      0.000     0.000     0.000       0.0
                B-month_name      0.000     0.000     0.000       0.0
                B-day_number      0.000     0.000     0.000       0.0
 I-return_date.date_relative      0.667     0.667     0.667       3.0
I-return_date.today_relative      0.000     0.000     0.000       0.0
      B-stoploc.airport_name      0.000     0.000     0.000       0.0
             B-time_relative      0.000     0.000     0.000       0.0
                      I-time      0.000     0.000     0.000       0.0
    I-return_date.day_number      0.000     0.000     0.000       0.0
          I-meal_description      0.000     0.000     0.000       0.0
B-return_date.today_relative      0.000     0.000     0.000       0.0
      B-return_date.day_name      0.000     0.000     0.000       2.0

                   micro avg      0.957     0.959     0.958    3657.0
                   macro avg      0.697     0.659     0.662    3657.0
                weighted avg      0.961     0.959     0.955    3657.0

I0716 04:16:15.437776 139795883440000 interactiveshell.py:2882] Best Slot F1: 0.959
I0716 04:16:15.439440 139795883440000 estimator_training.py:186] Not using Distribute Coordinator.
I0716 04:16:15.441732 139795883440000 training.py:612] Running training and evaluation locally (non-distributed).
I0716 04:16:15.446295 139795883440000 training.py:700] Start train and evaluate loop. The evaluate will happen after every checkpoint. Checkpoint frequency is determined based on RunConfig arguments: save_checkpoints_steps 312 or save_checkpoints_secs None.
I0716 04:16:15.507980 139795883440000 estimator.py:1145] Calling model_fn.
I0716 04:16:17.302391 139795883440000 saver.py:1499] Saver not created because there are no variables in the graph to restore
I0716 04:16:18.061816 139795883440000 <ipython-input-6-d6747b40360b>:55] 
[<tf.Variable 'embedding:0' shape=(750, 300) dtype=float32_ref>,
 <tf.Variable 'module/aggregation/weights:0' shape=(3,) dtype=float32>,
 <tf.Variable 'module/aggregation/scaling:0' shape=() dtype=float32>,
 <tf.Variable 'dense/kernel:0' shape=(1324, 300) dtype=float32_ref>,
 <tf.Variable 'dense/bias:0' shape=(300,) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/gates/kernel:0' shape=(600, 600) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/gates/bias:0' shape=(600,) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/candidate/kernel:0' shape=(600, 300) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/candidate/bias:0' shape=(300,) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/gates/kernel:0' shape=(600, 600) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/gates/bias:0' shape=(600,) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/candidate/kernel:0' shape=(600, 300) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/candidate/bias:0' shape=(300,) dtype=float32_ref>,
 <tf.Variable 'dense_1/kernel:0' shape=(600, 23) dtype=float32_ref>,
 <tf.Variable 'dense_1/bias:0' shape=(23,) dtype=float32_ref>,
 <tf.Variable 'dense_2/kernel:0' shape=(600, 122) dtype=float32_ref>,
 <tf.Variable 'dense_2/bias:0' shape=(122,) dtype=float32_ref>]
I0716 04:16:19.134381 139795883440000 estimator.py:1147] Done calling model_fn.
I0716 04:16:19.138150 139795883440000 basic_session_run_hooks.py:541] Create CheckpointSaverHook.
I0716 04:16:19.467717 139795883440000 monitored_session.py:240] Graph was finalized.
I0716 04:16:19.484432 139795883440000 saver.py:1280] Restoring parameters from ../model/elmo_bigru/model.ckpt-6552
I0716 04:16:20.499225 139795883440000 session_manager.py:500] Running local_init_op.
I0716 04:16:20.567731 139795883440000 session_manager.py:502] Done running local_init_op.
I0716 04:16:23.578376 139795883440000 basic_session_run_hooks.py:606] Saving checkpoints for 6552 into ../model/elmo_bigru/model.ckpt.
Reading ../data/atis.train.w-intent.iob
I0716 04:16:28.719417 139795883440000 basic_session_run_hooks.py:262] loss = 0.018550156, step = 6552
I0716 04:16:28.725748 139795883440000 basic_session_run_hooks.py:262] lr = 0.00015042433
I0716 04:16:56.900079 139795883440000 basic_session_run_hooks.py:692] global_step/sec: 3.54847
I0716 04:16:56.905190 139795883440000 basic_session_run_hooks.py:260] loss = 0.015438979, step = 6652 (28.186 sec)
I0716 04:16:56.908043 139795883440000 basic_session_run_hooks.py:260] lr = 0.00014884777 (28.182 sec)
I0716 04:17:23.714182 139795883440000 basic_session_run_hooks.py:692] global_step/sec: 3.72937
I0716 04:17:23.717083 139795883440000 basic_session_run_hooks.py:260] loss = 0.015352417, step = 6752 (26.812 sec)
I0716 04:17:23.719132 139795883440000 basic_session_run_hooks.py:260] lr = 0.00014728773 (26.811 sec)
I0716 04:17:49.155670 139795883440000 basic_session_run_hooks.py:692] global_step/sec: 3.93059
I0716 04:17:49.160705 139795883440000 basic_session_run_hooks.py:260] loss = 0.023017865, step = 6852 (25.444 sec)
I0716 04:17:49.164227 139795883440000 basic_session_run_hooks.py:260] lr = 0.00014574405 (25.445 sec)
I0716 04:17:51.754682 139795883440000 basic_session_run_hooks.py:606] Saving checkpoints for 6864 into ../model/elmo_bigru/model.ckpt.
I0716 04:17:53.686567 139795883440000 estimator.py:1145] Calling model_fn.
I0716 04:17:55.038545 139795883440000 saver.py:1499] Saver not created because there are no variables in the graph to restore
I0716 04:17:55.623387 139795883440000 estimator.py:1147] Done calling model_fn.
I0716 04:17:55.653599 139795883440000 evaluation.py:255] Starting evaluation at 2019-07-16T04:17:55Z
I0716 04:17:55.876722 139795883440000 monitored_session.py:240] Graph was finalized.
I0716 04:17:55.893433 139795883440000 saver.py:1280] Restoring parameters from ../model/elmo_bigru/model.ckpt-6864
I0716 04:17:56.822394 139795883440000 session_manager.py:500] Running local_init_op.
I0716 04:17:56.880496 139795883440000 session_manager.py:502] Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
I0716 04:18:55.726617 139795883440000 evaluation.py:275] Finished evaluation at 2019-07-16-04:18:55
I0716 04:18:55.728466 139795883440000 estimator.py:2039] Saving dict for global step 6864: global_step = 6864, loss = 0.33906943
I0716 04:18:55.733350 139795883440000 estimator.py:2099] Saving 'checkpoint_path' summary for global step 6864: ../model/elmo_bigru/model.ckpt-6864
I0716 04:18:55.811212 139795883440000 estimator.py:368] Loss for final step: 0.0032379918.
Reading ../data/atis.test.w-intent.iob
I0716 04:18:56.430192 139795883440000 estimator.py:1145] Calling model_fn.
I0716 04:18:58.101979 139795883440000 saver.py:1499] Saver not created because there are no variables in the graph to restore
I0716 04:18:58.792852 139795883440000 estimator.py:1147] Done calling model_fn.
I0716 04:18:59.017447 139795883440000 monitored_session.py:240] Graph was finalized.
I0716 04:18:59.033003 139795883440000 saver.py:1280] Restoring parameters from ../model/elmo_bigru/model.ckpt-6864
I0716 04:18:59.925140 139795883440000 session_manager.py:500] Running local_init_op.
I0716 04:18:59.971752 139795883440000 session_manager.py:502] Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.
  'precision', 'predicted', average, warn_for)
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1439: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples.
  'recall', 'true', average, warn_for)
I0716 04:19:57.884124 139795883440000 interactiveshell.py:2882] 
                                          precision    recall  f1-score   support

                             atis_flight      0.986     0.992     0.989       632
                            atis_airfare      0.941     1.000     0.970        48
                     atis_ground_service      0.973     1.000     0.986        36
                            atis_airline      1.000     1.000     1.000        38
                       atis_abbreviation      0.943     1.000     0.971        33
                           atis_aircraft      1.000     0.889     0.941         9
                        atis_flight_time      0.500     1.000     0.667         1
                           atis_quantity      0.429     1.000     0.600         3
                atis_flight#atis_airfare      0.833     0.417     0.556        12
                            atis_airport      1.000     0.944     0.971        18
                           atis_distance      1.000     1.000     1.000        10
                               atis_city      0.667     0.667     0.667         6
                        atis_ground_fare      1.000     0.857     0.923         7
                           atis_capacity      0.952     0.952     0.952        21
                          atis_flight_no      0.889     1.000     0.941         8
                               atis_meal      1.000     0.667     0.800         6
                        atis_restriction      0.000     0.000     0.000         0
             atis_airline#atis_flight_no      0.000     0.000     0.000         0
    atis_ground_service#atis_ground_fare      0.000     0.000     0.000         0
           atis_airfare#atis_flight_time      0.000     0.000     0.000         0
                           atis_cheapest      0.000     0.000     0.000         0
atis_aircraft#atis_flight#atis_flight_no      0.000     0.000     0.000         0

                               micro avg      0.972     0.977     0.975       888
                               macro avg      0.641     0.654     0.633       888
                            weighted avg      0.974     0.977     0.974       888

I0716 04:19:57.920375 139795883440000 interactiveshell.py:2882] 
                              precision    recall  f1-score   support

                           O      0.000     0.000     0.000       0.0
           B-toloc.city_name      0.977     0.994     0.985     716.0
         B-fromloc.city_name      0.992     0.999     0.995     704.0
           I-toloc.city_name      0.981     0.992     0.987     265.0
      B-depart_date.day_name      0.991     0.991     0.991     212.0
              B-airline_name      1.000     1.000     1.000     101.0
         I-fromloc.city_name      0.983     1.000     0.992     177.0
 B-depart_time.period_of_day      0.984     0.923     0.952     130.0
              I-airline_name      0.985     1.000     0.992      65.0
    B-depart_date.day_number      0.981     0.964     0.972      55.0
    B-depart_date.month_name      0.982     0.964     0.973      56.0
          B-depart_time.time      0.864     1.000     0.927      57.0
                B-round_trip      1.000     1.000     1.000      73.0
             B-cost_relative      0.973     0.973     0.973      37.0
                I-round_trip      1.000     1.000     1.000      71.0
                B-flight_mod      0.828     1.000     0.906      24.0
 B-depart_time.time_relative      0.955     0.969     0.962      65.0
          I-depart_time.time      0.945     1.000     0.972      52.0
         B-stoploc.city_name      0.870     1.000     0.930      20.0
                 B-city_name      0.902     0.649     0.755      57.0
                B-class_type      0.960     1.000     0.980      24.0
          B-arrive_time.time      0.971     0.971     0.971      34.0
 B-arrive_time.time_relative      0.967     0.935     0.951      31.0
                I-class_type      1.000     1.000     1.000      17.0
               B-flight_stop      1.000     1.000     1.000      21.0
          I-arrive_time.time      1.000     0.971     0.986      35.0
              B-airline_code      0.895     1.000     0.944      34.0
    I-depart_date.day_number      1.000     0.933     0.966      15.0
      I-fromloc.airport_name      0.441     1.000     0.612      15.0
      B-fromloc.airport_name      0.429     1.000     0.600      12.0
      B-arrive_date.day_name      0.786     1.000     0.880      11.0
          B-toloc.state_code      1.000     1.000     1.000      18.0
B-depart_date.today_relative      1.000     0.889     0.941       9.0
             B-flight_number      0.733     1.000     0.846      11.0
 B-depart_date.date_relative      0.944     1.000     0.971      17.0
          B-toloc.state_name      0.839     0.929     0.881      28.0
           B-fare_basis_code      0.850     1.000     0.919      17.0
               B-flight_time      1.000     1.000     1.000       1.0
                        B-or      1.000     1.000     1.000       3.0
 B-arrive_time.period_of_day      0.750     1.000     0.857       6.0
          B-meal_description      1.000     0.900     0.947      10.0
             I-cost_relative      1.000     0.667     0.800       3.0
              I-airport_name      0.833     0.345     0.488      29.0
               B-fare_amount      1.000     1.000     1.000       2.0
               I-fare_amount      1.000     1.000     1.000       2.0
                 I-city_name      0.950     0.633     0.760      30.0
        I-toloc.airport_name      1.000     1.000     1.000       3.0
            B-transport_type      1.000     1.000     1.000      10.0
    B-arrive_date.month_name      0.714     0.833     0.769       6.0
    B-arrive_date.day_number      0.714     0.833     0.769       6.0
         I-stoploc.city_name      1.000     1.000     1.000      10.0
                      B-meal      1.000     1.000     1.000      16.0
        B-fromloc.state_code      1.000     1.000     1.000      23.0
    B-depart_time.period_mod      0.833     1.000     0.909       5.0
                   B-connect      1.000     1.000     1.000       6.0
               B-flight_days      1.000     1.000     1.000      10.0
        B-toloc.airport_name      1.000     1.000     1.000       3.0
        B-fromloc.state_name      1.000     1.000     1.000      17.0
              B-airport_name      0.636     0.333     0.437      21.0
                   B-economy      1.000     1.000     1.000       6.0
               I-flight_time      1.000     1.000     1.000       1.0
             B-aircraft_code      1.000     0.818     0.900      33.0
                       B-mod      1.000     0.500     0.667       2.0
              B-airport_code      1.000     0.556     0.714       9.0
    B-depart_time.start_time      1.000     0.667     0.800       3.0
      B-depart_time.end_time      1.000     0.333     0.500       3.0
          B-depart_date.year      1.000     1.000     1.000       3.0
            I-transport_type      1.000     1.000     1.000       1.0
          B-restriction_code      1.000     1.000     1.000       4.0
    B-arrive_time.start_time      0.889     1.000     0.941       8.0
        B-toloc.airport_code      1.000     0.750     0.857       4.0
      B-arrive_time.end_time      0.889     1.000     0.941       8.0
      I-arrive_time.end_time      0.889     1.000     0.941       8.0
      I-depart_time.end_time      1.000     0.333     0.500       3.0
               I-flight_stop      0.000     0.000     0.000       0.0
      B-fromloc.airport_code      0.714     1.000     0.833       5.0
          I-restriction_code      1.000     1.000     1.000       3.0
    I-depart_time.start_time      1.000     1.000     1.000       1.0
          I-toloc.state_name      1.000     1.000     1.000       1.0
I-depart_date.today_relative      0.000     0.000     0.000       0.0
 B-arrive_date.date_relative      1.000     1.000     1.000       2.0
                I-flight_mod      1.000     0.167     0.286       6.0
                   I-economy      0.000     0.000     0.000       0.0
 B-return_date.date_relative      1.000     0.333     0.500       3.0
        I-fromloc.state_name      1.000     1.000     1.000       1.0
                B-state_code      1.000     1.000     1.000       1.0
    I-arrive_time.start_time      1.000     1.000     1.000       1.0
    I-arrive_date.day_number      0.000     0.000     0.000       0.0
                 B-meal_code      0.000     0.000     0.000       1.0
 I-depart_time.period_of_day      0.000     0.000     0.000       1.0
                  B-day_name      1.000     0.500     0.667       2.0
             B-period_of_day      1.000     0.750     0.857       4.0
        B-stoploc.state_code      0.000     0.000     0.000       0.0
    B-return_date.month_name      0.000     0.000     0.000       0.0
    B-return_date.day_number      0.000     0.000     0.000       0.0
    B-arrive_time.period_mod      0.000     0.000     0.000       0.0
                 I-meal_code      0.000     0.000     0.000       0.0
        B-toloc.country_name      0.000     0.000     0.000       1.0
                 B-days_code      0.000     0.000     0.000       1.0
 I-arrive_time.period_of_day      0.000     0.000     0.000       0.0
            I-today_relative      0.000     0.000     0.000       0.0
 B-return_time.period_of_day      0.000     0.000     0.000       0.0
                      B-time      0.000     0.000     0.000       0.0
           I-fare_basis_code      0.000     0.000     0.000       0.0
 I-arrive_time.time_relative      1.000     1.000     1.000       4.0
 I-depart_time.time_relative      0.000     0.000     0.000       1.0
            B-today_relative      0.000     0.000     0.000       0.0
                B-state_name      0.000     0.000     0.000       9.0
B-arrive_date.today_relative      0.000     0.000     0.000       0.0
    B-return_time.period_mod      0.000     0.000     0.000       0.0
                B-month_name      0.000     0.000     0.000       0.0
                B-day_number      0.000     0.000     0.000       0.0
 I-return_date.date_relative      0.667     0.667     0.667       3.0
I-return_date.today_relative      0.000     0.000     0.000       0.0
      B-stoploc.airport_name      0.000     0.000     0.000       0.0
             B-time_relative      0.000     0.000     0.000       0.0
                      I-time      0.000     0.000     0.000       0.0
    I-return_date.day_number      0.000     0.000     0.000       0.0
          I-meal_description      0.000     0.000     0.000       0.0
B-return_date.today_relative      0.000     0.000     0.000       0.0
      B-return_date.day_name      0.000     0.000     0.000       2.0

                   micro avg      0.959     0.960     0.960    3657.0
                   macro avg      0.673     0.644     0.645    3657.0
                weighted avg      0.962     0.960     0.957    3657.0

I0716 04:19:57.940826 139795883440000 interactiveshell.py:2882] Best Slot F1: 0.960
I0716 04:19:57.942459 139795883440000 estimator_training.py:186] Not using Distribute Coordinator.
I0716 04:19:57.943920 139795883440000 training.py:612] Running training and evaluation locally (non-distributed).
I0716 04:19:57.947064 139795883440000 training.py:700] Start train and evaluate loop. The evaluate will happen after every checkpoint. Checkpoint frequency is determined based on RunConfig arguments: save_checkpoints_steps 312 or save_checkpoints_secs None.
I0716 04:19:58.011204 139795883440000 estimator.py:1145] Calling model_fn.
I0716 04:19:59.334115 139795883440000 saver.py:1499] Saver not created because there are no variables in the graph to restore
I0716 04:20:00.090726 139795883440000 <ipython-input-6-d6747b40360b>:55] 
[<tf.Variable 'embedding:0' shape=(750, 300) dtype=float32_ref>,
 <tf.Variable 'module/aggregation/weights:0' shape=(3,) dtype=float32>,
 <tf.Variable 'module/aggregation/scaling:0' shape=() dtype=float32>,
 <tf.Variable 'dense/kernel:0' shape=(1324, 300) dtype=float32_ref>,
 <tf.Variable 'dense/bias:0' shape=(300,) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/gates/kernel:0' shape=(600, 600) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/gates/bias:0' shape=(600,) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/candidate/kernel:0' shape=(600, 300) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/candidate/bias:0' shape=(300,) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/gates/kernel:0' shape=(600, 600) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/gates/bias:0' shape=(600,) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/candidate/kernel:0' shape=(600, 300) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/candidate/bias:0' shape=(300,) dtype=float32_ref>,
 <tf.Variable 'dense_1/kernel:0' shape=(600, 23) dtype=float32_ref>,
 <tf.Variable 'dense_1/bias:0' shape=(23,) dtype=float32_ref>,
 <tf.Variable 'dense_2/kernel:0' shape=(600, 122) dtype=float32_ref>,
 <tf.Variable 'dense_2/bias:0' shape=(122,) dtype=float32_ref>]
I0716 04:20:01.162770 139795883440000 estimator.py:1147] Done calling model_fn.
I0716 04:20:01.166483 139795883440000 basic_session_run_hooks.py:541] Create CheckpointSaverHook.
I0716 04:20:01.490742 139795883440000 monitored_session.py:240] Graph was finalized.
I0716 04:20:01.508144 139795883440000 saver.py:1280] Restoring parameters from ../model/elmo_bigru/model.ckpt-6864
I0716 04:20:02.552333 139795883440000 session_manager.py:500] Running local_init_op.
I0716 04:20:02.620016 139795883440000 session_manager.py:502] Done running local_init_op.
I0716 04:20:05.599907 139795883440000 basic_session_run_hooks.py:606] Saving checkpoints for 6864 into ../model/elmo_bigru/model.ckpt.
Reading ../data/atis.train.w-intent.iob
I0716 04:20:10.308010 139795883440000 basic_session_run_hooks.py:262] loss = 0.016606176, step = 6864
I0716 04:20:10.309966 139795883440000 basic_session_run_hooks.py:262] lr = 0.0001455599
I0716 04:20:37.941588 139795883440000 basic_session_run_hooks.py:692] global_step/sec: 3.6187
I0716 04:20:37.946143 139795883440000 basic_session_run_hooks.py:260] loss = 0.018260527, step = 6964 (27.638 sec)
I0716 04:20:37.947947 139795883440000 basic_session_run_hooks.py:260] lr = 0.00014403433 (27.638 sec)
I0716 04:21:04.987621 139795883440000 basic_session_run_hooks.py:692] global_step/sec: 3.69741
I0716 04:21:04.993425 139795883440000 basic_session_run_hooks.py:260] loss = 0.013289442, step = 7064 (27.047 sec)
I0716 04:21:04.996516 139795883440000 basic_session_run_hooks.py:260] lr = 0.00014252475 (27.049 sec)
I0716 04:21:31.147312 139795883440000 basic_session_run_hooks.py:692] global_step/sec: 3.82267
I0716 04:21:31.152855 139795883440000 basic_session_run_hooks.py:260] loss = 0.012278922, step = 7164 (26.159 sec)
I0716 04:21:31.154504 139795883440000 basic_session_run_hooks.py:260] lr = 0.00014103096 (26.158 sec)
I0716 04:21:33.702331 139795883440000 basic_session_run_hooks.py:606] Saving checkpoints for 7176 into ../model/elmo_bigru/model.ckpt.
I0716 04:21:35.939595 139795883440000 estimator.py:1145] Calling model_fn.
I0716 04:21:37.311266 139795883440000 saver.py:1499] Saver not created because there are no variables in the graph to restore
I0716 04:21:37.893917 139795883440000 estimator.py:1147] Done calling model_fn.
I0716 04:21:37.924747 139795883440000 evaluation.py:255] Starting evaluation at 2019-07-16T04:21:37Z
I0716 04:21:38.176059 139795883440000 monitored_session.py:240] Graph was finalized.
I0716 04:21:38.188971 139795883440000 saver.py:1280] Restoring parameters from ../model/elmo_bigru/model.ckpt-7176
I0716 04:21:39.124279 139795883440000 session_manager.py:500] Running local_init_op.
I0716 04:21:39.181013 139795883440000 session_manager.py:502] Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
I0716 04:22:38.135489 139795883440000 evaluation.py:275] Finished evaluation at 2019-07-16-04:22:38
I0716 04:22:38.137200 139795883440000 estimator.py:2039] Saving dict for global step 7176: global_step = 7176, loss = 0.3438473
I0716 04:22:38.143747 139795883440000 estimator.py:2099] Saving 'checkpoint_path' summary for global step 7176: ../model/elmo_bigru/model.ckpt-7176
I0716 04:22:38.224134 139795883440000 estimator.py:368] Loss for final step: 0.0030283371.
Reading ../data/atis.test.w-intent.iob
I0716 04:22:38.820212 139795883440000 estimator.py:1145] Calling model_fn.
I0716 04:22:40.462757 139795883440000 saver.py:1499] Saver not created because there are no variables in the graph to restore
I0716 04:22:40.978199 139795883440000 estimator.py:1147] Done calling model_fn.
I0716 04:22:41.382744 139795883440000 monitored_session.py:240] Graph was finalized.
I0716 04:22:41.397099 139795883440000 saver.py:1280] Restoring parameters from ../model/elmo_bigru/model.ckpt-7176
I0716 04:22:42.377617 139795883440000 session_manager.py:500] Running local_init_op.
I0716 04:22:42.428126 139795883440000 session_manager.py:502] Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.
  'precision', 'predicted', average, warn_for)
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1439: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples.
  'recall', 'true', average, warn_for)
I0716 04:23:40.307400 139795883440000 interactiveshell.py:2882] 
                                          precision    recall  f1-score   support

                             atis_flight      0.986     0.992     0.989       632
                            atis_airfare      0.941     1.000     0.970        48
                     atis_ground_service      1.000     1.000     1.000        36
                            atis_airline      1.000     1.000     1.000        38
                       atis_abbreviation      0.943     1.000     0.971        33
                           atis_aircraft      1.000     0.889     0.941         9
                        atis_flight_time      0.500     1.000     0.667         1
                           atis_quantity      0.375     1.000     0.545         3
                atis_flight#atis_airfare      0.833     0.417     0.556        12
                            atis_airport      1.000     1.000     1.000        18
                           atis_distance      1.000     1.000     1.000        10
                               atis_city      0.667     0.667     0.667         6
                        atis_ground_fare      1.000     0.857     0.923         7
                           atis_capacity      1.000     0.952     0.976        21
                          atis_flight_no      0.889     1.000     0.941         8
                               atis_meal      1.000     0.667     0.800         6
                        atis_restriction      0.000     0.000     0.000         0
             atis_airline#atis_flight_no      0.000     0.000     0.000         0
    atis_ground_service#atis_ground_fare      0.000     0.000     0.000         0
           atis_airfare#atis_flight_time      0.000     0.000     0.000         0
                           atis_cheapest      0.000     0.000     0.000         0
atis_aircraft#atis_flight#atis_flight_no      0.000     0.000     0.000         0

                               micro avg      0.973     0.979     0.976       888
                               macro avg      0.642     0.656     0.634       888
                            weighted avg      0.976     0.979     0.976       888

I0716 04:23:40.341473 139795883440000 interactiveshell.py:2882] 
                              precision    recall  f1-score   support

                           O      0.000     0.000     0.000       0.0
           B-toloc.city_name      0.978     0.993     0.985     716.0
         B-fromloc.city_name      0.992     0.999     0.995     704.0
           I-toloc.city_name      0.978     0.992     0.985     265.0
      B-depart_date.day_name      0.991     0.991     0.991     212.0
              B-airline_name      1.000     1.000     1.000     101.0
         I-fromloc.city_name      0.983     1.000     0.992     177.0
 B-depart_time.period_of_day      0.992     0.923     0.956     130.0
              I-airline_name      1.000     1.000     1.000      65.0
    B-depart_date.day_number      0.981     0.964     0.972      55.0
    B-depart_date.month_name      0.982     0.964     0.973      56.0
          B-depart_time.time      0.864     1.000     0.927      57.0
                B-round_trip      1.000     1.000     1.000      73.0
             B-cost_relative      0.973     0.973     0.973      37.0
                I-round_trip      1.000     1.000     1.000      71.0
                B-flight_mod      0.828     1.000     0.906      24.0
 B-depart_time.time_relative      0.955     0.985     0.970      65.0
          I-depart_time.time      0.945     1.000     0.972      52.0
         B-stoploc.city_name      0.833     1.000     0.909      20.0
                 B-city_name      0.905     0.667     0.768      57.0
                B-class_type      0.960     1.000     0.980      24.0
          B-arrive_time.time      0.971     0.971     0.971      34.0
 B-arrive_time.time_relative      0.967     0.935     0.951      31.0
                I-class_type      1.000     1.000     1.000      17.0
               B-flight_stop      1.000     1.000     1.000      21.0
          I-arrive_time.time      1.000     0.971     0.986      35.0
              B-airline_code      0.895     1.000     0.944      34.0
    I-depart_date.day_number      1.000     0.933     0.966      15.0
      I-fromloc.airport_name      0.441     1.000     0.612      15.0
      B-fromloc.airport_name      0.429     1.000     0.600      12.0
      B-arrive_date.day_name      0.786     1.000     0.880      11.0
          B-toloc.state_code      1.000     1.000     1.000      18.0
B-depart_date.today_relative      1.000     0.889     0.941       9.0
             B-flight_number      0.733     1.000     0.846      11.0
 B-depart_date.date_relative      0.944     1.000     0.971      17.0
          B-toloc.state_name      0.839     0.929     0.881      28.0
           B-fare_basis_code      0.895     1.000     0.944      17.0
               B-flight_time      1.000     1.000     1.000       1.0
                        B-or      1.000     1.000     1.000       3.0
 B-arrive_time.period_of_day      0.750     1.000     0.857       6.0
          B-meal_description      1.000     0.900     0.947      10.0
             I-cost_relative      1.000     0.667     0.800       3.0
              I-airport_name      0.846     0.379     0.524      29.0
               B-fare_amount      1.000     1.000     1.000       2.0
               I-fare_amount      1.000     1.000     1.000       2.0
                 I-city_name      1.000     0.600     0.750      30.0
        I-toloc.airport_name      1.000     1.000     1.000       3.0
            B-transport_type      1.000     1.000     1.000      10.0
    B-arrive_date.month_name      0.714     0.833     0.769       6.0
    B-arrive_date.day_number      0.714     0.833     0.769       6.0
         I-stoploc.city_name      1.000     1.000     1.000      10.0
                      B-meal      1.000     1.000     1.000      16.0
        B-fromloc.state_code      1.000     1.000     1.000      23.0
    B-depart_time.period_mod      1.000     1.000     1.000       5.0
                   B-connect      1.000     1.000     1.000       6.0
               B-flight_days      1.000     1.000     1.000      10.0
        B-toloc.airport_name      1.000     1.000     1.000       3.0
        B-fromloc.state_name      1.000     1.000     1.000      17.0
              B-airport_name      0.636     0.333     0.437      21.0
                   B-economy      1.000     1.000     1.000       6.0
               I-flight_time      1.000     1.000     1.000       1.0
             B-aircraft_code      1.000     0.818     0.900      33.0
                       B-mod      1.000     0.500     0.667       2.0
              B-airport_code      1.000     0.444     0.615       9.0
    B-depart_time.start_time      1.000     0.667     0.800       3.0
      B-depart_time.end_time      1.000     0.333     0.500       3.0
          B-depart_date.year      1.000     1.000     1.000       3.0
            I-transport_type      0.000     0.000     0.000       1.0
          B-restriction_code      1.000     1.000     1.000       4.0
    B-arrive_time.start_time      0.889     1.000     0.941       8.0
        B-toloc.airport_code      1.000     0.750     0.857       4.0
      B-arrive_time.end_time      0.889     1.000     0.941       8.0
      I-arrive_time.end_time      0.889     1.000     0.941       8.0
      I-depart_time.end_time      1.000     0.333     0.500       3.0
               I-flight_stop      0.000     0.000     0.000       0.0
      B-fromloc.airport_code      0.625     1.000     0.769       5.0
          I-restriction_code      1.000     1.000     1.000       3.0
    I-depart_time.start_time      1.000     1.000     1.000       1.0
          I-toloc.state_name      1.000     1.000     1.000       1.0
I-depart_date.today_relative      0.000     0.000     0.000       0.0
 B-arrive_date.date_relative      1.000     1.000     1.000       2.0
                I-flight_mod      1.000     0.167     0.286       6.0
                   I-economy      0.000     0.000     0.000       0.0
 B-return_date.date_relative      1.000     0.667     0.800       3.0
        I-fromloc.state_name      1.000     1.000     1.000       1.0
                B-state_code      1.000     1.000     1.000       1.0
    I-arrive_time.start_time      1.000     1.000     1.000       1.0
    I-arrive_date.day_number      0.000     0.000     0.000       0.0
                 B-meal_code      0.000     0.000     0.000       1.0
 I-depart_time.period_of_day      1.000     1.000     1.000       1.0
                  B-day_name      1.000     0.500     0.667       2.0
             B-period_of_day      1.000     0.750     0.857       4.0
        B-stoploc.state_code      0.000     0.000     0.000       0.0
    B-return_date.month_name      0.000     0.000     0.000       0.0
    B-return_date.day_number      0.000     0.000     0.000       0.0
    B-arrive_time.period_mod      0.000     0.000     0.000       0.0
                 I-meal_code      0.000     0.000     0.000       0.0
        B-toloc.country_name      1.000     1.000     1.000       1.0
                 B-days_code      1.000     1.000     1.000       1.0
 I-arrive_time.period_of_day      0.000     0.000     0.000       0.0
            I-today_relative      0.000     0.000     0.000       0.0
 B-return_time.period_of_day      0.000     0.000     0.000       0.0
                      B-time      0.000     0.000     0.000       0.0
           I-fare_basis_code      0.000     0.000     0.000       0.0
 I-arrive_time.time_relative      1.000     1.000     1.000       4.0
 I-depart_time.time_relative      0.000     0.000     0.000       1.0
            B-today_relative      0.000     0.000     0.000       0.0
                B-state_name      0.000     0.000     0.000       9.0
B-arrive_date.today_relative      0.000     0.000     0.000       0.0
    B-return_time.period_mod      0.000     0.000     0.000       0.0
                B-month_name      0.000     0.000     0.000       0.0
                B-day_number      0.000     0.000     0.000       0.0
 I-return_date.date_relative      1.000     0.667     0.800       3.0
I-return_date.today_relative      0.000     0.000     0.000       0.0
      B-stoploc.airport_name      0.000     0.000     0.000       0.0
             B-time_relative      0.000     0.000     0.000       0.0
                      I-time      0.000     0.000     0.000       0.0
    I-return_date.day_number      0.000     0.000     0.000       0.0
          I-meal_description      0.000     0.000     0.000       0.0
B-return_date.today_relative      0.000     0.000     0.000       0.0
      B-return_date.day_name      0.000     0.000     0.000       2.0

                   micro avg      0.960     0.961     0.960    3657.0
                   macro avg      0.694     0.663     0.665    3657.0
                weighted avg      0.964     0.961     0.958    3657.0

I0716 04:23:40.360459 139795883440000 interactiveshell.py:2882] Best Slot F1: 0.960
I0716 04:23:40.362138 139795883440000 estimator_training.py:186] Not using Distribute Coordinator.
I0716 04:23:40.364122 139795883440000 training.py:612] Running training and evaluation locally (non-distributed).
I0716 04:23:40.365844 139795883440000 training.py:700] Start train and evaluate loop. The evaluate will happen after every checkpoint. Checkpoint frequency is determined based on RunConfig arguments: save_checkpoints_steps 312 or save_checkpoints_secs None.
I0716 04:23:40.436918 139795883440000 estimator.py:1145] Calling model_fn.
I0716 04:23:41.739951 139795883440000 saver.py:1499] Saver not created because there are no variables in the graph to restore
I0716 04:23:42.492683 139795883440000 <ipython-input-6-d6747b40360b>:55] 
[<tf.Variable 'embedding:0' shape=(750, 300) dtype=float32_ref>,
 <tf.Variable 'module/aggregation/weights:0' shape=(3,) dtype=float32>,
 <tf.Variable 'module/aggregation/scaling:0' shape=() dtype=float32>,
 <tf.Variable 'dense/kernel:0' shape=(1324, 300) dtype=float32_ref>,
 <tf.Variable 'dense/bias:0' shape=(300,) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/gates/kernel:0' shape=(600, 600) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/gates/bias:0' shape=(600,) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/candidate/kernel:0' shape=(600, 300) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/candidate/bias:0' shape=(300,) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/gates/kernel:0' shape=(600, 600) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/gates/bias:0' shape=(600,) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/candidate/kernel:0' shape=(600, 300) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/candidate/bias:0' shape=(300,) dtype=float32_ref>,
 <tf.Variable 'dense_1/kernel:0' shape=(600, 23) dtype=float32_ref>,
 <tf.Variable 'dense_1/bias:0' shape=(23,) dtype=float32_ref>,
 <tf.Variable 'dense_2/kernel:0' shape=(600, 122) dtype=float32_ref>,
 <tf.Variable 'dense_2/bias:0' shape=(122,) dtype=float32_ref>]
I0716 04:23:43.572311 139795883440000 estimator.py:1147] Done calling model_fn.
I0716 04:23:43.579196 139795883440000 basic_session_run_hooks.py:541] Create CheckpointSaverHook.
I0716 04:23:43.895811 139795883440000 monitored_session.py:240] Graph was finalized.
I0716 04:23:43.913872 139795883440000 saver.py:1280] Restoring parameters from ../model/elmo_bigru/model.ckpt-7176
I0716 04:23:44.946667 139795883440000 session_manager.py:500] Running local_init_op.
I0716 04:23:45.012770 139795883440000 session_manager.py:502] Done running local_init_op.
I0716 04:23:47.791606 139795883440000 basic_session_run_hooks.py:606] Saving checkpoints for 7176 into ../model/elmo_bigru/model.ckpt.
Reading ../data/atis.train.w-intent.iob
I0716 04:23:52.395349 139795883440000 basic_session_run_hooks.py:262] loss = 0.012449772, step = 7176
I0716 04:23:52.397220 139795883440000 basic_session_run_hooks.py:262] lr = 0.00014085279
I0716 04:24:19.545236 139795883440000 basic_session_run_hooks.py:692] global_step/sec: 3.68319
I0716 04:24:19.550419 139795883440000 basic_session_run_hooks.py:260] loss = 0.012549496, step = 7276 (27.155 sec)
I0716 04:24:19.552397 139795883440000 basic_session_run_hooks.py:260] lr = 0.00013937654 (27.155 sec)
I0716 04:24:45.943143 139795883440000 basic_session_run_hooks.py:692] global_step/sec: 3.78818
I0716 04:24:45.951433 139795883440000 basic_session_run_hooks.py:260] loss = 0.016497286, step = 7376 (26.401 sec)
I0716 04:24:45.953580 139795883440000 basic_session_run_hooks.py:260] lr = 0.00013791576 (26.401 sec)
I0716 04:25:11.899698 139795883440000 basic_session_run_hooks.py:692] global_step/sec: 3.85259
I0716 04:25:11.905721 139795883440000 basic_session_run_hooks.py:260] loss = 0.011051845, step = 7476 (25.954 sec)
I0716 04:25:11.907590 139795883440000 basic_session_run_hooks.py:260] lr = 0.0001364703 (25.954 sec)
I0716 04:25:14.569300 139795883440000 basic_session_run_hooks.py:606] Saving checkpoints for 7488 into ../model/elmo_bigru/model.ckpt.
I0716 04:25:16.719005 139795883440000 estimator.py:1145] Calling model_fn.
I0716 04:25:18.079175 139795883440000 saver.py:1499] Saver not created because there are no variables in the graph to restore
I0716 04:25:18.678386 139795883440000 estimator.py:1147] Done calling model_fn.
I0716 04:25:18.706707 139795883440000 evaluation.py:255] Starting evaluation at 2019-07-16T04:25:18Z
I0716 04:25:18.926704 139795883440000 monitored_session.py:240] Graph was finalized.
I0716 04:25:18.943488 139795883440000 saver.py:1280] Restoring parameters from ../model/elmo_bigru/model.ckpt-7488
I0716 04:25:19.838812 139795883440000 session_manager.py:500] Running local_init_op.
I0716 04:25:19.898500 139795883440000 session_manager.py:502] Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
I0716 04:26:19.402054 139795883440000 evaluation.py:275] Finished evaluation at 2019-07-16-04:26:19
I0716 04:26:19.403770 139795883440000 estimator.py:2039] Saving dict for global step 7488: global_step = 7488, loss = 0.35254303
I0716 04:26:19.412193 139795883440000 estimator.py:2099] Saving 'checkpoint_path' summary for global step 7488: ../model/elmo_bigru/model.ckpt-7488
I0716 04:26:19.497845 139795883440000 estimator.py:368] Loss for final step: 0.005028704.
Reading ../data/atis.test.w-intent.iob
I0716 04:26:20.104184 139795883440000 estimator.py:1145] Calling model_fn.
I0716 04:26:21.773334 139795883440000 saver.py:1499] Saver not created because there are no variables in the graph to restore
I0716 04:26:22.289934 139795883440000 estimator.py:1147] Done calling model_fn.
I0716 04:26:22.500285 139795883440000 monitored_session.py:240] Graph was finalized.
I0716 04:26:22.516782 139795883440000 saver.py:1280] Restoring parameters from ../model/elmo_bigru/model.ckpt-7488
I0716 04:26:23.414255 139795883440000 session_manager.py:500] Running local_init_op.
I0716 04:26:23.474438 139795883440000 session_manager.py:502] Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.
  'precision', 'predicted', average, warn_for)
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1439: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples.
  'recall', 'true', average, warn_for)
I0716 04:27:21.630802 139795883440000 interactiveshell.py:2882] 
                                          precision    recall  f1-score   support

                             atis_flight      0.986     0.992     0.989       632
                            atis_airfare      0.941     1.000     0.970        48
                     atis_ground_service      1.000     1.000     1.000        36
                            atis_airline      0.974     1.000     0.987        38
                       atis_abbreviation      0.943     1.000     0.971        33
                           atis_aircraft      1.000     0.889     0.941         9
                        atis_flight_time      0.500     1.000     0.667         1
                           atis_quantity      0.429     1.000     0.600         3
                atis_flight#atis_airfare      0.833     0.417     0.556        12
                            atis_airport      1.000     0.944     0.971        18
                           atis_distance      1.000     1.000     1.000        10
                               atis_city      0.667     0.667     0.667         6
                        atis_ground_fare      1.000     0.857     0.923         7
                           atis_capacity      0.952     0.952     0.952        21
                          atis_flight_no      0.889     1.000     0.941         8
                               atis_meal      1.000     0.667     0.800         6
                        atis_restriction      0.000     0.000     0.000         0
             atis_airline#atis_flight_no      0.000     0.000     0.000         0
    atis_ground_service#atis_ground_fare      0.000     0.000     0.000         0
           atis_airfare#atis_flight_time      0.000     0.000     0.000         0
                           atis_cheapest      0.000     0.000     0.000         0
atis_aircraft#atis_flight#atis_flight_no      0.000     0.000     0.000         0

                               micro avg      0.972     0.977     0.975       888
                               macro avg      0.642     0.654     0.633       888
                            weighted avg      0.974     0.977     0.974       888

I0716 04:27:21.669584 139795883440000 interactiveshell.py:2882] 
                              precision    recall  f1-score   support

                           O      0.000     0.000     0.000       0.0
           B-toloc.city_name      0.975     0.992     0.983     716.0
         B-fromloc.city_name      0.986     0.999     0.992     704.0
           I-toloc.city_name      0.978     0.992     0.985     265.0
      B-depart_date.day_name      0.991     0.991     0.991     212.0
              B-airline_name      1.000     1.000     1.000     101.0
         I-fromloc.city_name      0.983     1.000     0.992     177.0
 B-depart_time.period_of_day      0.992     0.923     0.956     130.0
              I-airline_name      1.000     1.000     1.000      65.0
    B-depart_date.day_number      0.981     0.964     0.972      55.0
    B-depart_date.month_name      0.982     0.964     0.973      56.0
          B-depart_time.time      0.864     1.000     0.927      57.0
                B-round_trip      1.000     1.000     1.000      73.0
             B-cost_relative      0.973     0.973     0.973      37.0
                I-round_trip      1.000     1.000     1.000      71.0
                B-flight_mod      0.828     1.000     0.906      24.0
 B-depart_time.time_relative      0.955     0.969     0.962      65.0
          I-depart_time.time      0.945     1.000     0.972      52.0
         B-stoploc.city_name      0.909     1.000     0.952      20.0
                 B-city_name      0.881     0.649     0.747      57.0
                B-class_type      0.960     1.000     0.980      24.0
          B-arrive_time.time      0.971     0.971     0.971      34.0
 B-arrive_time.time_relative      0.906     0.935     0.921      31.0
                I-class_type      1.000     1.000     1.000      17.0
               B-flight_stop      1.000     1.000     1.000      21.0
          I-arrive_time.time      1.000     0.971     0.986      35.0
              B-airline_code      0.944     1.000     0.971      34.0
    I-depart_date.day_number      1.000     0.933     0.966      15.0
      I-fromloc.airport_name      0.429     1.000     0.600      15.0
      B-fromloc.airport_name      0.429     1.000     0.600      12.0
      B-arrive_date.day_name      0.786     1.000     0.880      11.0
          B-toloc.state_code      1.000     1.000     1.000      18.0
B-depart_date.today_relative      1.000     0.889     0.941       9.0
             B-flight_number      0.733     1.000     0.846      11.0
 B-depart_date.date_relative      0.944     1.000     0.971      17.0
          B-toloc.state_name      0.862     0.893     0.877      28.0
           B-fare_basis_code      0.773     1.000     0.872      17.0
               B-flight_time      1.000     1.000     1.000       1.0
                        B-or      1.000     1.000     1.000       3.0
 B-arrive_time.period_of_day      0.750     1.000     0.857       6.0
          B-meal_description      1.000     0.900     0.947      10.0
             I-cost_relative      1.000     0.667     0.800       3.0
              I-airport_name      0.818     0.310     0.450      29.0
               B-fare_amount      1.000     1.000     1.000       2.0
               I-fare_amount      1.000     1.000     1.000       2.0
                 I-city_name      0.944     0.567     0.708      30.0
        I-toloc.airport_name      1.000     1.000     1.000       3.0
            B-transport_type      1.000     1.000     1.000      10.0
    B-arrive_date.month_name      0.714     0.833     0.769       6.0
    B-arrive_date.day_number      0.714     0.833     0.769       6.0
         I-stoploc.city_name      1.000     1.000     1.000      10.0
                      B-meal      1.000     1.000     1.000      16.0
        B-fromloc.state_code      1.000     1.000     1.000      23.0
    B-depart_time.period_mod      0.833     1.000     0.909       5.0
                   B-connect      1.000     1.000     1.000       6.0
               B-flight_days      1.000     1.000     1.000      10.0
        B-toloc.airport_name      1.000     1.000     1.000       3.0
        B-fromloc.state_name      0.944     1.000     0.971      17.0
              B-airport_name      0.600     0.286     0.387      21.0
                   B-economy      1.000     1.000     1.000       6.0
               I-flight_time      1.000     1.000     1.000       1.0
             B-aircraft_code      1.000     0.818     0.900      33.0
                       B-mod      1.000     0.500     0.667       2.0
              B-airport_code      1.000     0.444     0.615       9.0
    B-depart_time.start_time      1.000     0.667     0.800       3.0
      B-depart_time.end_time      1.000     0.333     0.500       3.0
          B-depart_date.year      1.000     1.000     1.000       3.0
            I-transport_type      0.000     0.000     0.000       1.0
          B-restriction_code      1.000     1.000     1.000       4.0
    B-arrive_time.start_time      0.889     1.000     0.941       8.0
        B-toloc.airport_code      1.000     0.750     0.857       4.0
      B-arrive_time.end_time      0.889     1.000     0.941       8.0
      I-arrive_time.end_time      0.889     1.000     0.941       8.0
      I-depart_time.end_time      1.000     0.333     0.500       3.0
               I-flight_stop      0.000     0.000     0.000       0.0
      B-fromloc.airport_code      0.625     1.000     0.769       5.0
          I-restriction_code      1.000     1.000     1.000       3.0
    I-depart_time.start_time      1.000     1.000     1.000       1.0
          I-toloc.state_name      1.000     1.000     1.000       1.0
I-depart_date.today_relative      0.000     0.000     0.000       0.0
 B-arrive_date.date_relative      1.000     1.000     1.000       2.0
                I-flight_mod      1.000     0.167     0.286       6.0
                   I-economy      0.000     0.000     0.000       0.0
 B-return_date.date_relative      1.000     0.667     0.800       3.0
        I-fromloc.state_name      1.000     1.000     1.000       1.0
                B-state_code      1.000     1.000     1.000       1.0
    I-arrive_time.start_time      1.000     1.000     1.000       1.0
    I-arrive_date.day_number      0.000     0.000     0.000       0.0
                 B-meal_code      0.000     0.000     0.000       1.0
 I-depart_time.period_of_day      1.000     1.000     1.000       1.0
                  B-day_name      1.000     0.500     0.667       2.0
             B-period_of_day      1.000     0.750     0.857       4.0
        B-stoploc.state_code      0.000     0.000     0.000       0.0
    B-return_date.month_name      0.000     0.000     0.000       0.0
    B-return_date.day_number      0.000     0.000     0.000       0.0
    B-arrive_time.period_mod      0.000     0.000     0.000       0.0
                 I-meal_code      0.000     0.000     0.000       0.0
        B-toloc.country_name      1.000     1.000     1.000       1.0
                 B-days_code      0.000     0.000     0.000       1.0
 I-arrive_time.period_of_day      0.000     0.000     0.000       0.0
            I-today_relative      0.000     0.000     0.000       0.0
 B-return_time.period_of_day      0.000     0.000     0.000       0.0
                      B-time      0.000     0.000     0.000       0.0
           I-fare_basis_code      0.000     0.000     0.000       0.0
 I-arrive_time.time_relative      1.000     0.500     0.667       4.0
 I-depart_time.time_relative      0.000     0.000     0.000       1.0
            B-today_relative      0.000     0.000     0.000       0.0
                B-state_name      0.000     0.000     0.000       9.0
B-arrive_date.today_relative      0.000     0.000     0.000       0.0
    B-return_time.period_mod      0.000     0.000     0.000       0.0
                B-month_name      0.000     0.000     0.000       0.0
                B-day_number      0.000     0.000     0.000       0.0
 I-return_date.date_relative      1.000     0.667     0.800       3.0
I-return_date.today_relative      0.000     0.000     0.000       0.0
      B-stoploc.airport_name      0.000     0.000     0.000       0.0
             B-time_relative      0.000     0.000     0.000       0.0
                      I-time      0.000     0.000     0.000       0.0
    I-return_date.day_number      0.000     0.000     0.000       0.0
          I-meal_description      0.000     0.000     0.000       0.0
B-return_date.today_relative      0.000     0.000     0.000       0.0
      B-return_date.day_name      0.000     0.000     0.000       2.0

                   micro avg      0.957     0.958     0.957    3657.0
                   macro avg      0.682     0.649     0.651    3657.0
                weighted avg      0.960     0.958     0.954    3657.0

I0716 04:27:21.691767 139795883440000 interactiveshell.py:2882] Best Slot F1: 0.960
I0716 04:27:21.693325 139795883440000 estimator_training.py:186] Not using Distribute Coordinator.
I0716 04:27:21.695086 139795883440000 training.py:612] Running training and evaluation locally (non-distributed).
I0716 04:27:21.697174 139795883440000 training.py:700] Start train and evaluate loop. The evaluate will happen after every checkpoint. Checkpoint frequency is determined based on RunConfig arguments: save_checkpoints_steps 312 or save_checkpoints_secs None.
I0716 04:27:21.766440 139795883440000 estimator.py:1145] Calling model_fn.
I0716 04:27:23.241953 139795883440000 saver.py:1499] Saver not created because there are no variables in the graph to restore
I0716 04:27:23.843529 139795883440000 <ipython-input-6-d6747b40360b>:55] 
[<tf.Variable 'embedding:0' shape=(750, 300) dtype=float32_ref>,
 <tf.Variable 'module/aggregation/weights:0' shape=(3,) dtype=float32>,
 <tf.Variable 'module/aggregation/scaling:0' shape=() dtype=float32>,
 <tf.Variable 'dense/kernel:0' shape=(1324, 300) dtype=float32_ref>,
 <tf.Variable 'dense/bias:0' shape=(300,) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/gates/kernel:0' shape=(600, 600) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/gates/bias:0' shape=(600,) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/candidate/kernel:0' shape=(600, 300) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/candidate/bias:0' shape=(300,) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/gates/kernel:0' shape=(600, 600) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/gates/bias:0' shape=(600,) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/candidate/kernel:0' shape=(600, 300) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/candidate/bias:0' shape=(300,) dtype=float32_ref>,
 <tf.Variable 'dense_1/kernel:0' shape=(600, 23) dtype=float32_ref>,
 <tf.Variable 'dense_1/bias:0' shape=(23,) dtype=float32_ref>,
 <tf.Variable 'dense_2/kernel:0' shape=(600, 122) dtype=float32_ref>,
 <tf.Variable 'dense_2/bias:0' shape=(122,) dtype=float32_ref>]
I0716 04:27:25.118303 139795883440000 estimator.py:1147] Done calling model_fn.
I0716 04:27:25.121737 139795883440000 basic_session_run_hooks.py:541] Create CheckpointSaverHook.
I0716 04:27:25.438082 139795883440000 monitored_session.py:240] Graph was finalized.
I0716 04:27:25.456156 139795883440000 saver.py:1280] Restoring parameters from ../model/elmo_bigru/model.ckpt-7488
I0716 04:27:26.499037 139795883440000 session_manager.py:500] Running local_init_op.
I0716 04:27:26.571258 139795883440000 session_manager.py:502] Done running local_init_op.
I0716 04:27:29.367456 139795883440000 basic_session_run_hooks.py:606] Saving checkpoints for 7488 into ../model/elmo_bigru/model.ckpt.
Reading ../data/atis.train.w-intent.iob
I0716 04:27:34.189820 139795883440000 basic_session_run_hooks.py:262] loss = 0.020746749, step = 7488
I0716 04:27:34.191736 139795883440000 basic_session_run_hooks.py:262] lr = 0.00013629787
I0716 04:28:01.006389 139795883440000 basic_session_run_hooks.py:692] global_step/sec: 3.72894
I0716 04:28:01.014677 139795883440000 basic_session_run_hooks.py:260] loss = 0.014712622, step = 7588 (26.825 sec)
I0716 04:28:01.016867 139795883440000 basic_session_run_hooks.py:260] lr = 0.00013486936 (26.825 sec)
I0716 04:28:26.477072 139795883440000 basic_session_run_hooks.py:692] global_step/sec: 3.92609
I0716 04:28:26.480191 139795883440000 basic_session_run_hooks.py:260] loss = 0.012815632, step = 7688 (25.466 sec)
I0716 04:28:26.481969 139795883440000 basic_session_run_hooks.py:260] lr = 0.00013345583 (25.465 sec)
I0716 04:28:54.003954 139795883440000 basic_session_run_hooks.py:692] global_step/sec: 3.63281
I0716 04:28:54.011584 139795883440000 basic_session_run_hooks.py:260] loss = 0.015086541, step = 7788 (27.531 sec)
I0716 04:28:54.014548 139795883440000 basic_session_run_hooks.py:260] lr = 0.00013205713 (27.533 sec)
I0716 04:28:57.070349 139795883440000 basic_session_run_hooks.py:606] Saving checkpoints for 7800 into ../model/elmo_bigru/model.ckpt.
I0716 04:28:59.224375 139795883440000 estimator.py:1145] Calling model_fn.
I0716 04:29:00.839649 139795883440000 saver.py:1499] Saver not created because there are no variables in the graph to restore
I0716 04:29:01.399063 139795883440000 estimator.py:1147] Done calling model_fn.
I0716 04:29:01.428005 139795883440000 evaluation.py:255] Starting evaluation at 2019-07-16T04:29:01Z
I0716 04:29:01.650110 139795883440000 monitored_session.py:240] Graph was finalized.
I0716 04:29:01.671104 139795883440000 saver.py:1280] Restoring parameters from ../model/elmo_bigru/model.ckpt-7800
I0716 04:29:02.555532 139795883440000 session_manager.py:500] Running local_init_op.
I0716 04:29:02.610178 139795883440000 session_manager.py:502] Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
I0716 04:30:01.353968 139795883440000 evaluation.py:275] Finished evaluation at 2019-07-16-04:30:01
I0716 04:30:01.355541 139795883440000 estimator.py:2039] Saving dict for global step 7800: global_step = 7800, loss = 0.34348074
I0716 04:30:01.362978 139795883440000 estimator.py:2099] Saving 'checkpoint_path' summary for global step 7800: ../model/elmo_bigru/model.ckpt-7800
I0716 04:30:01.447452 139795883440000 estimator.py:368] Loss for final step: 0.009322032.
Reading ../data/atis.test.w-intent.iob
I0716 04:30:02.041821 139795883440000 estimator.py:1145] Calling model_fn.
I0716 04:30:03.569986 139795883440000 saver.py:1499] Saver not created because there are no variables in the graph to restore
I0716 04:30:04.087395 139795883440000 estimator.py:1147] Done calling model_fn.
I0716 04:30:04.313645 139795883440000 monitored_session.py:240] Graph was finalized.
I0716 04:30:04.329914 139795883440000 saver.py:1280] Restoring parameters from ../model/elmo_bigru/model.ckpt-7800
I0716 04:30:05.232841 139795883440000 session_manager.py:500] Running local_init_op.
I0716 04:30:05.289027 139795883440000 session_manager.py:502] Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.
  'precision', 'predicted', average, warn_for)
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1439: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples.
  'recall', 'true', average, warn_for)
I0716 04:31:02.991110 139795883440000 interactiveshell.py:2882] 
                                          precision    recall  f1-score   support

                             atis_flight      0.987     0.992     0.990       632
                            atis_airfare      0.941     1.000     0.970        48
                     atis_ground_service      1.000     1.000     1.000        36
                            atis_airline      0.974     1.000     0.987        38
                       atis_abbreviation      0.943     1.000     0.971        33
                           atis_aircraft      1.000     0.889     0.941         9
                        atis_flight_time      0.333     1.000     0.500         1
                           atis_quantity      0.429     1.000     0.600         3
                atis_flight#atis_airfare      0.833     0.417     0.556        12
                            atis_airport      1.000     0.944     0.971        18
                           atis_distance      1.000     1.000     1.000        10
                               atis_city      0.800     0.667     0.727         6
                        atis_ground_fare      1.000     0.857     0.923         7
                           atis_capacity      0.952     0.952     0.952        21
                          atis_flight_no      0.889     1.000     0.941         8
                               atis_meal      1.000     0.833     0.909         6
                        atis_restriction      0.000     0.000     0.000         0
             atis_airline#atis_flight_no      0.000     0.000     0.000         0
    atis_ground_service#atis_ground_fare      0.000     0.000     0.000         0
           atis_airfare#atis_flight_time      0.000     0.000     0.000         0
                           atis_cheapest      0.000     0.000     0.000         0
atis_aircraft#atis_flight#atis_flight_no      0.000     0.000     0.000         0

                               micro avg      0.973     0.979     0.976       888
                               macro avg      0.640     0.661     0.634       888
                            weighted avg      0.976     0.979     0.976       888

I0716 04:31:03.029418 139795883440000 interactiveshell.py:2882] 
                              precision    recall  f1-score   support

                           O      0.000     0.000     0.000       0.0
           B-toloc.city_name      0.978     0.994     0.986     716.0
         B-fromloc.city_name      0.992     0.999     0.995     704.0
           I-toloc.city_name      0.978     0.992     0.985     265.0
      B-depart_date.day_name      0.991     0.991     0.991     212.0
              B-airline_name      1.000     1.000     1.000     101.0
         I-fromloc.city_name      0.983     1.000     0.992     177.0
 B-depart_time.period_of_day      0.992     0.923     0.956     130.0
              I-airline_name      0.985     1.000     0.992      65.0
    B-depart_date.day_number      0.981     0.964     0.972      55.0
    B-depart_date.month_name      0.982     0.964     0.973      56.0
          B-depart_time.time      0.864     1.000     0.927      57.0
                B-round_trip      1.000     1.000     1.000      73.0
             B-cost_relative      0.973     0.973     0.973      37.0
                I-round_trip      1.000     1.000     1.000      71.0
                B-flight_mod      0.828     1.000     0.906      24.0
 B-depart_time.time_relative      0.970     0.985     0.977      65.0
          I-depart_time.time      0.945     1.000     0.972      52.0
         B-stoploc.city_name      0.870     1.000     0.930      20.0
                 B-city_name      0.864     0.667     0.752      57.0
                B-class_type      0.960     1.000     0.980      24.0
          B-arrive_time.time      0.971     0.971     0.971      34.0
 B-arrive_time.time_relative      0.906     0.935     0.921      31.0
                I-class_type      1.000     1.000     1.000      17.0
               B-flight_stop      1.000     1.000     1.000      21.0
          I-arrive_time.time      1.000     0.971     0.986      35.0
              B-airline_code      0.895     1.000     0.944      34.0
    I-depart_date.day_number      1.000     0.933     0.966      15.0
      I-fromloc.airport_name      0.441     1.000     0.612      15.0
      B-fromloc.airport_name      0.444     1.000     0.615      12.0
      B-arrive_date.day_name      0.786     1.000     0.880      11.0
          B-toloc.state_code      1.000     1.000     1.000      18.0
B-depart_date.today_relative      1.000     0.889     0.941       9.0
             B-flight_number      0.733     1.000     0.846      11.0
 B-depart_date.date_relative      0.944     1.000     0.971      17.0
          B-toloc.state_name      0.788     0.929     0.852      28.0
           B-fare_basis_code      0.850     1.000     0.919      17.0
               B-flight_time      1.000     1.000     1.000       1.0
                        B-or      1.000     1.000     1.000       3.0
 B-arrive_time.period_of_day      0.750     1.000     0.857       6.0
          B-meal_description      1.000     0.900     0.947      10.0
             I-cost_relative      1.000     0.667     0.800       3.0
              I-airport_name      0.846     0.379     0.524      29.0
               B-fare_amount      1.000     1.000     1.000       2.0
               I-fare_amount      1.000     1.000     1.000       2.0
                 I-city_name      1.000     0.600     0.750      30.0
        I-toloc.airport_name      1.000     1.000     1.000       3.0
            B-transport_type      1.000     1.000     1.000      10.0
    B-arrive_date.month_name      0.714     0.833     0.769       6.0
    B-arrive_date.day_number      0.714     0.833     0.769       6.0
         I-stoploc.city_name      1.000     1.000     1.000      10.0
                      B-meal      0.941     1.000     0.970      16.0
        B-fromloc.state_code      1.000     1.000     1.000      23.0
    B-depart_time.period_mod      1.000     1.000     1.000       5.0
                   B-connect      1.000     1.000     1.000       6.0
               B-flight_days      1.000     1.000     1.000      10.0
        B-toloc.airport_name      1.000     1.000     1.000       3.0
        B-fromloc.state_name      1.000     1.000     1.000      17.0
              B-airport_name      0.750     0.286     0.414      21.0
                   B-economy      1.000     1.000     1.000       6.0
               I-flight_time      1.000     1.000     1.000       1.0
             B-aircraft_code      1.000     0.818     0.900      33.0
                       B-mod      1.000     0.500     0.667       2.0
              B-airport_code      1.000     0.444     0.615       9.0
    B-depart_time.start_time      1.000     0.667     0.800       3.0
      B-depart_time.end_time      1.000     0.333     0.500       3.0
          B-depart_date.year      1.000     1.000     1.000       3.0
            I-transport_type      0.000     0.000     0.000       1.0
          B-restriction_code      1.000     1.000     1.000       4.0
    B-arrive_time.start_time      0.889     1.000     0.941       8.0
        B-toloc.airport_code      1.000     0.750     0.857       4.0
      B-arrive_time.end_time      0.889     1.000     0.941       8.0
      I-arrive_time.end_time      0.889     1.000     0.941       8.0
      I-depart_time.end_time      1.000     0.333     0.500       3.0
               I-flight_stop      0.000     0.000     0.000       0.0
      B-fromloc.airport_code      0.625     1.000     0.769       5.0
          I-restriction_code      1.000     1.000     1.000       3.0
    I-depart_time.start_time      1.000     1.000     1.000       1.0
          I-toloc.state_name      1.000     1.000     1.000       1.0
I-depart_date.today_relative      0.000     0.000     0.000       0.0
 B-arrive_date.date_relative      1.000     1.000     1.000       2.0
                I-flight_mod      1.000     0.167     0.286       6.0
                   I-economy      0.000     0.000     0.000       0.0
 B-return_date.date_relative      1.000     0.667     0.800       3.0
        I-fromloc.state_name      1.000     1.000     1.000       1.0
                B-state_code      1.000     1.000     1.000       1.0
    I-arrive_time.start_time      1.000     1.000     1.000       1.0
    I-arrive_date.day_number      0.000     0.000     0.000       0.0
                 B-meal_code      0.000     0.000     0.000       1.0
 I-depart_time.period_of_day      1.000     1.000     1.000       1.0
                  B-day_name      1.000     0.500     0.667       2.0
             B-period_of_day      1.000     0.750     0.857       4.0
        B-stoploc.state_code      0.000     0.000     0.000       0.0
    B-return_date.month_name      0.000     0.000     0.000       0.0
    B-return_date.day_number      0.000     0.000     0.000       0.0
    B-arrive_time.period_mod      0.000     0.000     0.000       0.0
                 I-meal_code      0.000     0.000     0.000       0.0
        B-toloc.country_name      0.000     0.000     0.000       1.0
                 B-days_code      0.000     0.000     0.000       1.0
 I-arrive_time.period_of_day      0.000     0.000     0.000       0.0
            I-today_relative      0.000     0.000     0.000       0.0
 B-return_time.period_of_day      0.000     0.000     0.000       0.0
                      B-time      0.000     0.000     0.000       0.0
           I-fare_basis_code      0.000     0.000     0.000       0.0
 I-arrive_time.time_relative      1.000     0.500     0.667       4.0
 I-depart_time.time_relative      0.000     0.000     0.000       1.0
            B-today_relative      0.000     0.000     0.000       0.0
                B-state_name      0.000     0.000     0.000       9.0
B-arrive_date.today_relative      0.000     0.000     0.000       0.0
    B-return_time.period_mod      0.000     0.000     0.000       0.0
                B-month_name      0.000     0.000     0.000       0.0
                B-day_number      0.000     0.000     0.000       0.0
 I-return_date.date_relative      1.000     0.667     0.800       3.0
I-return_date.today_relative      0.000     0.000     0.000       0.0
      B-stoploc.airport_name      0.000     0.000     0.000       0.0
             B-time_relative      0.000     0.000     0.000       0.0
                      I-time      0.000     0.000     0.000       0.0
    I-return_date.day_number      0.000     0.000     0.000       0.0
          I-meal_description      0.000     0.000     0.000       0.0
B-return_date.today_relative      0.000     0.000     0.000       0.0
      B-return_date.day_name      0.000     0.000     0.000       2.0

                   micro avg      0.959     0.960     0.959    3657.0
                   macro avg      0.677     0.642     0.645    3657.0
                weighted avg      0.962     0.960     0.956    3657.0

I0716 04:31:03.051072 139795883440000 interactiveshell.py:2882] Best Slot F1: 0.960
I0716 04:31:03.052510 139795883440000 estimator_training.py:186] Not using Distribute Coordinator.
I0716 04:31:03.054505 139795883440000 training.py:612] Running training and evaluation locally (non-distributed).
I0716 04:31:03.056043 139795883440000 training.py:700] Start train and evaluate loop. The evaluate will happen after every checkpoint. Checkpoint frequency is determined based on RunConfig arguments: save_checkpoints_steps 312 or save_checkpoints_secs None.
I0716 04:31:03.125035 139795883440000 estimator.py:1145] Calling model_fn.
I0716 04:31:04.600318 139795883440000 saver.py:1499] Saver not created because there are no variables in the graph to restore
I0716 04:31:05.180009 139795883440000 <ipython-input-6-d6747b40360b>:55] 
[<tf.Variable 'embedding:0' shape=(750, 300) dtype=float32_ref>,
 <tf.Variable 'module/aggregation/weights:0' shape=(3,) dtype=float32>,
 <tf.Variable 'module/aggregation/scaling:0' shape=() dtype=float32>,
 <tf.Variable 'dense/kernel:0' shape=(1324, 300) dtype=float32_ref>,
 <tf.Variable 'dense/bias:0' shape=(300,) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/gates/kernel:0' shape=(600, 600) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/gates/bias:0' shape=(600,) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/candidate/kernel:0' shape=(600, 300) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/candidate/bias:0' shape=(300,) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/gates/kernel:0' shape=(600, 600) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/gates/bias:0' shape=(600,) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/candidate/kernel:0' shape=(600, 300) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/candidate/bias:0' shape=(300,) dtype=float32_ref>,
 <tf.Variable 'dense_1/kernel:0' shape=(600, 23) dtype=float32_ref>,
 <tf.Variable 'dense_1/bias:0' shape=(23,) dtype=float32_ref>,
 <tf.Variable 'dense_2/kernel:0' shape=(600, 122) dtype=float32_ref>,
 <tf.Variable 'dense_2/bias:0' shape=(122,) dtype=float32_ref>]
I0716 04:31:06.442768 139795883440000 estimator.py:1147] Done calling model_fn.
I0716 04:31:06.446258 139795883440000 basic_session_run_hooks.py:541] Create CheckpointSaverHook.
I0716 04:31:06.777996 139795883440000 monitored_session.py:240] Graph was finalized.
I0716 04:31:06.797566 139795883440000 saver.py:1280] Restoring parameters from ../model/elmo_bigru/model.ckpt-7800
I0716 04:31:07.846698 139795883440000 session_manager.py:500] Running local_init_op.
I0716 04:31:07.916826 139795883440000 session_manager.py:502] Done running local_init_op.
I0716 04:31:10.738055 139795883440000 basic_session_run_hooks.py:606] Saving checkpoints for 7800 into ../model/elmo_bigru/model.ckpt.
Reading ../data/atis.train.w-intent.iob
I0716 04:31:15.473568 139795883440000 basic_session_run_hooks.py:262] loss = 0.010955091, step = 7800
I0716 04:31:15.475627 139795883440000 basic_session_run_hooks.py:262] lr = 0.00013189027
I0716 04:31:42.469512 139795883440000 basic_session_run_hooks.py:692] global_step/sec: 3.7042
I0716 04:31:42.475643 139795883440000 basic_session_run_hooks.py:260] loss = 0.016697695, step = 7900 (27.002 sec)
I0716 04:31:42.478532 139795883440000 basic_session_run_hooks.py:260] lr = 0.00013050795 (27.003 sec)
I0716 04:32:09.090202 139795883440000 basic_session_run_hooks.py:692] global_step/sec: 3.75649
I0716 04:32:09.097246 139795883440000 basic_session_run_hooks.py:260] loss = 0.017720591, step = 8000 (26.622 sec)
I0716 04:32:09.100723 139795883440000 basic_session_run_hooks.py:260] lr = 0.00012914014 (26.622 sec)
I0716 04:32:36.386731 139795883440000 basic_session_run_hooks.py:692] global_step/sec: 3.66345
I0716 04:32:36.389632 139795883440000 basic_session_run_hooks.py:260] loss = 0.00943367, step = 8100 (27.292 sec)
I0716 04:32:36.396331 139795883440000 basic_session_run_hooks.py:260] lr = 0.00012778665 (27.296 sec)
I0716 04:32:39.250842 139795883440000 basic_session_run_hooks.py:606] Saving checkpoints for 8112 into ../model/elmo_bigru/model.ckpt.
I0716 04:32:41.199497 139795883440000 estimator.py:1145] Calling model_fn.
I0716 04:32:42.815036 139795883440000 saver.py:1499] Saver not created because there are no variables in the graph to restore
I0716 04:32:43.378526 139795883440000 estimator.py:1147] Done calling model_fn.
I0716 04:32:43.407763 139795883440000 evaluation.py:255] Starting evaluation at 2019-07-16T04:32:43Z
I0716 04:32:43.633741 139795883440000 monitored_session.py:240] Graph was finalized.
I0716 04:32:43.653753 139795883440000 saver.py:1280] Restoring parameters from ../model/elmo_bigru/model.ckpt-8112
I0716 04:32:44.570884 139795883440000 session_manager.py:500] Running local_init_op.
I0716 04:32:44.623782 139795883440000 session_manager.py:502] Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
I0716 04:33:43.256713 139795883440000 evaluation.py:275] Finished evaluation at 2019-07-16-04:33:43
I0716 04:33:43.258413 139795883440000 estimator.py:2039] Saving dict for global step 8112: global_step = 8112, loss = 0.3477777
I0716 04:33:43.267474 139795883440000 estimator.py:2099] Saving 'checkpoint_path' summary for global step 8112: ../model/elmo_bigru/model.ckpt-8112
I0716 04:33:43.352654 139795883440000 estimator.py:368] Loss for final step: 0.011134683.
Reading ../data/atis.test.w-intent.iob
I0716 04:33:43.942719 139795883440000 estimator.py:1145] Calling model_fn.
I0716 04:33:45.503614 139795883440000 saver.py:1499] Saver not created because there are no variables in the graph to restore
I0716 04:33:46.019409 139795883440000 estimator.py:1147] Done calling model_fn.
I0716 04:33:46.249564 139795883440000 monitored_session.py:240] Graph was finalized.
I0716 04:33:46.265639 139795883440000 saver.py:1280] Restoring parameters from ../model/elmo_bigru/model.ckpt-8112
I0716 04:33:47.177340 139795883440000 session_manager.py:500] Running local_init_op.
I0716 04:33:47.228640 139795883440000 session_manager.py:502] Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.
  'precision', 'predicted', average, warn_for)
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1439: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples.
  'recall', 'true', average, warn_for)
I0716 04:34:45.102535 139795883440000 interactiveshell.py:2882] 
                                          precision    recall  f1-score   support

                             atis_flight      0.984     0.992     0.988       632
                            atis_airfare      0.941     1.000     0.970        48
                     atis_ground_service      1.000     1.000     1.000        36
                            atis_airline      1.000     1.000     1.000        38
                       atis_abbreviation      0.943     1.000     0.971        33
                           atis_aircraft      1.000     0.889     0.941         9
                        atis_flight_time      0.500     1.000     0.667         1
                           atis_quantity      0.429     1.000     0.600         3
                atis_flight#atis_airfare      0.833     0.417     0.556        12
                            atis_airport      1.000     1.000     1.000        18
                           atis_distance      1.000     1.000     1.000        10
                               atis_city      0.750     0.500     0.600         6
                        atis_ground_fare      1.000     0.857     0.923         7
                           atis_capacity      0.952     0.952     0.952        21
                          atis_flight_no      0.889     1.000     0.941         8
                               atis_meal      1.000     0.833     0.909         6
                        atis_restriction      0.000     0.000     0.000         0
             atis_airline#atis_flight_no      0.000     0.000     0.000         0
    atis_ground_service#atis_ground_fare      0.000     0.000     0.000         0
           atis_airfare#atis_flight_time      0.000     0.000     0.000         0
                           atis_cheapest      0.000     0.000     0.000         0
atis_aircraft#atis_flight#atis_flight_no      0.000     0.000     0.000         0

                               micro avg      0.973     0.979     0.976       888
                               macro avg      0.646     0.656     0.637       888
                            weighted avg      0.975     0.979     0.975       888

I0716 04:34:45.141074 139795883440000 interactiveshell.py:2882] 
                              precision    recall  f1-score   support

                           O      0.000     0.000     0.000       0.0
           B-toloc.city_name      0.978     0.990     0.984     716.0
         B-fromloc.city_name      0.983     1.000     0.992     704.0
           I-toloc.city_name      0.981     0.977     0.979     265.0
      B-depart_date.day_name      0.991     0.991     0.991     212.0
              B-airline_name      1.000     1.000     1.000     101.0
         I-fromloc.city_name      0.957     1.000     0.978     177.0
 B-depart_time.period_of_day      0.992     0.923     0.956     130.0
              I-airline_name      1.000     1.000     1.000      65.0
    B-depart_date.day_number      0.981     0.964     0.972      55.0
    B-depart_date.month_name      0.982     0.964     0.973      56.0
          B-depart_time.time      0.864     1.000     0.927      57.0
                B-round_trip      1.000     1.000     1.000      73.0
             B-cost_relative      0.973     0.973     0.973      37.0
                I-round_trip      1.000     1.000     1.000      71.0
                B-flight_mod      0.828     1.000     0.906      24.0
 B-depart_time.time_relative      0.955     0.985     0.970      65.0
          I-depart_time.time      0.945     1.000     0.972      52.0
         B-stoploc.city_name      0.952     1.000     0.976      20.0
                 B-city_name      0.905     0.667     0.768      57.0
                B-class_type      0.960     1.000     0.980      24.0
          B-arrive_time.time      0.971     0.971     0.971      34.0
 B-arrive_time.time_relative      0.967     0.935     0.951      31.0
                I-class_type      1.000     1.000     1.000      17.0
               B-flight_stop      1.000     1.000     1.000      21.0
          I-arrive_time.time      1.000     0.971     0.986      35.0
              B-airline_code      0.895     1.000     0.944      34.0
    I-depart_date.day_number      1.000     0.933     0.966      15.0
      I-fromloc.airport_name      0.429     1.000     0.600      15.0
      B-fromloc.airport_name      0.429     1.000     0.600      12.0
      B-arrive_date.day_name      0.786     1.000     0.880      11.0
          B-toloc.state_code      1.000     1.000     1.000      18.0
B-depart_date.today_relative      1.000     0.889     0.941       9.0
             B-flight_number      0.733     1.000     0.846      11.0
 B-depart_date.date_relative      0.944     1.000     0.971      17.0
          B-toloc.state_name      0.893     0.893     0.893      28.0
           B-fare_basis_code      0.850     1.000     0.919      17.0
               B-flight_time      1.000     1.000     1.000       1.0
                        B-or      1.000     1.000     1.000       3.0
 B-arrive_time.period_of_day      0.750     1.000     0.857       6.0
          B-meal_description      1.000     0.900     0.947      10.0
             I-cost_relative      1.000     0.667     0.800       3.0
              I-airport_name      0.833     0.345     0.488      29.0
               B-fare_amount      1.000     1.000     1.000       2.0
               I-fare_amount      1.000     1.000     1.000       2.0
                 I-city_name      1.000     0.567     0.723      30.0
        I-toloc.airport_name      1.000     1.000     1.000       3.0
            B-transport_type      1.000     1.000     1.000      10.0
    B-arrive_date.month_name      0.714     0.833     0.769       6.0
    B-arrive_date.day_number      0.714     0.833     0.769       6.0
         I-stoploc.city_name      1.000     1.000     1.000      10.0
                      B-meal      0.941     1.000     0.970      16.0
        B-fromloc.state_code      1.000     1.000     1.000      23.0
    B-depart_time.period_mod      1.000     1.000     1.000       5.0
                   B-connect      1.000     1.000     1.000       6.0
               B-flight_days      1.000     1.000     1.000      10.0
        B-toloc.airport_name      1.000     1.000     1.000       3.0
        B-fromloc.state_name      0.944     1.000     0.971      17.0
              B-airport_name      0.583     0.333     0.424      21.0
                   B-economy      1.000     1.000     1.000       6.0
               I-flight_time      1.000     1.000     1.000       1.0
             B-aircraft_code      1.000     0.818     0.900      33.0
                       B-mod      1.000     0.500     0.667       2.0
              B-airport_code      1.000     0.444     0.615       9.0
    B-depart_time.start_time      1.000     0.667     0.800       3.0
      B-depart_time.end_time      1.000     0.333     0.500       3.0
          B-depart_date.year      1.000     1.000     1.000       3.0
            I-transport_type      1.000     1.000     1.000       1.0
          B-restriction_code      1.000     1.000     1.000       4.0
    B-arrive_time.start_time      0.889     1.000     0.941       8.0
        B-toloc.airport_code      1.000     0.750     0.857       4.0
      B-arrive_time.end_time      0.889     1.000     0.941       8.0
      I-arrive_time.end_time      0.889     1.000     0.941       8.0
      I-depart_time.end_time      1.000     0.333     0.500       3.0
               I-flight_stop      0.000     0.000     0.000       0.0
      B-fromloc.airport_code      0.625     1.000     0.769       5.0
          I-restriction_code      1.000     1.000     1.000       3.0
    I-depart_time.start_time      1.000     1.000     1.000       1.0
          I-toloc.state_name      1.000     1.000     1.000       1.0
I-depart_date.today_relative      0.000     0.000     0.000       0.0
 B-arrive_date.date_relative      1.000     1.000     1.000       2.0
                I-flight_mod      1.000     0.167     0.286       6.0
                   I-economy      0.000     0.000     0.000       0.0
 B-return_date.date_relative      1.000     0.667     0.800       3.0
        I-fromloc.state_name      1.000     1.000     1.000       1.0
                B-state_code      1.000     1.000     1.000       1.0
    I-arrive_time.start_time      1.000     1.000     1.000       1.0
    I-arrive_date.day_number      0.000     0.000     0.000       0.0
                 B-meal_code      0.000     0.000     0.000       1.0
 I-depart_time.period_of_day      1.000     1.000     1.000       1.0
                  B-day_name      1.000     0.500     0.667       2.0
             B-period_of_day      1.000     0.750     0.857       4.0
        B-stoploc.state_code      0.000     0.000     0.000       0.0
    B-return_date.month_name      0.000     0.000     0.000       0.0
    B-return_date.day_number      0.000     0.000     0.000       0.0
    B-arrive_time.period_mod      0.000     0.000     0.000       0.0
                 I-meal_code      0.000     0.000     0.000       0.0
        B-toloc.country_name      1.000     1.000     1.000       1.0
                 B-days_code      0.000     0.000     0.000       1.0
 I-arrive_time.period_of_day      0.000     0.000     0.000       0.0
            I-today_relative      0.000     0.000     0.000       0.0
 B-return_time.period_of_day      0.000     0.000     0.000       0.0
                      B-time      0.000     0.000     0.000       0.0
           I-fare_basis_code      0.000     0.000     0.000       0.0
 I-arrive_time.time_relative      1.000     1.000     1.000       4.0
 I-depart_time.time_relative      0.000     0.000     0.000       1.0
            B-today_relative      0.000     0.000     0.000       0.0
                B-state_name      0.000     0.000     0.000       9.0
B-arrive_date.today_relative      0.000     0.000     0.000       0.0
    B-return_time.period_mod      0.000     0.000     0.000       0.0
                B-month_name      0.000     0.000     0.000       0.0
                B-day_number      0.000     0.000     0.000       0.0
 I-return_date.date_relative      1.000     0.667     0.800       3.0
I-return_date.today_relative      0.000     0.000     0.000       0.0
      B-stoploc.airport_name      0.000     0.000     0.000       0.0
             B-time_relative      0.000     0.000     0.000       0.0
                      I-time      0.000     0.000     0.000       0.0
    I-return_date.day_number      0.000     0.000     0.000       0.0
          I-meal_description      0.000     0.000     0.000       0.0
B-return_date.today_relative      0.000     0.000     0.000       0.0
      B-return_date.day_name      0.000     0.000     0.000       2.0

                   micro avg      0.957     0.959     0.958    3657.0
                   macro avg      0.693     0.662     0.664    3657.0
                weighted avg      0.961     0.959     0.955    3657.0

I0716 04:34:45.163522 139795883440000 interactiveshell.py:2882] Best Slot F1: 0.960
I0716 04:34:45.165222 139795883440000 estimator_training.py:186] Not using Distribute Coordinator.
I0716 04:34:45.167124 139795883440000 training.py:612] Running training and evaluation locally (non-distributed).
I0716 04:34:45.168864 139795883440000 training.py:700] Start train and evaluate loop. The evaluate will happen after every checkpoint. Checkpoint frequency is determined based on RunConfig arguments: save_checkpoints_steps 312 or save_checkpoints_secs None.
I0716 04:34:45.237806 139795883440000 estimator.py:1145] Calling model_fn.
I0716 04:34:46.711839 139795883440000 saver.py:1499] Saver not created because there are no variables in the graph to restore
I0716 04:34:47.297554 139795883440000 <ipython-input-6-d6747b40360b>:55] 
[<tf.Variable 'embedding:0' shape=(750, 300) dtype=float32_ref>,
 <tf.Variable 'module/aggregation/weights:0' shape=(3,) dtype=float32>,
 <tf.Variable 'module/aggregation/scaling:0' shape=() dtype=float32>,
 <tf.Variable 'dense/kernel:0' shape=(1324, 300) dtype=float32_ref>,
 <tf.Variable 'dense/bias:0' shape=(300,) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/gates/kernel:0' shape=(600, 600) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/gates/bias:0' shape=(600,) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/candidate/kernel:0' shape=(600, 300) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/candidate/bias:0' shape=(300,) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/gates/kernel:0' shape=(600, 600) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/gates/bias:0' shape=(600,) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/candidate/kernel:0' shape=(600, 300) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/candidate/bias:0' shape=(300,) dtype=float32_ref>,
 <tf.Variable 'dense_1/kernel:0' shape=(600, 23) dtype=float32_ref>,
 <tf.Variable 'dense_1/bias:0' shape=(23,) dtype=float32_ref>,
 <tf.Variable 'dense_2/kernel:0' shape=(600, 122) dtype=float32_ref>,
 <tf.Variable 'dense_2/bias:0' shape=(122,) dtype=float32_ref>]
I0716 04:34:48.563750 139795883440000 estimator.py:1147] Done calling model_fn.
I0716 04:34:48.567470 139795883440000 basic_session_run_hooks.py:541] Create CheckpointSaverHook.
I0716 04:34:48.897768 139795883440000 monitored_session.py:240] Graph was finalized.
I0716 04:34:48.916313 139795883440000 saver.py:1280] Restoring parameters from ../model/elmo_bigru/model.ckpt-8112
I0716 04:34:49.950185 139795883440000 session_manager.py:500] Running local_init_op.
I0716 04:34:50.025686 139795883440000 session_manager.py:502] Done running local_init_op.
I0716 04:34:52.815325 139795883440000 basic_session_run_hooks.py:606] Saving checkpoints for 8112 into ../model/elmo_bigru/model.ckpt.
Reading ../data/atis.train.w-intent.iob
I0716 04:34:57.998856 139795883440000 basic_session_run_hooks.py:262] loss = 0.021866146, step = 8112
I0716 04:34:58.000585 139795883440000 basic_session_run_hooks.py:262] lr = 0.00012762519
I0716 04:35:25.058067 139795883440000 basic_session_run_hooks.py:692] global_step/sec: 3.69552
I0716 04:35:25.061380 139795883440000 basic_session_run_hooks.py:260] loss = 0.011125647, step = 8212 (27.063 sec)
I0716 04:35:25.066845 139795883440000 basic_session_run_hooks.py:260] lr = 0.00012628757 (27.066 sec)
I0716 04:35:51.005081 139795883440000 basic_session_run_hooks.py:692] global_step/sec: 3.85401
I0716 04:35:51.012858 139795883440000 basic_session_run_hooks.py:260] loss = 0.018105546, step = 8312 (25.951 sec)
I0716 04:35:51.015550 139795883440000 basic_session_run_hooks.py:260] lr = 0.00012496399 (25.949 sec)
I0716 04:36:18.147511 139795883440000 basic_session_run_hooks.py:692] global_step/sec: 3.68427
I0716 04:36:18.155609 139795883440000 basic_session_run_hooks.py:260] loss = 0.010910708, step = 8412 (27.143 sec)
I0716 04:36:18.158275 139795883440000 basic_session_run_hooks.py:260] lr = 0.00012365427 (27.143 sec)
I0716 04:36:20.829231 139795883440000 basic_session_run_hooks.py:606] Saving checkpoints for 8424 into ../model/elmo_bigru/model.ckpt.
I0716 04:36:22.884379 139795883440000 estimator.py:1145] Calling model_fn.
I0716 04:36:24.508466 139795883440000 saver.py:1499] Saver not created because there are no variables in the graph to restore
I0716 04:36:25.108688 139795883440000 estimator.py:1147] Done calling model_fn.
I0716 04:36:25.139392 139795883440000 evaluation.py:255] Starting evaluation at 2019-07-16T04:36:25Z
I0716 04:36:25.361651 139795883440000 monitored_session.py:240] Graph was finalized.
I0716 04:36:25.378874 139795883440000 saver.py:1280] Restoring parameters from ../model/elmo_bigru/model.ckpt-8424
I0716 04:36:26.279739 139795883440000 session_manager.py:500] Running local_init_op.
I0716 04:36:26.334746 139795883440000 session_manager.py:502] Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
I0716 04:37:25.359095 139795883440000 evaluation.py:275] Finished evaluation at 2019-07-16-04:37:25
I0716 04:37:25.361010 139795883440000 estimator.py:2039] Saving dict for global step 8424: global_step = 8424, loss = 0.35378975
I0716 04:37:25.370392 139795883440000 estimator.py:2099] Saving 'checkpoint_path' summary for global step 8424: ../model/elmo_bigru/model.ckpt-8424
I0716 04:37:25.453804 139795883440000 estimator.py:368] Loss for final step: 0.0015609998.
Reading ../data/atis.test.w-intent.iob
I0716 04:37:26.045480 139795883440000 estimator.py:1145] Calling model_fn.
I0716 04:37:27.562336 139795883440000 saver.py:1499] Saver not created because there are no variables in the graph to restore
I0716 04:37:28.074602 139795883440000 estimator.py:1147] Done calling model_fn.
I0716 04:37:28.298694 139795883440000 monitored_session.py:240] Graph was finalized.
I0716 04:37:28.319315 139795883440000 saver.py:1280] Restoring parameters from ../model/elmo_bigru/model.ckpt-8424
I0716 04:37:29.206162 139795883440000 session_manager.py:500] Running local_init_op.
I0716 04:37:29.255664 139795883440000 session_manager.py:502] Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.
  'precision', 'predicted', average, warn_for)
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1439: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples.
  'recall', 'true', average, warn_for)
I0716 04:38:27.078902 139795883440000 interactiveshell.py:2882] 
                                          precision    recall  f1-score   support

                             atis_flight      0.984     0.992     0.988       632
                            atis_airfare      0.941     1.000     0.970        48
                     atis_ground_service      1.000     1.000     1.000        36
                            atis_airline      1.000     1.000     1.000        38
                       atis_abbreviation      0.943     1.000     0.971        33
                           atis_aircraft      1.000     0.889     0.941         9
                        atis_flight_time      0.500     1.000     0.667         1
                           atis_quantity      0.429     1.000     0.600         3
                atis_flight#atis_airfare      0.833     0.417     0.556        12
                            atis_airport      1.000     1.000     1.000        18
                           atis_distance      1.000     1.000     1.000        10
                               atis_city      0.750     0.500     0.600         6
                        atis_ground_fare      1.000     0.857     0.923         7
                           atis_capacity      0.952     0.952     0.952        21
                          atis_flight_no      0.889     1.000     0.941         8
                               atis_meal      1.000     0.833     0.909         6
                        atis_restriction      0.000     0.000     0.000         0
             atis_airline#atis_flight_no      0.000     0.000     0.000         0
    atis_ground_service#atis_ground_fare      0.000     0.000     0.000         0
           atis_airfare#atis_flight_time      0.000     0.000     0.000         0
                           atis_cheapest      0.000     0.000     0.000         0
atis_aircraft#atis_flight#atis_flight_no      0.000     0.000     0.000         0

                               micro avg      0.973     0.979     0.976       888
                               macro avg      0.646     0.656     0.637       888
                            weighted avg      0.975     0.979     0.975       888

I0716 04:38:27.119267 139795883440000 interactiveshell.py:2882] 
                              precision    recall  f1-score   support

                           O      0.000     0.000     0.000       0.0
           B-toloc.city_name      0.977     0.993     0.985     716.0
         B-fromloc.city_name      0.992     0.999     0.995     704.0
           I-toloc.city_name      0.981     0.992     0.987     265.0
      B-depart_date.day_name      0.991     0.991     0.991     212.0
              B-airline_name      1.000     1.000     1.000     101.0
         I-fromloc.city_name      0.978     1.000     0.989     177.0
 B-depart_time.period_of_day      0.992     0.915     0.952     130.0
              I-airline_name      1.000     1.000     1.000      65.0
    B-depart_date.day_number      0.981     0.964     0.972      55.0
    B-depart_date.month_name      0.982     0.964     0.973      56.0
          B-depart_time.time      0.864     1.000     0.927      57.0
                B-round_trip      1.000     1.000     1.000      73.0
             B-cost_relative      0.973     0.973     0.973      37.0
                I-round_trip      1.000     1.000     1.000      71.0
                B-flight_mod      0.828     1.000     0.906      24.0
 B-depart_time.time_relative      0.970     0.985     0.977      65.0
          I-depart_time.time      0.929     1.000     0.963      52.0
         B-stoploc.city_name      0.833     1.000     0.909      20.0
                 B-city_name      0.844     0.667     0.745      57.0
                B-class_type      0.960     1.000     0.980      24.0
          B-arrive_time.time      0.971     0.971     0.971      34.0
 B-arrive_time.time_relative      0.906     0.935     0.921      31.0
                I-class_type      1.000     1.000     1.000      17.0
               B-flight_stop      1.000     1.000     1.000      21.0
          I-arrive_time.time      1.000     0.971     0.986      35.0
              B-airline_code      0.944     1.000     0.971      34.0
    I-depart_date.day_number      1.000     0.933     0.966      15.0
      I-fromloc.airport_name      0.441     1.000     0.612      15.0
      B-fromloc.airport_name      0.462     1.000     0.632      12.0
      B-arrive_date.day_name      0.786     1.000     0.880      11.0
          B-toloc.state_code      1.000     1.000     1.000      18.0
B-depart_date.today_relative      1.000     0.889     0.941       9.0
             B-flight_number      0.733     1.000     0.846      11.0
 B-depart_date.date_relative      0.944     1.000     0.971      17.0
          B-toloc.state_name      0.806     0.893     0.847      28.0
           B-fare_basis_code      0.850     1.000     0.919      17.0
               B-flight_time      1.000     1.000     1.000       1.0
                        B-or      1.000     1.000     1.000       3.0
 B-arrive_time.period_of_day      0.750     1.000     0.857       6.0
          B-meal_description      1.000     0.900     0.947      10.0
             I-cost_relative      1.000     0.667     0.800       3.0
              I-airport_name      0.846     0.379     0.524      29.0
               B-fare_amount      1.000     1.000     1.000       2.0
               I-fare_amount      1.000     1.000     1.000       2.0
                 I-city_name      1.000     0.600     0.750      30.0
        I-toloc.airport_name      1.000     1.000     1.000       3.0
            B-transport_type      1.000     1.000     1.000      10.0
    B-arrive_date.month_name      0.714     0.833     0.769       6.0
    B-arrive_date.day_number      0.714     0.833     0.769       6.0
         I-stoploc.city_name      1.000     1.000     1.000      10.0
                      B-meal      0.941     1.000     0.970      16.0
        B-fromloc.state_code      1.000     1.000     1.000      23.0
    B-depart_time.period_mod      1.000     1.000     1.000       5.0
                   B-connect      1.000     1.000     1.000       6.0
               B-flight_days      1.000     1.000     1.000      10.0
        B-toloc.airport_name      1.000     1.000     1.000       3.0
        B-fromloc.state_name      1.000     1.000     1.000      17.0
              B-airport_name      0.750     0.286     0.414      21.0
                   B-economy      1.000     1.000     1.000       6.0
               I-flight_time      1.000     1.000     1.000       1.0
             B-aircraft_code      1.000     0.879     0.935      33.0
                       B-mod      1.000     0.500     0.667       2.0
              B-airport_code      1.000     0.444     0.615       9.0
    B-depart_time.start_time      1.000     0.667     0.800       3.0
      B-depart_time.end_time      1.000     0.333     0.500       3.0
          B-depart_date.year      1.000     1.000     1.000       3.0
            I-transport_type      0.000     0.000     0.000       1.0
          B-restriction_code      1.000     1.000     1.000       4.0
    B-arrive_time.start_time      0.889     1.000     0.941       8.0
        B-toloc.airport_code      1.000     0.750     0.857       4.0
      B-arrive_time.end_time      0.889     1.000     0.941       8.0
      I-arrive_time.end_time      0.889     1.000     0.941       8.0
      I-depart_time.end_time      1.000     0.333     0.500       3.0
               I-flight_stop      0.000     0.000     0.000       0.0
      B-fromloc.airport_code      0.556     1.000     0.714       5.0
          I-restriction_code      1.000     1.000     1.000       3.0
    I-depart_time.start_time      1.000     1.000     1.000       1.0
          I-toloc.state_name      1.000     1.000     1.000       1.0
I-depart_date.today_relative      0.000     0.000     0.000       0.0
 B-arrive_date.date_relative      1.000     1.000     1.000       2.0
                I-flight_mod      1.000     0.167     0.286       6.0
                   I-economy      0.000     0.000     0.000       0.0
 B-return_date.date_relative      1.000     0.667     0.800       3.0
        I-fromloc.state_name      1.000     1.000     1.000       1.0
                B-state_code      1.000     1.000     1.000       1.0
    I-arrive_time.start_time      1.000     1.000     1.000       1.0
    I-arrive_date.day_number      0.000     0.000     0.000       0.0
                 B-meal_code      0.000     0.000     0.000       1.0
 I-depart_time.period_of_day      1.000     1.000     1.000       1.0
                  B-day_name      1.000     0.500     0.667       2.0
             B-period_of_day      1.000     0.750     0.857       4.0
        B-stoploc.state_code      0.000     0.000     0.000       0.0
    B-return_date.month_name      0.000     0.000     0.000       0.0
    B-return_date.day_number      0.000     0.000     0.000       0.0
    B-arrive_time.period_mod      0.000     0.000     0.000       0.0
                 I-meal_code      0.000     0.000     0.000       0.0
        B-toloc.country_name      1.000     1.000     1.000       1.0
                 B-days_code      0.000     0.000     0.000       1.0
 I-arrive_time.period_of_day      0.000     0.000     0.000       0.0
            I-today_relative      0.000     0.000     0.000       0.0
 B-return_time.period_of_day      0.000     0.000     0.000       0.0
                      B-time      0.000     0.000     0.000       0.0
           I-fare_basis_code      0.000     0.000     0.000       0.0
 I-arrive_time.time_relative      1.000     0.500     0.667       4.0
 I-depart_time.time_relative      0.000     0.000     0.000       1.0
            B-today_relative      0.000     0.000     0.000       0.0
                B-state_name      0.000     0.000     0.000       9.0
B-arrive_date.today_relative      0.000     0.000     0.000       0.0
    B-return_time.period_mod      0.000     0.000     0.000       0.0
                B-month_name      0.000     0.000     0.000       0.0
                B-day_number      0.000     0.000     0.000       0.0
 I-return_date.date_relative      1.000     0.667     0.800       3.0
I-return_date.today_relative      0.000     0.000     0.000       0.0
      B-stoploc.airport_name      0.000     0.000     0.000       0.0
             B-time_relative      0.000     0.000     0.000       0.0
                      I-time      0.000     0.000     0.000       0.0
    I-return_date.day_number      0.000     0.000     0.000       0.0
          I-meal_description      0.000     0.000     0.000       0.0
B-return_date.today_relative      0.000     0.000     0.000       0.0
      B-return_date.day_name      0.000     0.000     0.000       2.0

                   micro avg      0.959     0.960     0.959    3657.0
                   macro avg      0.685     0.650     0.653    3657.0
                weighted avg      0.962     0.960     0.956    3657.0

I0716 04:38:27.141413 139795883440000 interactiveshell.py:2882] Best Slot F1: 0.960
I0716 04:38:27.142925 139795883440000 estimator_training.py:186] Not using Distribute Coordinator.
I0716 04:38:27.144620 139795883440000 training.py:612] Running training and evaluation locally (non-distributed).
I0716 04:38:27.146125 139795883440000 training.py:700] Start train and evaluate loop. The evaluate will happen after every checkpoint. Checkpoint frequency is determined based on RunConfig arguments: save_checkpoints_steps 312 or save_checkpoints_secs None.
I0716 04:38:27.216298 139795883440000 estimator.py:1145] Calling model_fn.
I0716 04:38:28.712872 139795883440000 saver.py:1499] Saver not created because there are no variables in the graph to restore
I0716 04:38:29.477923 139795883440000 <ipython-input-6-d6747b40360b>:55] 
[<tf.Variable 'embedding:0' shape=(750, 300) dtype=float32_ref>,
 <tf.Variable 'module/aggregation/weights:0' shape=(3,) dtype=float32>,
 <tf.Variable 'module/aggregation/scaling:0' shape=() dtype=float32>,
 <tf.Variable 'dense/kernel:0' shape=(1324, 300) dtype=float32_ref>,
 <tf.Variable 'dense/bias:0' shape=(300,) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/gates/kernel:0' shape=(600, 600) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/gates/bias:0' shape=(600,) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/candidate/kernel:0' shape=(600, 300) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/candidate/bias:0' shape=(300,) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/gates/kernel:0' shape=(600, 600) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/gates/bias:0' shape=(600,) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/candidate/kernel:0' shape=(600, 300) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/candidate/bias:0' shape=(300,) dtype=float32_ref>,
 <tf.Variable 'dense_1/kernel:0' shape=(600, 23) dtype=float32_ref>,
 <tf.Variable 'dense_1/bias:0' shape=(23,) dtype=float32_ref>,
 <tf.Variable 'dense_2/kernel:0' shape=(600, 122) dtype=float32_ref>,
 <tf.Variable 'dense_2/bias:0' shape=(122,) dtype=float32_ref>]
I0716 04:38:30.564863 139795883440000 estimator.py:1147] Done calling model_fn.
I0716 04:38:30.569037 139795883440000 basic_session_run_hooks.py:541] Create CheckpointSaverHook.
I0716 04:38:30.901015 139795883440000 monitored_session.py:240] Graph was finalized.
I0716 04:38:30.921924 139795883440000 saver.py:1280] Restoring parameters from ../model/elmo_bigru/model.ckpt-8424
I0716 04:38:31.968199 139795883440000 session_manager.py:500] Running local_init_op.
I0716 04:38:32.039877 139795883440000 session_manager.py:502] Done running local_init_op.
I0716 04:38:35.075495 139795883440000 basic_session_run_hooks.py:606] Saving checkpoints for 8424 into ../model/elmo_bigru/model.ckpt.
Reading ../data/atis.train.w-intent.iob
I0716 04:38:39.802695 139795883440000 basic_session_run_hooks.py:262] loss = 0.019132338, step = 8424
I0716 04:38:39.804921 139795883440000 basic_session_run_hooks.py:262] lr = 0.00012349803
I0716 04:39:07.285551 139795883440000 basic_session_run_hooks.py:692] global_step/sec: 3.63856
I0716 04:39:07.297929 139795883440000 basic_session_run_hooks.py:260] loss = 0.015347525, step = 8524 (27.495 sec)
I0716 04:39:07.299650 139795883440000 basic_session_run_hooks.py:260] lr = 0.00012220368 (27.495 sec)
I0716 04:39:34.417328 139795883440000 basic_session_run_hooks.py:692] global_step/sec: 3.68571
I0716 04:39:34.422420 139795883440000 basic_session_run_hooks.py:260] loss = 0.013917305, step = 8624 (27.124 sec)
I0716 04:39:34.427424 139795883440000 basic_session_run_hooks.py:260] lr = 0.000120922894 (27.128 sec)
I0716 04:40:00.334441 139795883440000 basic_session_run_hooks.py:692] global_step/sec: 3.85845
I0716 04:40:00.337281 139795883440000 basic_session_run_hooks.py:260] loss = 0.015168118, step = 8724 (25.915 sec)
I0716 04:40:00.344412 139795883440000 basic_session_run_hooks.py:260] lr = 0.00011965554 (25.917 sec)
I0716 04:40:03.324968 139795883440000 basic_session_run_hooks.py:606] Saving checkpoints for 8736 into ../model/elmo_bigru/model.ckpt.
I0716 04:40:05.298134 139795883440000 estimator.py:1145] Calling model_fn.
I0716 04:40:06.645496 139795883440000 saver.py:1499] Saver not created because there are no variables in the graph to restore
I0716 04:40:07.228574 139795883440000 estimator.py:1147] Done calling model_fn.
I0716 04:40:07.259759 139795883440000 evaluation.py:255] Starting evaluation at 2019-07-16T04:40:07Z
I0716 04:40:07.483759 139795883440000 monitored_session.py:240] Graph was finalized.
I0716 04:40:07.500017 139795883440000 saver.py:1280] Restoring parameters from ../model/elmo_bigru/model.ckpt-8736
I0716 04:40:08.392496 139795883440000 session_manager.py:500] Running local_init_op.
I0716 04:40:08.454483 139795883440000 session_manager.py:502] Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
I0716 04:41:07.242574 139795883440000 evaluation.py:275] Finished evaluation at 2019-07-16-04:41:07
I0716 04:41:07.244690 139795883440000 estimator.py:2039] Saving dict for global step 8736: global_step = 8736, loss = 0.3608753
I0716 04:41:07.253153 139795883440000 estimator.py:2099] Saving 'checkpoint_path' summary for global step 8736: ../model/elmo_bigru/model.ckpt-8736
I0716 04:41:07.336138 139795883440000 estimator.py:368] Loss for final step: 0.0033090645.
Reading ../data/atis.test.w-intent.iob
I0716 04:41:07.933838 139795883440000 estimator.py:1145] Calling model_fn.
I0716 04:41:09.617775 139795883440000 saver.py:1499] Saver not created because there are no variables in the graph to restore
I0716 04:41:10.127541 139795883440000 estimator.py:1147] Done calling model_fn.
I0716 04:41:10.352676 139795883440000 monitored_session.py:240] Graph was finalized.
I0716 04:41:10.369623 139795883440000 saver.py:1280] Restoring parameters from ../model/elmo_bigru/model.ckpt-8736
I0716 04:41:11.262585 139795883440000 session_manager.py:500] Running local_init_op.
I0716 04:41:11.308635 139795883440000 session_manager.py:502] Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.
  'precision', 'predicted', average, warn_for)
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1439: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples.
  'recall', 'true', average, warn_for)
I0716 04:42:09.205271 139795883440000 interactiveshell.py:2882] 
                                          precision    recall  f1-score   support

                             atis_flight      0.984     0.992     0.988       632
                            atis_airfare      0.941     1.000     0.970        48
                     atis_ground_service      1.000     1.000     1.000        36
                            atis_airline      1.000     1.000     1.000        38
                       atis_abbreviation      0.943     1.000     0.971        33
                           atis_aircraft      1.000     0.889     0.941         9
                        atis_flight_time      0.500     1.000     0.667         1
                           atis_quantity      0.429     1.000     0.600         3
                atis_flight#atis_airfare      0.833     0.417     0.556        12
                            atis_airport      1.000     1.000     1.000        18
                           atis_distance      1.000     1.000     1.000        10
                               atis_city      0.750     0.500     0.600         6
                        atis_ground_fare      1.000     0.857     0.923         7
                           atis_capacity      0.952     0.952     0.952        21
                          atis_flight_no      0.889     1.000     0.941         8
                               atis_meal      1.000     0.833     0.909         6
                        atis_restriction      0.000     0.000     0.000         0
             atis_airline#atis_flight_no      0.000     0.000     0.000         0
    atis_ground_service#atis_ground_fare      0.000     0.000     0.000         0
           atis_airfare#atis_flight_time      0.000     0.000     0.000         0
                           atis_cheapest      0.000     0.000     0.000         0
atis_aircraft#atis_flight#atis_flight_no      0.000     0.000     0.000         0

                               micro avg      0.973     0.979     0.976       888
                               macro avg      0.646     0.656     0.637       888
                            weighted avg      0.975     0.979     0.975       888

I0716 04:42:09.242614 139795883440000 interactiveshell.py:2882] 
                              precision    recall  f1-score   support

                           O      0.000     0.000     0.000       0.0
           B-toloc.city_name      0.975     0.993     0.984     716.0
         B-fromloc.city_name      0.990     0.999     0.994     704.0
           I-toloc.city_name      0.978     0.985     0.981     265.0
      B-depart_date.day_name      0.991     0.991     0.991     212.0
              B-airline_name      1.000     1.000     1.000     101.0
         I-fromloc.city_name      0.973     1.000     0.986     177.0
 B-depart_time.period_of_day      1.000     0.908     0.952     130.0
              I-airline_name      1.000     1.000     1.000      65.0
    B-depart_date.day_number      0.981     0.964     0.972      55.0
    B-depart_date.month_name      0.982     0.964     0.973      56.0
          B-depart_time.time      0.864     1.000     0.927      57.0
                B-round_trip      1.000     1.000     1.000      73.0
             B-cost_relative      0.973     0.973     0.973      37.0
                I-round_trip      1.000     1.000     1.000      71.0
                B-flight_mod      0.828     1.000     0.906      24.0
 B-depart_time.time_relative      0.955     0.985     0.970      65.0
          I-depart_time.time      0.912     1.000     0.954      52.0
         B-stoploc.city_name      0.870     1.000     0.930      20.0
                 B-city_name      0.860     0.649     0.740      57.0
                B-class_type      0.960     1.000     0.980      24.0
          B-arrive_time.time      0.971     0.971     0.971      34.0
 B-arrive_time.time_relative      0.906     0.935     0.921      31.0
                I-class_type      1.000     1.000     1.000      17.0
               B-flight_stop      1.000     1.000     1.000      21.0
          I-arrive_time.time      1.000     0.971     0.986      35.0
              B-airline_code      0.971     1.000     0.986      34.0
    I-depart_date.day_number      1.000     0.933     0.966      15.0
      I-fromloc.airport_name      0.441     1.000     0.612      15.0
      B-fromloc.airport_name      0.444     1.000     0.615      12.0
      B-arrive_date.day_name      0.786     1.000     0.880      11.0
          B-toloc.state_code      1.000     1.000     1.000      18.0
B-depart_date.today_relative      1.000     0.889     0.941       9.0
             B-flight_number      0.733     1.000     0.846      11.0
 B-depart_date.date_relative      0.944     1.000     0.971      17.0
          B-toloc.state_name      0.781     0.893     0.833      28.0
           B-fare_basis_code      0.850     1.000     0.919      17.0
               B-flight_time      1.000     1.000     1.000       1.0
                        B-or      1.000     1.000     1.000       3.0
 B-arrive_time.period_of_day      0.750     1.000     0.857       6.0
          B-meal_description      1.000     0.900     0.947      10.0
             I-cost_relative      1.000     0.667     0.800       3.0
              I-airport_name      0.846     0.379     0.524      29.0
               B-fare_amount      1.000     1.000     1.000       2.0
               I-fare_amount      1.000     1.000     1.000       2.0
                 I-city_name      1.000     0.567     0.723      30.0
        I-toloc.airport_name      1.000     1.000     1.000       3.0
            B-transport_type      1.000     1.000     1.000      10.0
    B-arrive_date.month_name      0.714     0.833     0.769       6.0
    B-arrive_date.day_number      0.714     0.833     0.769       6.0
         I-stoploc.city_name      1.000     1.000     1.000      10.0
                      B-meal      1.000     1.000     1.000      16.0
        B-fromloc.state_code      1.000     1.000     1.000      23.0
    B-depart_time.period_mod      1.000     1.000     1.000       5.0
                   B-connect      1.000     1.000     1.000       6.0
               B-flight_days      1.000     1.000     1.000      10.0
        B-toloc.airport_name      1.000     1.000     1.000       3.0
        B-fromloc.state_name      1.000     1.000     1.000      17.0
              B-airport_name      0.750     0.286     0.414      21.0
                   B-economy      1.000     1.000     1.000       6.0
               I-flight_time      1.000     1.000     1.000       1.0
             B-aircraft_code      1.000     0.879     0.935      33.0
                       B-mod      1.000     0.500     0.667       2.0
              B-airport_code      1.000     0.444     0.615       9.0
    B-depart_time.start_time      1.000     0.667     0.800       3.0
      B-depart_time.end_time      1.000     0.333     0.500       3.0
          B-depart_date.year      1.000     1.000     1.000       3.0
            I-transport_type      0.000     0.000     0.000       1.0
          B-restriction_code      0.800     1.000     0.889       4.0
    B-arrive_time.start_time      0.889     1.000     0.941       8.0
        B-toloc.airport_code      1.000     0.750     0.857       4.0
      B-arrive_time.end_time      0.889     1.000     0.941       8.0
      I-arrive_time.end_time      0.889     1.000     0.941       8.0
      I-depart_time.end_time      1.000     0.333     0.500       3.0
               I-flight_stop      0.000     0.000     0.000       0.0
      B-fromloc.airport_code      0.625     1.000     0.769       5.0
          I-restriction_code      1.000     1.000     1.000       3.0
    I-depart_time.start_time      1.000     1.000     1.000       1.0
          I-toloc.state_name      1.000     1.000     1.000       1.0
I-depart_date.today_relative      0.000     0.000     0.000       0.0
 B-arrive_date.date_relative      1.000     1.000     1.000       2.0
                I-flight_mod      1.000     0.167     0.286       6.0
                   I-economy      0.000     0.000     0.000       0.0
 B-return_date.date_relative      1.000     0.333     0.500       3.0
        I-fromloc.state_name      1.000     1.000     1.000       1.0
                B-state_code      1.000     1.000     1.000       1.0
    I-arrive_time.start_time      1.000     1.000     1.000       1.0
    I-arrive_date.day_number      0.000     0.000     0.000       0.0
                 B-meal_code      0.000     0.000     0.000       1.0
 I-depart_time.period_of_day      1.000     1.000     1.000       1.0
                  B-day_name      1.000     0.500     0.667       2.0
             B-period_of_day      1.000     0.750     0.857       4.0
        B-stoploc.state_code      0.000     0.000     0.000       0.0
    B-return_date.month_name      0.000     0.000     0.000       0.0
    B-return_date.day_number      0.000     0.000     0.000       0.0
    B-arrive_time.period_mod      0.000     0.000     0.000       0.0
                 I-meal_code      0.000     0.000     0.000       0.0
        B-toloc.country_name      1.000     1.000     1.000       1.0
                 B-days_code      0.000     0.000     0.000       1.0
 I-arrive_time.period_of_day      0.000     0.000     0.000       0.0
            I-today_relative      0.000     0.000     0.000       0.0
 B-return_time.period_of_day      0.000     0.000     0.000       0.0
                      B-time      0.000     0.000     0.000       0.0
           I-fare_basis_code      0.000     0.000     0.000       0.0
 I-arrive_time.time_relative      1.000     0.500     0.667       4.0
 I-depart_time.time_relative      0.000     0.000     0.000       1.0
            B-today_relative      0.000     0.000     0.000       0.0
                B-state_name      0.000     0.000     0.000       9.0
B-arrive_date.today_relative      0.000     0.000     0.000       0.0
    B-return_time.period_mod      0.000     0.000     0.000       0.0
                B-month_name      0.000     0.000     0.000       0.0
                B-day_number      0.000     0.000     0.000       0.0
 I-return_date.date_relative      0.667     0.667     0.667       3.0
I-return_date.today_relative      0.000     0.000     0.000       0.0
      B-stoploc.airport_name      0.000     0.000     0.000       0.0
             B-time_relative      0.000     0.000     0.000       0.0
                      I-time      0.000     0.000     0.000       0.0
    I-return_date.day_number      0.000     0.000     0.000       0.0
          I-meal_description      0.000     0.000     0.000       0.0
B-return_date.today_relative      0.000     0.000     0.000       0.0
      B-return_date.day_name      0.000     0.000     0.000       2.0

                   micro avg      0.957     0.958     0.958    3657.0
                   macro avg      0.681     0.647     0.648    3657.0
                weighted avg      0.961     0.958     0.955    3657.0

I0716 04:42:09.264512 139795883440000 interactiveshell.py:2882] Best Slot F1: 0.960
I0716 04:42:09.265876 139795883440000 estimator_training.py:186] Not using Distribute Coordinator.
I0716 04:42:09.267675 139795883440000 training.py:612] Running training and evaluation locally (non-distributed).
I0716 04:42:09.269614 139795883440000 training.py:700] Start train and evaluate loop. The evaluate will happen after every checkpoint. Checkpoint frequency is determined based on RunConfig arguments: save_checkpoints_steps 312 or save_checkpoints_secs None.
I0716 04:42:09.341445 139795883440000 estimator.py:1145] Calling model_fn.
I0716 04:42:10.994044 139795883440000 saver.py:1499] Saver not created because there are no variables in the graph to restore
I0716 04:42:11.572091 139795883440000 <ipython-input-6-d6747b40360b>:55] 
[<tf.Variable 'embedding:0' shape=(750, 300) dtype=float32_ref>,
 <tf.Variable 'module/aggregation/weights:0' shape=(3,) dtype=float32>,
 <tf.Variable 'module/aggregation/scaling:0' shape=() dtype=float32>,
 <tf.Variable 'dense/kernel:0' shape=(1324, 300) dtype=float32_ref>,
 <tf.Variable 'dense/bias:0' shape=(300,) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/gates/kernel:0' shape=(600, 600) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/gates/bias:0' shape=(600,) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/candidate/kernel:0' shape=(600, 300) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/candidate/bias:0' shape=(300,) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/gates/kernel:0' shape=(600, 600) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/gates/bias:0' shape=(600,) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/candidate/kernel:0' shape=(600, 300) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/candidate/bias:0' shape=(300,) dtype=float32_ref>,
 <tf.Variable 'dense_1/kernel:0' shape=(600, 23) dtype=float32_ref>,
 <tf.Variable 'dense_1/bias:0' shape=(23,) dtype=float32_ref>,
 <tf.Variable 'dense_2/kernel:0' shape=(600, 122) dtype=float32_ref>,
 <tf.Variable 'dense_2/bias:0' shape=(122,) dtype=float32_ref>]
I0716 04:42:12.658575 139795883440000 estimator.py:1147] Done calling model_fn.
I0716 04:42:12.662549 139795883440000 basic_session_run_hooks.py:541] Create CheckpointSaverHook.
I0716 04:42:12.995083 139795883440000 monitored_session.py:240] Graph was finalized.
I0716 04:42:13.012826 139795883440000 saver.py:1280] Restoring parameters from ../model/elmo_bigru/model.ckpt-8736
I0716 04:42:14.061174 139795883440000 session_manager.py:500] Running local_init_op.
I0716 04:42:14.132613 139795883440000 session_manager.py:502] Done running local_init_op.
I0716 04:42:17.146679 139795883440000 basic_session_run_hooks.py:606] Saving checkpoints for 8736 into ../model/elmo_bigru/model.ckpt.
Reading ../data/atis.train.w-intent.iob
I0716 04:42:22.121123 139795883440000 basic_session_run_hooks.py:262] loss = 0.0142422775, step = 8736
I0716 04:42:22.122906 139795883440000 basic_session_run_hooks.py:262] lr = 0.00011950435
I0716 04:42:49.735462 139795883440000 basic_session_run_hooks.py:692] global_step/sec: 3.62123
I0716 04:42:49.744070 139795883440000 basic_session_run_hooks.py:260] loss = 0.012751075, step = 8836 (27.623 sec)
I0716 04:42:49.746460 139795883440000 basic_session_run_hooks.py:260] lr = 0.00011825185 (27.624 sec)
I0716 04:43:15.778196 139795883440000 basic_session_run_hooks.py:692] global_step/sec: 3.83984
I0716 04:43:15.785495 139795883440000 basic_session_run_hooks.py:260] loss = 0.016511267, step = 8936 (26.041 sec)
I0716 04:43:15.788418 139795883440000 basic_session_run_hooks.py:260] lr = 0.000117012474 (26.042 sec)
I0716 04:43:42.846633 139795883440000 basic_session_run_hooks.py:692] global_step/sec: 3.69434
I0716 04:43:42.852755 139795883440000 basic_session_run_hooks.py:260] loss = 0.01867107, step = 9036 (27.067 sec)
I0716 04:43:42.855791 139795883440000 basic_session_run_hooks.py:260] lr = 0.00011578611 (27.067 sec)
I0716 04:43:45.541065 139795883440000 basic_session_run_hooks.py:606] Saving checkpoints for 9048 into ../model/elmo_bigru/model.ckpt.
I0716 04:43:47.526752 139795883440000 estimator.py:1145] Calling model_fn.
I0716 04:43:48.916526 139795883440000 saver.py:1499] Saver not created because there are no variables in the graph to restore
I0716 04:43:49.481549 139795883440000 estimator.py:1147] Done calling model_fn.
I0716 04:43:49.511429 139795883440000 evaluation.py:255] Starting evaluation at 2019-07-16T04:43:49Z
I0716 04:43:49.733474 139795883440000 monitored_session.py:240] Graph was finalized.
I0716 04:43:49.750476 139795883440000 saver.py:1280] Restoring parameters from ../model/elmo_bigru/model.ckpt-9048
I0716 04:43:50.664488 139795883440000 session_manager.py:500] Running local_init_op.
I0716 04:43:50.723493 139795883440000 session_manager.py:502] Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
I0716 04:44:49.735225 139795883440000 evaluation.py:275] Finished evaluation at 2019-07-16-04:44:49
I0716 04:44:49.737258 139795883440000 estimator.py:2039] Saving dict for global step 9048: global_step = 9048, loss = 0.37418377
I0716 04:44:49.746189 139795883440000 estimator.py:2099] Saving 'checkpoint_path' summary for global step 9048: ../model/elmo_bigru/model.ckpt-9048
I0716 04:44:49.833694 139795883440000 estimator.py:368] Loss for final step: 0.0026269325.
Reading ../data/atis.test.w-intent.iob
I0716 04:44:50.443303 139795883440000 estimator.py:1145] Calling model_fn.
I0716 04:44:52.121428 139795883440000 saver.py:1499] Saver not created because there are no variables in the graph to restore
I0716 04:44:52.830277 139795883440000 estimator.py:1147] Done calling model_fn.
I0716 04:44:53.050785 139795883440000 monitored_session.py:240] Graph was finalized.
I0716 04:44:53.068392 139795883440000 saver.py:1280] Restoring parameters from ../model/elmo_bigru/model.ckpt-9048
I0716 04:44:53.968281 139795883440000 session_manager.py:500] Running local_init_op.
I0716 04:44:54.016930 139795883440000 session_manager.py:502] Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.
  'precision', 'predicted', average, warn_for)
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1439: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples.
  'recall', 'true', average, warn_for)
I0716 04:45:52.206138 139795883440000 interactiveshell.py:2882] 
                                          precision    recall  f1-score   support

                             atis_flight      0.984     0.992     0.988       632
                            atis_airfare      0.941     1.000     0.970        48
                     atis_ground_service      1.000     1.000     1.000        36
                            atis_airline      0.974     1.000     0.987        38
                       atis_abbreviation      0.943     1.000     0.971        33
                           atis_aircraft      1.000     0.889     0.941         9
                        atis_flight_time      0.500     1.000     0.667         1
                           atis_quantity      0.429     1.000     0.600         3
                atis_flight#atis_airfare      0.833     0.417     0.556        12
                            atis_airport      1.000     0.944     0.971        18
                           atis_distance      1.000     1.000     1.000        10
                               atis_city      0.750     0.500     0.600         6
                        atis_ground_fare      1.000     0.857     0.923         7
                           atis_capacity      0.952     0.952     0.952        21
                          atis_flight_no      0.889     1.000     0.941         8
                               atis_meal      1.000     0.833     0.909         6
                        atis_restriction      0.000     0.000     0.000         0
             atis_airline#atis_flight_no      0.000     0.000     0.000         0
    atis_ground_service#atis_ground_fare      0.000     0.000     0.000         0
           atis_airfare#atis_flight_time      0.000     0.000     0.000         0
                           atis_cheapest      0.000     0.000     0.000         0
atis_aircraft#atis_flight#atis_flight_no      0.000     0.000     0.000         0

                               micro avg      0.972     0.977     0.975       888
                               macro avg      0.645     0.654     0.635       888
                            weighted avg      0.974     0.977     0.974       888

I0716 04:45:52.241688 139795883440000 interactiveshell.py:2882] 
                              precision    recall  f1-score   support

                           O      0.000     0.000     0.000       0.0
           B-toloc.city_name      0.977     0.993     0.985     716.0
         B-fromloc.city_name      0.989     0.999     0.994     704.0
           I-toloc.city_name      0.977     0.981     0.979     265.0
      B-depart_date.day_name      0.986     0.991     0.988     212.0
              B-airline_name      1.000     1.000     1.000     101.0
         I-fromloc.city_name      0.962     1.000     0.981     177.0
 B-depart_time.period_of_day      0.992     0.923     0.956     130.0
              I-airline_name      1.000     1.000     1.000      65.0
    B-depart_date.day_number      0.981     0.964     0.972      55.0
    B-depart_date.month_name      0.982     0.964     0.973      56.0
          B-depart_time.time      0.864     1.000     0.927      57.0
                B-round_trip      1.000     1.000     1.000      73.0
             B-cost_relative      0.973     0.973     0.973      37.0
                I-round_trip      1.000     1.000     1.000      71.0
                B-flight_mod      0.828     1.000     0.906      24.0
 B-depart_time.time_relative      0.955     0.969     0.962      65.0
          I-depart_time.time      0.945     1.000     0.972      52.0
         B-stoploc.city_name      0.909     1.000     0.952      20.0
                 B-city_name      0.902     0.649     0.755      57.0
                B-class_type      0.960     1.000     0.980      24.0
          B-arrive_time.time      0.971     0.971     0.971      34.0
 B-arrive_time.time_relative      0.967     0.935     0.951      31.0
                I-class_type      1.000     1.000     1.000      17.0
               B-flight_stop      1.000     1.000     1.000      21.0
          I-arrive_time.time      1.000     0.971     0.986      35.0
              B-airline_code      0.895     1.000     0.944      34.0
    I-depart_date.day_number      1.000     1.000     1.000      15.0
      I-fromloc.airport_name      0.441     1.000     0.612      15.0
      B-fromloc.airport_name      0.444     1.000     0.615      12.0
      B-arrive_date.day_name      0.769     0.909     0.833      11.0
          B-toloc.state_code      1.000     1.000     1.000      18.0
B-depart_date.today_relative      1.000     1.000     1.000       9.0
             B-flight_number      0.733     1.000     0.846      11.0
 B-depart_date.date_relative      0.944     1.000     0.971      17.0
          B-toloc.state_name      0.788     0.929     0.852      28.0
           B-fare_basis_code      0.850     1.000     0.919      17.0
               B-flight_time      1.000     1.000     1.000       1.0
                        B-or      1.000     1.000     1.000       3.0
 B-arrive_time.period_of_day      0.750     1.000     0.857       6.0
          B-meal_description      1.000     0.900     0.947      10.0
             I-cost_relative      1.000     0.667     0.800       3.0
              I-airport_name      0.846     0.379     0.524      29.0
               B-fare_amount      1.000     1.000     1.000       2.0
               I-fare_amount      1.000     1.000     1.000       2.0
                 I-city_name      1.000     0.533     0.696      30.0
        I-toloc.airport_name      1.000     1.000     1.000       3.0
            B-transport_type      1.000     1.000     1.000      10.0
    B-arrive_date.month_name      0.714     0.833     0.769       6.0
    B-arrive_date.day_number      0.714     0.833     0.769       6.0
         I-stoploc.city_name      1.000     1.000     1.000      10.0
                      B-meal      0.941     1.000     0.970      16.0
        B-fromloc.state_code      1.000     1.000     1.000      23.0
    B-depart_time.period_mod      0.833     1.000     0.909       5.0
                   B-connect      1.000     1.000     1.000       6.0
               B-flight_days      1.000     1.000     1.000      10.0
        B-toloc.airport_name      1.000     1.000     1.000       3.0
        B-fromloc.state_name      0.944     1.000     0.971      17.0
              B-airport_name      0.778     0.333     0.467      21.0
                   B-economy      1.000     1.000     1.000       6.0
               I-flight_time      1.000     1.000     1.000       1.0
             B-aircraft_code      1.000     0.818     0.900      33.0
                       B-mod      1.000     0.500     0.667       2.0
              B-airport_code      1.000     0.444     0.615       9.0
    B-depart_time.start_time      0.750     1.000     0.857       3.0
      B-depart_time.end_time      0.667     0.667     0.667       3.0
          B-depart_date.year      1.000     1.000     1.000       3.0
            I-transport_type      1.000     1.000     1.000       1.0
          B-restriction_code      1.000     1.000     1.000       4.0
    B-arrive_time.start_time      1.000     0.875     0.933       8.0
        B-toloc.airport_code      1.000     0.750     0.857       4.0
      B-arrive_time.end_time      1.000     0.875     0.933       8.0
      I-arrive_time.end_time      1.000     1.000     1.000       8.0
      I-depart_time.end_time      1.000     0.667     0.800       3.0
               I-flight_stop      0.000     0.000     0.000       0.0
      B-fromloc.airport_code      0.625     1.000     0.769       5.0
          I-restriction_code      1.000     1.000     1.000       3.0
    I-depart_time.start_time      1.000     1.000     1.000       1.0
          I-toloc.state_name      1.000     1.000     1.000       1.0
I-depart_date.today_relative      0.000     0.000     0.000       0.0
 B-arrive_date.date_relative      1.000     1.000     1.000       2.0
                I-flight_mod      1.000     0.167     0.286       6.0
                   I-economy      0.000     0.000     0.000       0.0
 B-return_date.date_relative      1.000     0.667     0.800       3.0
        I-fromloc.state_name      1.000     1.000     1.000       1.0
                B-state_code      1.000     1.000     1.000       1.0
    I-arrive_time.start_time      1.000     1.000     1.000       1.0
    I-arrive_date.day_number      0.000     0.000     0.000       0.0
                 B-meal_code      0.000     0.000     0.000       1.0
 I-depart_time.period_of_day      0.000     0.000     0.000       1.0
                  B-day_name      1.000     0.500     0.667       2.0
             B-period_of_day      1.000     0.750     0.857       4.0
        B-stoploc.state_code      0.000     0.000     0.000       0.0
    B-return_date.month_name      0.000     0.000     0.000       0.0
    B-return_date.day_number      0.000     0.000     0.000       0.0
    B-arrive_time.period_mod      0.000     0.000     0.000       0.0
                 I-meal_code      0.000     0.000     0.000       0.0
        B-toloc.country_name      1.000     1.000     1.000       1.0
                 B-days_code      0.000     0.000     0.000       1.0
 I-arrive_time.period_of_day      0.000     0.000     0.000       0.0
            I-today_relative      0.000     0.000     0.000       0.0
 B-return_time.period_of_day      0.000     0.000     0.000       0.0
                      B-time      0.000     0.000     0.000       0.0
           I-fare_basis_code      0.000     0.000     0.000       0.0
 I-arrive_time.time_relative      1.000     1.000     1.000       4.0
 I-depart_time.time_relative      0.000     0.000     0.000       1.0
            B-today_relative      0.000     0.000     0.000       0.0
                B-state_name      0.000     0.000     0.000       9.0
B-arrive_date.today_relative      0.000     0.000     0.000       0.0
    B-return_time.period_mod      0.000     0.000     0.000       0.0
                B-month_name      0.000     0.000     0.000       0.0
                B-day_number      0.000     0.000     0.000       0.0
 I-return_date.date_relative      1.000     0.667     0.800       3.0
I-return_date.today_relative      0.000     0.000     0.000       0.0
      B-stoploc.airport_name      0.000     0.000     0.000       0.0
             B-time_relative      0.000     0.000     0.000       0.0
                      I-time      0.000     0.000     0.000       0.0
    I-return_date.day_number      0.000     0.000     0.000       0.0
          I-meal_description      0.000     0.000     0.000       0.0
B-return_date.today_relative      0.000     0.000     0.000       0.0
      B-return_date.day_name      0.000     0.000     0.000       2.0

                   micro avg      0.958     0.960     0.959    3657.0
                   macro avg      0.682     0.661     0.660    3657.0
                weighted avg      0.961     0.960     0.956    3657.0

I0716 04:45:52.263083 139795883440000 interactiveshell.py:2882] Best Slot F1: 0.960
I0716 04:45:52.264766 139795883440000 estimator_training.py:186] Not using Distribute Coordinator.
I0716 04:45:52.266667 139795883440000 training.py:612] Running training and evaluation locally (non-distributed).
I0716 04:45:52.268577 139795883440000 training.py:700] Start train and evaluate loop. The evaluate will happen after every checkpoint. Checkpoint frequency is determined based on RunConfig arguments: save_checkpoints_steps 312 or save_checkpoints_secs None.
I0716 04:45:52.337149 139795883440000 estimator.py:1145] Calling model_fn.
I0716 04:45:53.673756 139795883440000 saver.py:1499] Saver not created because there are no variables in the graph to restore
I0716 04:45:54.459498 139795883440000 <ipython-input-6-d6747b40360b>:55] 
[<tf.Variable 'embedding:0' shape=(750, 300) dtype=float32_ref>,
 <tf.Variable 'module/aggregation/weights:0' shape=(3,) dtype=float32>,
 <tf.Variable 'module/aggregation/scaling:0' shape=() dtype=float32>,
 <tf.Variable 'dense/kernel:0' shape=(1324, 300) dtype=float32_ref>,
 <tf.Variable 'dense/bias:0' shape=(300,) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/gates/kernel:0' shape=(600, 600) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/gates/bias:0' shape=(600,) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/candidate/kernel:0' shape=(600, 300) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/candidate/bias:0' shape=(300,) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/gates/kernel:0' shape=(600, 600) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/gates/bias:0' shape=(600,) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/candidate/kernel:0' shape=(600, 300) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/candidate/bias:0' shape=(300,) dtype=float32_ref>,
 <tf.Variable 'dense_1/kernel:0' shape=(600, 23) dtype=float32_ref>,
 <tf.Variable 'dense_1/bias:0' shape=(23,) dtype=float32_ref>,
 <tf.Variable 'dense_2/kernel:0' shape=(600, 122) dtype=float32_ref>,
 <tf.Variable 'dense_2/bias:0' shape=(122,) dtype=float32_ref>]
I0716 04:45:55.540261 139795883440000 estimator.py:1147] Done calling model_fn.
I0716 04:45:55.543845 139795883440000 basic_session_run_hooks.py:541] Create CheckpointSaverHook.
I0716 04:45:55.876288 139795883440000 monitored_session.py:240] Graph was finalized.
I0716 04:45:55.897523 139795883440000 saver.py:1280] Restoring parameters from ../model/elmo_bigru/model.ckpt-9048
I0716 04:45:56.945858 139795883440000 session_manager.py:500] Running local_init_op.
I0716 04:45:57.013511 139795883440000 session_manager.py:502] Done running local_init_op.
I0716 04:46:00.010006 139795883440000 basic_session_run_hooks.py:606] Saving checkpoints for 9048 into ../model/elmo_bigru/model.ckpt.
Reading ../data/atis.train.w-intent.iob
I0716 04:46:04.935000 139795883440000 basic_session_run_hooks.py:262] loss = 0.010518936, step = 9048
I0716 04:46:04.937154 139795883440000 basic_session_run_hooks.py:262] lr = 0.00011563981
I0716 04:46:33.369325 139795883440000 basic_session_run_hooks.py:692] global_step/sec: 3.51678
I0716 04:46:33.371953 139795883440000 basic_session_run_hooks.py:260] loss = 0.012888418, step = 9148 (28.437 sec)
I0716 04:46:33.378367 139795883440000 basic_session_run_hooks.py:260] lr = 0.00011442781 (28.441 sec)
I0716 04:46:59.021768 139795883440000 basic_session_run_hooks.py:692] global_step/sec: 3.89827
I0716 04:46:59.028245 139795883440000 basic_session_run_hooks.py:260] loss = 0.0126974005, step = 9248 (25.656 sec)
I0716 04:46:59.030674 139795883440000 basic_session_run_hooks.py:260] lr = 0.00011322854 (25.652 sec)
I0716 04:47:25.371392 139795883440000 basic_session_run_hooks.py:692] global_step/sec: 3.79512
I0716 04:47:25.377985 139795883440000 basic_session_run_hooks.py:260] loss = 0.011571307, step = 9348 (26.350 sec)
I0716 04:47:25.380642 139795883440000 basic_session_run_hooks.py:260] lr = 0.0001120418 (26.350 sec)
I0716 04:47:27.977454 139795883440000 basic_session_run_hooks.py:606] Saving checkpoints for 9360 into ../model/elmo_bigru/model.ckpt.
I0716 04:47:30.206533 139795883440000 estimator.py:1145] Calling model_fn.
I0716 04:47:31.614375 139795883440000 saver.py:1499] Saver not created because there are no variables in the graph to restore
I0716 04:47:32.210842 139795883440000 estimator.py:1147] Done calling model_fn.
I0716 04:47:32.241384 139795883440000 evaluation.py:255] Starting evaluation at 2019-07-16T04:47:32Z
I0716 04:47:32.468497 139795883440000 monitored_session.py:240] Graph was finalized.
I0716 04:47:32.487444 139795883440000 saver.py:1280] Restoring parameters from ../model/elmo_bigru/model.ckpt-9360
I0716 04:47:33.419073 139795883440000 session_manager.py:500] Running local_init_op.
I0716 04:47:33.478233 139795883440000 session_manager.py:502] Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
I0716 04:48:32.354332 139795883440000 evaluation.py:275] Finished evaluation at 2019-07-16-04:48:32
I0716 04:48:32.356114 139795883440000 estimator.py:2039] Saving dict for global step 9360: global_step = 9360, loss = 0.36635378
I0716 04:48:32.363076 139795883440000 estimator.py:2099] Saving 'checkpoint_path' summary for global step 9360: ../model/elmo_bigru/model.ckpt-9360
I0716 04:48:32.447439 139795883440000 estimator.py:368] Loss for final step: 0.002667467.
Reading ../data/atis.test.w-intent.iob
I0716 04:48:33.036143 139795883440000 estimator.py:1145] Calling model_fn.
I0716 04:48:34.747920 139795883440000 saver.py:1499] Saver not created because there are no variables in the graph to restore
I0716 04:48:35.261643 139795883440000 estimator.py:1147] Done calling model_fn.
I0716 04:48:35.486868 139795883440000 monitored_session.py:240] Graph was finalized.
I0716 04:48:35.502614 139795883440000 saver.py:1280] Restoring parameters from ../model/elmo_bigru/model.ckpt-9360
I0716 04:48:36.388677 139795883440000 session_manager.py:500] Running local_init_op.
I0716 04:48:36.440654 139795883440000 session_manager.py:502] Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.
  'precision', 'predicted', average, warn_for)
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1439: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples.
  'recall', 'true', average, warn_for)
I0716 04:49:34.259678 139795883440000 interactiveshell.py:2882] 
                                          precision    recall  f1-score   support

                             atis_flight      0.984     0.992     0.988       632
                            atis_airfare      0.941     1.000     0.970        48
                     atis_ground_service      1.000     1.000     1.000        36
                            atis_airline      0.974     1.000     0.987        38
                       atis_abbreviation      0.943     1.000     0.971        33
                           atis_aircraft      1.000     0.889     0.941         9
                        atis_flight_time      0.500     1.000     0.667         1
                           atis_quantity      0.429     1.000     0.600         3
                atis_flight#atis_airfare      0.833     0.417     0.556        12
                            atis_airport      1.000     0.944     0.971        18
                           atis_distance      1.000     1.000     1.000        10
                               atis_city      0.600     0.500     0.545         6
                        atis_ground_fare      1.000     0.857     0.923         7
                           atis_capacity      0.952     0.952     0.952        21
                          atis_flight_no      0.889     1.000     0.941         8
                               atis_meal      1.000     0.667     0.800         6
                        atis_restriction      0.000     0.000     0.000         0
             atis_airline#atis_flight_no      0.000     0.000     0.000         0
    atis_ground_service#atis_ground_fare      0.000     0.000     0.000         0
           atis_airfare#atis_flight_time      0.000     0.000     0.000         0
                           atis_cheapest      0.000     0.000     0.000         0
atis_aircraft#atis_flight#atis_flight_no      0.000     0.000     0.000         0

                               micro avg      0.971     0.976     0.974       888
                               macro avg      0.638     0.646     0.628       888
                            weighted avg      0.973     0.976     0.973       888

I0716 04:49:34.296104 139795883440000 interactiveshell.py:2882] 
                              precision    recall  f1-score   support

                           O      0.000     0.000     0.000       0.0
           B-toloc.city_name      0.977     0.993     0.985     716.0
         B-fromloc.city_name      0.992     0.999     0.995     704.0
           I-toloc.city_name      0.981     0.981     0.981     265.0
      B-depart_date.day_name      0.991     0.991     0.991     212.0
              B-airline_name      1.000     1.000     1.000     101.0
         I-fromloc.city_name      0.973     1.000     0.986     177.0
 B-depart_time.period_of_day      1.000     0.923     0.960     130.0
              I-airline_name      1.000     1.000     1.000      65.0
    B-depart_date.day_number      0.981     0.927     0.953      55.0
    B-depart_date.month_name      0.982     0.964     0.973      56.0
          B-depart_time.time      0.864     1.000     0.927      57.0
                B-round_trip      1.000     1.000     1.000      73.0
             B-cost_relative      0.973     0.973     0.973      37.0
                I-round_trip      1.000     1.000     1.000      71.0
                B-flight_mod      0.828     1.000     0.906      24.0
 B-depart_time.time_relative      0.955     0.985     0.970      65.0
          I-depart_time.time      0.945     1.000     0.972      52.0
         B-stoploc.city_name      0.833     1.000     0.909      20.0
                 B-city_name      0.844     0.667     0.745      57.0
                B-class_type      0.889     1.000     0.941      24.0
          B-arrive_time.time      0.971     0.971     0.971      34.0
 B-arrive_time.time_relative      0.906     0.935     0.921      31.0
                I-class_type      1.000     1.000     1.000      17.0
               B-flight_stop      1.000     1.000     1.000      21.0
          I-arrive_time.time      1.000     0.971     0.986      35.0
              B-airline_code      0.895     1.000     0.944      34.0
    I-depart_date.day_number      1.000     0.933     0.966      15.0
      I-fromloc.airport_name      0.441     1.000     0.612      15.0
      B-fromloc.airport_name      0.462     1.000     0.632      12.0
      B-arrive_date.day_name      0.786     1.000     0.880      11.0
          B-toloc.state_code      1.000     1.000     1.000      18.0
B-depart_date.today_relative      1.000     1.000     1.000       9.0
             B-flight_number      0.733     1.000     0.846      11.0
 B-depart_date.date_relative      0.944     1.000     0.971      17.0
          B-toloc.state_name      0.806     0.893     0.847      28.0
           B-fare_basis_code      0.850     1.000     0.919      17.0
               B-flight_time      1.000     1.000     1.000       1.0
                        B-or      1.000     1.000     1.000       3.0
 B-arrive_time.period_of_day      0.750     1.000     0.857       6.0
          B-meal_description      1.000     0.900     0.947      10.0
             I-cost_relative      1.000     0.667     0.800       3.0
              I-airport_name      0.833     0.345     0.488      29.0
               B-fare_amount      1.000     1.000     1.000       2.0
               I-fare_amount      1.000     1.000     1.000       2.0
                 I-city_name      0.950     0.633     0.760      30.0
        I-toloc.airport_name      1.000     1.000     1.000       3.0
            B-transport_type      1.000     1.000     1.000      10.0
    B-arrive_date.month_name      0.714     0.833     0.769       6.0
    B-arrive_date.day_number      0.714     0.833     0.769       6.0
         I-stoploc.city_name      0.909     1.000     0.952      10.0
                      B-meal      0.941     1.000     0.970      16.0
        B-fromloc.state_code      1.000     1.000     1.000      23.0
    B-depart_time.period_mod      1.000     1.000     1.000       5.0
                   B-connect      1.000     1.000     1.000       6.0
               B-flight_days      1.000     1.000     1.000      10.0
        B-toloc.airport_name      1.000     1.000     1.000       3.0
        B-fromloc.state_name      1.000     1.000     1.000      17.0
              B-airport_name      0.750     0.286     0.414      21.0
                   B-economy      1.000     1.000     1.000       6.0
               I-flight_time      1.000     1.000     1.000       1.0
             B-aircraft_code      1.000     0.818     0.900      33.0
                       B-mod      1.000     0.500     0.667       2.0
              B-airport_code      1.000     0.444     0.615       9.0
    B-depart_time.start_time      1.000     0.667     0.800       3.0
      B-depart_time.end_time      1.000     0.333     0.500       3.0
          B-depart_date.year      1.000     1.000     1.000       3.0
            I-transport_type      1.000     1.000     1.000       1.0
          B-restriction_code      1.000     1.000     1.000       4.0
    B-arrive_time.start_time      0.889     1.000     0.941       8.0
        B-toloc.airport_code      1.000     0.750     0.857       4.0
      B-arrive_time.end_time      0.889     1.000     0.941       8.0
      I-arrive_time.end_time      0.889     1.000     0.941       8.0
      I-depart_time.end_time      1.000     0.333     0.500       3.0
               I-flight_stop      0.000     0.000     0.000       0.0
      B-fromloc.airport_code      0.625     1.000     0.769       5.0
          I-restriction_code      1.000     1.000     1.000       3.0
    I-depart_time.start_time      1.000     1.000     1.000       1.0
          I-toloc.state_name      1.000     1.000     1.000       1.0
I-depart_date.today_relative      0.000     0.000     0.000       0.0
 B-arrive_date.date_relative      1.000     1.000     1.000       2.0
                I-flight_mod      1.000     0.167     0.286       6.0
                   I-economy      0.000     0.000     0.000       0.0
 B-return_date.date_relative      1.000     0.333     0.500       3.0
        I-fromloc.state_name      1.000     1.000     1.000       1.0
                B-state_code      1.000     1.000     1.000       1.0
    I-arrive_time.start_time      1.000     1.000     1.000       1.0
    I-arrive_date.day_number      0.000     0.000     0.000       0.0
                 B-meal_code      0.000     0.000     0.000       1.0
 I-depart_time.period_of_day      1.000     1.000     1.000       1.0
                  B-day_name      1.000     0.500     0.667       2.0
             B-period_of_day      1.000     0.750     0.857       4.0
        B-stoploc.state_code      0.000     0.000     0.000       0.0
    B-return_date.month_name      0.000     0.000     0.000       0.0
    B-return_date.day_number      0.000     0.000     0.000       0.0
    B-arrive_time.period_mod      0.000     0.000     0.000       0.0
                 I-meal_code      0.000     0.000     0.000       0.0
        B-toloc.country_name      1.000     1.000     1.000       1.0
                 B-days_code      0.000     0.000     0.000       1.0
 I-arrive_time.period_of_day      0.000     0.000     0.000       0.0
            I-today_relative      0.000     0.000     0.000       0.0
 B-return_time.period_of_day      0.000     0.000     0.000       0.0
                      B-time      0.000     0.000     0.000       0.0
           I-fare_basis_code      0.000     0.000     0.000       0.0
 I-arrive_time.time_relative      1.000     0.500     0.667       4.0
 I-depart_time.time_relative      0.000     0.000     0.000       1.0
            B-today_relative      0.000     0.000     0.000       0.0
                B-state_name      0.000     0.000     0.000       9.0
B-arrive_date.today_relative      0.000     0.000     0.000       0.0
    B-return_time.period_mod      0.000     0.000     0.000       0.0
                B-month_name      0.000     0.000     0.000       0.0
                B-day_number      0.000     0.000     0.000       0.0
 I-return_date.date_relative      0.667     0.667     0.667       3.0
I-return_date.today_relative      0.000     0.000     0.000       0.0
      B-stoploc.airport_name      0.000     0.000     0.000       0.0
             B-time_relative      0.000     0.000     0.000       0.0
                      I-time      0.000     0.000     0.000       0.0
    I-return_date.day_number      0.000     0.000     0.000       0.0
          I-meal_description      0.000     0.000     0.000       0.0
B-return_date.today_relative      0.000     0.000     0.000       0.0
      B-return_date.day_name      0.000     0.000     0.000       2.0

                   micro avg      0.957     0.959     0.958    3657.0
                   macro avg      0.689     0.656     0.657    3657.0
                weighted avg      0.960     0.959     0.955    3657.0

I0716 04:49:34.317782 139795883440000 interactiveshell.py:2882] Best Slot F1: 0.960
I0716 04:49:34.319455 139795883440000 estimator_training.py:186] Not using Distribute Coordinator.
I0716 04:49:34.322131 139795883440000 training.py:612] Running training and evaluation locally (non-distributed).
I0716 04:49:34.325804 139795883440000 training.py:700] Start train and evaluate loop. The evaluate will happen after every checkpoint. Checkpoint frequency is determined based on RunConfig arguments: save_checkpoints_steps 312 or save_checkpoints_secs None.
I0716 04:49:34.390808 139795883440000 estimator.py:1145] Calling model_fn.
I0716 04:49:35.879683 139795883440000 saver.py:1499] Saver not created because there are no variables in the graph to restore
I0716 04:49:36.453143 139795883440000 <ipython-input-6-d6747b40360b>:55] 
[<tf.Variable 'embedding:0' shape=(750, 300) dtype=float32_ref>,
 <tf.Variable 'module/aggregation/weights:0' shape=(3,) dtype=float32>,
 <tf.Variable 'module/aggregation/scaling:0' shape=() dtype=float32>,
 <tf.Variable 'dense/kernel:0' shape=(1324, 300) dtype=float32_ref>,
 <tf.Variable 'dense/bias:0' shape=(300,) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/gates/kernel:0' shape=(600, 600) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/gates/bias:0' shape=(600,) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/candidate/kernel:0' shape=(600, 300) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/candidate/bias:0' shape=(300,) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/gates/kernel:0' shape=(600, 600) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/gates/bias:0' shape=(600,) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/candidate/kernel:0' shape=(600, 300) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/candidate/bias:0' shape=(300,) dtype=float32_ref>,
 <tf.Variable 'dense_1/kernel:0' shape=(600, 23) dtype=float32_ref>,
 <tf.Variable 'dense_1/bias:0' shape=(23,) dtype=float32_ref>,
 <tf.Variable 'dense_2/kernel:0' shape=(600, 122) dtype=float32_ref>,
 <tf.Variable 'dense_2/bias:0' shape=(122,) dtype=float32_ref>]
I0716 04:49:37.719774 139795883440000 estimator.py:1147] Done calling model_fn.
I0716 04:49:37.724008 139795883440000 basic_session_run_hooks.py:541] Create CheckpointSaverHook.
I0716 04:49:38.057800 139795883440000 monitored_session.py:240] Graph was finalized.
I0716 04:49:38.076627 139795883440000 saver.py:1280] Restoring parameters from ../model/elmo_bigru/model.ckpt-9360
I0716 04:49:39.114448 139795883440000 session_manager.py:500] Running local_init_op.
I0716 04:49:39.180683 139795883440000 session_manager.py:502] Done running local_init_op.
I0716 04:49:41.989519 139795883440000 basic_session_run_hooks.py:606] Saving checkpoints for 9360 into ../model/elmo_bigru/model.ckpt.
Reading ../data/atis.train.w-intent.iob
I0716 04:49:46.790730 139795883440000 basic_session_run_hooks.py:262] loss = 0.01551132, step = 9360
I0716 04:49:46.792497 139795883440000 basic_session_run_hooks.py:262] lr = 0.00011190023
I0716 04:50:14.630988 139795883440000 basic_session_run_hooks.py:692] global_step/sec: 3.59184
I0716 04:50:14.636307 139795883440000 basic_session_run_hooks.py:260] loss = 0.011015689, step = 9460 (27.846 sec)
I0716 04:50:14.639242 139795883440000 basic_session_run_hooks.py:260] lr = 0.00011072745 (27.847 sec)
I0716 04:50:41.028852 139795883440000 basic_session_run_hooks.py:692] global_step/sec: 3.78819
I0716 04:50:41.037099 139795883440000 basic_session_run_hooks.py:260] loss = 0.013117211, step = 9560 (26.401 sec)
I0716 04:50:41.039311 139795883440000 basic_session_run_hooks.py:260] lr = 0.00010956693 (26.400 sec)
I0716 04:51:06.975775 139795883440000 basic_session_run_hooks.py:692] global_step/sec: 3.85402
I0716 04:51:06.979199 139795883440000 basic_session_run_hooks.py:260] loss = 0.015385278, step = 9660 (25.942 sec)
I0716 04:51:06.983596 139795883440000 basic_session_run_hooks.py:260] lr = 0.00010841859 (25.944 sec)
I0716 04:51:09.733838 139795883440000 basic_session_run_hooks.py:606] Saving checkpoints for 9672 into ../model/elmo_bigru/model.ckpt.
I0716 04:51:11.702174 139795883440000 estimator.py:1145] Calling model_fn.
I0716 04:51:13.322754 139795883440000 saver.py:1499] Saver not created because there are no variables in the graph to restore
I0716 04:51:13.908298 139795883440000 estimator.py:1147] Done calling model_fn.
I0716 04:51:13.939200 139795883440000 evaluation.py:255] Starting evaluation at 2019-07-16T04:51:13Z
I0716 04:51:14.166243 139795883440000 monitored_session.py:240] Graph was finalized.
I0716 04:51:14.183458 139795883440000 saver.py:1280] Restoring parameters from ../model/elmo_bigru/model.ckpt-9672
I0716 04:51:15.079905 139795883440000 session_manager.py:500] Running local_init_op.
I0716 04:51:15.133797 139795883440000 session_manager.py:502] Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
I0716 04:52:13.750108 139795883440000 evaluation.py:275] Finished evaluation at 2019-07-16-04:52:13
I0716 04:52:13.751840 139795883440000 estimator.py:2039] Saving dict for global step 9672: global_step = 9672, loss = 0.36763337
I0716 04:52:13.758711 139795883440000 estimator.py:2099] Saving 'checkpoint_path' summary for global step 9672: ../model/elmo_bigru/model.ckpt-9672
I0716 04:52:13.845093 139795883440000 estimator.py:368] Loss for final step: 0.00026629955.
Reading ../data/atis.test.w-intent.iob
I0716 04:52:14.444941 139795883440000 estimator.py:1145] Calling model_fn.
I0716 04:52:15.972132 139795883440000 saver.py:1499] Saver not created because there are no variables in the graph to restore
I0716 04:52:16.502604 139795883440000 estimator.py:1147] Done calling model_fn.
I0716 04:52:16.716974 139795883440000 monitored_session.py:240] Graph was finalized.
I0716 04:52:16.734670 139795883440000 saver.py:1280] Restoring parameters from ../model/elmo_bigru/model.ckpt-9672
I0716 04:52:17.638433 139795883440000 session_manager.py:500] Running local_init_op.
I0716 04:52:17.689142 139795883440000 session_manager.py:502] Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.
  'precision', 'predicted', average, warn_for)
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1439: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples.
  'recall', 'true', average, warn_for)
I0716 04:53:15.658815 139795883440000 interactiveshell.py:2882] 
                                          precision    recall  f1-score   support

                             atis_flight      0.984     0.991     0.987       632
                            atis_airfare      0.941     1.000     0.970        48
                     atis_ground_service      1.000     1.000     1.000        36
                            atis_airline      1.000     1.000     1.000        38
                       atis_abbreviation      0.943     1.000     0.971        33
                           atis_aircraft      1.000     0.889     0.941         9
                        atis_flight_time      0.500     1.000     0.667         1
                           atis_quantity      0.429     1.000     0.600         3
                atis_flight#atis_airfare      0.833     0.417     0.556        12
                            atis_airport      0.947     1.000     0.973        18
                           atis_distance      1.000     1.000     1.000        10
                               atis_city      0.750     0.500     0.600         6
                        atis_ground_fare      1.000     0.857     0.923         7
                           atis_capacity      0.952     0.952     0.952        21
                          atis_flight_no      0.889     1.000     0.941         8
                               atis_meal      1.000     0.833     0.909         6
                        atis_restriction      0.000     0.000     0.000         0
             atis_airline#atis_flight_no      0.000     0.000     0.000         0
    atis_ground_service#atis_ground_fare      0.000     0.000     0.000         0
           atis_airfare#atis_flight_time      0.000     0.000     0.000         0
                           atis_cheapest      0.000     0.000     0.000         0
atis_aircraft#atis_flight#atis_flight_no      0.000     0.000     0.000         0

                               micro avg      0.972     0.977     0.975       888
                               macro avg      0.644     0.656     0.636       888
                            weighted avg      0.974     0.977     0.974       888

I0716 04:53:15.698364 139795883440000 interactiveshell.py:2882] 
                              precision    recall  f1-score   support

                           O      0.000     0.000     0.000       0.0
           B-toloc.city_name      0.979     0.993     0.986     716.0
         B-fromloc.city_name      0.992     0.999     0.995     704.0
           I-toloc.city_name      0.981     0.981     0.981     265.0
      B-depart_date.day_name      0.991     0.991     0.991     212.0
              B-airline_name      1.000     1.000     1.000     101.0
         I-fromloc.city_name      0.973     1.000     0.986     177.0
 B-depart_time.period_of_day      0.992     0.908     0.948     130.0
              I-airline_name      1.000     1.000     1.000      65.0
    B-depart_date.day_number      0.981     0.964     0.972      55.0
    B-depart_date.month_name      0.982     0.964     0.973      56.0
          B-depart_time.time      0.864     1.000     0.927      57.0
                B-round_trip      1.000     1.000     1.000      73.0
             B-cost_relative      0.973     0.973     0.973      37.0
                I-round_trip      1.000     1.000     1.000      71.0
                B-flight_mod      0.828     1.000     0.906      24.0
 B-depart_time.time_relative      0.955     0.985     0.970      65.0
          I-depart_time.time      0.912     1.000     0.954      52.0
         B-stoploc.city_name      0.833     1.000     0.909      20.0
                 B-city_name      0.902     0.649     0.755      57.0
                B-class_type      0.960     1.000     0.980      24.0
          B-arrive_time.time      0.971     0.971     0.971      34.0
 B-arrive_time.time_relative      0.967     0.935     0.951      31.0
                I-class_type      1.000     1.000     1.000      17.0
               B-flight_stop      1.000     1.000     1.000      21.0
          I-arrive_time.time      1.000     0.971     0.986      35.0
              B-airline_code      0.971     1.000     0.986      34.0
    I-depart_date.day_number      1.000     0.933     0.966      15.0
      I-fromloc.airport_name      0.441     1.000     0.612      15.0
      B-fromloc.airport_name      0.462     1.000     0.632      12.0
      B-arrive_date.day_name      0.786     1.000     0.880      11.0
          B-toloc.state_code      1.000     1.000     1.000      18.0
B-depart_date.today_relative      1.000     0.889     0.941       9.0
             B-flight_number      0.733     1.000     0.846      11.0
 B-depart_date.date_relative      0.944     1.000     0.971      17.0
          B-toloc.state_name      0.794     0.964     0.871      28.0
           B-fare_basis_code      0.895     1.000     0.944      17.0
               B-flight_time      1.000     1.000     1.000       1.0
                        B-or      1.000     1.000     1.000       3.0
 B-arrive_time.period_of_day      0.750     1.000     0.857       6.0
          B-meal_description      1.000     0.900     0.947      10.0
             I-cost_relative      1.000     0.667     0.800       3.0
              I-airport_name      0.846     0.379     0.524      29.0
               B-fare_amount      1.000     1.000     1.000       2.0
               I-fare_amount      1.000     1.000     1.000       2.0
                 I-city_name      1.000     0.633     0.776      30.0
        I-toloc.airport_name      1.000     1.000     1.000       3.0
            B-transport_type      1.000     1.000     1.000      10.0
    B-arrive_date.month_name      0.714     0.833     0.769       6.0
    B-arrive_date.day_number      0.714     0.833     0.769       6.0
         I-stoploc.city_name      0.909     1.000     0.952      10.0
                      B-meal      0.941     1.000     0.970      16.0
        B-fromloc.state_code      1.000     1.000     1.000      23.0
    B-depart_time.period_mod      1.000     1.000     1.000       5.0
                   B-connect      1.000     1.000     1.000       6.0
               B-flight_days      1.000     1.000     1.000      10.0
        B-toloc.airport_name      1.000     1.000     1.000       3.0
        B-fromloc.state_name      1.000     1.000     1.000      17.0
              B-airport_name      0.778     0.333     0.467      21.0
                   B-economy      1.000     1.000     1.000       6.0
               I-flight_time      1.000     1.000     1.000       1.0
             B-aircraft_code      1.000     0.879     0.935      33.0
                       B-mod      1.000     0.500     0.667       2.0
              B-airport_code      1.000     0.444     0.615       9.0
    B-depart_time.start_time      1.000     0.667     0.800       3.0
      B-depart_time.end_time      1.000     0.333     0.500       3.0
          B-depart_date.year      1.000     1.000     1.000       3.0
            I-transport_type      0.000     0.000     0.000       1.0
          B-restriction_code      0.800     1.000     0.889       4.0
    B-arrive_time.start_time      0.889     1.000     0.941       8.0
        B-toloc.airport_code      1.000     1.000     1.000       4.0
      B-arrive_time.end_time      0.889     1.000     0.941       8.0
      I-arrive_time.end_time      0.889     1.000     0.941       8.0
      I-depart_time.end_time      1.000     0.333     0.500       3.0
               I-flight_stop      0.000     0.000     0.000       0.0
      B-fromloc.airport_code      0.556     1.000     0.714       5.0
          I-restriction_code      1.000     1.000     1.000       3.0
    I-depart_time.start_time      1.000     1.000     1.000       1.0
          I-toloc.state_name      1.000     1.000     1.000       1.0
I-depart_date.today_relative      0.000     0.000     0.000       0.0
 B-arrive_date.date_relative      1.000     1.000     1.000       2.0
                I-flight_mod      1.000     0.167     0.286       6.0
                   I-economy      0.000     0.000     0.000       0.0
 B-return_date.date_relative      1.000     0.333     0.500       3.0
        I-fromloc.state_name      1.000     1.000     1.000       1.0
                B-state_code      1.000     1.000     1.000       1.0
    I-arrive_time.start_time      1.000     1.000     1.000       1.0
    I-arrive_date.day_number      0.000     0.000     0.000       0.0
                 B-meal_code      0.000     0.000     0.000       1.0
 I-depart_time.period_of_day      1.000     1.000     1.000       1.0
                  B-day_name      1.000     0.500     0.667       2.0
             B-period_of_day      1.000     0.750     0.857       4.0
        B-stoploc.state_code      0.000     0.000     0.000       0.0
    B-return_date.month_name      0.000     0.000     0.000       0.0
    B-return_date.day_number      0.000     0.000     0.000       0.0
    B-arrive_time.period_mod      0.000     0.000     0.000       0.0
                 I-meal_code      0.000     0.000     0.000       0.0
        B-toloc.country_name      1.000     1.000     1.000       1.0
                 B-days_code      0.000     0.000     0.000       1.0
 I-arrive_time.period_of_day      0.000     0.000     0.000       0.0
            I-today_relative      0.000     0.000     0.000       0.0
 B-return_time.period_of_day      0.000     0.000     0.000       0.0
                      B-time      0.000     0.000     0.000       0.0
           I-fare_basis_code      0.000     0.000     0.000       0.0
 I-arrive_time.time_relative      1.000     1.000     1.000       4.0
 I-depart_time.time_relative      0.000     0.000     0.000       1.0
            B-today_relative      0.000     0.000     0.000       0.0
                B-state_name      0.000     0.000     0.000       9.0
B-arrive_date.today_relative      0.000     0.000     0.000       0.0
    B-return_time.period_mod      0.000     0.000     0.000       0.0
                B-month_name      0.000     0.000     0.000       0.0
                B-day_number      0.000     0.000     0.000       0.0
 I-return_date.date_relative      0.667     0.667     0.667       3.0
I-return_date.today_relative      0.000     0.000     0.000       0.0
      B-stoploc.airport_name      0.000     0.000     0.000       0.0
             B-time_relative      0.000     0.000     0.000       0.0
                      I-time      0.000     0.000     0.000       0.0
    I-return_date.day_number      0.000     0.000     0.000       0.0
          I-meal_description      0.000     0.000     0.000       0.0
B-return_date.today_relative      0.000     0.000     0.000       0.0
      B-return_date.day_name      0.000     0.000     0.000       2.0

                   micro avg      0.959     0.960     0.960    3657.0
                   macro avg      0.681     0.655     0.653    3657.0
                weighted avg      0.963     0.960     0.957    3657.0

I0716 04:53:15.721315 139795883440000 interactiveshell.py:2882] Best Slot F1: 0.960
I0716 04:53:15.723052 139795883440000 estimator_training.py:186] Not using Distribute Coordinator.
I0716 04:53:15.724583 139795883440000 training.py:612] Running training and evaluation locally (non-distributed).
I0716 04:53:15.726242 139795883440000 training.py:700] Start train and evaluate loop. The evaluate will happen after every checkpoint. Checkpoint frequency is determined based on RunConfig arguments: save_checkpoints_steps 312 or save_checkpoints_secs None.
I0716 04:53:15.794797 139795883440000 estimator.py:1145] Calling model_fn.
I0716 04:53:17.253643 139795883440000 saver.py:1499] Saver not created because there are no variables in the graph to restore
I0716 04:53:17.837841 139795883440000 <ipython-input-6-d6747b40360b>:55] 
[<tf.Variable 'embedding:0' shape=(750, 300) dtype=float32_ref>,
 <tf.Variable 'module/aggregation/weights:0' shape=(3,) dtype=float32>,
 <tf.Variable 'module/aggregation/scaling:0' shape=() dtype=float32>,
 <tf.Variable 'dense/kernel:0' shape=(1324, 300) dtype=float32_ref>,
 <tf.Variable 'dense/bias:0' shape=(300,) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/gates/kernel:0' shape=(600, 600) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/gates/bias:0' shape=(600,) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/candidate/kernel:0' shape=(600, 300) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/candidate/bias:0' shape=(300,) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/gates/kernel:0' shape=(600, 600) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/gates/bias:0' shape=(600,) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/candidate/kernel:0' shape=(600, 300) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/candidate/bias:0' shape=(300,) dtype=float32_ref>,
 <tf.Variable 'dense_1/kernel:0' shape=(600, 23) dtype=float32_ref>,
 <tf.Variable 'dense_1/bias:0' shape=(23,) dtype=float32_ref>,
 <tf.Variable 'dense_2/kernel:0' shape=(600, 122) dtype=float32_ref>,
 <tf.Variable 'dense_2/bias:0' shape=(122,) dtype=float32_ref>]
I0716 04:53:19.109848 139795883440000 estimator.py:1147] Done calling model_fn.
I0716 04:53:19.113912 139795883440000 basic_session_run_hooks.py:541] Create CheckpointSaverHook.
I0716 04:53:19.430395 139795883440000 monitored_session.py:240] Graph was finalized.
I0716 04:53:19.450618 139795883440000 saver.py:1280] Restoring parameters from ../model/elmo_bigru/model.ckpt-9672
I0716 04:53:20.499954 139795883440000 session_manager.py:500] Running local_init_op.
I0716 04:53:20.569100 139795883440000 session_manager.py:502] Done running local_init_op.
I0716 04:53:23.373302 139795883440000 basic_session_run_hooks.py:606] Saving checkpoints for 9672 into ../model/elmo_bigru/model.ckpt.
Reading ../data/atis.train.w-intent.iob
I0716 04:53:28.404636 139795883440000 basic_session_run_hooks.py:262] loss = 0.020569747, step = 9672
I0716 04:53:28.406489 139795883440000 basic_session_run_hooks.py:262] lr = 0.000108281594
I0716 04:53:55.312132 139795883440000 basic_session_run_hooks.py:692] global_step/sec: 3.71635
I0716 04:53:55.315697 139795883440000 basic_session_run_hooks.py:260] loss = 0.012076699, step = 9772 (26.911 sec)
I0716 04:53:55.319453 139795883440000 basic_session_run_hooks.py:260] lr = 0.000107146734 (26.913 sec)
I0716 04:54:21.683307 139795883440000 basic_session_run_hooks.py:692] global_step/sec: 3.79202
I0716 04:54:21.689350 139795883440000 basic_session_run_hooks.py:260] loss = 0.013083725, step = 9872 (26.374 sec)
I0716 04:54:21.691671 139795883440000 basic_session_run_hooks.py:260] lr = 0.00010602375 (26.372 sec)
I0716 04:54:48.945347 139795883440000 basic_session_run_hooks.py:692] global_step/sec: 3.66811
I0716 04:54:48.953207 139795883440000 basic_session_run_hooks.py:260] loss = 0.013616568, step = 9972 (27.264 sec)
I0716 04:54:48.955022 139795883440000 basic_session_run_hooks.py:260] lr = 0.00010491254 (27.263 sec)
I0716 04:54:51.518794 139795883440000 basic_session_run_hooks.py:606] Saving checkpoints for 9984 into ../model/elmo_bigru/model.ckpt.
I0716 04:54:53.406455 139795883440000 estimator.py:1145] Calling model_fn.
I0716 04:54:55.017331 139795883440000 saver.py:1499] Saver not created because there are no variables in the graph to restore
I0716 04:54:55.584659 139795883440000 estimator.py:1147] Done calling model_fn.
I0716 04:54:55.615095 139795883440000 evaluation.py:255] Starting evaluation at 2019-07-16T04:54:55Z
I0716 04:54:55.838979 139795883440000 monitored_session.py:240] Graph was finalized.
I0716 04:54:55.857573 139795883440000 saver.py:1280] Restoring parameters from ../model/elmo_bigru/model.ckpt-9984
I0716 04:54:56.831674 139795883440000 session_manager.py:500] Running local_init_op.
I0716 04:54:56.906322 139795883440000 session_manager.py:502] Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
I0716 04:55:56.055133 139795883440000 evaluation.py:275] Finished evaluation at 2019-07-16-04:55:56
I0716 04:55:56.056856 139795883440000 estimator.py:2039] Saving dict for global step 9984: global_step = 9984, loss = 0.38307405
I0716 04:55:56.065994 139795883440000 estimator.py:2099] Saving 'checkpoint_path' summary for global step 9984: ../model/elmo_bigru/model.ckpt-9984
I0716 04:55:56.149209 139795883440000 estimator.py:368] Loss for final step: 0.0013525934.
Reading ../data/atis.test.w-intent.iob
I0716 04:55:56.767248 139795883440000 estimator.py:1145] Calling model_fn.
I0716 04:55:58.302370 139795883440000 saver.py:1499] Saver not created because there are no variables in the graph to restore
I0716 04:55:58.829959 139795883440000 estimator.py:1147] Done calling model_fn.
I0716 04:55:59.043296 139795883440000 monitored_session.py:240] Graph was finalized.
I0716 04:55:59.062215 139795883440000 saver.py:1280] Restoring parameters from ../model/elmo_bigru/model.ckpt-9984
I0716 04:55:59.979383 139795883440000 session_manager.py:500] Running local_init_op.
I0716 04:56:00.027868 139795883440000 session_manager.py:502] Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.
  'precision', 'predicted', average, warn_for)
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1439: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples.
  'recall', 'true', average, warn_for)
I0716 04:56:57.841996 139795883440000 interactiveshell.py:2882] 
                                          precision    recall  f1-score   support

                             atis_flight      0.986     0.991     0.988       632
                            atis_airfare      0.941     1.000     0.970        48
                     atis_ground_service      1.000     1.000     1.000        36
                            atis_airline      0.974     1.000     0.987        38
                       atis_abbreviation      0.943     1.000     0.971        33
                           atis_aircraft      1.000     0.889     0.941         9
                        atis_flight_time      0.500     1.000     0.667         1
                           atis_quantity      0.429     1.000     0.600         3
                atis_flight#atis_airfare      0.833     0.417     0.556        12
                            atis_airport      0.944     0.944     0.944        18
                           atis_distance      1.000     1.000     1.000        10
                               atis_city      0.800     0.667     0.727         6
                        atis_ground_fare      1.000     0.857     0.923         7
                           atis_capacity      0.952     0.952     0.952        21
                          atis_flight_no      0.889     1.000     0.941         8
                               atis_meal      1.000     0.833     0.909         6
                        atis_restriction      0.000     0.000     0.000         0
             atis_airline#atis_flight_no      0.000     0.000     0.000         0
    atis_ground_service#atis_ground_fare      0.000     0.000     0.000         0
           atis_airfare#atis_flight_time      0.000     0.000     0.000         0
                           atis_cheapest      0.000     0.000     0.000         0
atis_aircraft#atis_flight#atis_flight_no      0.000     0.000     0.000         0

                               micro avg      0.972     0.977     0.975       888
                               macro avg      0.645     0.661     0.640       888
                            weighted avg      0.974     0.977     0.974       888

I0716 04:56:57.888020 139795883440000 interactiveshell.py:2882] 
                              precision    recall  f1-score   support

                           O      0.000     0.000     0.000       0.0
           B-toloc.city_name      0.975     0.994     0.985     716.0
         B-fromloc.city_name      0.990     0.999     0.994     704.0
           I-toloc.city_name      0.978     0.985     0.981     265.0
      B-depart_date.day_name      0.986     0.991     0.988     212.0
              B-airline_name      1.000     1.000     1.000     101.0
         I-fromloc.city_name      0.973     1.000     0.986     177.0
 B-depart_time.period_of_day      0.984     0.923     0.952     130.0
              I-airline_name      1.000     1.000     1.000      65.0
    B-depart_date.day_number      0.981     0.964     0.972      55.0
    B-depart_date.month_name      0.982     0.964     0.973      56.0
          B-depart_time.time      0.864     1.000     0.927      57.0
                B-round_trip      1.000     1.000     1.000      73.0
             B-cost_relative      0.973     0.973     0.973      37.0
                I-round_trip      1.000     1.000     1.000      71.0
                B-flight_mod      0.828     1.000     0.906      24.0
 B-depart_time.time_relative      0.955     0.985     0.970      65.0
          I-depart_time.time      0.945     1.000     0.972      52.0
         B-stoploc.city_name      0.909     1.000     0.952      20.0
                 B-city_name      0.881     0.649     0.747      57.0
                B-class_type      0.960     1.000     0.980      24.0
          B-arrive_time.time      0.971     0.971     0.971      34.0
 B-arrive_time.time_relative      0.967     0.935     0.951      31.0
                I-class_type      1.000     1.000     1.000      17.0
               B-flight_stop      1.000     1.000     1.000      21.0
          I-arrive_time.time      1.000     0.971     0.986      35.0
              B-airline_code      0.895     1.000     0.944      34.0
    I-depart_date.day_number      1.000     1.000     1.000      15.0
      I-fromloc.airport_name      0.441     1.000     0.612      15.0
      B-fromloc.airport_name      0.462     1.000     0.632      12.0
      B-arrive_date.day_name      0.769     0.909     0.833      11.0
          B-toloc.state_code      1.000     1.000     1.000      18.0
B-depart_date.today_relative      1.000     0.889     0.941       9.0
             B-flight_number      0.733     1.000     0.846      11.0
 B-depart_date.date_relative      0.944     1.000     0.971      17.0
          B-toloc.state_name      0.833     0.893     0.862      28.0
           B-fare_basis_code      0.850     1.000     0.919      17.0
               B-flight_time      1.000     1.000     1.000       1.0
                        B-or      1.000     1.000     1.000       3.0
 B-arrive_time.period_of_day      0.750     1.000     0.857       6.0
          B-meal_description      1.000     0.900     0.947      10.0
             I-cost_relative      1.000     0.667     0.800       3.0
              I-airport_name      0.846     0.379     0.524      29.0
               B-fare_amount      1.000     1.000     1.000       2.0
               I-fare_amount      1.000     1.000     1.000       2.0
                 I-city_name      1.000     0.600     0.750      30.0
        I-toloc.airport_name      1.000     1.000     1.000       3.0
            B-transport_type      1.000     1.000     1.000      10.0
    B-arrive_date.month_name      0.714     0.833     0.769       6.0
    B-arrive_date.day_number      0.714     0.833     0.769       6.0
         I-stoploc.city_name      1.000     1.000     1.000      10.0
                      B-meal      1.000     1.000     1.000      16.0
        B-fromloc.state_code      1.000     1.000     1.000      23.0
    B-depart_time.period_mod      1.000     1.000     1.000       5.0
                   B-connect      1.000     1.000     1.000       6.0
               B-flight_days      1.000     1.000     1.000      10.0
        B-toloc.airport_name      1.000     1.000     1.000       3.0
        B-fromloc.state_name      1.000     1.000     1.000      17.0
              B-airport_name      0.636     0.333     0.437      21.0
                   B-economy      1.000     1.000     1.000       6.0
               I-flight_time      1.000     1.000     1.000       1.0
             B-aircraft_code      1.000     0.818     0.900      33.0
                       B-mod      1.000     0.500     0.667       2.0
              B-airport_code      1.000     0.444     0.615       9.0
    B-depart_time.start_time      1.000     1.000     1.000       3.0
      B-depart_time.end_time      1.000     0.667     0.800       3.0
          B-depart_date.year      1.000     1.000     1.000       3.0
            I-transport_type      0.000     0.000     0.000       1.0
          B-restriction_code      1.000     1.000     1.000       4.0
    B-arrive_time.start_time      1.000     1.000     1.000       8.0
        B-toloc.airport_code      1.000     0.750     0.857       4.0
      B-arrive_time.end_time      1.000     1.000     1.000       8.0
      I-arrive_time.end_time      1.000     1.000     1.000       8.0
      I-depart_time.end_time      1.000     0.667     0.800       3.0
               I-flight_stop      0.000     0.000     0.000       0.0
      B-fromloc.airport_code      0.556     1.000     0.714       5.0
          I-restriction_code      1.000     1.000     1.000       3.0
    I-depart_time.start_time      1.000     1.000     1.000       1.0
          I-toloc.state_name      1.000     1.000     1.000       1.0
I-depart_date.today_relative      0.000     0.000     0.000       0.0
 B-arrive_date.date_relative      1.000     1.000     1.000       2.0
                I-flight_mod      1.000     0.167     0.286       6.0
                   I-economy      0.000     0.000     0.000       0.0
 B-return_date.date_relative      1.000     0.667     0.800       3.0
        I-fromloc.state_name      1.000     1.000     1.000       1.0
                B-state_code      1.000     1.000     1.000       1.0
    I-arrive_time.start_time      1.000     1.000     1.000       1.0
    I-arrive_date.day_number      0.000     0.000     0.000       0.0
                 B-meal_code      0.000     0.000     0.000       1.0
 I-depart_time.period_of_day      0.000     0.000     0.000       1.0
                  B-day_name      1.000     0.500     0.667       2.0
             B-period_of_day      1.000     0.750     0.857       4.0
        B-stoploc.state_code      0.000     0.000     0.000       0.0
    B-return_date.month_name      0.000     0.000     0.000       0.0
    B-return_date.day_number      0.000     0.000     0.000       0.0
    B-arrive_time.period_mod      0.000     0.000     0.000       0.0
                 I-meal_code      0.000     0.000     0.000       0.0
        B-toloc.country_name      1.000     1.000     1.000       1.0
                 B-days_code      0.000     0.000     0.000       1.0
 I-arrive_time.period_of_day      0.000     0.000     0.000       0.0
            I-today_relative      0.000     0.000     0.000       0.0
 B-return_time.period_of_day      0.000     0.000     0.000       0.0
                      B-time      0.000     0.000     0.000       0.0
           I-fare_basis_code      0.000     0.000     0.000       0.0
 I-arrive_time.time_relative      1.000     1.000     1.000       4.0
 I-depart_time.time_relative      0.000     0.000     0.000       1.0
            B-today_relative      0.000     0.000     0.000       0.0
                B-state_name      0.000     0.000     0.000       9.0
B-arrive_date.today_relative      0.000     0.000     0.000       0.0
    B-return_time.period_mod      0.000     0.000     0.000       0.0
                B-month_name      0.000     0.000     0.000       0.0
                B-day_number      0.000     0.000     0.000       0.0
 I-return_date.date_relative      1.000     0.667     0.800       3.0
I-return_date.today_relative      0.000     0.000     0.000       0.0
      B-stoploc.airport_name      0.000     0.000     0.000       0.0
             B-time_relative      0.000     0.000     0.000       0.0
                      I-time      0.000     0.000     0.000       0.0
    I-return_date.day_number      0.000     0.000     0.000       0.0
          I-meal_description      0.000     0.000     0.000       0.0
B-return_date.today_relative      0.000     0.000     0.000       0.0
      B-return_date.day_name      0.000     0.000     0.000       2.0

                   micro avg      0.959     0.961     0.960    3657.0
                   macro avg      0.680     0.654     0.656    3657.0
                weighted avg      0.962     0.961     0.957    3657.0

I0716 04:56:57.910744 139795883440000 interactiveshell.py:2882] Best Slot F1: 0.960
I0716 04:56:57.912216 139795883440000 estimator_training.py:186] Not using Distribute Coordinator.
I0716 04:56:57.915825 139795883440000 training.py:612] Running training and evaluation locally (non-distributed).
I0716 04:56:57.918539 139795883440000 training.py:700] Start train and evaluate loop. The evaluate will happen after every checkpoint. Checkpoint frequency is determined based on RunConfig arguments: save_checkpoints_steps 312 or save_checkpoints_secs None.
I0716 04:56:57.983797 139795883440000 estimator.py:1145] Calling model_fn.
I0716 04:56:59.463993 139795883440000 saver.py:1499] Saver not created because there are no variables in the graph to restore
I0716 04:57:00.242153 139795883440000 <ipython-input-6-d6747b40360b>:55] 
[<tf.Variable 'embedding:0' shape=(750, 300) dtype=float32_ref>,
 <tf.Variable 'module/aggregation/weights:0' shape=(3,) dtype=float32>,
 <tf.Variable 'module/aggregation/scaling:0' shape=() dtype=float32>,
 <tf.Variable 'dense/kernel:0' shape=(1324, 300) dtype=float32_ref>,
 <tf.Variable 'dense/bias:0' shape=(300,) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/gates/kernel:0' shape=(600, 600) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/gates/bias:0' shape=(600,) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/candidate/kernel:0' shape=(600, 300) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/candidate/bias:0' shape=(300,) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/gates/kernel:0' shape=(600, 600) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/gates/bias:0' shape=(600,) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/candidate/kernel:0' shape=(600, 300) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/candidate/bias:0' shape=(300,) dtype=float32_ref>,
 <tf.Variable 'dense_1/kernel:0' shape=(600, 23) dtype=float32_ref>,
 <tf.Variable 'dense_1/bias:0' shape=(23,) dtype=float32_ref>,
 <tf.Variable 'dense_2/kernel:0' shape=(600, 122) dtype=float32_ref>,
 <tf.Variable 'dense_2/bias:0' shape=(122,) dtype=float32_ref>]
I0716 04:57:01.325490 139795883440000 estimator.py:1147] Done calling model_fn.
I0716 04:57:01.329560 139795883440000 basic_session_run_hooks.py:541] Create CheckpointSaverHook.
I0716 04:57:01.651513 139795883440000 monitored_session.py:240] Graph was finalized.
I0716 04:57:01.670848 139795883440000 saver.py:1280] Restoring parameters from ../model/elmo_bigru/model.ckpt-9984
I0716 04:57:02.726634 139795883440000 session_manager.py:500] Running local_init_op.
I0716 04:57:02.803383 139795883440000 session_manager.py:502] Done running local_init_op.
I0716 04:57:05.796596 139795883440000 basic_session_run_hooks.py:606] Saving checkpoints for 9984 into ../model/elmo_bigru/model.ckpt.
Reading ../data/atis.train.w-intent.iob
I0716 04:57:10.651218 139795883440000 basic_session_run_hooks.py:262] loss = 0.016779646, step = 9984
I0716 04:57:10.653019 139795883440000 basic_session_run_hooks.py:262] lr = 0.00010477998
I0716 04:57:37.472784 139795883440000 basic_session_run_hooks.py:692] global_step/sec: 3.72826
I0716 04:57:37.478230 139795883440000 basic_session_run_hooks.py:260] loss = 0.012659789, step = 10084 (26.827 sec)
I0716 04:57:37.483809 139795883440000 basic_session_run_hooks.py:260] lr = 0.000103681814 (26.831 sec)
I0716 04:58:03.187721 139795883440000 basic_session_run_hooks.py:692] global_step/sec: 3.88878
I0716 04:58:03.193797 139795883440000 basic_session_run_hooks.py:260] loss = 0.00961412, step = 10184 (25.716 sec)
I0716 04:58:03.197678 139795883440000 basic_session_run_hooks.py:260] lr = 0.000102595164 (25.714 sec)
I0716 04:58:31.202835 139795883440000 basic_session_run_hooks.py:692] global_step/sec: 3.56951
I0716 04:58:31.212342 139795883440000 basic_session_run_hooks.py:260] loss = 0.016829388, step = 10284 (28.019 sec)
I0716 04:58:31.213922 139795883440000 basic_session_run_hooks.py:260] lr = 0.00010151988 (28.016 sec)
I0716 04:58:33.982954 139795883440000 basic_session_run_hooks.py:606] Saving checkpoints for 10296 into ../model/elmo_bigru/model.ckpt.
I0716 04:58:35.982879 139795883440000 estimator.py:1145] Calling model_fn.
I0716 04:58:37.380511 139795883440000 saver.py:1499] Saver not created because there are no variables in the graph to restore
I0716 04:58:37.954942 139795883440000 estimator.py:1147] Done calling model_fn.
I0716 04:58:37.984667 139795883440000 evaluation.py:255] Starting evaluation at 2019-07-16T04:58:37Z
I0716 04:58:38.220102 139795883440000 monitored_session.py:240] Graph was finalized.
I0716 04:58:38.235623 139795883440000 saver.py:1280] Restoring parameters from ../model/elmo_bigru/model.ckpt-10296
I0716 04:58:39.129740 139795883440000 session_manager.py:500] Running local_init_op.
I0716 04:58:39.201298 139795883440000 session_manager.py:502] Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
I0716 04:59:38.152055 139795883440000 evaluation.py:275] Finished evaluation at 2019-07-16-04:59:38
I0716 04:59:38.153831 139795883440000 estimator.py:2039] Saving dict for global step 10296: global_step = 10296, loss = 0.37506437
I0716 04:59:38.162429 139795883440000 estimator.py:2099] Saving 'checkpoint_path' summary for global step 10296: ../model/elmo_bigru/model.ckpt-10296
I0716 04:59:38.247560 139795883440000 estimator.py:368] Loss for final step: 0.01134271.
Reading ../data/atis.test.w-intent.iob
I0716 04:59:38.860542 139795883440000 estimator.py:1145] Calling model_fn.
I0716 04:59:40.533974 139795883440000 saver.py:1499] Saver not created because there are no variables in the graph to restore
I0716 04:59:41.061809 139795883440000 estimator.py:1147] Done calling model_fn.
I0716 04:59:41.465726 139795883440000 monitored_session.py:240] Graph was finalized.
I0716 04:59:41.481816 139795883440000 saver.py:1280] Restoring parameters from ../model/elmo_bigru/model.ckpt-10296
I0716 04:59:42.392911 139795883440000 session_manager.py:500] Running local_init_op.
I0716 04:59:42.440152 139795883440000 session_manager.py:502] Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.
  'precision', 'predicted', average, warn_for)
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1439: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples.
  'recall', 'true', average, warn_for)
I0716 05:00:40.164161 139795883440000 interactiveshell.py:2882] 
                                          precision    recall  f1-score   support

                             atis_flight      0.984     0.992     0.988       632
                            atis_airfare      0.941     1.000     0.970        48
                     atis_ground_service      1.000     1.000     1.000        36
                            atis_airline      1.000     1.000     1.000        38
                       atis_abbreviation      0.943     1.000     0.971        33
                           atis_aircraft      1.000     0.889     0.941         9
                        atis_flight_time      0.500     1.000     0.667         1
                           atis_quantity      0.429     1.000     0.600         3
                atis_flight#atis_airfare      0.833     0.417     0.556        12
                            atis_airport      1.000     1.000     1.000        18
                           atis_distance      1.000     1.000     1.000        10
                               atis_city      0.600     0.500     0.545         6
                        atis_ground_fare      1.000     0.857     0.923         7
                           atis_capacity      0.952     0.952     0.952        21
                          atis_flight_no      0.889     1.000     0.941         8
                               atis_meal      1.000     0.667     0.800         6
                        atis_restriction      0.000     0.000     0.000         0
             atis_airline#atis_flight_no      0.000     0.000     0.000         0
    atis_ground_service#atis_ground_fare      0.000     0.000     0.000         0
           atis_airfare#atis_flight_time      0.000     0.000     0.000         0
                           atis_cheapest      0.000     0.000     0.000         0
atis_aircraft#atis_flight#atis_flight_no      0.000     0.000     0.000         0

                               micro avg      0.972     0.977     0.975       888
                               macro avg      0.640     0.649     0.630       888
                            weighted avg      0.974     0.977     0.974       888

I0716 05:00:40.202049 139795883440000 interactiveshell.py:2882] 
                              precision    recall  f1-score   support

                           O      0.000     0.000     0.000       0.0
           B-toloc.city_name      0.975     0.993     0.984     716.0
         B-fromloc.city_name      0.989     0.999     0.994     704.0
           I-toloc.city_name      0.978     0.985     0.981     265.0
      B-depart_date.day_name      0.991     0.991     0.991     212.0
              B-airline_name      1.000     1.000     1.000     101.0
         I-fromloc.city_name      0.967     1.000     0.983     177.0
 B-depart_time.period_of_day      0.992     0.923     0.956     130.0
              I-airline_name      1.000     1.000     1.000      65.0
    B-depart_date.day_number      0.981     0.964     0.972      55.0
    B-depart_date.month_name      0.982     0.964     0.973      56.0
          B-depart_time.time      0.864     1.000     0.927      57.0
                B-round_trip      1.000     1.000     1.000      73.0
             B-cost_relative      0.973     0.973     0.973      37.0
                I-round_trip      1.000     1.000     1.000      71.0
                B-flight_mod      0.828     1.000     0.906      24.0
 B-depart_time.time_relative      0.955     0.985     0.970      65.0
          I-depart_time.time      0.945     1.000     0.972      52.0
         B-stoploc.city_name      0.870     1.000     0.930      20.0
                 B-city_name      0.949     0.649     0.771      57.0
                B-class_type      0.960     1.000     0.980      24.0
          B-arrive_time.time      0.971     0.971     0.971      34.0
 B-arrive_time.time_relative      0.967     0.935     0.951      31.0
                I-class_type      1.000     1.000     1.000      17.0
               B-flight_stop      1.000     1.000     1.000      21.0
          I-arrive_time.time      1.000     0.971     0.986      35.0
              B-airline_code      0.944     1.000     0.971      34.0
    I-depart_date.day_number      1.000     0.933     0.966      15.0
      I-fromloc.airport_name      0.441     1.000     0.612      15.0
      B-fromloc.airport_name      0.462     1.000     0.632      12.0
      B-arrive_date.day_name      0.786     1.000     0.880      11.0
          B-toloc.state_code      1.000     1.000     1.000      18.0
B-depart_date.today_relative      1.000     0.889     0.941       9.0
             B-flight_number      0.733     1.000     0.846      11.0
 B-depart_date.date_relative      0.944     1.000     0.971      17.0
          B-toloc.state_name      0.833     0.893     0.862      28.0
           B-fare_basis_code      0.895     1.000     0.944      17.0
               B-flight_time      1.000     1.000     1.000       1.0
                        B-or      1.000     1.000     1.000       3.0
 B-arrive_time.period_of_day      0.750     1.000     0.857       6.0
          B-meal_description      1.000     0.900     0.947      10.0
             I-cost_relative      1.000     0.667     0.800       3.0
              I-airport_name      0.846     0.379     0.524      29.0
               B-fare_amount      1.000     1.000     1.000       2.0
               I-fare_amount      1.000     1.000     1.000       2.0
                 I-city_name      1.000     0.567     0.723      30.0
        I-toloc.airport_name      1.000     1.000     1.000       3.0
            B-transport_type      1.000     1.000     1.000      10.0
    B-arrive_date.month_name      0.714     0.833     0.769       6.0
    B-arrive_date.day_number      0.714     0.833     0.769       6.0
         I-stoploc.city_name      1.000     1.000     1.000      10.0
                      B-meal      1.000     1.000     1.000      16.0
        B-fromloc.state_code      1.000     1.000     1.000      23.0
    B-depart_time.period_mod      1.000     1.000     1.000       5.0
                   B-connect      1.000     1.000     1.000       6.0
               B-flight_days      1.000     1.000     1.000      10.0
        B-toloc.airport_name      1.000     1.000     1.000       3.0
        B-fromloc.state_name      0.944     1.000     0.971      17.0
              B-airport_name      0.667     0.381     0.485      21.0
                   B-economy      1.000     1.000     1.000       6.0
               I-flight_time      1.000     1.000     1.000       1.0
             B-aircraft_code      1.000     0.879     0.935      33.0
                       B-mod      1.000     0.500     0.667       2.0
              B-airport_code      1.000     0.444     0.615       9.0
    B-depart_time.start_time      1.000     1.000     1.000       3.0
      B-depart_time.end_time      1.000     0.667     0.800       3.0
          B-depart_date.year      1.000     1.000     1.000       3.0
            I-transport_type      0.000     0.000     0.000       1.0
          B-restriction_code      1.000     1.000     1.000       4.0
    B-arrive_time.start_time      1.000     1.000     1.000       8.0
        B-toloc.airport_code      1.000     0.750     0.857       4.0
      B-arrive_time.end_time      1.000     1.000     1.000       8.0
      I-arrive_time.end_time      0.889     1.000     0.941       8.0
      I-depart_time.end_time      1.000     0.333     0.500       3.0
               I-flight_stop      0.000     0.000     0.000       0.0
      B-fromloc.airport_code      0.556     1.000     0.714       5.0
          I-restriction_code      1.000     1.000     1.000       3.0
    I-depart_time.start_time      1.000     1.000     1.000       1.0
          I-toloc.state_name      1.000     1.000     1.000       1.0
I-depart_date.today_relative      0.000     0.000     0.000       0.0
 B-arrive_date.date_relative      1.000     1.000     1.000       2.0
                I-flight_mod      1.000     0.167     0.286       6.0
                   I-economy      0.000     0.000     0.000       0.0
 B-return_date.date_relative      1.000     0.667     0.800       3.0
        I-fromloc.state_name      1.000     1.000     1.000       1.0
                B-state_code      1.000     1.000     1.000       1.0
    I-arrive_time.start_time      1.000     1.000     1.000       1.0
    I-arrive_date.day_number      0.000     0.000     0.000       0.0
                 B-meal_code      0.000     0.000     0.000       1.0
 I-depart_time.period_of_day      1.000     1.000     1.000       1.0
                  B-day_name      1.000     0.500     0.667       2.0
             B-period_of_day      1.000     0.750     0.857       4.0
        B-stoploc.state_code      0.000     0.000     0.000       0.0
    B-return_date.month_name      0.000     0.000     0.000       0.0
    B-return_date.day_number      0.000     0.000     0.000       0.0
    B-arrive_time.period_mod      0.000     0.000     0.000       0.0
                 I-meal_code      0.000     0.000     0.000       0.0
        B-toloc.country_name      1.000     1.000     1.000       1.0
                 B-days_code      1.000     1.000     1.000       1.0
 I-arrive_time.period_of_day      0.000     0.000     0.000       0.0
            I-today_relative      0.000     0.000     0.000       0.0
 B-return_time.period_of_day      0.000     0.000     0.000       0.0
                      B-time      0.000     0.000     0.000       0.0
           I-fare_basis_code      0.000     0.000     0.000       0.0
 I-arrive_time.time_relative      1.000     1.000     1.000       4.0
 I-depart_time.time_relative      0.000     0.000     0.000       1.0
            B-today_relative      0.000     0.000     0.000       0.0
                B-state_name      0.000     0.000     0.000       9.0
B-arrive_date.today_relative      0.000     0.000     0.000       0.0
    B-return_time.period_mod      0.000     0.000     0.000       0.0
                B-month_name      0.000     0.000     0.000       0.0
                B-day_number      0.000     0.000     0.000       0.0
 I-return_date.date_relative      1.000     0.667     0.800       3.0
I-return_date.today_relative      0.000     0.000     0.000       0.0
      B-stoploc.airport_name      0.000     0.000     0.000       0.0
             B-time_relative      0.000     0.000     0.000       0.0
                      I-time      0.000     0.000     0.000       0.0
    I-return_date.day_number      0.000     0.000     0.000       0.0
          I-meal_description      0.000     0.000     0.000       0.0
B-return_date.today_relative      0.000     0.000     0.000       0.0
      B-return_date.day_name      0.000     0.000     0.000       2.0

                   micro avg      0.960     0.961     0.960    3657.0
                   macro avg      0.696     0.669     0.670    3657.0
                weighted avg      0.963     0.961     0.958    3657.0

I0716 05:00:40.224560 139795883440000 interactiveshell.py:2882] Best Slot F1: 0.960
I0716 05:00:40.226076 139795883440000 estimator_training.py:186] Not using Distribute Coordinator.
I0716 05:00:40.228124 139795883440000 training.py:612] Running training and evaluation locally (non-distributed).
I0716 05:00:40.230159 139795883440000 training.py:700] Start train and evaluate loop. The evaluate will happen after every checkpoint. Checkpoint frequency is determined based on RunConfig arguments: save_checkpoints_steps 312 or save_checkpoints_secs None.
I0716 05:00:40.298190 139795883440000 estimator.py:1145] Calling model_fn.
I0716 05:00:41.621490 139795883440000 saver.py:1499] Saver not created because there are no variables in the graph to restore
I0716 05:00:42.228225 139795883440000 <ipython-input-6-d6747b40360b>:55] 
[<tf.Variable 'embedding:0' shape=(750, 300) dtype=float32_ref>,
 <tf.Variable 'module/aggregation/weights:0' shape=(3,) dtype=float32>,
 <tf.Variable 'module/aggregation/scaling:0' shape=() dtype=float32>,
 <tf.Variable 'dense/kernel:0' shape=(1324, 300) dtype=float32_ref>,
 <tf.Variable 'dense/bias:0' shape=(300,) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/gates/kernel:0' shape=(600, 600) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/gates/bias:0' shape=(600,) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/candidate/kernel:0' shape=(600, 300) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/candidate/bias:0' shape=(300,) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/gates/kernel:0' shape=(600, 600) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/gates/bias:0' shape=(600,) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/candidate/kernel:0' shape=(600, 300) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/candidate/bias:0' shape=(300,) dtype=float32_ref>,
 <tf.Variable 'dense_1/kernel:0' shape=(600, 23) dtype=float32_ref>,
 <tf.Variable 'dense_1/bias:0' shape=(23,) dtype=float32_ref>,
 <tf.Variable 'dense_2/kernel:0' shape=(600, 122) dtype=float32_ref>,
 <tf.Variable 'dense_2/bias:0' shape=(122,) dtype=float32_ref>]
I0716 05:00:43.507210 139795883440000 estimator.py:1147] Done calling model_fn.
I0716 05:00:43.511086 139795883440000 basic_session_run_hooks.py:541] Create CheckpointSaverHook.
I0716 05:00:43.832129 139795883440000 monitored_session.py:240] Graph was finalized.
I0716 05:00:43.852826 139795883440000 saver.py:1280] Restoring parameters from ../model/elmo_bigru/model.ckpt-10296
I0716 05:00:44.897862 139795883440000 session_manager.py:500] Running local_init_op.
I0716 05:00:44.963641 139795883440000 session_manager.py:502] Done running local_init_op.
I0716 05:00:47.794221 139795883440000 basic_session_run_hooks.py:606] Saving checkpoints for 10296 into ../model/elmo_bigru/model.ckpt.
Reading ../data/atis.train.w-intent.iob
I0716 05:00:52.672652 139795883440000 basic_session_run_hooks.py:262] loss = 0.01384804, step = 10296
I0716 05:00:52.675022 139795883440000 basic_session_run_hooks.py:262] lr = 0.00010139161
I0716 05:01:19.914961 139795883440000 basic_session_run_hooks.py:692] global_step/sec: 3.67065
I0716 05:01:19.922856 139795883440000 basic_session_run_hooks.py:260] loss = 0.011046371, step = 10396 (27.250 sec)
I0716 05:01:19.925280 139795883440000 basic_session_run_hooks.py:260] lr = 0.000100328936 (27.250 sec)
I0716 05:01:46.527574 139795883440000 basic_session_run_hooks.py:692] global_step/sec: 3.75762
I0716 05:01:46.532375 139795883440000 basic_session_run_hooks.py:260] loss = 0.011879209, step = 10496 (26.610 sec)
I0716 05:01:46.536274 139795883440000 basic_session_run_hooks.py:260] lr = 9.927743e-05 (26.611 sec)
I0716 05:02:13.149505 139795883440000 basic_session_run_hooks.py:692] global_step/sec: 3.75629
I0716 05:02:13.151699 139795883440000 basic_session_run_hooks.py:260] loss = 0.01011859, step = 10596 (26.619 sec)
I0716 05:02:13.154321 139795883440000 basic_session_run_hooks.py:260] lr = 9.823692e-05 (26.618 sec)
I0716 05:02:15.924441 139795883440000 basic_session_run_hooks.py:606] Saving checkpoints for 10608 into ../model/elmo_bigru/model.ckpt.
I0716 05:02:18.027540 139795883440000 estimator.py:1145] Calling model_fn.
I0716 05:02:19.667282 139795883440000 saver.py:1499] Saver not created because there are no variables in the graph to restore
I0716 05:02:20.244174 139795883440000 estimator.py:1147] Done calling model_fn.
I0716 05:02:20.274494 139795883440000 evaluation.py:255] Starting evaluation at 2019-07-16T05:02:20Z
I0716 05:02:20.502266 139795883440000 monitored_session.py:240] Graph was finalized.
I0716 05:02:20.528296 139795883440000 saver.py:1280] Restoring parameters from ../model/elmo_bigru/model.ckpt-10608
I0716 05:02:21.418348 139795883440000 session_manager.py:500] Running local_init_op.
I0716 05:02:21.476039 139795883440000 session_manager.py:502] Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
I0716 05:03:20.553541 139795883440000 evaluation.py:275] Finished evaluation at 2019-07-16-05:03:20
I0716 05:03:20.555404 139795883440000 estimator.py:2039] Saving dict for global step 10608: global_step = 10608, loss = 0.36588457
I0716 05:03:20.564051 139795883440000 estimator.py:2099] Saving 'checkpoint_path' summary for global step 10608: ../model/elmo_bigru/model.ckpt-10608
I0716 05:03:20.651795 139795883440000 estimator.py:368] Loss for final step: 0.0058385767.
Reading ../data/atis.test.w-intent.iob
I0716 05:03:21.270508 139795883440000 estimator.py:1145] Calling model_fn.
I0716 05:03:22.821979 139795883440000 saver.py:1499] Saver not created because there are no variables in the graph to restore
I0716 05:03:23.353141 139795883440000 estimator.py:1147] Done calling model_fn.
I0716 05:03:23.571113 139795883440000 monitored_session.py:240] Graph was finalized.
I0716 05:03:23.588160 139795883440000 saver.py:1280] Restoring parameters from ../model/elmo_bigru/model.ckpt-10608
I0716 05:03:24.530436 139795883440000 session_manager.py:500] Running local_init_op.
I0716 05:03:24.580191 139795883440000 session_manager.py:502] Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.
  'precision', 'predicted', average, warn_for)
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1439: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples.
  'recall', 'true', average, warn_for)
I0716 05:04:22.721977 139795883440000 interactiveshell.py:2882] 
                                          precision    recall  f1-score   support

                             atis_flight      0.984     0.992     0.988       632
                            atis_airfare      0.941     1.000     0.970        48
                     atis_ground_service      1.000     1.000     1.000        36
                            atis_airline      1.000     1.000     1.000        38
                       atis_abbreviation      0.943     1.000     0.971        33
                           atis_aircraft      1.000     0.889     0.941         9
                        atis_flight_time      0.500     1.000     0.667         1
                           atis_quantity      0.429     1.000     0.600         3
                atis_flight#atis_airfare      0.833     0.417     0.556        12
                            atis_airport      1.000     1.000     1.000        18
                           atis_distance      1.000     1.000     1.000        10
                               atis_city      0.600     0.500     0.545         6
                        atis_ground_fare      1.000     0.857     0.923         7
                           atis_capacity      0.952     0.952     0.952        21
                          atis_flight_no      0.889     1.000     0.941         8
                               atis_meal      1.000     0.667     0.800         6
                        atis_restriction      0.000     0.000     0.000         0
             atis_airline#atis_flight_no      0.000     0.000     0.000         0
    atis_ground_service#atis_ground_fare      0.000     0.000     0.000         0
           atis_airfare#atis_flight_time      0.000     0.000     0.000         0
                           atis_cheapest      0.000     0.000     0.000         0
atis_aircraft#atis_flight#atis_flight_no      0.000     0.000     0.000         0

                               micro avg      0.972     0.977     0.975       888
                               macro avg      0.640     0.649     0.630       888
                            weighted avg      0.974     0.977     0.974       888

I0716 05:04:22.761313 139795883440000 interactiveshell.py:2882] 
                              precision    recall  f1-score   support

                           O      0.000     0.000     0.000       0.0
           B-toloc.city_name      0.977     0.993     0.985     716.0
         B-fromloc.city_name      0.990     0.999     0.994     704.0
           I-toloc.city_name      0.977     0.981     0.979     265.0
      B-depart_date.day_name      0.991     0.991     0.991     212.0
              B-airline_name      1.000     1.000     1.000     101.0
         I-fromloc.city_name      0.973     1.000     0.986     177.0
 B-depart_time.period_of_day      0.992     0.923     0.956     130.0
              I-airline_name      1.000     1.000     1.000      65.0
    B-depart_date.day_number      0.981     0.964     0.972      55.0
    B-depart_date.month_name      0.982     0.964     0.973      56.0
          B-depart_time.time      0.864     1.000     0.927      57.0
                B-round_trip      1.000     1.000     1.000      73.0
             B-cost_relative      0.973     0.973     0.973      37.0
                I-round_trip      1.000     1.000     1.000      71.0
                B-flight_mod      0.828     1.000     0.906      24.0
 B-depart_time.time_relative      0.955     0.985     0.970      65.0
          I-depart_time.time      0.945     1.000     0.972      52.0
         B-stoploc.city_name      0.870     1.000     0.930      20.0
                 B-city_name      0.902     0.649     0.755      57.0
                B-class_type      0.960     1.000     0.980      24.0
          B-arrive_time.time      0.971     0.971     0.971      34.0
 B-arrive_time.time_relative      0.967     0.935     0.951      31.0
                I-class_type      1.000     1.000     1.000      17.0
               B-flight_stop      1.000     1.000     1.000      21.0
          I-arrive_time.time      1.000     0.971     0.986      35.0
              B-airline_code      0.944     1.000     0.971      34.0
    I-depart_date.day_number      1.000     1.000     1.000      15.0
      I-fromloc.airport_name      0.441     1.000     0.612      15.0
      B-fromloc.airport_name      0.462     1.000     0.632      12.0
      B-arrive_date.day_name      0.786     1.000     0.880      11.0
          B-toloc.state_code      1.000     1.000     1.000      18.0
B-depart_date.today_relative      1.000     0.889     0.941       9.0
             B-flight_number      0.733     1.000     0.846      11.0
 B-depart_date.date_relative      0.944     1.000     0.971      17.0
          B-toloc.state_name      0.788     0.929     0.852      28.0
           B-fare_basis_code      0.895     1.000     0.944      17.0
               B-flight_time      1.000     1.000     1.000       1.0
                        B-or      1.000     1.000     1.000       3.0
 B-arrive_time.period_of_day      0.750     1.000     0.857       6.0
          B-meal_description      1.000     0.900     0.947      10.0
             I-cost_relative      1.000     0.667     0.800       3.0
              I-airport_name      0.846     0.379     0.524      29.0
               B-fare_amount      1.000     1.000     1.000       2.0
               I-fare_amount      1.000     1.000     1.000       2.0
                 I-city_name      1.000     0.600     0.750      30.0
        I-toloc.airport_name      1.000     1.000     1.000       3.0
            B-transport_type      1.000     1.000     1.000      10.0
    B-arrive_date.month_name      0.714     0.833     0.769       6.0
    B-arrive_date.day_number      0.714     0.833     0.769       6.0
         I-stoploc.city_name      1.000     1.000     1.000      10.0
                      B-meal      1.000     1.000     1.000      16.0
        B-fromloc.state_code      1.000     1.000     1.000      23.0
    B-depart_time.period_mod      1.000     1.000     1.000       5.0
                   B-connect      1.000     1.000     1.000       6.0
               B-flight_days      1.000     1.000     1.000      10.0
        B-toloc.airport_name      1.000     1.000     1.000       3.0
        B-fromloc.state_name      0.944     1.000     0.971      17.0
              B-airport_name      0.778     0.333     0.467      21.0
                   B-economy      1.000     1.000     1.000       6.0
               I-flight_time      1.000     1.000     1.000       1.0
             B-aircraft_code      1.000     0.879     0.935      33.0
                       B-mod      1.000     0.500     0.667       2.0
              B-airport_code      1.000     0.444     0.615       9.0
    B-depart_time.start_time      1.000     1.000     1.000       3.0
      B-depart_time.end_time      1.000     0.667     0.800       3.0
          B-depart_date.year      1.000     1.000     1.000       3.0
            I-transport_type      0.000     0.000     0.000       1.0
          B-restriction_code      1.000     1.000     1.000       4.0
    B-arrive_time.start_time      1.000     1.000     1.000       8.0
        B-toloc.airport_code      1.000     0.750     0.857       4.0
      B-arrive_time.end_time      1.000     1.000     1.000       8.0
      I-arrive_time.end_time      0.889     1.000     0.941       8.0
      I-depart_time.end_time      1.000     0.333     0.500       3.0
               I-flight_stop      0.000     0.000     0.000       0.0
      B-fromloc.airport_code      0.556     1.000     0.714       5.0
          I-restriction_code      1.000     1.000     1.000       3.0
    I-depart_time.start_time      1.000     1.000     1.000       1.0
          I-toloc.state_name      1.000     1.000     1.000       1.0
I-depart_date.today_relative      0.000     0.000     0.000       0.0
 B-arrive_date.date_relative      1.000     1.000     1.000       2.0
                I-flight_mod      1.000     0.167     0.286       6.0
                   I-economy      0.000     0.000     0.000       0.0
 B-return_date.date_relative      1.000     0.333     0.500       3.0
        I-fromloc.state_name      1.000     1.000     1.000       1.0
                B-state_code      1.000     1.000     1.000       1.0
    I-arrive_time.start_time      1.000     1.000     1.000       1.0
    I-arrive_date.day_number      0.000     0.000     0.000       0.0
                 B-meal_code      0.000     0.000     0.000       1.0
 I-depart_time.period_of_day      1.000     1.000     1.000       1.0
                  B-day_name      1.000     0.500     0.667       2.0
             B-period_of_day      1.000     0.750     0.857       4.0
        B-stoploc.state_code      0.000     0.000     0.000       0.0
    B-return_date.month_name      0.000     0.000     0.000       0.0
    B-return_date.day_number      0.000     0.000     0.000       0.0
    B-arrive_time.period_mod      0.000     0.000     0.000       0.0
                 I-meal_code      0.000     0.000     0.000       0.0
        B-toloc.country_name      1.000     1.000     1.000       1.0
                 B-days_code      1.000     1.000     1.000       1.0
 I-arrive_time.period_of_day      0.000     0.000     0.000       0.0
            I-today_relative      0.000     0.000     0.000       0.0
 B-return_time.period_of_day      0.000     0.000     0.000       0.0
                      B-time      0.000     0.000     0.000       0.0
           I-fare_basis_code      0.000     0.000     0.000       0.0
 I-arrive_time.time_relative      1.000     1.000     1.000       4.0
 I-depart_time.time_relative      0.000     0.000     0.000       1.0
            B-today_relative      0.000     0.000     0.000       0.0
                B-state_name      0.000     0.000     0.000       9.0
B-arrive_date.today_relative      0.000     0.000     0.000       0.0
    B-return_time.period_mod      0.000     0.000     0.000       0.0
                B-month_name      0.000     0.000     0.000       0.0
                B-day_number      0.000     0.000     0.000       0.0
 I-return_date.date_relative      0.667     0.667     0.667       3.0
I-return_date.today_relative      0.000     0.000     0.000       0.0
      B-stoploc.airport_name      0.000     0.000     0.000       0.0
             B-time_relative      0.000     0.000     0.000       0.0
                      I-time      0.000     0.000     0.000       0.0
    I-return_date.day_number      0.000     0.000     0.000       0.0
          I-meal_description      0.000     0.000     0.000       0.0
B-return_date.today_relative      0.000     0.000     0.000       0.0
      B-return_date.day_name      0.000     0.000     0.000       2.0

                   micro avg      0.960     0.961     0.960    3657.0
                   macro avg      0.694     0.667     0.667    3657.0
                weighted avg      0.964     0.961     0.958    3657.0

I0716 05:04:22.784583 139795883440000 interactiveshell.py:2882] Best Slot F1: 0.960
I0716 05:04:22.786408 139795883440000 estimator_training.py:186] Not using Distribute Coordinator.
I0716 05:04:22.788550 139795883440000 training.py:612] Running training and evaluation locally (non-distributed).
I0716 05:04:22.790685 139795883440000 training.py:700] Start train and evaluate loop. The evaluate will happen after every checkpoint. Checkpoint frequency is determined based on RunConfig arguments: save_checkpoints_steps 312 or save_checkpoints_secs None.
I0716 05:04:22.860918 139795883440000 estimator.py:1145] Calling model_fn.
I0716 05:04:24.386466 139795883440000 saver.py:1499] Saver not created because there are no variables in the graph to restore
I0716 05:04:25.001788 139795883440000 <ipython-input-6-d6747b40360b>:55] 
[<tf.Variable 'embedding:0' shape=(750, 300) dtype=float32_ref>,
 <tf.Variable 'module/aggregation/weights:0' shape=(3,) dtype=float32>,
 <tf.Variable 'module/aggregation/scaling:0' shape=() dtype=float32>,
 <tf.Variable 'dense/kernel:0' shape=(1324, 300) dtype=float32_ref>,
 <tf.Variable 'dense/bias:0' shape=(300,) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/gates/kernel:0' shape=(600, 600) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/gates/bias:0' shape=(600,) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/candidate/kernel:0' shape=(600, 300) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/candidate/bias:0' shape=(300,) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/gates/kernel:0' shape=(600, 600) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/gates/bias:0' shape=(600,) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/candidate/kernel:0' shape=(600, 300) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/candidate/bias:0' shape=(300,) dtype=float32_ref>,
 <tf.Variable 'dense_1/kernel:0' shape=(600, 23) dtype=float32_ref>,
 <tf.Variable 'dense_1/bias:0' shape=(23,) dtype=float32_ref>,
 <tf.Variable 'dense_2/kernel:0' shape=(600, 122) dtype=float32_ref>,
 <tf.Variable 'dense_2/bias:0' shape=(122,) dtype=float32_ref>]
I0716 05:04:26.290348 139795883440000 estimator.py:1147] Done calling model_fn.
I0716 05:04:26.294201 139795883440000 basic_session_run_hooks.py:541] Create CheckpointSaverHook.
I0716 05:04:26.632357 139795883440000 monitored_session.py:240] Graph was finalized.
I0716 05:04:26.651122 139795883440000 saver.py:1280] Restoring parameters from ../model/elmo_bigru/model.ckpt-10608
I0716 05:04:27.718711 139795883440000 session_manager.py:500] Running local_init_op.
I0716 05:04:27.792746 139795883440000 session_manager.py:502] Done running local_init_op.
I0716 05:04:30.619942 139795883440000 basic_session_run_hooks.py:606] Saving checkpoints for 10608 into ../model/elmo_bigru/model.ckpt.
Reading ../data/atis.train.w-intent.iob
I0716 05:04:35.721859 139795883440000 basic_session_run_hooks.py:262] loss = 0.0103135025, step = 10608
I0716 05:04:35.723819 139795883440000 basic_session_run_hooks.py:262] lr = 9.8112796e-05
I0716 05:05:03.612728 139795883440000 basic_session_run_hooks.py:692] global_step/sec: 3.58534
I0716 05:05:03.621054 139795883440000 basic_session_run_hooks.py:260] loss = 0.0095020505, step = 10708 (27.899 sec)
I0716 05:05:03.624384 139795883440000 basic_session_run_hooks.py:260] lr = 9.708451e-05 (27.901 sec)
I0716 05:05:30.273659 139795883440000 basic_session_run_hooks.py:692] global_step/sec: 3.7508
I0716 05:05:30.280654 139795883440000 basic_session_run_hooks.py:260] loss = 0.011504768, step = 10808 (26.659 sec)
I0716 05:05:30.284494 139795883440000 basic_session_run_hooks.py:260] lr = 9.6066986e-05 (26.660 sec)
I0716 05:05:56.251807 139795883440000 basic_session_run_hooks.py:692] global_step/sec: 3.84939
I0716 05:05:56.258651 139795883440000 basic_session_run_hooks.py:260] loss = 0.009077469, step = 10908 (25.978 sec)
I0716 05:05:56.261323 139795883440000 basic_session_run_hooks.py:260] lr = 9.506013e-05 (25.977 sec)
I0716 05:05:59.056719 139795883440000 basic_session_run_hooks.py:606] Saving checkpoints for 10920 into ../model/elmo_bigru/model.ckpt.
I0716 05:06:00.985701 139795883440000 estimator.py:1145] Calling model_fn.
I0716 05:06:02.611641 139795883440000 saver.py:1499] Saver not created because there are no variables in the graph to restore
I0716 05:06:03.195277 139795883440000 estimator.py:1147] Done calling model_fn.
I0716 05:06:03.226158 139795883440000 evaluation.py:255] Starting evaluation at 2019-07-16T05:06:03Z
I0716 05:06:03.456297 139795883440000 monitored_session.py:240] Graph was finalized.
I0716 05:06:03.473925 139795883440000 saver.py:1280] Restoring parameters from ../model/elmo_bigru/model.ckpt-10920
I0716 05:06:04.373917 139795883440000 session_manager.py:500] Running local_init_op.
I0716 05:06:04.427072 139795883440000 session_manager.py:502] Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
I0716 05:07:03.300335 139795883440000 evaluation.py:275] Finished evaluation at 2019-07-16-05:07:03
I0716 05:07:03.302104 139795883440000 estimator.py:2039] Saving dict for global step 10920: global_step = 10920, loss = 0.36773965
I0716 05:07:03.312190 139795883440000 estimator.py:2099] Saving 'checkpoint_path' summary for global step 10920: ../model/elmo_bigru/model.ckpt-10920
I0716 05:07:03.396445 139795883440000 estimator.py:368] Loss for final step: 0.004308995.
Reading ../data/atis.test.w-intent.iob
I0716 05:07:04.000622 139795883440000 estimator.py:1145] Calling model_fn.
I0716 05:07:05.549351 139795883440000 saver.py:1499] Saver not created because there are no variables in the graph to restore
I0716 05:07:06.064721 139795883440000 estimator.py:1147] Done calling model_fn.
I0716 05:07:06.476282 139795883440000 monitored_session.py:240] Graph was finalized.
I0716 05:07:06.488260 139795883440000 saver.py:1280] Restoring parameters from ../model/elmo_bigru/model.ckpt-10920
I0716 05:07:07.387382 139795883440000 session_manager.py:500] Running local_init_op.
I0716 05:07:07.434978 139795883440000 session_manager.py:502] Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.
  'precision', 'predicted', average, warn_for)
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1439: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples.
  'recall', 'true', average, warn_for)
I0716 05:08:05.408970 139795883440000 interactiveshell.py:2882] 
                                          precision    recall  f1-score   support

                             atis_flight      0.986     0.991     0.988       632
                            atis_airfare      0.941     1.000     0.970        48
                     atis_ground_service      1.000     1.000     1.000        36
                            atis_airline      1.000     1.000     1.000        38
                       atis_abbreviation      0.943     1.000     0.971        33
                           atis_aircraft      1.000     0.889     0.941         9
                        atis_flight_time      0.500     1.000     0.667         1
                           atis_quantity      0.429     1.000     0.600         3
                atis_flight#atis_airfare      0.833     0.417     0.556        12
                            atis_airport      0.947     1.000     0.973        18
                           atis_distance      1.000     1.000     1.000        10
                               atis_city      0.667     0.667     0.667         6
                        atis_ground_fare      1.000     0.857     0.923         7
                           atis_capacity      0.952     0.952     0.952        21
                          atis_flight_no      0.889     1.000     0.941         8
                               atis_meal      1.000     0.667     0.800         6
                        atis_restriction      0.000     0.000     0.000         0
             atis_airline#atis_flight_no      0.000     0.000     0.000         0
    atis_ground_service#atis_ground_fare      0.000     0.000     0.000         0
           atis_airfare#atis_flight_time      0.000     0.000     0.000         0
                           atis_cheapest      0.000     0.000     0.000         0
atis_aircraft#atis_flight#atis_flight_no      0.000     0.000     0.000         0

                               micro avg      0.972     0.977     0.975       888
                               macro avg      0.640     0.656     0.634       888
                            weighted avg      0.974     0.977     0.974       888

I0716 05:08:05.447309 139795883440000 interactiveshell.py:2882] 
                              precision    recall  f1-score   support

                           O      0.000     0.000     0.000       0.0
           B-toloc.city_name      0.974     0.994     0.984     716.0
         B-fromloc.city_name      0.992     0.999     0.995     704.0
           I-toloc.city_name      0.978     0.992     0.985     265.0
      B-depart_date.day_name      0.991     0.991     0.991     212.0
              B-airline_name      1.000     1.000     1.000     101.0
         I-fromloc.city_name      0.983     1.000     0.992     177.0
 B-depart_time.period_of_day      0.992     0.923     0.956     130.0
              I-airline_name      1.000     1.000     1.000      65.0
    B-depart_date.day_number      0.981     0.964     0.972      55.0
    B-depart_date.month_name      0.981     0.946     0.964      56.0
          B-depart_time.time      0.864     1.000     0.927      57.0
                B-round_trip      1.000     1.000     1.000      73.0
             B-cost_relative      0.973     0.973     0.973      37.0
                I-round_trip      1.000     1.000     1.000      71.0
                B-flight_mod      0.828     1.000     0.906      24.0
 B-depart_time.time_relative      0.955     0.985     0.970      65.0
          I-depart_time.time      0.945     1.000     0.972      52.0
         B-stoploc.city_name      0.870     1.000     0.930      20.0
                 B-city_name      0.900     0.632     0.742      57.0
                B-class_type      0.960     1.000     0.980      24.0
          B-arrive_time.time      0.971     0.971     0.971      34.0
 B-arrive_time.time_relative      0.967     0.935     0.951      31.0
                I-class_type      1.000     1.000     1.000      17.0
               B-flight_stop      1.000     1.000     1.000      21.0
          I-arrive_time.time      1.000     0.971     0.986      35.0
              B-airline_code      0.944     1.000     0.971      34.0
    I-depart_date.day_number      1.000     0.933     0.966      15.0
      I-fromloc.airport_name      0.441     1.000     0.612      15.0
      B-fromloc.airport_name      0.462     1.000     0.632      12.0
      B-arrive_date.day_name      0.786     1.000     0.880      11.0
          B-toloc.state_code      1.000     1.000     1.000      18.0
B-depart_date.today_relative      1.000     0.889     0.941       9.0
             B-flight_number      0.733     1.000     0.846      11.0
 B-depart_date.date_relative      0.944     1.000     0.971      17.0
          B-toloc.state_name      0.781     0.893     0.833      28.0
           B-fare_basis_code      0.850     1.000     0.919      17.0
               B-flight_time      1.000     1.000     1.000       1.0
                        B-or      1.000     1.000     1.000       3.0
 B-arrive_time.period_of_day      0.750     1.000     0.857       6.0
          B-meal_description      1.000     0.900     0.947      10.0
             I-cost_relative      1.000     0.667     0.800       3.0
              I-airport_name      0.846     0.379     0.524      29.0
               B-fare_amount      1.000     1.000     1.000       2.0
               I-fare_amount      1.000     1.000     1.000       2.0
                 I-city_name      1.000     0.600     0.750      30.0
        I-toloc.airport_name      1.000     1.000     1.000       3.0
            B-transport_type      1.000     1.000     1.000      10.0
    B-arrive_date.month_name      0.625     0.833     0.714       6.0
    B-arrive_date.day_number      0.714     0.833     0.769       6.0
         I-stoploc.city_name      0.909     1.000     0.952      10.0
                      B-meal      1.000     1.000     1.000      16.0
        B-fromloc.state_code      1.000     1.000     1.000      23.0
    B-depart_time.period_mod      1.000     1.000     1.000       5.0
                   B-connect      1.000     1.000     1.000       6.0
               B-flight_days      1.000     1.000     1.000      10.0
        B-toloc.airport_name      1.000     1.000     1.000       3.0
        B-fromloc.state_name      1.000     1.000     1.000      17.0
              B-airport_name      0.778     0.333     0.467      21.0
                   B-economy      1.000     1.000     1.000       6.0
               I-flight_time      1.000     1.000     1.000       1.0
             B-aircraft_code      1.000     0.879     0.935      33.0
                       B-mod      1.000     0.500     0.667       2.0
              B-airport_code      1.000     0.444     0.615       9.0
    B-depart_time.start_time      1.000     0.667     0.800       3.0
      B-depart_time.end_time      1.000     0.333     0.500       3.0
          B-depart_date.year      1.000     1.000     1.000       3.0
            I-transport_type      0.000     0.000     0.000       1.0
          B-restriction_code      1.000     1.000     1.000       4.0
    B-arrive_time.start_time      0.889     1.000     0.941       8.0
        B-toloc.airport_code      1.000     0.750     0.857       4.0
      B-arrive_time.end_time      0.889     1.000     0.941       8.0
      I-arrive_time.end_time      0.889     1.000     0.941       8.0
      I-depart_time.end_time      1.000     0.333     0.500       3.0
               I-flight_stop      0.000     0.000     0.000       0.0
      B-fromloc.airport_code      0.556     1.000     0.714       5.0
          I-restriction_code      1.000     1.000     1.000       3.0
    I-depart_time.start_time      1.000     1.000     1.000       1.0
          I-toloc.state_name      1.000     1.000     1.000       1.0
I-depart_date.today_relative      0.000     0.000     0.000       0.0
 B-arrive_date.date_relative      1.000     1.000     1.000       2.0
                I-flight_mod      1.000     0.167     0.286       6.0
                   I-economy      0.000     0.000     0.000       0.0
 B-return_date.date_relative      1.000     0.333     0.500       3.0
        I-fromloc.state_name      1.000     1.000     1.000       1.0
                B-state_code      1.000     1.000     1.000       1.0
    I-arrive_time.start_time      1.000     1.000     1.000       1.0
    I-arrive_date.day_number      0.000     0.000     0.000       0.0
                 B-meal_code      0.000     0.000     0.000       1.0
 I-depart_time.period_of_day      1.000     1.000     1.000       1.0
                  B-day_name      1.000     0.500     0.667       2.0
             B-period_of_day      1.000     0.750     0.857       4.0
        B-stoploc.state_code      0.000     0.000     0.000       0.0
    B-return_date.month_name      0.000     0.000     0.000       0.0
    B-return_date.day_number      0.000     0.000     0.000       0.0
    B-arrive_time.period_mod      0.000     0.000     0.000       0.0
                 I-meal_code      0.000     0.000     0.000       0.0
        B-toloc.country_name      1.000     1.000     1.000       1.0
                 B-days_code      0.000     0.000     0.000       1.0
 I-arrive_time.period_of_day      0.000     0.000     0.000       0.0
            I-today_relative      0.000     0.000     0.000       0.0
 B-return_time.period_of_day      0.000     0.000     0.000       0.0
                      B-time      0.000     0.000     0.000       0.0
           I-fare_basis_code      0.000     0.000     0.000       0.0
 I-arrive_time.time_relative      1.000     1.000     1.000       4.0
 I-depart_time.time_relative      0.000     0.000     0.000       1.0
            B-today_relative      0.000     0.000     0.000       0.0
                B-state_name      0.000     0.000     0.000       9.0
B-arrive_date.today_relative      0.000     0.000     0.000       0.0
    B-return_time.period_mod      0.000     0.000     0.000       0.0
                B-month_name      0.000     0.000     0.000       0.0
                B-day_number      0.000     0.000     0.000       0.0
 I-return_date.date_relative      0.667     0.667     0.667       3.0
I-return_date.today_relative      0.000     0.000     0.000       0.0
      B-stoploc.airport_name      0.000     0.000     0.000       0.0
             B-time_relative      0.000     0.000     0.000       0.0
                      I-time      0.000     0.000     0.000       0.0
    I-return_date.day_number      0.000     0.000     0.000       0.0
          I-meal_description      0.000     0.000     0.000       0.0
B-return_date.today_relative      0.000     0.000     0.000       0.0
      B-return_date.day_name      0.000     0.000     0.000       2.0

                   micro avg      0.959     0.960     0.960    3657.0
                   macro avg      0.682     0.652     0.652    3657.0
                weighted avg      0.963     0.960     0.957    3657.0

I0716 05:08:05.473785 139795883440000 interactiveshell.py:2882] Best Slot F1: 0.960
I0716 05:08:05.481377 139795883440000 estimator_training.py:186] Not using Distribute Coordinator.
I0716 05:08:05.482621 139795883440000 training.py:612] Running training and evaluation locally (non-distributed).
I0716 05:08:05.485518 139795883440000 training.py:700] Start train and evaluate loop. The evaluate will happen after every checkpoint. Checkpoint frequency is determined based on RunConfig arguments: save_checkpoints_steps 312 or save_checkpoints_secs None.
I0716 05:08:05.558955 139795883440000 estimator.py:1145] Calling model_fn.
I0716 05:08:06.878779 139795883440000 saver.py:1499] Saver not created because there are no variables in the graph to restore
I0716 05:08:07.455803 139795883440000 <ipython-input-6-d6747b40360b>:55] 
[<tf.Variable 'embedding:0' shape=(750, 300) dtype=float32_ref>,
 <tf.Variable 'module/aggregation/weights:0' shape=(3,) dtype=float32>,
 <tf.Variable 'module/aggregation/scaling:0' shape=() dtype=float32>,
 <tf.Variable 'dense/kernel:0' shape=(1324, 300) dtype=float32_ref>,
 <tf.Variable 'dense/bias:0' shape=(300,) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/gates/kernel:0' shape=(600, 600) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/gates/bias:0' shape=(600,) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/candidate/kernel:0' shape=(600, 300) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/candidate/bias:0' shape=(300,) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/gates/kernel:0' shape=(600, 600) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/gates/bias:0' shape=(600,) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/candidate/kernel:0' shape=(600, 300) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/candidate/bias:0' shape=(300,) dtype=float32_ref>,
 <tf.Variable 'dense_1/kernel:0' shape=(600, 23) dtype=float32_ref>,
 <tf.Variable 'dense_1/bias:0' shape=(23,) dtype=float32_ref>,
 <tf.Variable 'dense_2/kernel:0' shape=(600, 122) dtype=float32_ref>,
 <tf.Variable 'dense_2/bias:0' shape=(122,) dtype=float32_ref>]
I0716 05:08:08.773962 139795883440000 estimator.py:1147] Done calling model_fn.
I0716 05:08:08.777846 139795883440000 basic_session_run_hooks.py:541] Create CheckpointSaverHook.
I0716 05:08:09.101681 139795883440000 monitored_session.py:240] Graph was finalized.
I0716 05:08:09.123294 139795883440000 saver.py:1280] Restoring parameters from ../model/elmo_bigru/model.ckpt-10920
I0716 05:08:10.187270 139795883440000 session_manager.py:500] Running local_init_op.
I0716 05:08:10.259480 139795883440000 session_manager.py:502] Done running local_init_op.
I0716 05:08:13.139483 139795883440000 basic_session_run_hooks.py:606] Saving checkpoints for 10920 into ../model/elmo_bigru/model.ckpt.
Reading ../data/atis.train.w-intent.iob
I0716 05:08:18.349324 139795883440000 basic_session_run_hooks.py:262] loss = 0.017390745, step = 10920
I0716 05:08:18.351059 139795883440000 basic_session_run_hooks.py:262] lr = 9.494002e-05
I0716 05:08:46.093603 139795883440000 basic_session_run_hooks.py:692] global_step/sec: 3.60425
I0716 05:08:46.101212 139795883440000 basic_session_run_hooks.py:260] loss = 0.012770555, step = 11020 (27.752 sec)
I0716 05:08:46.103015 139795883440000 basic_session_run_hooks.py:260] lr = 9.394498e-05 (27.752 sec)
I0716 05:09:12.375688 139795883440000 basic_session_run_hooks.py:692] global_step/sec: 3.80488
I0716 05:09:12.383651 139795883440000 basic_session_run_hooks.py:260] loss = 0.014521266, step = 11120 (26.282 sec)
I0716 05:09:12.385815 139795883440000 basic_session_run_hooks.py:260] lr = 9.296037e-05 (26.283 sec)
I0716 05:09:38.238720 139795883440000 basic_session_run_hooks.py:692] global_step/sec: 3.86651
I0716 05:09:38.245971 139795883440000 basic_session_run_hooks.py:260] loss = 0.014142913, step = 11220 (25.862 sec)
I0716 05:09:38.248379 139795883440000 basic_session_run_hooks.py:260] lr = 9.198607e-05 (25.863 sec)
I0716 05:09:41.196772 139795883440000 basic_session_run_hooks.py:606] Saving checkpoints for 11232 into ../model/elmo_bigru/model.ckpt.
I0716 05:09:43.256902 139795883440000 estimator.py:1145] Calling model_fn.
I0716 05:09:44.903710 139795883440000 saver.py:1499] Saver not created because there are no variables in the graph to restore
I0716 05:09:45.473230 139795883440000 estimator.py:1147] Done calling model_fn.
I0716 05:09:45.503853 139795883440000 evaluation.py:255] Starting evaluation at 2019-07-16T05:09:45Z
I0716 05:09:45.734034 139795883440000 monitored_session.py:240] Graph was finalized.
I0716 05:09:45.750732 139795883440000 saver.py:1280] Restoring parameters from ../model/elmo_bigru/model.ckpt-11232
I0716 05:09:46.685195 139795883440000 session_manager.py:500] Running local_init_op.
I0716 05:09:46.751442 139795883440000 session_manager.py:502] Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
I0716 05:10:45.692678 139795883440000 evaluation.py:275] Finished evaluation at 2019-07-16-05:10:45
I0716 05:10:45.694537 139795883440000 estimator.py:2039] Saving dict for global step 11232: global_step = 11232, loss = 0.383035
I0716 05:10:45.701206 139795883440000 estimator.py:2099] Saving 'checkpoint_path' summary for global step 11232: ../model/elmo_bigru/model.ckpt-11232
I0716 05:10:45.789470 139795883440000 estimator.py:368] Loss for final step: 0.0029675646.
Reading ../data/atis.test.w-intent.iob
I0716 05:10:46.391677 139795883440000 estimator.py:1145] Calling model_fn.
I0716 05:10:47.934705 139795883440000 saver.py:1499] Saver not created because there are no variables in the graph to restore
I0716 05:10:48.634458 139795883440000 estimator.py:1147] Done calling model_fn.
I0716 05:10:48.851438 139795883440000 monitored_session.py:240] Graph was finalized.
I0716 05:10:48.867738 139795883440000 saver.py:1280] Restoring parameters from ../model/elmo_bigru/model.ckpt-11232
I0716 05:10:49.769472 139795883440000 session_manager.py:500] Running local_init_op.
I0716 05:10:49.824869 139795883440000 session_manager.py:502] Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.
  'precision', 'predicted', average, warn_for)
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1439: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples.
  'recall', 'true', average, warn_for)
I0716 05:11:47.888665 139795883440000 interactiveshell.py:2882] 
                                          precision    recall  f1-score   support

                             atis_flight      0.984     0.992     0.988       632
                            atis_airfare      0.941     1.000     0.970        48
                     atis_ground_service      1.000     1.000     1.000        36
                            atis_airline      1.000     1.000     1.000        38
                       atis_abbreviation      0.943     1.000     0.971        33
                           atis_aircraft      1.000     0.889     0.941         9
                        atis_flight_time      0.500     1.000     0.667         1
                           atis_quantity      0.429     1.000     0.600         3
                atis_flight#atis_airfare      0.833     0.417     0.556        12
                            atis_airport      1.000     1.000     1.000        18
                           atis_distance      1.000     1.000     1.000        10
                               atis_city      0.600     0.500     0.545         6
                        atis_ground_fare      1.000     0.857     0.923         7
                           atis_capacity      0.952     0.952     0.952        21
                          atis_flight_no      0.889     1.000     0.941         8
                               atis_meal      1.000     0.667     0.800         6
                        atis_restriction      0.000     0.000     0.000         0
             atis_airline#atis_flight_no      0.000     0.000     0.000         0
    atis_ground_service#atis_ground_fare      0.000     0.000     0.000         0
           atis_airfare#atis_flight_time      0.000     0.000     0.000         0
                           atis_cheapest      0.000     0.000     0.000         0
atis_aircraft#atis_flight#atis_flight_no      0.000     0.000     0.000         0

                               micro avg      0.972     0.977     0.975       888
                               macro avg      0.640     0.649     0.630       888
                            weighted avg      0.974     0.977     0.974       888

I0716 05:11:47.926136 139795883440000 interactiveshell.py:2882] 
                              precision    recall  f1-score   support

                           O      0.000     0.000     0.000       0.0
           B-toloc.city_name      0.977     0.992     0.984     716.0
         B-fromloc.city_name      0.987     0.999     0.993     704.0
           I-toloc.city_name      0.978     0.985     0.981     265.0
      B-depart_date.day_name      0.991     0.991     0.991     212.0
              B-airline_name      1.000     1.000     1.000     101.0
         I-fromloc.city_name      0.973     1.000     0.986     177.0
 B-depart_time.period_of_day      1.000     0.923     0.960     130.0
              I-airline_name      1.000     1.000     1.000      65.0
    B-depart_date.day_number      0.981     0.964     0.972      55.0
    B-depart_date.month_name      0.981     0.946     0.964      56.0
          B-depart_time.time      0.864     1.000     0.927      57.0
                B-round_trip      1.000     0.986     0.993      73.0
             B-cost_relative      0.973     0.973     0.973      37.0
                I-round_trip      1.000     1.000     1.000      71.0
                B-flight_mod      0.828     1.000     0.906      24.0
 B-depart_time.time_relative      0.955     0.985     0.970      65.0
          I-depart_time.time      0.945     1.000     0.972      52.0
         B-stoploc.city_name      0.909     1.000     0.952      20.0
                 B-city_name      0.902     0.649     0.755      57.0
                B-class_type      0.960     1.000     0.980      24.0
          B-arrive_time.time      0.971     0.971     0.971      34.0
 B-arrive_time.time_relative      0.935     0.935     0.935      31.0
                I-class_type      1.000     1.000     1.000      17.0
               B-flight_stop      1.000     1.000     1.000      21.0
          I-arrive_time.time      1.000     0.971     0.986      35.0
              B-airline_code      0.944     1.000     0.971      34.0
    I-depart_date.day_number      1.000     0.933     0.966      15.0
      I-fromloc.airport_name      0.441     1.000     0.612      15.0
      B-fromloc.airport_name      0.462     1.000     0.632      12.0
      B-arrive_date.day_name      0.786     1.000     0.880      11.0
          B-toloc.state_code      1.000     1.000     1.000      18.0
B-depart_date.today_relative      1.000     0.889     0.941       9.0
             B-flight_number      0.733     1.000     0.846      11.0
 B-depart_date.date_relative      0.944     1.000     0.971      17.0
          B-toloc.state_name      0.788     0.929     0.852      28.0
           B-fare_basis_code      0.850     1.000     0.919      17.0
               B-flight_time      1.000     1.000     1.000       1.0
                        B-or      1.000     1.000     1.000       3.0
 B-arrive_time.period_of_day      0.750     1.000     0.857       6.0
          B-meal_description      1.000     0.900     0.947      10.0
             I-cost_relative      1.000     0.667     0.800       3.0
              I-airport_name      0.846     0.379     0.524      29.0
               B-fare_amount      1.000     1.000     1.000       2.0
               I-fare_amount      1.000     1.000     1.000       2.0
                 I-city_name      1.000     0.600     0.750      30.0
        I-toloc.airport_name      1.000     1.000     1.000       3.0
            B-transport_type      1.000     1.000     1.000      10.0
    B-arrive_date.month_name      0.625     0.833     0.714       6.0
    B-arrive_date.day_number      0.714     0.833     0.769       6.0
         I-stoploc.city_name      1.000     1.000     1.000      10.0
                      B-meal      1.000     1.000     1.000      16.0
        B-fromloc.state_code      1.000     1.000     1.000      23.0
    B-depart_time.period_mod      1.000     1.000     1.000       5.0
                   B-connect      1.000     1.000     1.000       6.0
               B-flight_days      1.000     1.000     1.000      10.0
        B-toloc.airport_name      1.000     1.000     1.000       3.0
        B-fromloc.state_name      0.944     1.000     0.971      17.0
              B-airport_name      0.778     0.333     0.467      21.0
                   B-economy      1.000     1.000     1.000       6.0
               I-flight_time      1.000     1.000     1.000       1.0
             B-aircraft_code      1.000     0.879     0.935      33.0
                       B-mod      1.000     0.500     0.667       2.0
              B-airport_code      1.000     0.444     0.615       9.0
    B-depart_time.start_time      1.000     0.667     0.800       3.0
      B-depart_time.end_time      1.000     0.333     0.500       3.0
          B-depart_date.year      1.000     1.000     1.000       3.0
            I-transport_type      0.000     0.000     0.000       1.0
          B-restriction_code      1.000     1.000     1.000       4.0
    B-arrive_time.start_time      0.889     1.000     0.941       8.0
        B-toloc.airport_code      1.000     0.750     0.857       4.0
      B-arrive_time.end_time      0.889     1.000     0.941       8.0
      I-arrive_time.end_time      0.889     1.000     0.941       8.0
      I-depart_time.end_time      1.000     0.333     0.500       3.0
               I-flight_stop      0.000     0.000     0.000       0.0
      B-fromloc.airport_code      0.556     1.000     0.714       5.0
          I-restriction_code      1.000     1.000     1.000       3.0
    I-depart_time.start_time      1.000     1.000     1.000       1.0
          I-toloc.state_name      1.000     1.000     1.000       1.0
I-depart_date.today_relative      0.000     0.000     0.000       0.0
 B-arrive_date.date_relative      1.000     1.000     1.000       2.0
                I-flight_mod      1.000     0.167     0.286       6.0
                   I-economy      0.000     0.000     0.000       0.0
 B-return_date.date_relative      1.000     0.333     0.500       3.0
        I-fromloc.state_name      1.000     1.000     1.000       1.0
                B-state_code      1.000     1.000     1.000       1.0
    I-arrive_time.start_time      1.000     1.000     1.000       1.0
    I-arrive_date.day_number      0.000     0.000     0.000       0.0
                 B-meal_code      0.000     0.000     0.000       1.0
 I-depart_time.period_of_day      1.000     1.000     1.000       1.0
                  B-day_name      1.000     0.500     0.667       2.0
             B-period_of_day      1.000     0.750     0.857       4.0
        B-stoploc.state_code      0.000     0.000     0.000       0.0
    B-return_date.month_name      0.000     0.000     0.000       0.0
    B-return_date.day_number      0.000     0.000     0.000       0.0
    B-arrive_time.period_mod      0.000     0.000     0.000       0.0
                 I-meal_code      0.000     0.000     0.000       0.0
        B-toloc.country_name      1.000     1.000     1.000       1.0
                 B-days_code      0.000     0.000     0.000       1.0
 I-arrive_time.period_of_day      0.000     0.000     0.000       0.0
            I-today_relative      0.000     0.000     0.000       0.0
 B-return_time.period_of_day      0.000     0.000     0.000       0.0
                      B-time      0.000     0.000     0.000       0.0
           I-fare_basis_code      0.000     0.000     0.000       0.0
 I-arrive_time.time_relative      1.000     0.750     0.857       4.0
 I-depart_time.time_relative      0.000     0.000     0.000       1.0
            B-today_relative      0.000     0.000     0.000       0.0
                B-state_name      0.000     0.000     0.000       9.0
B-arrive_date.today_relative      0.000     0.000     0.000       0.0
    B-return_time.period_mod      0.000     0.000     0.000       0.0
                B-month_name      0.000     0.000     0.000       0.0
                B-day_number      0.000     0.000     0.000       0.0
 I-return_date.date_relative      0.667     0.667     0.667       3.0
I-return_date.today_relative      0.000     0.000     0.000       0.0
      B-stoploc.airport_name      0.000     0.000     0.000       0.0
             B-time_relative      0.000     0.000     0.000       0.0
                      I-time      0.000     0.000     0.000       0.0
    I-return_date.day_number      0.000     0.000     0.000       0.0
          I-meal_description      0.000     0.000     0.000       0.0
B-return_date.today_relative      0.000     0.000     0.000       0.0
      B-return_date.day_name      0.000     0.000     0.000       2.0

                   micro avg      0.958     0.959     0.958    3657.0
                   macro avg      0.683     0.650     0.651    3657.0
                weighted avg      0.962     0.959     0.956    3657.0

I0716 05:11:47.950561 139795883440000 interactiveshell.py:2882] Best Slot F1: 0.960
I0716 05:11:47.952192 139795883440000 estimator_training.py:186] Not using Distribute Coordinator.
I0716 05:11:47.954119 139795883440000 training.py:612] Running training and evaluation locally (non-distributed).
I0716 05:11:47.956347 139795883440000 training.py:700] Start train and evaluate loop. The evaluate will happen after every checkpoint. Checkpoint frequency is determined based on RunConfig arguments: save_checkpoints_steps 312 or save_checkpoints_secs None.
I0716 05:11:48.025987 139795883440000 estimator.py:1145] Calling model_fn.
I0716 05:11:49.363261 139795883440000 saver.py:1499] Saver not created because there are no variables in the graph to restore
I0716 05:11:50.152054 139795883440000 <ipython-input-6-d6747b40360b>:55] 
[<tf.Variable 'embedding:0' shape=(750, 300) dtype=float32_ref>,
 <tf.Variable 'module/aggregation/weights:0' shape=(3,) dtype=float32>,
 <tf.Variable 'module/aggregation/scaling:0' shape=() dtype=float32>,
 <tf.Variable 'dense/kernel:0' shape=(1324, 300) dtype=float32_ref>,
 <tf.Variable 'dense/bias:0' shape=(300,) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/gates/kernel:0' shape=(600, 600) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/gates/bias:0' shape=(600,) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/candidate/kernel:0' shape=(600, 300) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/candidate/bias:0' shape=(300,) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/gates/kernel:0' shape=(600, 600) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/gates/bias:0' shape=(600,) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/candidate/kernel:0' shape=(600, 300) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/candidate/bias:0' shape=(300,) dtype=float32_ref>,
 <tf.Variable 'dense_1/kernel:0' shape=(600, 23) dtype=float32_ref>,
 <tf.Variable 'dense_1/bias:0' shape=(23,) dtype=float32_ref>,
 <tf.Variable 'dense_2/kernel:0' shape=(600, 122) dtype=float32_ref>,
 <tf.Variable 'dense_2/bias:0' shape=(122,) dtype=float32_ref>]
I0716 05:11:51.244380 139795883440000 estimator.py:1147] Done calling model_fn.
I0716 05:11:51.248389 139795883440000 basic_session_run_hooks.py:541] Create CheckpointSaverHook.
I0716 05:11:51.570209 139795883440000 monitored_session.py:240] Graph was finalized.
I0716 05:11:51.590957 139795883440000 saver.py:1280] Restoring parameters from ../model/elmo_bigru/model.ckpt-11232
I0716 05:11:52.627373 139795883440000 session_manager.py:500] Running local_init_op.
I0716 05:11:52.704813 139795883440000 session_manager.py:502] Done running local_init_op.
I0716 05:11:55.754058 139795883440000 basic_session_run_hooks.py:606] Saving checkpoints for 11232 into ../model/elmo_bigru/model.ckpt.
Reading ../data/atis.train.w-intent.iob
I0716 05:12:00.553711 139795883440000 basic_session_run_hooks.py:262] loss = 0.008925883, step = 11232
I0716 05:12:00.555503 139795883440000 basic_session_run_hooks.py:262] lr = 9.186985e-05
I0716 05:12:27.661089 139795883440000 basic_session_run_hooks.py:692] global_step/sec: 3.68895
I0716 05:12:27.664352 139795883440000 basic_session_run_hooks.py:260] loss = 0.013533504, step = 11332 (27.111 sec)
I0716 05:12:27.669419 139795883440000 basic_session_run_hooks.py:260] lr = 9.090697e-05 (27.114 sec)
I0716 05:12:54.545852 139795883440000 basic_session_run_hooks.py:692] global_step/sec: 3.71959
I0716 05:12:54.552693 139795883440000 basic_session_run_hooks.py:260] loss = 0.0067935092, step = 11432 (26.888 sec)
I0716 05:12:54.556052 139795883440000 basic_session_run_hooks.py:260] lr = 8.9954214e-05 (26.887 sec)
I0716 05:13:21.051222 139795883440000 basic_session_run_hooks.py:692] global_step/sec: 3.77281
I0716 05:13:21.059669 139795883440000 basic_session_run_hooks.py:260] loss = 0.008703114, step = 11532 (26.507 sec)
I0716 05:13:21.062155 139795883440000 basic_session_run_hooks.py:260] lr = 8.9011424e-05 (26.506 sec)
I0716 05:13:24.135333 139795883440000 basic_session_run_hooks.py:606] Saving checkpoints for 11544 into ../model/elmo_bigru/model.ckpt.
I0716 05:13:26.381620 139795883440000 estimator.py:1145] Calling model_fn.
I0716 05:13:27.781203 139795883440000 saver.py:1499] Saver not created because there are no variables in the graph to restore
I0716 05:13:28.389799 139795883440000 estimator.py:1147] Done calling model_fn.
I0716 05:13:28.420340 139795883440000 evaluation.py:255] Starting evaluation at 2019-07-16T05:13:28Z
I0716 05:13:28.651257 139795883440000 monitored_session.py:240] Graph was finalized.
I0716 05:13:28.673435 139795883440000 saver.py:1280] Restoring parameters from ../model/elmo_bigru/model.ckpt-11544
I0716 05:13:29.603673 139795883440000 session_manager.py:500] Running local_init_op.
I0716 05:13:29.663183 139795883440000 session_manager.py:502] Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
I0716 05:14:28.770346 139795883440000 evaluation.py:275] Finished evaluation at 2019-07-16-05:14:28
I0716 05:14:28.772348 139795883440000 estimator.py:2039] Saving dict for global step 11544: global_step = 11544, loss = 0.38771486
I0716 05:14:28.782020 139795883440000 estimator.py:2099] Saving 'checkpoint_path' summary for global step 11544: ../model/elmo_bigru/model.ckpt-11544
I0716 05:14:28.863938 139795883440000 estimator.py:368] Loss for final step: 0.0041209282.
Reading ../data/atis.test.w-intent.iob
I0716 05:14:29.477226 139795883440000 estimator.py:1145] Calling model_fn.
I0716 05:14:31.202950 139795883440000 saver.py:1499] Saver not created because there are no variables in the graph to restore
I0716 05:14:31.746677 139795883440000 estimator.py:1147] Done calling model_fn.
I0716 05:14:31.969483 139795883440000 monitored_session.py:240] Graph was finalized.
I0716 05:14:31.987271 139795883440000 saver.py:1280] Restoring parameters from ../model/elmo_bigru/model.ckpt-11544
I0716 05:14:32.905600 139795883440000 session_manager.py:500] Running local_init_op.
I0716 05:14:32.958049 139795883440000 session_manager.py:502] Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.
  'precision', 'predicted', average, warn_for)
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1439: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples.
  'recall', 'true', average, warn_for)
I0716 05:15:31.219921 139795883440000 interactiveshell.py:2882] 
                                          precision    recall  f1-score   support

                             atis_flight      0.984     0.992     0.988       632
                            atis_airfare      0.941     1.000     0.970        48
                     atis_ground_service      1.000     1.000     1.000        36
                            atis_airline      0.974     1.000     0.987        38
                       atis_abbreviation      0.943     1.000     0.971        33
                           atis_aircraft      1.000     0.889     0.941         9
                        atis_flight_time      0.500     1.000     0.667         1
                           atis_quantity      0.429     1.000     0.600         3
                atis_flight#atis_airfare      0.833     0.417     0.556        12
                            atis_airport      1.000     0.944     0.971        18
                           atis_distance      1.000     1.000     1.000        10
                               atis_city      0.750     0.500     0.600         6
                        atis_ground_fare      1.000     0.857     0.923         7
                           atis_capacity      0.952     0.952     0.952        21
                          atis_flight_no      0.889     1.000     0.941         8
                               atis_meal      1.000     0.833     0.909         6
                        atis_restriction      0.000     0.000     0.000         0
             atis_airline#atis_flight_no      0.000     0.000     0.000         0
    atis_ground_service#atis_ground_fare      0.000     0.000     0.000         0
           atis_airfare#atis_flight_time      0.000     0.000     0.000         0
                           atis_cheapest      0.000     0.000     0.000         0
atis_aircraft#atis_flight#atis_flight_no      0.000     0.000     0.000         0

                               micro avg      0.972     0.977     0.975       888
                               macro avg      0.645     0.654     0.635       888
                            weighted avg      0.974     0.977     0.974       888

I0716 05:15:31.258357 139795883440000 interactiveshell.py:2882] 
                              precision    recall  f1-score   support

                           O      0.000     0.000     0.000       0.0
           B-toloc.city_name      0.973     0.994     0.983     716.0
         B-fromloc.city_name      0.992     0.999     0.995     704.0
           I-toloc.city_name      0.978     0.989     0.983     265.0
      B-depart_date.day_name      0.991     0.991     0.991     212.0
              B-airline_name      1.000     1.000     1.000     101.0
         I-fromloc.city_name      0.973     1.000     0.986     177.0
 B-depart_time.period_of_day      1.000     0.923     0.960     130.0
              I-airline_name      1.000     1.000     1.000      65.0
    B-depart_date.day_number      0.981     0.964     0.972      55.0
    B-depart_date.month_name      0.981     0.946     0.964      56.0
          B-depart_time.time      0.864     1.000     0.927      57.0
                B-round_trip      1.000     1.000     1.000      73.0
             B-cost_relative      0.973     0.973     0.973      37.0
                I-round_trip      1.000     1.000     1.000      71.0
                B-flight_mod      0.828     1.000     0.906      24.0
 B-depart_time.time_relative      0.955     0.985     0.970      65.0
          I-depart_time.time      0.945     1.000     0.972      52.0
         B-stoploc.city_name      0.870     1.000     0.930      20.0
                 B-city_name      0.923     0.632     0.750      57.0
                B-class_type      0.960     1.000     0.980      24.0
          B-arrive_time.time      0.971     0.971     0.971      34.0
 B-arrive_time.time_relative      0.967     0.935     0.951      31.0
                I-class_type      1.000     1.000     1.000      17.0
               B-flight_stop      1.000     1.000     1.000      21.0
          I-arrive_time.time      1.000     0.971     0.986      35.0
              B-airline_code      0.944     1.000     0.971      34.0
    I-depart_date.day_number      1.000     0.933     0.966      15.0
      I-fromloc.airport_name      0.441     1.000     0.612      15.0
      B-fromloc.airport_name      0.462     1.000     0.632      12.0
      B-arrive_date.day_name      0.786     1.000     0.880      11.0
          B-toloc.state_code      1.000     1.000     1.000      18.0
B-depart_date.today_relative      1.000     0.889     0.941       9.0
             B-flight_number      0.733     1.000     0.846      11.0
 B-depart_date.date_relative      0.944     1.000     0.971      17.0
          B-toloc.state_name      0.828     0.857     0.842      28.0
           B-fare_basis_code      0.850     1.000     0.919      17.0
               B-flight_time      1.000     1.000     1.000       1.0
                        B-or      1.000     1.000     1.000       3.0
 B-arrive_time.period_of_day      0.750     1.000     0.857       6.0
          B-meal_description      1.000     0.900     0.947      10.0
             I-cost_relative      1.000     0.667     0.800       3.0
              I-airport_name      0.846     0.379     0.524      29.0
               B-fare_amount      1.000     1.000     1.000       2.0
               I-fare_amount      1.000     1.000     1.000       2.0
                 I-city_name      1.000     0.533     0.696      30.0
        I-toloc.airport_name      1.000     1.000     1.000       3.0
            B-transport_type      1.000     1.000     1.000      10.0
    B-arrive_date.month_name      0.625     0.833     0.714       6.0
    B-arrive_date.day_number      0.714     0.833     0.769       6.0
         I-stoploc.city_name      1.000     1.000     1.000      10.0
                      B-meal      1.000     1.000     1.000      16.0
        B-fromloc.state_code      1.000     1.000     1.000      23.0
    B-depart_time.period_mod      1.000     1.000     1.000       5.0
                   B-connect      1.000     1.000     1.000       6.0
               B-flight_days      1.000     1.000     1.000      10.0
        B-toloc.airport_name      1.000     1.000     1.000       3.0
        B-fromloc.state_name      0.944     1.000     0.971      17.0
              B-airport_name      0.667     0.381     0.485      21.0
                   B-economy      1.000     1.000     1.000       6.0
               I-flight_time      1.000     1.000     1.000       1.0
             B-aircraft_code      1.000     0.879     0.935      33.0
                       B-mod      1.000     0.500     0.667       2.0
              B-airport_code      1.000     0.444     0.615       9.0
    B-depart_time.start_time      1.000     0.667     0.800       3.0
      B-depart_time.end_time      1.000     0.333     0.500       3.0
          B-depart_date.year      1.000     1.000     1.000       3.0
            I-transport_type      0.000     0.000     0.000       1.0
          B-restriction_code      1.000     1.000     1.000       4.0
    B-arrive_time.start_time      0.889     1.000     0.941       8.0
        B-toloc.airport_code      1.000     0.750     0.857       4.0
      B-arrive_time.end_time      0.889     1.000     0.941       8.0
      I-arrive_time.end_time      0.889     1.000     0.941       8.0
      I-depart_time.end_time      1.000     0.333     0.500       3.0
               I-flight_stop      0.000     0.000     0.000       0.0
      B-fromloc.airport_code      0.556     1.000     0.714       5.0
          I-restriction_code      1.000     1.000     1.000       3.0
    I-depart_time.start_time      1.000     1.000     1.000       1.0
          I-toloc.state_name      1.000     1.000     1.000       1.0
I-depart_date.today_relative      0.000     0.000     0.000       0.0
 B-arrive_date.date_relative      1.000     1.000     1.000       2.0
                I-flight_mod      1.000     0.167     0.286       6.0
                   I-economy      0.000     0.000     0.000       0.0
 B-return_date.date_relative      1.000     0.667     0.800       3.0
        I-fromloc.state_name      1.000     1.000     1.000       1.0
                B-state_code      1.000     1.000     1.000       1.0
    I-arrive_time.start_time      1.000     1.000     1.000       1.0
    I-arrive_date.day_number      0.000     0.000     0.000       0.0
                 B-meal_code      0.000     0.000     0.000       1.0
 I-depart_time.period_of_day      1.000     1.000     1.000       1.0
                  B-day_name      1.000     0.500     0.667       2.0
             B-period_of_day      1.000     0.750     0.857       4.0
        B-stoploc.state_code      0.000     0.000     0.000       0.0
    B-return_date.month_name      0.000     0.000     0.000       0.0
    B-return_date.day_number      0.000     0.000     0.000       0.0
    B-arrive_time.period_mod      0.000     0.000     0.000       0.0
                 I-meal_code      0.000     0.000     0.000       0.0
        B-toloc.country_name      1.000     1.000     1.000       1.0
                 B-days_code      0.000     0.000     0.000       1.0
 I-arrive_time.period_of_day      0.000     0.000     0.000       0.0
            I-today_relative      0.000     0.000     0.000       0.0
 B-return_time.period_of_day      0.000     0.000     0.000       0.0
                      B-time      0.000     0.000     0.000       0.0
           I-fare_basis_code      0.000     0.000     0.000       0.0
 I-arrive_time.time_relative      1.000     1.000     1.000       4.0
 I-depart_time.time_relative      0.000     0.000     0.000       1.0
            B-today_relative      0.000     0.000     0.000       0.0
                B-state_name      0.000     0.000     0.000       9.0
B-arrive_date.today_relative      0.000     0.000     0.000       0.0
    B-return_time.period_mod      0.000     0.000     0.000       0.0
                B-month_name      0.000     0.000     0.000       0.0
                B-day_number      0.000     0.000     0.000       0.0
 I-return_date.date_relative      1.000     0.667     0.800       3.0
I-return_date.today_relative      0.000     0.000     0.000       0.0
      B-stoploc.airport_name      0.000     0.000     0.000       0.0
             B-time_relative      0.000     0.000     0.000       0.0
                      I-time      0.000     0.000     0.000       0.0
    I-return_date.day_number      0.000     0.000     0.000       0.0
          I-meal_description      0.000     0.000     0.000       0.0
B-return_date.today_relative      0.000     0.000     0.000       0.0
      B-return_date.day_name      0.000     0.000     0.000       2.0

                   micro avg      0.958     0.960     0.959    3657.0
                   macro avg      0.685     0.654     0.655    3657.0
                weighted avg      0.963     0.960     0.957    3657.0

I0716 05:15:31.287286 139795883440000 interactiveshell.py:2882] Best Slot F1: 0.960
I0716 05:15:31.288828 139795883440000 estimator_training.py:186] Not using Distribute Coordinator.
I0716 05:15:31.290701 139795883440000 training.py:612] Running training and evaluation locally (non-distributed).
I0716 05:15:31.292778 139795883440000 training.py:700] Start train and evaluate loop. The evaluate will happen after every checkpoint. Checkpoint frequency is determined based on RunConfig arguments: save_checkpoints_steps 312 or save_checkpoints_secs None.
I0716 05:15:31.363937 139795883440000 estimator.py:1145] Calling model_fn.
I0716 05:15:32.893659 139795883440000 saver.py:1499] Saver not created because there are no variables in the graph to restore
I0716 05:15:33.484046 139795883440000 <ipython-input-6-d6747b40360b>:55] 
[<tf.Variable 'embedding:0' shape=(750, 300) dtype=float32_ref>,
 <tf.Variable 'module/aggregation/weights:0' shape=(3,) dtype=float32>,
 <tf.Variable 'module/aggregation/scaling:0' shape=() dtype=float32>,
 <tf.Variable 'dense/kernel:0' shape=(1324, 300) dtype=float32_ref>,
 <tf.Variable 'dense/bias:0' shape=(300,) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/gates/kernel:0' shape=(600, 600) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/gates/bias:0' shape=(600,) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/candidate/kernel:0' shape=(600, 300) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/candidate/bias:0' shape=(300,) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/gates/kernel:0' shape=(600, 600) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/gates/bias:0' shape=(600,) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/candidate/kernel:0' shape=(600, 300) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/candidate/bias:0' shape=(300,) dtype=float32_ref>,
 <tf.Variable 'dense_1/kernel:0' shape=(600, 23) dtype=float32_ref>,
 <tf.Variable 'dense_1/bias:0' shape=(23,) dtype=float32_ref>,
 <tf.Variable 'dense_2/kernel:0' shape=(600, 122) dtype=float32_ref>,
 <tf.Variable 'dense_2/bias:0' shape=(122,) dtype=float32_ref>]
I0716 05:15:34.796094 139795883440000 estimator.py:1147] Done calling model_fn.
I0716 05:15:34.800277 139795883440000 basic_session_run_hooks.py:541] Create CheckpointSaverHook.
I0716 05:15:35.143608 139795883440000 monitored_session.py:240] Graph was finalized.
I0716 05:15:35.165673 139795883440000 saver.py:1280] Restoring parameters from ../model/elmo_bigru/model.ckpt-11544
I0716 05:15:36.244131 139795883440000 session_manager.py:500] Running local_init_op.
I0716 05:15:36.319446 139795883440000 session_manager.py:502] Done running local_init_op.
I0716 05:15:39.263292 139795883440000 basic_session_run_hooks.py:606] Saving checkpoints for 11544 into ../model/elmo_bigru/model.ckpt.
Reading ../data/atis.train.w-intent.iob
I0716 05:15:44.056493 139795883440000 basic_session_run_hooks.py:262] loss = 0.007963607, step = 11544
I0716 05:15:44.058551 139795883440000 basic_session_run_hooks.py:262] lr = 8.889895e-05
I0716 05:16:11.216505 139795883440000 basic_session_run_hooks.py:692] global_step/sec: 3.68181
I0716 05:16:11.223562 139795883440000 basic_session_run_hooks.py:260] loss = 0.00936635, step = 11644 (27.167 sec)
I0716 05:16:11.227022 139795883440000 basic_session_run_hooks.py:260] lr = 8.7967215e-05 (27.168 sec)
I0716 05:16:37.509724 139795883440000 basic_session_run_hooks.py:692] global_step/sec: 3.80326
I0716 05:16:37.513396 139795883440000 basic_session_run_hooks.py:260] loss = 0.01121197, step = 11744 (26.290 sec)
I0716 05:16:37.519301 139795883440000 basic_session_run_hooks.py:260] lr = 8.704526e-05 (26.291 sec)
I0716 05:17:04.397915 139795883440000 basic_session_run_hooks.py:692] global_step/sec: 3.71911
I0716 05:17:04.406521 139795883440000 basic_session_run_hooks.py:260] loss = 0.014993166, step = 11844 (26.893 sec)
I0716 05:17:04.408484 139795883440000 basic_session_run_hooks.py:260] lr = 8.613297e-05 (26.890 sec)
I0716 05:17:07.297354 139795883440000 basic_session_run_hooks.py:606] Saving checkpoints for 11856 into ../model/elmo_bigru/model.ckpt.
I0716 05:17:09.314856 139795883440000 estimator.py:1145] Calling model_fn.
I0716 05:17:10.967811 139795883440000 saver.py:1499] Saver not created because there are no variables in the graph to restore
I0716 05:17:11.550818 139795883440000 estimator.py:1147] Done calling model_fn.
I0716 05:17:11.581848 139795883440000 evaluation.py:255] Starting evaluation at 2019-07-16T05:17:11Z
I0716 05:17:11.808008 139795883440000 monitored_session.py:240] Graph was finalized.
I0716 05:17:11.826797 139795883440000 saver.py:1280] Restoring parameters from ../model/elmo_bigru/model.ckpt-11856
I0716 05:17:12.744184 139795883440000 session_manager.py:500] Running local_init_op.
I0716 05:17:12.809362 139795883440000 session_manager.py:502] Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
I0716 05:18:11.775803 139795883440000 evaluation.py:275] Finished evaluation at 2019-07-16-05:18:11
I0716 05:18:11.777727 139795883440000 estimator.py:2039] Saving dict for global step 11856: global_step = 11856, loss = 0.39094597
I0716 05:18:11.788371 139795883440000 estimator.py:2099] Saving 'checkpoint_path' summary for global step 11856: ../model/elmo_bigru/model.ckpt-11856
I0716 05:18:11.875290 139795883440000 estimator.py:368] Loss for final step: 0.0013453001.
Reading ../data/atis.test.w-intent.iob
I0716 05:18:12.474029 139795883440000 estimator.py:1145] Calling model_fn.
I0716 05:18:14.052869 139795883440000 saver.py:1499] Saver not created because there are no variables in the graph to restore
I0716 05:18:14.570518 139795883440000 estimator.py:1147] Done calling model_fn.
I0716 05:18:14.795010 139795883440000 monitored_session.py:240] Graph was finalized.
I0716 05:18:14.810380 139795883440000 saver.py:1280] Restoring parameters from ../model/elmo_bigru/model.ckpt-11856
I0716 05:18:15.705967 139795883440000 session_manager.py:500] Running local_init_op.
I0716 05:18:15.757150 139795883440000 session_manager.py:502] Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.
  'precision', 'predicted', average, warn_for)
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1439: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples.
  'recall', 'true', average, warn_for)
I0716 05:19:13.985109 139795883440000 interactiveshell.py:2882] 
                                          precision    recall  f1-score   support

                             atis_flight      0.984     0.992     0.988       632
                            atis_airfare      0.960     1.000     0.980        48
                     atis_ground_service      1.000     1.000     1.000        36
                            atis_airline      1.000     1.000     1.000        38
                       atis_abbreviation      0.943     1.000     0.971        33
                           atis_aircraft      1.000     0.889     0.941         9
                        atis_flight_time      0.500     1.000     0.667         1
                           atis_quantity      0.375     1.000     0.545         3
                atis_flight#atis_airfare      0.833     0.417     0.556        12
                            atis_airport      1.000     1.000     1.000        18
                           atis_distance      1.000     1.000     1.000        10
                               atis_city      0.750     0.500     0.600         6
                        atis_ground_fare      1.000     0.857     0.923         7
                           atis_capacity      0.952     0.952     0.952        21
                          atis_flight_no      0.889     1.000     0.941         8
                               atis_meal      1.000     0.833     0.909         6
                        atis_restriction      0.000     0.000     0.000         0
             atis_airline#atis_flight_no      0.000     0.000     0.000         0
    atis_ground_service#atis_ground_fare      0.000     0.000     0.000         0
           atis_airfare#atis_flight_time      0.000     0.000     0.000         0
                           atis_cheapest      0.000     0.000     0.000         0
atis_aircraft#atis_flight#atis_flight_no      0.000     0.000     0.000         0

                               micro avg      0.973     0.979     0.976       888
                               macro avg      0.645     0.656     0.635       888
                            weighted avg      0.976     0.979     0.975       888

I0716 05:19:14.023535 139795883440000 interactiveshell.py:2882] 
                              precision    recall  f1-score   support

                           O      0.000     0.000     0.000       0.0
           B-toloc.city_name      0.979     0.993     0.986     716.0
         B-fromloc.city_name      0.989     0.999     0.994     704.0
           I-toloc.city_name      0.978     0.989     0.983     265.0
      B-depart_date.day_name      0.991     0.991     0.991     212.0
              B-airline_name      1.000     1.000     1.000     101.0
         I-fromloc.city_name      0.973     1.000     0.986     177.0
 B-depart_time.period_of_day      0.992     0.923     0.956     130.0
              I-airline_name      1.000     1.000     1.000      65.0
    B-depart_date.day_number      0.981     0.964     0.972      55.0
    B-depart_date.month_name      0.982     0.964     0.973      56.0
          B-depart_time.time      0.864     1.000     0.927      57.0
                B-round_trip      1.000     1.000     1.000      73.0
             B-cost_relative      0.973     0.973     0.973      37.0
                I-round_trip      1.000     1.000     1.000      71.0
                B-flight_mod      0.828     1.000     0.906      24.0
 B-depart_time.time_relative      0.955     0.985     0.970      65.0
          I-depart_time.time      0.945     1.000     0.972      52.0
         B-stoploc.city_name      0.909     1.000     0.952      20.0
                 B-city_name      0.923     0.632     0.750      57.0
                B-class_type      0.960     1.000     0.980      24.0
          B-arrive_time.time      0.971     0.971     0.971      34.0
 B-arrive_time.time_relative      0.967     0.935     0.951      31.0
                I-class_type      1.000     1.000     1.000      17.0
               B-flight_stop      1.000     1.000     1.000      21.0
          I-arrive_time.time      1.000     0.971     0.986      35.0
              B-airline_code      0.971     1.000     0.986      34.0
    I-depart_date.day_number      1.000     0.933     0.966      15.0
      I-fromloc.airport_name      0.441     1.000     0.612      15.0
      B-fromloc.airport_name      0.462     1.000     0.632      12.0
      B-arrive_date.day_name      0.786     1.000     0.880      11.0
          B-toloc.state_code      1.000     1.000     1.000      18.0
B-depart_date.today_relative      1.000     0.889     0.941       9.0
             B-flight_number      0.733     1.000     0.846      11.0
 B-depart_date.date_relative      0.944     1.000     0.971      17.0
          B-toloc.state_name      0.800     1.000     0.889      28.0
           B-fare_basis_code      0.850     1.000     0.919      17.0
               B-flight_time      1.000     1.000     1.000       1.0
                        B-or      1.000     1.000     1.000       3.0
 B-arrive_time.period_of_day      0.750     1.000     0.857       6.0
          B-meal_description      1.000     0.900     0.947      10.0
             I-cost_relative      1.000     0.667     0.800       3.0
              I-airport_name      0.846     0.379     0.524      29.0
               B-fare_amount      1.000     1.000     1.000       2.0
               I-fare_amount      1.000     1.000     1.000       2.0
                 I-city_name      1.000     0.533     0.696      30.0
        I-toloc.airport_name      1.000     1.000     1.000       3.0
            B-transport_type      1.000     1.000     1.000      10.0
    B-arrive_date.month_name      0.714     0.833     0.769       6.0
    B-arrive_date.day_number      0.714     0.833     0.769       6.0
         I-stoploc.city_name      1.000     1.000     1.000      10.0
                      B-meal      1.000     1.000     1.000      16.0
        B-fromloc.state_code      1.000     1.000     1.000      23.0
    B-depart_time.period_mod      1.000     1.000     1.000       5.0
                   B-connect      1.000     1.000     1.000       6.0
               B-flight_days      1.000     1.000     1.000      10.0
        B-toloc.airport_name      1.000     1.000     1.000       3.0
        B-fromloc.state_name      0.944     1.000     0.971      17.0
              B-airport_name      0.800     0.381     0.516      21.0
                   B-economy      1.000     1.000     1.000       6.0
               I-flight_time      1.000     1.000     1.000       1.0
             B-aircraft_code      1.000     0.879     0.935      33.0
                       B-mod      1.000     0.500     0.667       2.0
              B-airport_code      1.000     0.444     0.615       9.0
    B-depart_time.start_time      1.000     0.667     0.800       3.0
      B-depart_time.end_time      1.000     0.333     0.500       3.0
          B-depart_date.year      1.000     1.000     1.000       3.0
            I-transport_type      0.000     0.000     0.000       1.0
          B-restriction_code      0.800     1.000     0.889       4.0
    B-arrive_time.start_time      0.889     1.000     0.941       8.0
        B-toloc.airport_code      1.000     1.000     1.000       4.0
      B-arrive_time.end_time      0.889     1.000     0.941       8.0
      I-arrive_time.end_time      0.889     1.000     0.941       8.0
      I-depart_time.end_time      1.000     0.333     0.500       3.0
               I-flight_stop      0.000     0.000     0.000       0.0
      B-fromloc.airport_code      0.556     1.000     0.714       5.0
          I-restriction_code      1.000     1.000     1.000       3.0
    I-depart_time.start_time      1.000     1.000     1.000       1.0
          I-toloc.state_name      1.000     1.000     1.000       1.0
I-depart_date.today_relative      0.000     0.000     0.000       0.0
 B-arrive_date.date_relative      1.000     1.000     1.000       2.0
                I-flight_mod      1.000     0.167     0.286       6.0
                   I-economy      0.000     0.000     0.000       0.0
 B-return_date.date_relative      1.000     0.333     0.500       3.0
        I-fromloc.state_name      1.000     1.000     1.000       1.0
                B-state_code      1.000     1.000     1.000       1.0
    I-arrive_time.start_time      1.000     1.000     1.000       1.0
    I-arrive_date.day_number      0.000     0.000     0.000       0.0
                 B-meal_code      0.000     0.000     0.000       1.0
 I-depart_time.period_of_day      1.000     1.000     1.000       1.0
                  B-day_name      1.000     0.500     0.667       2.0
             B-period_of_day      1.000     0.750     0.857       4.0
        B-stoploc.state_code      0.000     0.000     0.000       0.0
    B-return_date.month_name      0.000     0.000     0.000       0.0
    B-return_date.day_number      0.000     0.000     0.000       0.0
    B-arrive_time.period_mod      0.000     0.000     0.000       0.0
                 I-meal_code      0.000     0.000     0.000       0.0
        B-toloc.country_name      1.000     1.000     1.000       1.0
                 B-days_code      0.000     0.000     0.000       1.0
 I-arrive_time.period_of_day      0.000     0.000     0.000       0.0
            I-today_relative      0.000     0.000     0.000       0.0
 B-return_time.period_of_day      0.000     0.000     0.000       0.0
                      B-time      0.000     0.000     0.000       0.0
           I-fare_basis_code      0.000     0.000     0.000       0.0
 I-arrive_time.time_relative      1.000     1.000     1.000       4.0
 I-depart_time.time_relative      0.000     0.000     0.000       1.0
            B-today_relative      0.000     0.000     0.000       0.0
                B-state_name      0.000     0.000     0.000       9.0
B-arrive_date.today_relative      0.000     0.000     0.000       0.0
    B-return_time.period_mod      0.000     0.000     0.000       0.0
                B-month_name      0.000     0.000     0.000       0.0
                B-day_number      0.000     0.000     0.000       0.0
 I-return_date.date_relative      0.667     0.667     0.667       3.0
I-return_date.today_relative      0.000     0.000     0.000       0.0
      B-stoploc.airport_name      0.000     0.000     0.000       0.0
             B-time_relative      0.000     0.000     0.000       0.0
                      I-time      0.000     0.000     0.000       0.0
    I-return_date.day_number      0.000     0.000     0.000       0.0
          I-meal_description      0.000     0.000     0.000       0.0
B-return_date.today_relative      0.000     0.000     0.000       0.0
      B-return_date.day_name      0.000     0.000     0.000       2.0

                   micro avg      0.959     0.961     0.960    3657.0
                   macro avg      0.683     0.655     0.654    3657.0
                weighted avg      0.964     0.961     0.957    3657.0

I0716 05:19:14.047439 139795883440000 interactiveshell.py:2882] Best Slot F1: 0.960
I0716 05:19:14.049042 139795883440000 estimator_training.py:186] Not using Distribute Coordinator.
I0716 05:19:14.054128 139795883440000 training.py:612] Running training and evaluation locally (non-distributed).
I0716 05:19:14.056412 139795883440000 training.py:700] Start train and evaluate loop. The evaluate will happen after every checkpoint. Checkpoint frequency is determined based on RunConfig arguments: save_checkpoints_steps 312 or save_checkpoints_secs None.
I0716 05:19:14.123416 139795883440000 estimator.py:1145] Calling model_fn.
I0716 05:19:15.632527 139795883440000 saver.py:1499] Saver not created because there are no variables in the graph to restore
I0716 05:19:16.231479 139795883440000 <ipython-input-6-d6747b40360b>:55] 
[<tf.Variable 'embedding:0' shape=(750, 300) dtype=float32_ref>,
 <tf.Variable 'module/aggregation/weights:0' shape=(3,) dtype=float32>,
 <tf.Variable 'module/aggregation/scaling:0' shape=() dtype=float32>,
 <tf.Variable 'dense/kernel:0' shape=(1324, 300) dtype=float32_ref>,
 <tf.Variable 'dense/bias:0' shape=(300,) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/gates/kernel:0' shape=(600, 600) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/gates/bias:0' shape=(600,) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/candidate/kernel:0' shape=(600, 300) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/candidate/bias:0' shape=(300,) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/gates/kernel:0' shape=(600, 600) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/gates/bias:0' shape=(600,) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/candidate/kernel:0' shape=(600, 300) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/candidate/bias:0' shape=(300,) dtype=float32_ref>,
 <tf.Variable 'dense_1/kernel:0' shape=(600, 23) dtype=float32_ref>,
 <tf.Variable 'dense_1/bias:0' shape=(23,) dtype=float32_ref>,
 <tf.Variable 'dense_2/kernel:0' shape=(600, 122) dtype=float32_ref>,
 <tf.Variable 'dense_2/bias:0' shape=(122,) dtype=float32_ref>]
I0716 05:19:17.529751 139795883440000 estimator.py:1147] Done calling model_fn.
I0716 05:19:17.533969 139795883440000 basic_session_run_hooks.py:541] Create CheckpointSaverHook.
I0716 05:19:17.854725 139795883440000 monitored_session.py:240] Graph was finalized.
I0716 05:19:17.874505 139795883440000 saver.py:1280] Restoring parameters from ../model/elmo_bigru/model.ckpt-11856
I0716 05:19:18.951483 139795883440000 session_manager.py:500] Running local_init_op.
I0716 05:19:19.021305 139795883440000 session_manager.py:502] Done running local_init_op.
I0716 05:19:21.874391 139795883440000 basic_session_run_hooks.py:606] Saving checkpoints for 11856 into ../model/elmo_bigru/model.ckpt.
Reading ../data/atis.train.w-intent.iob
I0716 05:19:26.661842 139795883440000 basic_session_run_hooks.py:262] loss = 0.010607904, step = 11856
I0716 05:19:26.663809 139795883440000 basic_session_run_hooks.py:262] lr = 8.6024134e-05
I0716 05:19:54.552931 139795883440000 basic_session_run_hooks.py:692] global_step/sec: 3.58531
I0716 05:19:54.556187 139795883440000 basic_session_run_hooks.py:260] loss = 0.011345856, step = 11956 (27.894 sec)
I0716 05:19:54.558486 139795883440000 basic_session_run_hooks.py:260] lr = 8.5122534e-05 (27.895 sec)
I0716 05:20:20.401957 139795883440000 basic_session_run_hooks.py:692] global_step/sec: 3.86861
I0716 05:20:20.410420 139795883440000 basic_session_run_hooks.py:260] loss = 0.01699819, step = 12056 (25.854 sec)
I0716 05:20:20.412324 139795883440000 basic_session_run_hooks.py:260] lr = 8.4230385e-05 (25.854 sec)
I0716 05:20:47.246008 139795883440000 basic_session_run_hooks.py:692] global_step/sec: 3.72522
I0716 05:20:47.254420 139795883440000 basic_session_run_hooks.py:260] loss = 0.008648333, step = 12156 (26.844 sec)
I0716 05:20:47.256663 139795883440000 basic_session_run_hooks.py:260] lr = 8.334759e-05 (26.844 sec)
I0716 05:20:50.029184 139795883440000 basic_session_run_hooks.py:606] Saving checkpoints for 12168 into ../model/elmo_bigru/model.ckpt.
I0716 05:20:52.050083 139795883440000 estimator.py:1145] Calling model_fn.
I0716 05:20:53.702356 139795883440000 saver.py:1499] Saver not created because there are no variables in the graph to restore
I0716 05:20:54.303798 139795883440000 estimator.py:1147] Done calling model_fn.
I0716 05:20:54.333423 139795883440000 evaluation.py:255] Starting evaluation at 2019-07-16T05:20:54Z
I0716 05:20:54.558538 139795883440000 monitored_session.py:240] Graph was finalized.
I0716 05:20:54.576385 139795883440000 saver.py:1280] Restoring parameters from ../model/elmo_bigru/model.ckpt-12168
I0716 05:20:55.496074 139795883440000 session_manager.py:500] Running local_init_op.
I0716 05:20:55.558856 139795883440000 session_manager.py:502] Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
I0716 05:21:54.515105 139795883440000 evaluation.py:275] Finished evaluation at 2019-07-16-05:21:54
I0716 05:21:54.517024 139795883440000 estimator.py:2039] Saving dict for global step 12168: global_step = 12168, loss = 0.38762906
I0716 05:21:54.524751 139795883440000 estimator.py:2099] Saving 'checkpoint_path' summary for global step 12168: ../model/elmo_bigru/model.ckpt-12168
I0716 05:21:54.619015 139795883440000 estimator.py:368] Loss for final step: 0.0029825827.
Reading ../data/atis.test.w-intent.iob
I0716 05:21:55.235663 139795883440000 estimator.py:1145] Calling model_fn.
I0716 05:21:56.801426 139795883440000 saver.py:1499] Saver not created because there are no variables in the graph to restore
I0716 05:21:57.311077 139795883440000 estimator.py:1147] Done calling model_fn.
I0716 05:21:57.526308 139795883440000 monitored_session.py:240] Graph was finalized.
I0716 05:21:57.553237 139795883440000 saver.py:1280] Restoring parameters from ../model/elmo_bigru/model.ckpt-12168
I0716 05:21:58.451761 139795883440000 session_manager.py:500] Running local_init_op.
I0716 05:21:58.505653 139795883440000 session_manager.py:502] Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.
  'precision', 'predicted', average, warn_for)
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1439: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples.
  'recall', 'true', average, warn_for)
I0716 05:22:56.483352 139795883440000 interactiveshell.py:2882] 
                                          precision    recall  f1-score   support

                             atis_flight      0.984     0.992     0.988       632
                            atis_airfare      0.960     1.000     0.980        48
                     atis_ground_service      1.000     1.000     1.000        36
                            atis_airline      0.974     1.000     0.987        38
                       atis_abbreviation      0.943     1.000     0.971        33
                           atis_aircraft      1.000     0.889     0.941         9
                        atis_flight_time      0.500     1.000     0.667         1
                           atis_quantity      0.375     1.000     0.545         3
                atis_flight#atis_airfare      0.833     0.417     0.556        12
                            atis_airport      1.000     0.944     0.971        18
                           atis_distance      1.000     1.000     1.000        10
                               atis_city      0.750     0.500     0.600         6
                        atis_ground_fare      1.000     0.857     0.923         7
                           atis_capacity      0.952     0.952     0.952        21
                          atis_flight_no      0.889     1.000     0.941         8
                               atis_meal      1.000     0.833     0.909         6
                        atis_restriction      0.000     0.000     0.000         0
             atis_airline#atis_flight_no      0.000     0.000     0.000         0
    atis_ground_service#atis_ground_fare      0.000     0.000     0.000         0
           atis_airfare#atis_flight_time      0.000     0.000     0.000         0
                           atis_cheapest      0.000     0.000     0.000         0
atis_aircraft#atis_flight#atis_flight_no      0.000     0.000     0.000         0

                               micro avg      0.972     0.977     0.975       888
                               macro avg      0.644     0.654     0.633       888
                            weighted avg      0.975     0.977     0.974       888

I0716 05:22:56.522810 139795883440000 interactiveshell.py:2882] 
                              precision    recall  f1-score   support

                           O      0.000     0.000     0.000       0.0
           B-toloc.city_name      0.982     0.992     0.987     716.0
         B-fromloc.city_name      0.987     0.999     0.993     704.0
           I-toloc.city_name      0.977     0.981     0.979     265.0
      B-depart_date.day_name      0.991     0.991     0.991     212.0
              B-airline_name      1.000     1.000     1.000     101.0
         I-fromloc.city_name      0.967     1.000     0.983     177.0
 B-depart_time.period_of_day      1.000     0.923     0.960     130.0
              I-airline_name      1.000     1.000     1.000      65.0
    B-depart_date.day_number      0.981     0.964     0.972      55.0
    B-depart_date.month_name      0.982     0.964     0.973      56.0
          B-depart_time.time      0.864     1.000     0.927      57.0
                B-round_trip      1.000     1.000     1.000      73.0
             B-cost_relative      0.973     0.973     0.973      37.0
                I-round_trip      1.000     1.000     1.000      71.0
                B-flight_mod      0.828     1.000     0.906      24.0
 B-depart_time.time_relative      0.970     0.985     0.977      65.0
          I-depart_time.time      0.945     1.000     0.972      52.0
         B-stoploc.city_name      0.909     1.000     0.952      20.0
                 B-city_name      0.927     0.667     0.776      57.0
                B-class_type      0.960     1.000     0.980      24.0
          B-arrive_time.time      0.971     0.971     0.971      34.0
 B-arrive_time.time_relative      0.967     0.935     0.951      31.0
                I-class_type      1.000     1.000     1.000      17.0
               B-flight_stop      1.000     1.000     1.000      21.0
          I-arrive_time.time      1.000     0.971     0.986      35.0
              B-airline_code      0.971     1.000     0.986      34.0
    I-depart_date.day_number      1.000     0.933     0.966      15.0
      I-fromloc.airport_name      0.441     1.000     0.612      15.0
      B-fromloc.airport_name      0.462     1.000     0.632      12.0
      B-arrive_date.day_name      0.786     1.000     0.880      11.0
          B-toloc.state_code      1.000     1.000     1.000      18.0
B-depart_date.today_relative      1.000     0.889     0.941       9.0
             B-flight_number      0.733     1.000     0.846      11.0
 B-depart_date.date_relative      0.944     1.000     0.971      17.0
          B-toloc.state_name      0.800     1.000     0.889      28.0
           B-fare_basis_code      0.850     1.000     0.919      17.0
               B-flight_time      1.000     1.000     1.000       1.0
                        B-or      1.000     1.000     1.000       3.0
 B-arrive_time.period_of_day      0.750     1.000     0.857       6.0
          B-meal_description      1.000     0.900     0.947      10.0
             I-cost_relative      1.000     0.667     0.800       3.0
              I-airport_name      0.846     0.379     0.524      29.0
               B-fare_amount      1.000     1.000     1.000       2.0
               I-fare_amount      1.000     1.000     1.000       2.0
                 I-city_name      1.000     0.533     0.696      30.0
        I-toloc.airport_name      1.000     1.000     1.000       3.0
            B-transport_type      1.000     1.000     1.000      10.0
    B-arrive_date.month_name      0.714     0.833     0.769       6.0
    B-arrive_date.day_number      0.714     0.833     0.769       6.0
         I-stoploc.city_name      1.000     1.000     1.000      10.0
                      B-meal      0.941     1.000     0.970      16.0
        B-fromloc.state_code      1.000     1.000     1.000      23.0
    B-depart_time.period_mod      1.000     1.000     1.000       5.0
                   B-connect      1.000     1.000     1.000       6.0
               B-flight_days      1.000     1.000     1.000      10.0
        B-toloc.airport_name      1.000     1.000     1.000       3.0
        B-fromloc.state_name      0.944     1.000     0.971      17.0
              B-airport_name      0.800     0.381     0.516      21.0
                   B-economy      1.000     1.000     1.000       6.0
               I-flight_time      1.000     1.000     1.000       1.0
             B-aircraft_code      1.000     0.879     0.935      33.0
                       B-mod      1.000     0.500     0.667       2.0
              B-airport_code      1.000     0.444     0.615       9.0
    B-depart_time.start_time      1.000     0.667     0.800       3.0
      B-depart_time.end_time      1.000     0.333     0.500       3.0
          B-depart_date.year      1.000     1.000     1.000       3.0
            I-transport_type      0.000     0.000     0.000       1.0
          B-restriction_code      0.800     1.000     0.889       4.0
    B-arrive_time.start_time      0.889     1.000     0.941       8.0
        B-toloc.airport_code      1.000     1.000     1.000       4.0
      B-arrive_time.end_time      0.889     1.000     0.941       8.0
      I-arrive_time.end_time      0.889     1.000     0.941       8.0
      I-depart_time.end_time      1.000     0.333     0.500       3.0
               I-flight_stop      0.000     0.000     0.000       0.0
      B-fromloc.airport_code      0.556     1.000     0.714       5.0
          I-restriction_code      1.000     1.000     1.000       3.0
    I-depart_time.start_time      1.000     1.000     1.000       1.0
          I-toloc.state_name      1.000     1.000     1.000       1.0
I-depart_date.today_relative      0.000     0.000     0.000       0.0
 B-arrive_date.date_relative      1.000     1.000     1.000       2.0
                I-flight_mod      1.000     0.167     0.286       6.0
                   I-economy      0.000     0.000     0.000       0.0
 B-return_date.date_relative      1.000     0.333     0.500       3.0
        I-fromloc.state_name      1.000     1.000     1.000       1.0
                B-state_code      1.000     1.000     1.000       1.0
    I-arrive_time.start_time      1.000     1.000     1.000       1.0
    I-arrive_date.day_number      0.000     0.000     0.000       0.0
                 B-meal_code      0.000     0.000     0.000       1.0
 I-depart_time.period_of_day      1.000     1.000     1.000       1.0
                  B-day_name      1.000     0.500     0.667       2.0
             B-period_of_day      1.000     0.750     0.857       4.0
        B-stoploc.state_code      0.000     0.000     0.000       0.0
    B-return_date.month_name      0.000     0.000     0.000       0.0
    B-return_date.day_number      0.000     0.000     0.000       0.0
    B-arrive_time.period_mod      0.000     0.000     0.000       0.0
                 I-meal_code      0.000     0.000     0.000       0.0
        B-toloc.country_name      1.000     1.000     1.000       1.0
                 B-days_code      0.000     0.000     0.000       1.0
 I-arrive_time.period_of_day      0.000     0.000     0.000       0.0
            I-today_relative      0.000     0.000     0.000       0.0
 B-return_time.period_of_day      0.000     0.000     0.000       0.0
                      B-time      0.000     0.000     0.000       0.0
           I-fare_basis_code      0.000     0.000     0.000       0.0
 I-arrive_time.time_relative      1.000     1.000     1.000       4.0
 I-depart_time.time_relative      0.000     0.000     0.000       1.0
            B-today_relative      0.000     0.000     0.000       0.0
                B-state_name      0.000     0.000     0.000       9.0
B-arrive_date.today_relative      0.000     0.000     0.000       0.0
    B-return_time.period_mod      0.000     0.000     0.000       0.0
                B-month_name      0.000     0.000     0.000       0.0
                B-day_number      0.000     0.000     0.000       0.0
 I-return_date.date_relative      0.667     0.667     0.667       3.0
I-return_date.today_relative      0.000     0.000     0.000       0.0
      B-stoploc.airport_name      0.000     0.000     0.000       0.0
             B-time_relative      0.000     0.000     0.000       0.0
                      I-time      0.000     0.000     0.000       0.0
    I-return_date.day_number      0.000     0.000     0.000       0.0
          I-meal_description      0.000     0.000     0.000       0.0
B-return_date.today_relative      0.000     0.000     0.000       0.0
      B-return_date.day_name      0.000     0.000     0.000       2.0

                   micro avg      0.959     0.961     0.960    3657.0
                   macro avg      0.682     0.655     0.654    3657.0
                weighted avg      0.964     0.961     0.957    3657.0

I0716 05:22:56.546641 139795883440000 interactiveshell.py:2882] Best Slot F1: 0.960
I0716 05:22:56.548236 139795883440000 estimator_training.py:186] Not using Distribute Coordinator.
I0716 05:22:56.550099 139795883440000 training.py:612] Running training and evaluation locally (non-distributed).
I0716 05:22:56.553230 139795883440000 training.py:700] Start train and evaluate loop. The evaluate will happen after every checkpoint. Checkpoint frequency is determined based on RunConfig arguments: save_checkpoints_steps 312 or save_checkpoints_secs None.
I0716 05:22:56.622224 139795883440000 estimator.py:1145] Calling model_fn.
I0716 05:22:58.115841 139795883440000 saver.py:1499] Saver not created because there are no variables in the graph to restore
I0716 05:22:58.702072 139795883440000 <ipython-input-6-d6747b40360b>:55] 
[<tf.Variable 'embedding:0' shape=(750, 300) dtype=float32_ref>,
 <tf.Variable 'module/aggregation/weights:0' shape=(3,) dtype=float32>,
 <tf.Variable 'module/aggregation/scaling:0' shape=() dtype=float32>,
 <tf.Variable 'dense/kernel:0' shape=(1324, 300) dtype=float32_ref>,
 <tf.Variable 'dense/bias:0' shape=(300,) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/gates/kernel:0' shape=(600, 600) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/gates/bias:0' shape=(600,) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/candidate/kernel:0' shape=(600, 300) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/candidate/bias:0' shape=(300,) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/gates/kernel:0' shape=(600, 600) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/gates/bias:0' shape=(600,) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/candidate/kernel:0' shape=(600, 300) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/candidate/bias:0' shape=(300,) dtype=float32_ref>,
 <tf.Variable 'dense_1/kernel:0' shape=(600, 23) dtype=float32_ref>,
 <tf.Variable 'dense_1/bias:0' shape=(23,) dtype=float32_ref>,
 <tf.Variable 'dense_2/kernel:0' shape=(600, 122) dtype=float32_ref>,
 <tf.Variable 'dense_2/bias:0' shape=(122,) dtype=float32_ref>]
I0716 05:22:59.997102 139795883440000 estimator.py:1147] Done calling model_fn.
I0716 05:23:00.000872 139795883440000 basic_session_run_hooks.py:541] Create CheckpointSaverHook.
I0716 05:23:00.332017 139795883440000 monitored_session.py:240] Graph was finalized.
I0716 05:23:00.355240 139795883440000 saver.py:1280] Restoring parameters from ../model/elmo_bigru/model.ckpt-12168
I0716 05:23:01.405848 139795883440000 session_manager.py:500] Running local_init_op.
I0716 05:23:01.476660 139795883440000 session_manager.py:502] Done running local_init_op.
I0716 05:23:04.277058 139795883440000 basic_session_run_hooks.py:606] Saving checkpoints for 12168 into ../model/elmo_bigru/model.ckpt.
Reading ../data/atis.train.w-intent.iob
I0716 05:23:08.941863 139795883440000 basic_session_run_hooks.py:262] loss = 0.009021113, step = 12168
I0716 05:23:08.943840 139795883440000 basic_session_run_hooks.py:262] lr = 8.324228e-05
I0716 05:23:36.479554 139795883440000 basic_session_run_hooks.py:692] global_step/sec: 3.63132
I0716 05:23:36.483439 139795883440000 basic_session_run_hooks.py:260] loss = 0.008717981, step = 12268 (27.542 sec)
I0716 05:23:36.488025 139795883440000 basic_session_run_hooks.py:260] lr = 8.2369836e-05 (27.544 sec)
I0716 05:24:02.972101 139795883440000 basic_session_run_hooks.py:692] global_step/sec: 3.77464
I0716 05:24:02.974956 139795883440000 basic_session_run_hooks.py:260] loss = 0.012833698, step = 12368 (26.492 sec)
I0716 05:24:02.982555 139795883440000 basic_session_run_hooks.py:260] lr = 8.150654e-05 (26.495 sec)
I0716 05:24:29.164611 139795883440000 basic_session_run_hooks.py:692] global_step/sec: 3.81788
I0716 05:24:29.168652 139795883440000 basic_session_run_hooks.py:260] loss = 0.013801475, step = 12468 (26.194 sec)
I0716 05:24:29.172936 139795883440000 basic_session_run_hooks.py:260] lr = 8.0652295e-05 (26.190 sec)
I0716 05:24:32.146581 139795883440000 basic_session_run_hooks.py:606] Saving checkpoints for 12480 into ../model/elmo_bigru/model.ckpt.
I0716 05:24:34.259977 139795883440000 estimator.py:1145] Calling model_fn.
I0716 05:24:35.882333 139795883440000 saver.py:1499] Saver not created because there are no variables in the graph to restore
I0716 05:24:36.459211 139795883440000 estimator.py:1147] Done calling model_fn.
I0716 05:24:36.488796 139795883440000 evaluation.py:255] Starting evaluation at 2019-07-16T05:24:36Z
I0716 05:24:36.715009 139795883440000 monitored_session.py:240] Graph was finalized.
I0716 05:24:36.728446 139795883440000 saver.py:1280] Restoring parameters from ../model/elmo_bigru/model.ckpt-12480
I0716 05:24:37.638752 139795883440000 session_manager.py:500] Running local_init_op.
I0716 05:24:37.698906 139795883440000 session_manager.py:502] Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
I0716 05:25:36.709956 139795883440000 evaluation.py:275] Finished evaluation at 2019-07-16-05:25:36
I0716 05:25:36.711731 139795883440000 estimator.py:2039] Saving dict for global step 12480: global_step = 12480, loss = 0.3937179
I0716 05:25:36.718591 139795883440000 estimator.py:2099] Saving 'checkpoint_path' summary for global step 12480: ../model/elmo_bigru/model.ckpt-12480
I0716 05:25:36.809600 139795883440000 estimator.py:368] Loss for final step: 0.0027113266.
Reading ../data/atis.test.w-intent.iob
I0716 05:25:37.415125 139795883440000 estimator.py:1145] Calling model_fn.
I0716 05:25:38.997050 139795883440000 saver.py:1499] Saver not created because there are no variables in the graph to restore
I0716 05:25:39.525370 139795883440000 estimator.py:1147] Done calling model_fn.
I0716 05:25:39.755532 139795883440000 monitored_session.py:240] Graph was finalized.
I0716 05:25:39.773118 139795883440000 saver.py:1280] Restoring parameters from ../model/elmo_bigru/model.ckpt-12480
I0716 05:25:40.659859 139795883440000 session_manager.py:500] Running local_init_op.
I0716 05:25:40.732056 139795883440000 session_manager.py:502] Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.
  'precision', 'predicted', average, warn_for)
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1439: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples.
  'recall', 'true', average, warn_for)
I0716 05:26:38.619279 139795883440000 interactiveshell.py:2882] 
                                          precision    recall  f1-score   support

                             atis_flight      0.984     0.992     0.988       632
                            atis_airfare      0.941     1.000     0.970        48
                     atis_ground_service      1.000     1.000     1.000        36
                            atis_airline      0.974     1.000     0.987        38
                       atis_abbreviation      0.943     1.000     0.971        33
                           atis_aircraft      1.000     0.889     0.941         9
                        atis_flight_time      0.500     1.000     0.667         1
                           atis_quantity      0.429     1.000     0.600         3
                atis_flight#atis_airfare      0.833     0.417     0.556        12
                            atis_airport      1.000     0.944     0.971        18
                           atis_distance      1.000     1.000     1.000        10
                               atis_city      0.750     0.500     0.600         6
                        atis_ground_fare      1.000     0.857     0.923         7
                           atis_capacity      0.952     0.952     0.952        21
                          atis_flight_no      0.889     1.000     0.941         8
                               atis_meal      1.000     0.833     0.909         6
                        atis_restriction      0.000     0.000     0.000         0
             atis_airline#atis_flight_no      0.000     0.000     0.000         0
    atis_ground_service#atis_ground_fare      0.000     0.000     0.000         0
           atis_airfare#atis_flight_time      0.000     0.000     0.000         0
                           atis_cheapest      0.000     0.000     0.000         0
atis_aircraft#atis_flight#atis_flight_no      0.000     0.000     0.000         0

                               micro avg      0.972     0.977     0.975       888
                               macro avg      0.645     0.654     0.635       888
                            weighted avg      0.974     0.977     0.974       888

I0716 05:26:38.657084 139795883440000 interactiveshell.py:2882] 
                              precision    recall  f1-score   support

                           O      0.000     0.000     0.000       0.0
           B-toloc.city_name      0.977     0.994     0.985     716.0
         B-fromloc.city_name      0.990     0.999     0.994     704.0
           I-toloc.city_name      0.978     0.989     0.983     265.0
      B-depart_date.day_name      0.991     0.991     0.991     212.0
              B-airline_name      1.000     1.000     1.000     101.0
         I-fromloc.city_name      0.973     1.000     0.986     177.0
 B-depart_time.period_of_day      1.000     0.923     0.960     130.0
              I-airline_name      1.000     1.000     1.000      65.0
    B-depart_date.day_number      0.981     0.964     0.972      55.0
    B-depart_date.month_name      0.981     0.946     0.964      56.0
          B-depart_time.time      0.864     1.000     0.927      57.0
                B-round_trip      1.000     1.000     1.000      73.0
             B-cost_relative      0.973     0.973     0.973      37.0
                I-round_trip      1.000     1.000     1.000      71.0
                B-flight_mod      0.828     1.000     0.906      24.0
 B-depart_time.time_relative      0.955     0.985     0.970      65.0
          I-depart_time.time      0.945     1.000     0.972      52.0
         B-stoploc.city_name      0.909     1.000     0.952      20.0
                 B-city_name      0.925     0.649     0.763      57.0
                B-class_type      0.960     1.000     0.980      24.0
          B-arrive_time.time      0.971     0.971     0.971      34.0
 B-arrive_time.time_relative      0.967     0.935     0.951      31.0
                I-class_type      1.000     1.000     1.000      17.0
               B-flight_stop      1.000     1.000     1.000      21.0
          I-arrive_time.time      1.000     0.971     0.986      35.0
              B-airline_code      0.971     1.000     0.986      34.0
    I-depart_date.day_number      1.000     0.933     0.966      15.0
      I-fromloc.airport_name      0.441     1.000     0.612      15.0
      B-fromloc.airport_name      0.462     1.000     0.632      12.0
      B-arrive_date.day_name      0.786     1.000     0.880      11.0
          B-toloc.state_code      1.000     1.000     1.000      18.0
B-depart_date.today_relative      1.000     0.889     0.941       9.0
             B-flight_number      0.733     1.000     0.846      11.0
 B-depart_date.date_relative      0.944     1.000     0.971      17.0
          B-toloc.state_name      0.788     0.929     0.852      28.0
           B-fare_basis_code      0.944     1.000     0.971      17.0
               B-flight_time      1.000     1.000     1.000       1.0
                        B-or      1.000     1.000     1.000       3.0
 B-arrive_time.period_of_day      0.750     1.000     0.857       6.0
          B-meal_description      1.000     0.900     0.947      10.0
             I-cost_relative      1.000     0.667     0.800       3.0
              I-airport_name      0.846     0.379     0.524      29.0
               B-fare_amount      1.000     1.000     1.000       2.0
               I-fare_amount      1.000     1.000     1.000       2.0
                 I-city_name      1.000     0.567     0.723      30.0
        I-toloc.airport_name      1.000     1.000     1.000       3.0
            B-transport_type      1.000     1.000     1.000      10.0
    B-arrive_date.month_name      0.625     0.833     0.714       6.0
    B-arrive_date.day_number      0.714     0.833     0.769       6.0
         I-stoploc.city_name      1.000     1.000     1.000      10.0
                      B-meal      1.000     1.000     1.000      16.0
        B-fromloc.state_code      1.000     1.000     1.000      23.0
    B-depart_time.period_mod      1.000     1.000     1.000       5.0
                   B-connect      1.000     1.000     1.000       6.0
               B-flight_days      1.000     1.000     1.000      10.0
        B-toloc.airport_name      1.000     1.000     1.000       3.0
        B-fromloc.state_name      0.944     1.000     0.971      17.0
              B-airport_name      0.800     0.381     0.516      21.0
                   B-economy      1.000     1.000     1.000       6.0
               I-flight_time      1.000     1.000     1.000       1.0
             B-aircraft_code      1.000     0.879     0.935      33.0
                       B-mod      1.000     0.500     0.667       2.0
              B-airport_code      1.000     0.444     0.615       9.0
    B-depart_time.start_time      1.000     0.667     0.800       3.0
      B-depart_time.end_time      1.000     0.333     0.500       3.0
          B-depart_date.year      1.000     1.000     1.000       3.0
            I-transport_type      0.000     0.000     0.000       1.0
          B-restriction_code      0.800     1.000     0.889       4.0
    B-arrive_time.start_time      0.889     1.000     0.941       8.0
        B-toloc.airport_code      1.000     0.750     0.857       4.0
      B-arrive_time.end_time      0.889     1.000     0.941       8.0
      I-arrive_time.end_time      0.889     1.000     0.941       8.0
      I-depart_time.end_time      1.000     0.333     0.500       3.0
               I-flight_stop      0.000     0.000     0.000       0.0
      B-fromloc.airport_code      0.556     1.000     0.714       5.0
          I-restriction_code      1.000     1.000     1.000       3.0
    I-depart_time.start_time      1.000     1.000     1.000       1.0
          I-toloc.state_name      1.000     1.000     1.000       1.0
I-depart_date.today_relative      0.000     0.000     0.000       0.0
 B-arrive_date.date_relative      1.000     1.000     1.000       2.0
                I-flight_mod      1.000     0.167     0.286       6.0
                   I-economy      0.000     0.000     0.000       0.0
 B-return_date.date_relative      1.000     0.333     0.500       3.0
        I-fromloc.state_name      1.000     1.000     1.000       1.0
                B-state_code      1.000     1.000     1.000       1.0
    I-arrive_time.start_time      1.000     1.000     1.000       1.0
    I-arrive_date.day_number      0.000     0.000     0.000       0.0
                 B-meal_code      0.000     0.000     0.000       1.0
 I-depart_time.period_of_day      1.000     1.000     1.000       1.0
                  B-day_name      1.000     0.500     0.667       2.0
             B-period_of_day      1.000     0.750     0.857       4.0
        B-stoploc.state_code      0.000     0.000     0.000       0.0
    B-return_date.month_name      0.000     0.000     0.000       0.0
    B-return_date.day_number      0.000     0.000     0.000       0.0
    B-arrive_time.period_mod      0.000     0.000     0.000       0.0
                 I-meal_code      0.000     0.000     0.000       0.0
        B-toloc.country_name      1.000     1.000     1.000       1.0
                 B-days_code      1.000     1.000     1.000       1.0
 I-arrive_time.period_of_day      0.000     0.000     0.000       0.0
            I-today_relative      0.000     0.000     0.000       0.0
 B-return_time.period_of_day      0.000     0.000     0.000       0.0
                      B-time      0.000     0.000     0.000       0.0
           I-fare_basis_code      0.000     0.000     0.000       0.0
 I-arrive_time.time_relative      1.000     1.000     1.000       4.0
 I-depart_time.time_relative      0.000     0.000     0.000       1.0
            B-today_relative      0.000     0.000     0.000       0.0
                B-state_name      0.000     0.000     0.000       9.0
B-arrive_date.today_relative      0.000     0.000     0.000       0.0
    B-return_time.period_mod      0.000     0.000     0.000       0.0
                B-month_name      0.000     0.000     0.000       0.0
                B-day_number      0.000     0.000     0.000       0.0
 I-return_date.date_relative      0.750     1.000     0.857       3.0
I-return_date.today_relative      0.000     0.000     0.000       0.0
      B-stoploc.airport_name      0.000     0.000     0.000       0.0
             B-time_relative      0.000     0.000     0.000       0.0
                      I-time      0.000     0.000     0.000       0.0
    I-return_date.day_number      0.000     0.000     0.000       0.0
          I-meal_description      0.000     0.000     0.000       0.0
B-return_date.today_relative      0.000     0.000     0.000       0.0
      B-return_date.day_name      0.000     0.000     0.000       2.0

                   micro avg      0.960     0.961     0.960    3657.0
                   macro avg      0.692     0.663     0.662    3657.0
                weighted avg      0.964     0.961     0.958    3657.0

I0716 05:26:38.680395 139795883440000 interactiveshell.py:2882] Best Slot F1: 0.960
I0716 05:26:38.682062 139795883440000 estimator_training.py:186] Not using Distribute Coordinator.
I0716 05:26:38.684017 139795883440000 training.py:612] Running training and evaluation locally (non-distributed).
I0716 05:26:38.685970 139795883440000 training.py:700] Start train and evaluate loop. The evaluate will happen after every checkpoint. Checkpoint frequency is determined based on RunConfig arguments: save_checkpoints_steps 312 or save_checkpoints_secs None.
I0716 05:26:38.756810 139795883440000 estimator.py:1145] Calling model_fn.
I0716 05:26:40.255664 139795883440000 saver.py:1499] Saver not created because there are no variables in the graph to restore
I0716 05:26:40.830694 139795883440000 <ipython-input-6-d6747b40360b>:55] 
[<tf.Variable 'embedding:0' shape=(750, 300) dtype=float32_ref>,
 <tf.Variable 'module/aggregation/weights:0' shape=(3,) dtype=float32>,
 <tf.Variable 'module/aggregation/scaling:0' shape=() dtype=float32>,
 <tf.Variable 'dense/kernel:0' shape=(1324, 300) dtype=float32_ref>,
 <tf.Variable 'dense/bias:0' shape=(300,) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/gates/kernel:0' shape=(600, 600) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/gates/bias:0' shape=(600,) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/candidate/kernel:0' shape=(600, 300) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/candidate/bias:0' shape=(300,) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/gates/kernel:0' shape=(600, 600) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/gates/bias:0' shape=(600,) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/candidate/kernel:0' shape=(600, 300) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/candidate/bias:0' shape=(300,) dtype=float32_ref>,
 <tf.Variable 'dense_1/kernel:0' shape=(600, 23) dtype=float32_ref>,
 <tf.Variable 'dense_1/bias:0' shape=(23,) dtype=float32_ref>,
 <tf.Variable 'dense_2/kernel:0' shape=(600, 122) dtype=float32_ref>,
 <tf.Variable 'dense_2/bias:0' shape=(122,) dtype=float32_ref>]
I0716 05:26:42.128607 139795883440000 estimator.py:1147] Done calling model_fn.
I0716 05:26:42.132558 139795883440000 basic_session_run_hooks.py:541] Create CheckpointSaverHook.
I0716 05:26:42.452786 139795883440000 monitored_session.py:240] Graph was finalized.
I0716 05:26:42.471553 139795883440000 saver.py:1280] Restoring parameters from ../model/elmo_bigru/model.ckpt-12480
I0716 05:26:43.507312 139795883440000 session_manager.py:500] Running local_init_op.
I0716 05:26:43.579066 139795883440000 session_manager.py:502] Done running local_init_op.
I0716 05:26:46.398946 139795883440000 basic_session_run_hooks.py:606] Saving checkpoints for 12480 into ../model/elmo_bigru/model.ckpt.
Reading ../data/atis.train.w-intent.iob
I0716 05:26:51.132528 139795883440000 basic_session_run_hooks.py:262] loss = 0.013520321, step = 12480
I0716 05:26:51.134556 139795883440000 basic_session_run_hooks.py:262] lr = 8.055038e-05
I0716 05:27:19.365086 139795883440000 basic_session_run_hooks.py:692] global_step/sec: 3.54194
I0716 05:27:19.370056 139795883440000 basic_session_run_hooks.py:260] loss = 0.01258297, step = 12580 (28.238 sec)
I0716 05:27:19.375082 139795883440000 basic_session_run_hooks.py:260] lr = 7.970615e-05 (28.241 sec)
I0716 05:27:44.703674 139795883440000 basic_session_run_hooks.py:692] global_step/sec: 3.94662
I0716 05:27:44.711696 139795883440000 basic_session_run_hooks.py:260] loss = 0.005740271, step = 12680 (25.342 sec)
I0716 05:27:44.713462 139795883440000 basic_session_run_hooks.py:260] lr = 7.887078e-05 (25.338 sec)
I0716 05:28:10.674861 139795883440000 basic_session_run_hooks.py:692] global_step/sec: 3.85037
I0716 05:28:10.683306 139795883440000 basic_session_run_hooks.py:260] loss = 0.008855683, step = 12780 (25.972 sec)
I0716 05:28:10.684962 139795883440000 basic_session_run_hooks.py:260] lr = 7.804416e-05 (25.971 sec)
I0716 05:28:13.473150 139795883440000 basic_session_run_hooks.py:606] Saving checkpoints for 12792 into ../model/elmo_bigru/model.ckpt.
I0716 05:28:15.436150 139795883440000 estimator.py:1145] Calling model_fn.
I0716 05:28:17.078930 139795883440000 saver.py:1499] Saver not created because there are no variables in the graph to restore
I0716 05:28:17.668273 139795883440000 estimator.py:1147] Done calling model_fn.
I0716 05:28:17.698833 139795883440000 evaluation.py:255] Starting evaluation at 2019-07-16T05:28:17Z
I0716 05:28:17.923661 139795883440000 monitored_session.py:240] Graph was finalized.
I0716 05:28:17.941342 139795883440000 saver.py:1280] Restoring parameters from ../model/elmo_bigru/model.ckpt-12792
I0716 05:28:18.848675 139795883440000 session_manager.py:500] Running local_init_op.
I0716 05:28:18.905096 139795883440000 session_manager.py:502] Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
I0716 05:29:17.960201 139795883440000 evaluation.py:275] Finished evaluation at 2019-07-16-05:29:17
I0716 05:29:17.961978 139795883440000 estimator.py:2039] Saving dict for global step 12792: global_step = 12792, loss = 0.400475
I0716 05:29:17.968168 139795883440000 estimator.py:2099] Saving 'checkpoint_path' summary for global step 12792: ../model/elmo_bigru/model.ckpt-12792
I0716 05:29:18.057105 139795883440000 estimator.py:368] Loss for final step: 0.0023489797.
Reading ../data/atis.test.w-intent.iob
I0716 05:29:18.660959 139795883440000 estimator.py:1145] Calling model_fn.
I0716 05:29:20.220807 139795883440000 saver.py:1499] Saver not created because there are no variables in the graph to restore
I0716 05:29:20.743836 139795883440000 estimator.py:1147] Done calling model_fn.
I0716 05:29:20.963388 139795883440000 monitored_session.py:240] Graph was finalized.
I0716 05:29:20.982475 139795883440000 saver.py:1280] Restoring parameters from ../model/elmo_bigru/model.ckpt-12792
I0716 05:29:21.911297 139795883440000 session_manager.py:500] Running local_init_op.
I0716 05:29:21.957988 139795883440000 session_manager.py:502] Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.
  'precision', 'predicted', average, warn_for)
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1439: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples.
  'recall', 'true', average, warn_for)
I0716 05:30:19.955024 139795883440000 interactiveshell.py:2882] 
                                          precision    recall  f1-score   support

                             atis_flight      0.984     0.992     0.988       632
                            atis_airfare      0.941     1.000     0.970        48
                     atis_ground_service      1.000     1.000     1.000        36
                            atis_airline      0.974     1.000     0.987        38
                       atis_abbreviation      0.943     1.000     0.971        33
                           atis_aircraft      1.000     0.889     0.941         9
                        atis_flight_time      0.500     1.000     0.667         1
                           atis_quantity      0.429     1.000     0.600         3
                atis_flight#atis_airfare      0.833     0.417     0.556        12
                            atis_airport      1.000     0.944     0.971        18
                           atis_distance      1.000     1.000     1.000        10
                               atis_city      0.750     0.500     0.600         6
                        atis_ground_fare      1.000     0.857     0.923         7
                           atis_capacity      0.952     0.952     0.952        21
                          atis_flight_no      0.889     1.000     0.941         8
                               atis_meal      1.000     0.833     0.909         6
                        atis_restriction      0.000     0.000     0.000         0
             atis_airline#atis_flight_no      0.000     0.000     0.000         0
    atis_ground_service#atis_ground_fare      0.000     0.000     0.000         0
           atis_airfare#atis_flight_time      0.000     0.000     0.000         0
                           atis_cheapest      0.000     0.000     0.000         0
atis_aircraft#atis_flight#atis_flight_no      0.000     0.000     0.000         0

                               micro avg      0.972     0.977     0.975       888
                               macro avg      0.645     0.654     0.635       888
                            weighted avg      0.974     0.977     0.974       888

I0716 05:30:19.991873 139795883440000 interactiveshell.py:2882] 
                              precision    recall  f1-score   support

                           O      0.000     0.000     0.000       0.0
           B-toloc.city_name      0.978     0.992     0.985     716.0
         B-fromloc.city_name      0.987     0.999     0.993     704.0
           I-toloc.city_name      0.978     0.985     0.981     265.0
      B-depart_date.day_name      0.991     0.991     0.991     212.0
              B-airline_name      1.000     1.000     1.000     101.0
         I-fromloc.city_name      0.967     1.000     0.983     177.0
 B-depart_time.period_of_day      1.000     0.923     0.960     130.0
              I-airline_name      1.000     1.000     1.000      65.0
    B-depart_date.day_number      0.981     0.964     0.972      55.0
    B-depart_date.month_name      0.982     0.964     0.973      56.0
          B-depart_time.time      0.864     1.000     0.927      57.0
                B-round_trip      1.000     1.000     1.000      73.0
             B-cost_relative      0.973     0.973     0.973      37.0
                I-round_trip      1.000     1.000     1.000      71.0
                B-flight_mod      0.828     1.000     0.906      24.0
 B-depart_time.time_relative      0.955     0.985     0.970      65.0
          I-depart_time.time      0.945     1.000     0.972      52.0
         B-stoploc.city_name      0.909     1.000     0.952      20.0
                 B-city_name      0.923     0.632     0.750      57.0
                B-class_type      0.960     1.000     0.980      24.0
          B-arrive_time.time      0.971     0.971     0.971      34.0
 B-arrive_time.time_relative      0.967     0.935     0.951      31.0
                I-class_type      1.000     1.000     1.000      17.0
               B-flight_stop      1.000     1.000     1.000      21.0
          I-arrive_time.time      1.000     0.971     0.986      35.0
              B-airline_code      0.971     1.000     0.986      34.0
    I-depart_date.day_number      1.000     0.933     0.966      15.0
      I-fromloc.airport_name      0.441     1.000     0.612      15.0
      B-fromloc.airport_name      0.462     1.000     0.632      12.0
      B-arrive_date.day_name      0.786     1.000     0.880      11.0
          B-toloc.state_code      1.000     1.000     1.000      18.0
B-depart_date.today_relative      1.000     0.889     0.941       9.0
             B-flight_number      0.733     1.000     0.846      11.0
 B-depart_date.date_relative      0.944     1.000     0.971      17.0
          B-toloc.state_name      0.794     0.964     0.871      28.0
           B-fare_basis_code      0.850     1.000     0.919      17.0
               B-flight_time      1.000     1.000     1.000       1.0
                        B-or      1.000     1.000     1.000       3.0
 B-arrive_time.period_of_day      0.750     1.000     0.857       6.0
          B-meal_description      1.000     0.900     0.947      10.0
             I-cost_relative      1.000     0.667     0.800       3.0
              I-airport_name      0.846     0.379     0.524      29.0
               B-fare_amount      1.000     1.000     1.000       2.0
               I-fare_amount      1.000     1.000     1.000       2.0
                 I-city_name      1.000     0.567     0.723      30.0
        I-toloc.airport_name      1.000     1.000     1.000       3.0
            B-transport_type      1.000     1.000     1.000      10.0
    B-arrive_date.month_name      0.714     0.833     0.769       6.0
    B-arrive_date.day_number      0.714     0.833     0.769       6.0
         I-stoploc.city_name      1.000     1.000     1.000      10.0
                      B-meal      1.000     1.000     1.000      16.0
        B-fromloc.state_code      1.000     1.000     1.000      23.0
    B-depart_time.period_mod      1.000     1.000     1.000       5.0
                   B-connect      1.000     1.000     1.000       6.0
               B-flight_days      1.000     1.000     1.000      10.0
        B-toloc.airport_name      1.000     1.000     1.000       3.0
        B-fromloc.state_name      0.944     1.000     0.971      17.0
              B-airport_name      0.800     0.381     0.516      21.0
                   B-economy      1.000     1.000     1.000       6.0
               I-flight_time      1.000     1.000     1.000       1.0
             B-aircraft_code      1.000     0.879     0.935      33.0
                       B-mod      1.000     0.500     0.667       2.0
              B-airport_code      1.000     0.444     0.615       9.0
    B-depart_time.start_time      1.000     0.667     0.800       3.0
      B-depart_time.end_time      1.000     0.333     0.500       3.0
          B-depart_date.year      1.000     1.000     1.000       3.0
            I-transport_type      0.000     0.000     0.000       1.0
          B-restriction_code      0.800     1.000     0.889       4.0
    B-arrive_time.start_time      0.889     1.000     0.941       8.0
        B-toloc.airport_code      1.000     1.000     1.000       4.0
      B-arrive_time.end_time      0.889     1.000     0.941       8.0
      I-arrive_time.end_time      0.889     1.000     0.941       8.0
      I-depart_time.end_time      1.000     0.333     0.500       3.0
               I-flight_stop      0.000     0.000     0.000       0.0
      B-fromloc.airport_code      0.556     1.000     0.714       5.0
          I-restriction_code      1.000     1.000     1.000       3.0
    I-depart_time.start_time      1.000     1.000     1.000       1.0
          I-toloc.state_name      1.000     1.000     1.000       1.0
I-depart_date.today_relative      0.000     0.000     0.000       0.0
 B-arrive_date.date_relative      1.000     1.000     1.000       2.0
                I-flight_mod      1.000     0.167     0.286       6.0
                   I-economy      0.000     0.000     0.000       0.0
 B-return_date.date_relative      1.000     0.667     0.800       3.0
        I-fromloc.state_name      1.000     1.000     1.000       1.0
                B-state_code      1.000     1.000     1.000       1.0
    I-arrive_time.start_time      1.000     1.000     1.000       1.0
    I-arrive_date.day_number      0.000     0.000     0.000       0.0
                 B-meal_code      0.000     0.000     0.000       1.0
 I-depart_time.period_of_day      1.000     1.000     1.000       1.0
                  B-day_name      1.000     0.500     0.667       2.0
             B-period_of_day      1.000     0.750     0.857       4.0
        B-stoploc.state_code      0.000     0.000     0.000       0.0
    B-return_date.month_name      0.000     0.000     0.000       0.0
    B-return_date.day_number      0.000     0.000     0.000       0.0
    B-arrive_time.period_mod      0.000     0.000     0.000       0.0
                 I-meal_code      0.000     0.000     0.000       0.0
        B-toloc.country_name      1.000     1.000     1.000       1.0
                 B-days_code      0.000     0.000     0.000       1.0
 I-arrive_time.period_of_day      0.000     0.000     0.000       0.0
            I-today_relative      0.000     0.000     0.000       0.0
 B-return_time.period_of_day      0.000     0.000     0.000       0.0
                      B-time      0.000     0.000     0.000       0.0
           I-fare_basis_code      0.000     0.000     0.000       0.0
 I-arrive_time.time_relative      1.000     1.000     1.000       4.0
 I-depart_time.time_relative      0.000     0.000     0.000       1.0
            B-today_relative      0.000     0.000     0.000       0.0
                B-state_name      0.000     0.000     0.000       9.0
B-arrive_date.today_relative      0.000     0.000     0.000       0.0
    B-return_time.period_mod      0.000     0.000     0.000       0.0
                B-month_name      0.000     0.000     0.000       0.0
                B-day_number      0.000     0.000     0.000       0.0
 I-return_date.date_relative      1.000     0.667     0.800       3.0
I-return_date.today_relative      0.000     0.000     0.000       0.0
      B-stoploc.airport_name      0.000     0.000     0.000       0.0
             B-time_relative      0.000     0.000     0.000       0.0
                      I-time      0.000     0.000     0.000       0.0
    I-return_date.day_number      0.000     0.000     0.000       0.0
          I-meal_description      0.000     0.000     0.000       0.0
B-return_date.today_relative      0.000     0.000     0.000       0.0
      B-return_date.day_name      0.000     0.000     0.000       2.0

                   micro avg      0.959     0.961     0.960    3657.0
                   macro avg      0.685     0.657     0.657    3657.0
                weighted avg      0.963     0.961     0.957    3657.0

I0716 05:30:20.019577 139795883440000 interactiveshell.py:2882] Best Slot F1: 0.960
I0716 05:30:20.023614 139795883440000 estimator_training.py:186] Not using Distribute Coordinator.
I0716 05:30:20.025929 139795883440000 training.py:612] Running training and evaluation locally (non-distributed).
I0716 05:30:20.027685 139795883440000 training.py:700] Start train and evaluate loop. The evaluate will happen after every checkpoint. Checkpoint frequency is determined based on RunConfig arguments: save_checkpoints_steps 312 or save_checkpoints_secs None.
I0716 05:30:20.102035 139795883440000 estimator.py:1145] Calling model_fn.
I0716 05:30:21.588777 139795883440000 saver.py:1499] Saver not created because there are no variables in the graph to restore
I0716 05:30:22.181025 139795883440000 <ipython-input-6-d6747b40360b>:55] 
[<tf.Variable 'embedding:0' shape=(750, 300) dtype=float32_ref>,
 <tf.Variable 'module/aggregation/weights:0' shape=(3,) dtype=float32>,
 <tf.Variable 'module/aggregation/scaling:0' shape=() dtype=float32>,
 <tf.Variable 'dense/kernel:0' shape=(1324, 300) dtype=float32_ref>,
 <tf.Variable 'dense/bias:0' shape=(300,) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/gates/kernel:0' shape=(600, 600) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/gates/bias:0' shape=(600,) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/candidate/kernel:0' shape=(600, 300) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/candidate/bias:0' shape=(300,) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/gates/kernel:0' shape=(600, 600) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/gates/bias:0' shape=(600,) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/candidate/kernel:0' shape=(600, 300) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/candidate/bias:0' shape=(300,) dtype=float32_ref>,
 <tf.Variable 'dense_1/kernel:0' shape=(600, 23) dtype=float32_ref>,
 <tf.Variable 'dense_1/bias:0' shape=(23,) dtype=float32_ref>,
 <tf.Variable 'dense_2/kernel:0' shape=(600, 122) dtype=float32_ref>,
 <tf.Variable 'dense_2/bias:0' shape=(122,) dtype=float32_ref>]
I0716 05:30:23.469295 139795883440000 estimator.py:1147] Done calling model_fn.
I0716 05:30:23.473536 139795883440000 basic_session_run_hooks.py:541] Create CheckpointSaverHook.
I0716 05:30:23.794810 139795883440000 monitored_session.py:240] Graph was finalized.
I0716 05:30:23.816685 139795883440000 saver.py:1280] Restoring parameters from ../model/elmo_bigru/model.ckpt-12792
I0716 05:30:24.864028 139795883440000 session_manager.py:500] Running local_init_op.
I0716 05:30:24.939088 139795883440000 session_manager.py:502] Done running local_init_op.
I0716 05:30:27.726537 139795883440000 basic_session_run_hooks.py:606] Saving checkpoints for 12792 into ../model/elmo_bigru/model.ckpt.
Reading ../data/atis.train.w-intent.iob
I0716 05:30:32.478336 139795883440000 basic_session_run_hooks.py:262] loss = 0.010260903, step = 12792
I0716 05:30:32.480191 139795883440000 basic_session_run_hooks.py:262] lr = 7.7945544e-05
I0716 05:31:00.489519 139795883440000 basic_session_run_hooks.py:692] global_step/sec: 3.56993
I0716 05:31:00.497004 139795883440000 basic_session_run_hooks.py:260] loss = 0.009084362, step = 12892 (28.019 sec)
I0716 05:31:00.499659 139795883440000 basic_session_run_hooks.py:260] lr = 7.712862e-05 (28.019 sec)
I0716 05:31:26.556809 139795883440000 basic_session_run_hooks.py:692] global_step/sec: 3.83623
I0716 05:31:26.560199 139795883440000 basic_session_run_hooks.py:260] loss = 0.008441718, step = 12992 (26.063 sec)
I0716 05:31:26.567608 139795883440000 basic_session_run_hooks.py:260] lr = 7.632024e-05 (26.068 sec)
I0716 05:31:53.002555 139795883440000 basic_session_run_hooks.py:692] global_step/sec: 3.78133
I0716 05:31:53.007514 139795883440000 basic_session_run_hooks.py:260] loss = 0.012072688, step = 13092 (26.447 sec)
I0716 05:31:53.012902 139795883440000 basic_session_run_hooks.py:260] lr = 7.552035e-05 (26.445 sec)
I0716 05:31:55.986191 139795883440000 basic_session_run_hooks.py:606] Saving checkpoints for 13104 into ../model/elmo_bigru/model.ckpt.
I0716 05:31:58.048022 139795883440000 estimator.py:1145] Calling model_fn.
I0716 05:31:59.683301 139795883440000 saver.py:1499] Saver not created because there are no variables in the graph to restore
I0716 05:32:00.265349 139795883440000 estimator.py:1147] Done calling model_fn.
I0716 05:32:00.295489 139795883440000 evaluation.py:255] Starting evaluation at 2019-07-16T05:32:00Z
I0716 05:32:00.528330 139795883440000 monitored_session.py:240] Graph was finalized.
I0716 05:32:00.546332 139795883440000 saver.py:1280] Restoring parameters from ../model/elmo_bigru/model.ckpt-13104
I0716 05:32:01.450866 139795883440000 session_manager.py:500] Running local_init_op.
I0716 05:32:01.516671 139795883440000 session_manager.py:502] Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
I0716 05:33:00.227576 139795883440000 evaluation.py:275] Finished evaluation at 2019-07-16-05:33:00
I0716 05:33:00.229390 139795883440000 estimator.py:2039] Saving dict for global step 13104: global_step = 13104, loss = 0.40517667
I0716 05:33:00.236587 139795883440000 estimator.py:2099] Saving 'checkpoint_path' summary for global step 13104: ../model/elmo_bigru/model.ckpt-13104
I0716 05:33:00.323032 139795883440000 estimator.py:368] Loss for final step: 0.0011228888.
Reading ../data/atis.test.w-intent.iob
I0716 05:33:00.930331 139795883440000 estimator.py:1145] Calling model_fn.
I0716 05:33:02.460177 139795883440000 saver.py:1499] Saver not created because there are no variables in the graph to restore
I0716 05:33:02.986480 139795883440000 estimator.py:1147] Done calling model_fn.
I0716 05:33:03.196828 139795883440000 monitored_session.py:240] Graph was finalized.
I0716 05:33:03.215934 139795883440000 saver.py:1280] Restoring parameters from ../model/elmo_bigru/model.ckpt-13104
I0716 05:33:04.122166 139795883440000 session_manager.py:500] Running local_init_op.
I0716 05:33:04.172596 139795883440000 session_manager.py:502] Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.
  'precision', 'predicted', average, warn_for)
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1439: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples.
  'recall', 'true', average, warn_for)
I0716 05:34:02.142135 139795883440000 interactiveshell.py:2882] 
                                          precision    recall  f1-score   support

                             atis_flight      0.984     0.992     0.988       632
                            atis_airfare      0.960     1.000     0.980        48
                     atis_ground_service      1.000     1.000     1.000        36
                            atis_airline      0.974     1.000     0.987        38
                       atis_abbreviation      0.943     1.000     0.971        33
                           atis_aircraft      1.000     0.889     0.941         9
                        atis_flight_time      0.500     1.000     0.667         1
                           atis_quantity      0.375     1.000     0.545         3
                atis_flight#atis_airfare      0.833     0.417     0.556        12
                            atis_airport      1.000     0.944     0.971        18
                           atis_distance      1.000     1.000     1.000        10
                               atis_city      0.750     0.500     0.600         6
                        atis_ground_fare      1.000     0.857     0.923         7
                           atis_capacity      0.952     0.952     0.952        21
                          atis_flight_no      0.889     1.000     0.941         8
                               atis_meal      1.000     0.833     0.909         6
                        atis_restriction      0.000     0.000     0.000         0
             atis_airline#atis_flight_no      0.000     0.000     0.000         0
    atis_ground_service#atis_ground_fare      0.000     0.000     0.000         0
           atis_airfare#atis_flight_time      0.000     0.000     0.000         0
                           atis_cheapest      0.000     0.000     0.000         0
atis_aircraft#atis_flight#atis_flight_no      0.000     0.000     0.000         0

                               micro avg      0.972     0.977     0.975       888
                               macro avg      0.644     0.654     0.633       888
                            weighted avg      0.975     0.977     0.974       888

I0716 05:34:02.187218 139795883440000 interactiveshell.py:2882] 
                              precision    recall  f1-score   support

                           O      0.000     0.000     0.000       0.0
           B-toloc.city_name      0.974     0.994     0.984     716.0
         B-fromloc.city_name      0.990     0.999     0.994     704.0
           I-toloc.city_name      0.978     0.989     0.983     265.0
      B-depart_date.day_name      0.991     0.991     0.991     212.0
              B-airline_name      1.000     1.000     1.000     101.0
         I-fromloc.city_name      0.973     1.000     0.986     177.0
 B-depart_time.period_of_day      1.000     0.915     0.956     130.0
              I-airline_name      1.000     1.000     1.000      65.0
    B-depart_date.day_number      0.981     0.964     0.972      55.0
    B-depart_date.month_name      0.982     0.964     0.973      56.0
          B-depart_time.time      0.864     1.000     0.927      57.0
                B-round_trip      1.000     1.000     1.000      73.0
             B-cost_relative      0.973     0.973     0.973      37.0
                I-round_trip      1.000     1.000     1.000      71.0
                B-flight_mod      0.828     1.000     0.906      24.0
 B-depart_time.time_relative      0.970     0.985     0.977      65.0
          I-depart_time.time      0.929     1.000     0.963      52.0
         B-stoploc.city_name      0.909     1.000     0.952      20.0
                 B-city_name      0.923     0.632     0.750      57.0
                B-class_type      0.960     1.000     0.980      24.0
          B-arrive_time.time      0.971     0.971     0.971      34.0
 B-arrive_time.time_relative      0.967     0.935     0.951      31.0
                I-class_type      1.000     1.000     1.000      17.0
               B-flight_stop      1.000     1.000     1.000      21.0
          I-arrive_time.time      1.000     0.971     0.986      35.0
              B-airline_code      0.944     1.000     0.971      34.0
    I-depart_date.day_number      1.000     0.933     0.966      15.0
      I-fromloc.airport_name      0.441     1.000     0.612      15.0
      B-fromloc.airport_name      0.462     1.000     0.632      12.0
      B-arrive_date.day_name      0.786     1.000     0.880      11.0
          B-toloc.state_code      1.000     1.000     1.000      18.0
B-depart_date.today_relative      1.000     0.889     0.941       9.0
             B-flight_number      0.733     1.000     0.846      11.0
 B-depart_date.date_relative      0.944     1.000     0.971      17.0
          B-toloc.state_name      0.833     0.893     0.862      28.0
           B-fare_basis_code      0.850     1.000     0.919      17.0
               B-flight_time      1.000     1.000     1.000       1.0
                        B-or      1.000     1.000     1.000       3.0
 B-arrive_time.period_of_day      0.750     1.000     0.857       6.0
          B-meal_description      1.000     0.900     0.947      10.0
             I-cost_relative      1.000     0.667     0.800       3.0
              I-airport_name      0.846     0.379     0.524      29.0
               B-fare_amount      1.000     1.000     1.000       2.0
               I-fare_amount      1.000     1.000     1.000       2.0
                 I-city_name      1.000     0.533     0.696      30.0
        I-toloc.airport_name      1.000     1.000     1.000       3.0
            B-transport_type      1.000     1.000     1.000      10.0
    B-arrive_date.month_name      0.714     0.833     0.769       6.0
    B-arrive_date.day_number      0.714     0.833     0.769       6.0
         I-stoploc.city_name      1.000     1.000     1.000      10.0
                      B-meal      1.000     1.000     1.000      16.0
        B-fromloc.state_code      1.000     1.000     1.000      23.0
    B-depart_time.period_mod      1.000     1.000     1.000       5.0
                   B-connect      1.000     1.000     1.000       6.0
               B-flight_days      1.000     1.000     1.000      10.0
        B-toloc.airport_name      1.000     1.000     1.000       3.0
        B-fromloc.state_name      0.944     1.000     0.971      17.0
              B-airport_name      0.667     0.381     0.485      21.0
                   B-economy      1.000     1.000     1.000       6.0
               I-flight_time      1.000     1.000     1.000       1.0
             B-aircraft_code      1.000     0.879     0.935      33.0
                       B-mod      1.000     0.500     0.667       2.0
              B-airport_code      1.000     0.444     0.615       9.0
    B-depart_time.start_time      1.000     0.667     0.800       3.0
      B-depart_time.end_time      1.000     0.333     0.500       3.0
          B-depart_date.year      1.000     1.000     1.000       3.0
            I-transport_type      0.000     0.000     0.000       1.0
          B-restriction_code      1.000     1.000     1.000       4.0
    B-arrive_time.start_time      0.889     1.000     0.941       8.0
        B-toloc.airport_code      1.000     0.750     0.857       4.0
      B-arrive_time.end_time      0.889     1.000     0.941       8.0
      I-arrive_time.end_time      0.889     1.000     0.941       8.0
      I-depart_time.end_time      1.000     0.333     0.500       3.0
               I-flight_stop      0.000     0.000     0.000       0.0
      B-fromloc.airport_code      0.556     1.000     0.714       5.0
          I-restriction_code      1.000     1.000     1.000       3.0
    I-depart_time.start_time      1.000     1.000     1.000       1.0
          I-toloc.state_name      1.000     1.000     1.000       1.0
I-depart_date.today_relative      0.000     0.000     0.000       0.0
 B-arrive_date.date_relative      1.000     1.000     1.000       2.0
                I-flight_mod      1.000     0.167     0.286       6.0
                   I-economy      0.000     0.000     0.000       0.0
 B-return_date.date_relative      1.000     0.333     0.500       3.0
        I-fromloc.state_name      1.000     1.000     1.000       1.0
                B-state_code      1.000     1.000     1.000       1.0
    I-arrive_time.start_time      1.000     1.000     1.000       1.0
    I-arrive_date.day_number      0.000     0.000     0.000       0.0
                 B-meal_code      0.000     0.000     0.000       1.0
 I-depart_time.period_of_day      1.000     1.000     1.000       1.0
                  B-day_name      1.000     0.500     0.667       2.0
             B-period_of_day      1.000     0.750     0.857       4.0
        B-stoploc.state_code      0.000     0.000     0.000       0.0
    B-return_date.month_name      0.000     0.000     0.000       0.0
    B-return_date.day_number      0.000     0.000     0.000       0.0
    B-arrive_time.period_mod      0.000     0.000     0.000       0.0
                 I-meal_code      0.000     0.000     0.000       0.0
        B-toloc.country_name      1.000     1.000     1.000       1.0
                 B-days_code      0.000     0.000     0.000       1.0
 I-arrive_time.period_of_day      0.000     0.000     0.000       0.0
            I-today_relative      0.000     0.000     0.000       0.0
 B-return_time.period_of_day      0.000     0.000     0.000       0.0
                      B-time      0.000     0.000     0.000       0.0
           I-fare_basis_code      0.000     0.000     0.000       0.0
 I-arrive_time.time_relative      1.000     1.000     1.000       4.0
 I-depart_time.time_relative      0.000     0.000     0.000       1.0
            B-today_relative      0.000     0.000     0.000       0.0
                B-state_name      0.000     0.000     0.000       9.0
B-arrive_date.today_relative      0.000     0.000     0.000       0.0
    B-return_time.period_mod      0.000     0.000     0.000       0.0
                B-month_name      0.000     0.000     0.000       0.0
                B-day_number      0.000     0.000     0.000       0.0
 I-return_date.date_relative      0.667     0.667     0.667       3.0
I-return_date.today_relative      0.000     0.000     0.000       0.0
      B-stoploc.airport_name      0.000     0.000     0.000       0.0
             B-time_relative      0.000     0.000     0.000       0.0
                      I-time      0.000     0.000     0.000       0.0
    I-return_date.day_number      0.000     0.000     0.000       0.0
          I-meal_description      0.000     0.000     0.000       0.0
B-return_date.today_relative      0.000     0.000     0.000       0.0
      B-return_date.day_name      0.000     0.000     0.000       2.0

                   micro avg      0.958     0.960     0.959    3657.0
                   macro avg      0.683     0.652     0.653    3657.0
                weighted avg      0.963     0.960     0.957    3657.0

I0716 05:34:02.211748 139795883440000 interactiveshell.py:2882] Best Slot F1: 0.960
I0716 05:34:02.213319 139795883440000 estimator_training.py:186] Not using Distribute Coordinator.
I0716 05:34:02.216927 139795883440000 training.py:612] Running training and evaluation locally (non-distributed).
I0716 05:34:02.219686 139795883440000 training.py:700] Start train and evaluate loop. The evaluate will happen after every checkpoint. Checkpoint frequency is determined based on RunConfig arguments: save_checkpoints_steps 312 or save_checkpoints_secs None.
I0716 05:34:02.287338 139795883440000 estimator.py:1145] Calling model_fn.
I0716 05:34:03.786828 139795883440000 saver.py:1499] Saver not created because there are no variables in the graph to restore
I0716 05:34:04.382152 139795883440000 <ipython-input-6-d6747b40360b>:55] 
[<tf.Variable 'embedding:0' shape=(750, 300) dtype=float32_ref>,
 <tf.Variable 'module/aggregation/weights:0' shape=(3,) dtype=float32>,
 <tf.Variable 'module/aggregation/scaling:0' shape=() dtype=float32>,
 <tf.Variable 'dense/kernel:0' shape=(1324, 300) dtype=float32_ref>,
 <tf.Variable 'dense/bias:0' shape=(300,) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/gates/kernel:0' shape=(600, 600) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/gates/bias:0' shape=(600,) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/candidate/kernel:0' shape=(600, 300) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/candidate/bias:0' shape=(300,) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/gates/kernel:0' shape=(600, 600) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/gates/bias:0' shape=(600,) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/candidate/kernel:0' shape=(600, 300) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/candidate/bias:0' shape=(300,) dtype=float32_ref>,
 <tf.Variable 'dense_1/kernel:0' shape=(600, 23) dtype=float32_ref>,
 <tf.Variable 'dense_1/bias:0' shape=(23,) dtype=float32_ref>,
 <tf.Variable 'dense_2/kernel:0' shape=(600, 122) dtype=float32_ref>,
 <tf.Variable 'dense_2/bias:0' shape=(122,) dtype=float32_ref>]
I0716 05:34:05.677148 139795883440000 estimator.py:1147] Done calling model_fn.
I0716 05:34:05.680834 139795883440000 basic_session_run_hooks.py:541] Create CheckpointSaverHook.
I0716 05:34:06.006551 139795883440000 monitored_session.py:240] Graph was finalized.
I0716 05:34:06.029321 139795883440000 saver.py:1280] Restoring parameters from ../model/elmo_bigru/model.ckpt-13104
I0716 05:34:07.111840 139795883440000 session_manager.py:500] Running local_init_op.
I0716 05:34:07.186209 139795883440000 session_manager.py:502] Done running local_init_op.
I0716 05:34:10.109262 139795883440000 basic_session_run_hooks.py:606] Saving checkpoints for 13104 into ../model/elmo_bigru/model.ckpt.
Reading ../data/atis.train.w-intent.iob
I0716 05:34:15.083790 139795883440000 basic_session_run_hooks.py:262] loss = 0.008863335, step = 13104
I0716 05:34:15.085566 139795883440000 basic_session_run_hooks.py:262] lr = 7.542493e-05
I0716 05:34:42.543205 139795883440000 basic_session_run_hooks.py:692] global_step/sec: 3.64162
I0716 05:34:42.549776 139795883440000 basic_session_run_hooks.py:260] loss = 0.008364289, step = 13204 (27.466 sec)
I0716 05:34:42.552909 139795883440000 basic_session_run_hooks.py:260] lr = 7.463443e-05 (27.467 sec)
I0716 05:35:09.690642 139795883440000 basic_session_run_hooks.py:692] global_step/sec: 3.68359
I0716 05:35:09.699984 139795883440000 basic_session_run_hooks.py:260] loss = 0.011803856, step = 13304 (27.150 sec)
I0716 05:35:09.701951 139795883440000 basic_session_run_hooks.py:260] lr = 7.385219e-05 (27.149 sec)
I0716 05:35:35.376586 139795883440000 basic_session_run_hooks.py:692] global_step/sec: 3.89318
I0716 05:35:35.386376 139795883440000 basic_session_run_hooks.py:260] loss = 0.012592914, step = 13404 (25.686 sec)
I0716 05:35:35.388249 139795883440000 basic_session_run_hooks.py:260] lr = 7.3078176e-05 (25.686 sec)
I0716 05:35:38.137051 139795883440000 basic_session_run_hooks.py:606] Saving checkpoints for 13416 into ../model/elmo_bigru/model.ckpt.
I0716 05:35:40.099873 139795883440000 estimator.py:1145] Calling model_fn.
I0716 05:35:41.769351 139795883440000 saver.py:1499] Saver not created because there are no variables in the graph to restore
I0716 05:35:42.353270 139795883440000 estimator.py:1147] Done calling model_fn.
I0716 05:35:42.386612 139795883440000 evaluation.py:255] Starting evaluation at 2019-07-16T05:35:42Z
I0716 05:35:42.613936 139795883440000 monitored_session.py:240] Graph was finalized.
I0716 05:35:42.632848 139795883440000 saver.py:1280] Restoring parameters from ../model/elmo_bigru/model.ckpt-13416
I0716 05:35:43.552337 139795883440000 session_manager.py:500] Running local_init_op.
I0716 05:35:43.637071 139795883440000 session_manager.py:502] Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
I0716 05:36:42.807984 139795883440000 evaluation.py:275] Finished evaluation at 2019-07-16-05:36:42
I0716 05:36:42.809957 139795883440000 estimator.py:2039] Saving dict for global step 13416: global_step = 13416, loss = 0.41098255
I0716 05:36:42.819127 139795883440000 estimator.py:2099] Saving 'checkpoint_path' summary for global step 13416: ../model/elmo_bigru/model.ckpt-13416
I0716 05:36:42.902808 139795883440000 estimator.py:368] Loss for final step: 0.013912404.
Reading ../data/atis.test.w-intent.iob
I0716 05:36:43.498341 139795883440000 estimator.py:1145] Calling model_fn.
I0716 05:36:45.058119 139795883440000 saver.py:1499] Saver not created because there are no variables in the graph to restore
I0716 05:36:45.578801 139795883440000 estimator.py:1147] Done calling model_fn.
I0716 05:36:45.795963 139795883440000 monitored_session.py:240] Graph was finalized.
I0716 05:36:45.814599 139795883440000 saver.py:1280] Restoring parameters from ../model/elmo_bigru/model.ckpt-13416
I0716 05:36:46.731653 139795883440000 session_manager.py:500] Running local_init_op.
I0716 05:36:46.780838 139795883440000 session_manager.py:502] Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.
  'precision', 'predicted', average, warn_for)
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1439: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples.
  'recall', 'true', average, warn_for)
I0716 05:37:44.828861 139795883440000 interactiveshell.py:2882] 
                                          precision    recall  f1-score   support

                             atis_flight      0.986     0.992     0.989       632
                            atis_airfare      0.923     1.000     0.960        48
                     atis_ground_service      1.000     1.000     1.000        36
                            atis_airline      0.974     1.000     0.987        38
                       atis_abbreviation      0.943     1.000     0.971        33
                           atis_aircraft      1.000     0.889     0.941         9
                        atis_flight_time      0.500     1.000     0.667         1
                           atis_quantity      0.429     1.000     0.600         3
                atis_flight#atis_airfare      0.833     0.417     0.556        12
                            atis_airport      1.000     0.944     0.971        18
                           atis_distance      1.000     1.000     1.000        10
                               atis_city      0.750     0.500     0.600         6
                        atis_ground_fare      1.000     0.857     0.923         7
                           atis_capacity      0.952     0.952     0.952        21
                          atis_flight_no      0.889     1.000     0.941         8
                               atis_meal      1.000     0.833     0.909         6
                        atis_restriction      0.000     0.000     0.000         0
             atis_airline#atis_flight_no      0.000     0.000     0.000         0
    atis_ground_service#atis_ground_fare      0.000     0.000     0.000         0
           atis_airfare#atis_flight_time      0.000     0.000     0.000         0
                           atis_cheapest      0.000     0.000     0.000         0
atis_aircraft#atis_flight#atis_flight_no      0.000     0.000     0.000         0

                               micro avg      0.972     0.977     0.975       888
                               macro avg      0.645     0.654     0.635       888
                            weighted avg      0.974     0.977     0.974       888

I0716 05:37:44.868015 139795883440000 interactiveshell.py:2882] 
                              precision    recall  f1-score   support

                           O      0.000     0.000     0.000       0.0
           B-toloc.city_name      0.973     0.996     0.984     716.0
         B-fromloc.city_name      0.992     0.999     0.995     704.0
           I-toloc.city_name      0.978     0.992     0.985     265.0
      B-depart_date.day_name      0.991     0.991     0.991     212.0
              B-airline_name      1.000     1.000     1.000     101.0
         I-fromloc.city_name      0.983     1.000     0.992     177.0
 B-depart_time.period_of_day      0.992     0.923     0.956     130.0
              I-airline_name      1.000     1.000     1.000      65.0
    B-depart_date.day_number      0.981     0.964     0.972      55.0
    B-depart_date.month_name      0.982     0.964     0.973      56.0
          B-depart_time.time      0.864     1.000     0.927      57.0
                B-round_trip      1.000     0.986     0.993      73.0
             B-cost_relative      0.973     0.973     0.973      37.0
                I-round_trip      1.000     1.000     1.000      71.0
                B-flight_mod      0.828     1.000     0.906      24.0
 B-depart_time.time_relative      0.970     0.985     0.977      65.0
          I-depart_time.time      0.945     1.000     0.972      52.0
         B-stoploc.city_name      0.909     1.000     0.952      20.0
                 B-city_name      0.897     0.614     0.729      57.0
                B-class_type      0.960     1.000     0.980      24.0
          B-arrive_time.time      0.971     0.971     0.971      34.0
 B-arrive_time.time_relative      0.906     0.935     0.921      31.0
                I-class_type      1.000     1.000     1.000      17.0
               B-flight_stop      1.000     1.000     1.000      21.0
          I-arrive_time.time      1.000     0.971     0.986      35.0
              B-airline_code      0.944     1.000     0.971      34.0
    I-depart_date.day_number      1.000     0.933     0.966      15.0
      I-fromloc.airport_name      0.441     1.000     0.612      15.0
      B-fromloc.airport_name      0.462     1.000     0.632      12.0
      B-arrive_date.day_name      0.786     1.000     0.880      11.0
          B-toloc.state_code      1.000     1.000     1.000      18.0
B-depart_date.today_relative      1.000     0.889     0.941       9.0
             B-flight_number      0.733     1.000     0.846      11.0
 B-depart_date.date_relative      0.944     1.000     0.971      17.0
          B-toloc.state_name      0.833     0.893     0.862      28.0
           B-fare_basis_code      0.895     1.000     0.944      17.0
               B-flight_time      1.000     1.000     1.000       1.0
                        B-or      1.000     1.000     1.000       3.0
 B-arrive_time.period_of_day      0.750     1.000     0.857       6.0
          B-meal_description      1.000     0.900     0.947      10.0
             I-cost_relative      1.000     0.667     0.800       3.0
              I-airport_name      0.846     0.379     0.524      29.0
               B-fare_amount      1.000     1.000     1.000       2.0
               I-fare_amount      1.000     1.000     1.000       2.0
                 I-city_name      1.000     0.567     0.723      30.0
        I-toloc.airport_name      1.000     1.000     1.000       3.0
            B-transport_type      1.000     1.000     1.000      10.0
    B-arrive_date.month_name      0.714     0.833     0.769       6.0
    B-arrive_date.day_number      0.714     0.833     0.769       6.0
         I-stoploc.city_name      1.000     1.000     1.000      10.0
                      B-meal      1.000     1.000     1.000      16.0
        B-fromloc.state_code      1.000     1.000     1.000      23.0
    B-depart_time.period_mod      1.000     1.000     1.000       5.0
                   B-connect      1.000     1.000     1.000       6.0
               B-flight_days      1.000     1.000     1.000      10.0
        B-toloc.airport_name      1.000     1.000     1.000       3.0
        B-fromloc.state_name      0.944     1.000     0.971      17.0
              B-airport_name      0.636     0.333     0.437      21.0
                   B-economy      1.000     1.000     1.000       6.0
               I-flight_time      1.000     1.000     1.000       1.0
             B-aircraft_code      1.000     0.879     0.935      33.0
                       B-mod      1.000     0.500     0.667       2.0
              B-airport_code      1.000     0.444     0.615       9.0
    B-depart_time.start_time      1.000     0.667     0.800       3.0
      B-depart_time.end_time      1.000     0.333     0.500       3.0
          B-depart_date.year      1.000     1.000     1.000       3.0
            I-transport_type      0.000     0.000     0.000       1.0
          B-restriction_code      1.000     1.000     1.000       4.0
    B-arrive_time.start_time      0.889     1.000     0.941       8.0
        B-toloc.airport_code      1.000     0.750     0.857       4.0
      B-arrive_time.end_time      0.889     1.000     0.941       8.0
      I-arrive_time.end_time      0.889     1.000     0.941       8.0
      I-depart_time.end_time      1.000     0.333     0.500       3.0
               I-flight_stop      0.000     0.000     0.000       0.0
      B-fromloc.airport_code      0.556     1.000     0.714       5.0
          I-restriction_code      1.000     1.000     1.000       3.0
    I-depart_time.start_time      1.000     1.000     1.000       1.0
          I-toloc.state_name      1.000     1.000     1.000       1.0
I-depart_date.today_relative      0.000     0.000     0.000       0.0
 B-arrive_date.date_relative      1.000     1.000     1.000       2.0
                I-flight_mod      1.000     0.167     0.286       6.0
                   I-economy      0.000     0.000     0.000       0.0
 B-return_date.date_relative      1.000     0.333     0.500       3.0
        I-fromloc.state_name      1.000     1.000     1.000       1.0
                B-state_code      1.000     1.000     1.000       1.0
    I-arrive_time.start_time      1.000     1.000     1.000       1.0
    I-arrive_date.day_number      0.000     0.000     0.000       0.0
                 B-meal_code      0.000     0.000     0.000       1.0
 I-depart_time.period_of_day      1.000     1.000     1.000       1.0
                  B-day_name      1.000     0.500     0.667       2.0
             B-period_of_day      1.000     0.750     0.857       4.0
        B-stoploc.state_code      0.000     0.000     0.000       0.0
    B-return_date.month_name      0.000     0.000     0.000       0.0
    B-return_date.day_number      0.000     0.000     0.000       0.0
    B-arrive_time.period_mod      0.000     0.000     0.000       0.0
                 I-meal_code      0.000     0.000     0.000       0.0
        B-toloc.country_name      1.000     1.000     1.000       1.0
                 B-days_code      1.000     1.000     1.000       1.0
 I-arrive_time.period_of_day      0.000     0.000     0.000       0.0
            I-today_relative      0.000     0.000     0.000       0.0
 B-return_time.period_of_day      0.000     0.000     0.000       0.0
                      B-time      0.000     0.000     0.000       0.0
           I-fare_basis_code      0.000     0.000     0.000       0.0
 I-arrive_time.time_relative      1.000     0.500     0.667       4.0
 I-depart_time.time_relative      0.000     0.000     0.000       1.0
            B-today_relative      0.000     0.000     0.000       0.0
                B-state_name      0.000     0.000     0.000       9.0
B-arrive_date.today_relative      0.000     0.000     0.000       0.0
    B-return_time.period_mod      0.000     0.000     0.000       0.0
                B-month_name      0.000     0.000     0.000       0.0
                B-day_number      0.000     0.000     0.000       0.0
 I-return_date.date_relative      0.667     0.667     0.667       3.0
I-return_date.today_relative      0.000     0.000     0.000       0.0
      B-stoploc.airport_name      0.000     0.000     0.000       0.0
             B-time_relative      0.000     0.000     0.000       0.0
                      I-time      0.000     0.000     0.000       0.0
    I-return_date.day_number      0.000     0.000     0.000       0.0
          I-meal_description      0.000     0.000     0.000       0.0
B-return_date.today_relative      0.000     0.000     0.000       0.0
      B-return_date.day_name      0.000     0.000     0.000       2.0

                   micro avg      0.958     0.960     0.959    3657.0
                   macro avg      0.691     0.655     0.658    3657.0
                weighted avg      0.963     0.960     0.956    3657.0

I0716 05:37:44.891681 139795883440000 interactiveshell.py:2882] Best Slot F1: 0.960
I0716 05:37:44.893238 139795883440000 estimator_training.py:186] Not using Distribute Coordinator.
I0716 05:37:44.895620 139795883440000 training.py:612] Running training and evaluation locally (non-distributed).
I0716 05:37:44.898410 139795883440000 training.py:700] Start train and evaluate loop. The evaluate will happen after every checkpoint. Checkpoint frequency is determined based on RunConfig arguments: save_checkpoints_steps 312 or save_checkpoints_secs None.
I0716 05:37:44.966067 139795883440000 estimator.py:1145] Calling model_fn.
I0716 05:37:46.474880 139795883440000 saver.py:1499] Saver not created because there are no variables in the graph to restore
I0716 05:37:47.060688 139795883440000 <ipython-input-6-d6747b40360b>:55] 
[<tf.Variable 'embedding:0' shape=(750, 300) dtype=float32_ref>,
 <tf.Variable 'module/aggregation/weights:0' shape=(3,) dtype=float32>,
 <tf.Variable 'module/aggregation/scaling:0' shape=() dtype=float32>,
 <tf.Variable 'dense/kernel:0' shape=(1324, 300) dtype=float32_ref>,
 <tf.Variable 'dense/bias:0' shape=(300,) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/gates/kernel:0' shape=(600, 600) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/gates/bias:0' shape=(600,) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/candidate/kernel:0' shape=(600, 300) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/candidate/bias:0' shape=(300,) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/gates/kernel:0' shape=(600, 600) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/gates/bias:0' shape=(600,) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/candidate/kernel:0' shape=(600, 300) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/candidate/bias:0' shape=(300,) dtype=float32_ref>,
 <tf.Variable 'dense_1/kernel:0' shape=(600, 23) dtype=float32_ref>,
 <tf.Variable 'dense_1/bias:0' shape=(23,) dtype=float32_ref>,
 <tf.Variable 'dense_2/kernel:0' shape=(600, 122) dtype=float32_ref>,
 <tf.Variable 'dense_2/bias:0' shape=(122,) dtype=float32_ref>]
I0716 05:37:48.348798 139795883440000 estimator.py:1147] Done calling model_fn.
I0716 05:37:48.357215 139795883440000 basic_session_run_hooks.py:541] Create CheckpointSaverHook.
I0716 05:37:48.674329 139795883440000 monitored_session.py:240] Graph was finalized.
I0716 05:37:48.693415 139795883440000 saver.py:1280] Restoring parameters from ../model/elmo_bigru/model.ckpt-13416
I0716 05:37:49.772269 139795883440000 session_manager.py:500] Running local_init_op.
I0716 05:37:49.842147 139795883440000 session_manager.py:502] Done running local_init_op.
I0716 05:37:52.684488 139795883440000 basic_session_run_hooks.py:606] Saving checkpoints for 13416 into ../model/elmo_bigru/model.ckpt.
Reading ../data/atis.train.w-intent.iob
I0716 05:37:57.513536 139795883440000 basic_session_run_hooks.py:262] loss = 0.013887277, step = 13416
I0716 05:37:57.515413 139795883440000 basic_session_run_hooks.py:262] lr = 7.298584e-05
I0716 05:38:24.983728 139795883440000 basic_session_run_hooks.py:692] global_step/sec: 3.64024
I0716 05:38:24.987360 139795883440000 basic_session_run_hooks.py:260] loss = 0.012135924, step = 13516 (27.474 sec)
I0716 05:38:24.992357 139795883440000 basic_session_run_hooks.py:260] lr = 7.2220886e-05 (27.477 sec)
I0716 05:38:51.416158 139795883440000 basic_session_run_hooks.py:692] global_step/sec: 3.78322
I0716 05:38:51.424153 139795883440000 basic_session_run_hooks.py:260] loss = 0.01026399, step = 13616 (26.437 sec)
I0716 05:38:51.426458 139795883440000 basic_session_run_hooks.py:260] lr = 7.146397e-05 (26.434 sec)
I0716 05:39:18.190498 139795883440000 basic_session_run_hooks.py:692] global_step/sec: 3.73492
I0716 05:39:18.199103 139795883440000 basic_session_run_hooks.py:260] loss = 0.0123380385, step = 13716 (26.775 sec)
I0716 05:39:18.201380 139795883440000 basic_session_run_hooks.py:260] lr = 7.071497e-05 (26.775 sec)
I0716 05:39:20.919991 139795883440000 basic_session_run_hooks.py:606] Saving checkpoints for 13728 into ../model/elmo_bigru/model.ckpt.
I0716 05:39:23.175500 139795883440000 estimator.py:1145] Calling model_fn.
I0716 05:39:24.846983 139795883440000 saver.py:1499] Saver not created because there are no variables in the graph to restore
I0716 05:39:25.436039 139795883440000 estimator.py:1147] Done calling model_fn.
I0716 05:39:25.467047 139795883440000 evaluation.py:255] Starting evaluation at 2019-07-16T05:39:25Z
I0716 05:39:25.705051 139795883440000 monitored_session.py:240] Graph was finalized.
I0716 05:39:25.720287 139795883440000 saver.py:1280] Restoring parameters from ../model/elmo_bigru/model.ckpt-13728
I0716 05:39:26.682774 139795883440000 session_manager.py:500] Running local_init_op.
I0716 05:39:26.746172 139795883440000 session_manager.py:502] Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
I0716 05:40:25.841186 139795883440000 evaluation.py:275] Finished evaluation at 2019-07-16-05:40:25
I0716 05:40:25.843400 139795883440000 estimator.py:2039] Saving dict for global step 13728: global_step = 13728, loss = 0.40032935
I0716 05:40:25.851620 139795883440000 estimator.py:2099] Saving 'checkpoint_path' summary for global step 13728: ../model/elmo_bigru/model.ckpt-13728
I0716 05:40:25.939419 139795883440000 estimator.py:368] Loss for final step: 0.012668098.
Reading ../data/atis.test.w-intent.iob
I0716 05:40:26.538111 139795883440000 estimator.py:1145] Calling model_fn.
I0716 05:40:28.083170 139795883440000 saver.py:1499] Saver not created because there are no variables in the graph to restore
I0716 05:40:28.598452 139795883440000 estimator.py:1147] Done calling model_fn.
I0716 05:40:28.815289 139795883440000 monitored_session.py:240] Graph was finalized.
I0716 05:40:28.834128 139795883440000 saver.py:1280] Restoring parameters from ../model/elmo_bigru/model.ckpt-13728
I0716 05:40:29.743818 139795883440000 session_manager.py:500] Running local_init_op.
I0716 05:40:29.792791 139795883440000 session_manager.py:502] Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.
  'precision', 'predicted', average, warn_for)
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1439: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples.
  'recall', 'true', average, warn_for)
I0716 05:41:27.823527 139795883440000 interactiveshell.py:2882] 
                                          precision    recall  f1-score   support

                             atis_flight      0.987     0.991     0.989       632
                            atis_airfare      0.960     1.000     0.980        48
                     atis_ground_service      1.000     1.000     1.000        36
                            atis_airline      0.974     1.000     0.987        38
                       atis_abbreviation      0.943     1.000     0.971        33
                           atis_aircraft      1.000     0.889     0.941         9
                        atis_flight_time      0.333     1.000     0.500         1
                           atis_quantity      0.375     1.000     0.545         3
                atis_flight#atis_airfare      0.833     0.417     0.556        12
                            atis_airport      0.944     0.944     0.944        18
                           atis_distance      1.000     1.000     1.000        10
                               atis_city      0.800     0.667     0.727         6
                        atis_ground_fare      1.000     0.857     0.923         7
                           atis_capacity      0.952     0.952     0.952        21
                          atis_flight_no      0.889     1.000     0.941         8
                               atis_meal      1.000     0.833     0.909         6
                        atis_restriction      0.000     0.000     0.000         0
             atis_airline#atis_flight_no      0.000     0.000     0.000         0
    atis_ground_service#atis_ground_fare      0.000     0.000     0.000         0
           atis_airfare#atis_flight_time      0.000     0.000     0.000         0
                           atis_cheapest      0.000     0.000     0.000         0
atis_aircraft#atis_flight#atis_flight_no      0.000     0.000     0.000         0

                               micro avg      0.972     0.977     0.975       888
                               macro avg      0.636     0.661     0.630       888
                            weighted avg      0.976     0.977     0.975       888

I0716 05:41:27.862074 139795883440000 interactiveshell.py:2882] 
                              precision    recall  f1-score   support

                           O      0.000     0.000     0.000       0.0
           B-toloc.city_name      0.974     0.992     0.983     716.0
         B-fromloc.city_name      0.985     1.000     0.992     704.0
           I-toloc.city_name      0.978     0.985     0.981     265.0
      B-depart_date.day_name      0.991     0.991     0.991     212.0
              B-airline_name      1.000     1.000     1.000     101.0
         I-fromloc.city_name      0.967     1.000     0.983     177.0
 B-depart_time.period_of_day      0.992     0.923     0.956     130.0
              I-airline_name      0.985     1.000     0.992      65.0
    B-depart_date.day_number      0.981     0.964     0.972      55.0
    B-depart_date.month_name      0.982     0.964     0.973      56.0
          B-depart_time.time      0.864     1.000     0.927      57.0
                B-round_trip      1.000     1.000     1.000      73.0
             B-cost_relative      0.973     0.973     0.973      37.0
                I-round_trip      1.000     1.000     1.000      71.0
                B-flight_mod      0.828     1.000     0.906      24.0
 B-depart_time.time_relative      0.970     0.985     0.977      65.0
          I-depart_time.time      0.945     1.000     0.972      52.0
         B-stoploc.city_name      0.952     1.000     0.976      20.0
                 B-city_name      0.946     0.614     0.745      57.0
                B-class_type      0.960     1.000     0.980      24.0
          B-arrive_time.time      0.971     0.971     0.971      34.0
 B-arrive_time.time_relative      0.967     0.935     0.951      31.0
                I-class_type      1.000     1.000     1.000      17.0
               B-flight_stop      1.000     1.000     1.000      21.0
          I-arrive_time.time      1.000     0.971     0.986      35.0
              B-airline_code      0.944     1.000     0.971      34.0
    I-depart_date.day_number      1.000     0.933     0.966      15.0
      I-fromloc.airport_name      0.441     1.000     0.612      15.0
      B-fromloc.airport_name      0.462     1.000     0.632      12.0
      B-arrive_date.day_name      0.786     1.000     0.880      11.0
          B-toloc.state_code      1.000     1.000     1.000      18.0
B-depart_date.today_relative      1.000     0.889     0.941       9.0
             B-flight_number      0.733     1.000     0.846      11.0
 B-depart_date.date_relative      0.944     1.000     0.971      17.0
          B-toloc.state_name      0.862     0.893     0.877      28.0
           B-fare_basis_code      0.850     1.000     0.919      17.0
               B-flight_time      1.000     1.000     1.000       1.0
                        B-or      1.000     1.000     1.000       3.0
 B-arrive_time.period_of_day      0.750     1.000     0.857       6.0
          B-meal_description      1.000     0.900     0.947      10.0
             I-cost_relative      1.000     0.667     0.800       3.0
              I-airport_name      0.846     0.379     0.524      29.0
               B-fare_amount      1.000     1.000     1.000       2.0
               I-fare_amount      1.000     1.000     1.000       2.0
                 I-city_name      1.000     0.533     0.696      30.0
        I-toloc.airport_name      1.000     1.000     1.000       3.0
            B-transport_type      1.000     1.000     1.000      10.0
    B-arrive_date.month_name      0.714     0.833     0.769       6.0
    B-arrive_date.day_number      0.714     0.833     0.769       6.0
         I-stoploc.city_name      1.000     1.000     1.000      10.0
                      B-meal      1.000     1.000     1.000      16.0
        B-fromloc.state_code      1.000     1.000     1.000      23.0
    B-depart_time.period_mod      1.000     1.000     1.000       5.0
                   B-connect      1.000     1.000     1.000       6.0
               B-flight_days      1.000     1.000     1.000      10.0
        B-toloc.airport_name      1.000     1.000     1.000       3.0
        B-fromloc.state_name      0.895     1.000     0.944      17.0
              B-airport_name      0.667     0.381     0.485      21.0
                   B-economy      1.000     1.000     1.000       6.0
               I-flight_time      1.000     1.000     1.000       1.0
             B-aircraft_code      1.000     0.879     0.935      33.0
                       B-mod      1.000     0.500     0.667       2.0
              B-airport_code      1.000     0.444     0.615       9.0
    B-depart_time.start_time      1.000     0.667     0.800       3.0
      B-depart_time.end_time      1.000     0.333     0.500       3.0
          B-depart_date.year      1.000     1.000     1.000       3.0
            I-transport_type      0.000     0.000     0.000       1.0
          B-restriction_code      1.000     1.000     1.000       4.0
    B-arrive_time.start_time      0.889     1.000     0.941       8.0
        B-toloc.airport_code      1.000     0.750     0.857       4.0
      B-arrive_time.end_time      0.889     1.000     0.941       8.0
      I-arrive_time.end_time      0.889     1.000     0.941       8.0
      I-depart_time.end_time      1.000     0.333     0.500       3.0
               I-flight_stop      0.000     0.000     0.000       0.0
      B-fromloc.airport_code      0.556     1.000     0.714       5.0
          I-restriction_code      1.000     1.000     1.000       3.0
    I-depart_time.start_time      1.000     1.000     1.000       1.0
          I-toloc.state_name      1.000     1.000     1.000       1.0
I-depart_date.today_relative      0.000     0.000     0.000       0.0
 B-arrive_date.date_relative      1.000     1.000     1.000       2.0
                I-flight_mod      1.000     0.167     0.286       6.0
                   I-economy      0.000     0.000     0.000       0.0
 B-return_date.date_relative      1.000     0.333     0.500       3.0
        I-fromloc.state_name      1.000     1.000     1.000       1.0
                B-state_code      1.000     1.000     1.000       1.0
    I-arrive_time.start_time      1.000     1.000     1.000       1.0
    I-arrive_date.day_number      0.000     0.000     0.000       0.0
                 B-meal_code      0.000     0.000     0.000       1.0
 I-depart_time.period_of_day      1.000     1.000     1.000       1.0
                  B-day_name      1.000     0.500     0.667       2.0
             B-period_of_day      1.000     0.750     0.857       4.0
        B-stoploc.state_code      0.000     0.000     0.000       0.0
    B-return_date.month_name      0.000     0.000     0.000       0.0
    B-return_date.day_number      0.000     0.000     0.000       0.0
    B-arrive_time.period_mod      0.000     0.000     0.000       0.0
                 I-meal_code      0.000     0.000     0.000       0.0
        B-toloc.country_name      0.000     0.000     0.000       1.0
                 B-days_code      0.000     0.000     0.000       1.0
 I-arrive_time.period_of_day      0.000     0.000     0.000       0.0
            I-today_relative      0.000     0.000     0.000       0.0
 B-return_time.period_of_day      0.000     0.000     0.000       0.0
                      B-time      0.000     0.000     0.000       0.0
           I-fare_basis_code      0.000     0.000     0.000       0.0
 I-arrive_time.time_relative      1.000     1.000     1.000       4.0
 I-depart_time.time_relative      0.000     0.000     0.000       1.0
            B-today_relative      0.000     0.000     0.000       0.0
                B-state_name      0.000     0.000     0.000       9.0
B-arrive_date.today_relative      0.000     0.000     0.000       0.0
    B-return_time.period_mod      0.000     0.000     0.000       0.0
                B-month_name      0.000     0.000     0.000       0.0
                B-day_number      0.000     0.000     0.000       0.0
 I-return_date.date_relative      0.667     0.667     0.667       3.0
I-return_date.today_relative      0.000     0.000     0.000       0.0
      B-stoploc.airport_name      0.000     0.000     0.000       0.0
             B-time_relative      0.000     0.000     0.000       0.0
                      I-time      0.000     0.000     0.000       0.0
    I-return_date.day_number      0.000     0.000     0.000       0.0
          I-meal_description      0.000     0.000     0.000       0.0
B-return_date.today_relative      0.000     0.000     0.000       0.0
      B-return_date.day_name      0.000     0.000     0.000       2.0

                   micro avg      0.957     0.959     0.958    3657.0
                   macro avg      0.675     0.643     0.644    3657.0
                weighted avg      0.961     0.959     0.955    3657.0

I0716 05:41:27.884293 139795883440000 interactiveshell.py:2882] Best Slot F1: 0.960
I0716 05:41:27.885876 139795883440000 estimator_training.py:186] Not using Distribute Coordinator.
I0716 05:41:27.887674 139795883440000 training.py:612] Running training and evaluation locally (non-distributed).
I0716 05:41:27.889172 139795883440000 training.py:700] Start train and evaluate loop. The evaluate will happen after every checkpoint. Checkpoint frequency is determined based on RunConfig arguments: save_checkpoints_steps 312 or save_checkpoints_secs None.
I0716 05:41:27.958157 139795883440000 estimator.py:1145] Calling model_fn.
I0716 05:41:29.460845 139795883440000 saver.py:1499] Saver not created because there are no variables in the graph to restore
I0716 05:41:30.036725 139795883440000 <ipython-input-6-d6747b40360b>:55] 
[<tf.Variable 'embedding:0' shape=(750, 300) dtype=float32_ref>,
 <tf.Variable 'module/aggregation/weights:0' shape=(3,) dtype=float32>,
 <tf.Variable 'module/aggregation/scaling:0' shape=() dtype=float32>,
 <tf.Variable 'dense/kernel:0' shape=(1324, 300) dtype=float32_ref>,
 <tf.Variable 'dense/bias:0' shape=(300,) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/gates/kernel:0' shape=(600, 600) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/gates/bias:0' shape=(600,) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/candidate/kernel:0' shape=(600, 300) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/candidate/bias:0' shape=(300,) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/gates/kernel:0' shape=(600, 600) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/gates/bias:0' shape=(600,) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/candidate/kernel:0' shape=(600, 300) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/candidate/bias:0' shape=(300,) dtype=float32_ref>,
 <tf.Variable 'dense_1/kernel:0' shape=(600, 23) dtype=float32_ref>,
 <tf.Variable 'dense_1/bias:0' shape=(23,) dtype=float32_ref>,
 <tf.Variable 'dense_2/kernel:0' shape=(600, 122) dtype=float32_ref>,
 <tf.Variable 'dense_2/bias:0' shape=(122,) dtype=float32_ref>]
I0716 05:41:31.329411 139795883440000 estimator.py:1147] Done calling model_fn.
I0716 05:41:31.334978 139795883440000 basic_session_run_hooks.py:541] Create CheckpointSaverHook.
I0716 05:41:31.666571 139795883440000 monitored_session.py:240] Graph was finalized.
I0716 05:41:31.685984 139795883440000 saver.py:1280] Restoring parameters from ../model/elmo_bigru/model.ckpt-13728
I0716 05:41:32.754310 139795883440000 session_manager.py:500] Running local_init_op.
I0716 05:41:32.839430 139795883440000 session_manager.py:502] Done running local_init_op.
I0716 05:41:35.610599 139795883440000 basic_session_run_hooks.py:606] Saving checkpoints for 13728 into ../model/elmo_bigru/model.ckpt.
Reading ../data/atis.train.w-intent.iob
I0716 05:41:40.410586 139795883440000 basic_session_run_hooks.py:262] loss = 0.010003799, step = 13728
I0716 05:41:40.412563 139795883440000 basic_session_run_hooks.py:262] lr = 7.062562e-05
I0716 05:42:08.743339 139795883440000 basic_session_run_hooks.py:692] global_step/sec: 3.52939
I0716 05:42:08.751814 139795883440000 basic_session_run_hooks.py:260] loss = 0.009100836, step = 13828 (28.341 sec)
I0716 05:42:08.753760 139795883440000 basic_session_run_hooks.py:260] lr = 6.98854e-05 (28.341 sec)
I0716 05:42:35.140239 139795883440000 basic_session_run_hooks.py:692] global_step/sec: 3.78833
I0716 05:42:35.145344 139795883440000 basic_session_run_hooks.py:260] loss = 0.012343187, step = 13928 (26.394 sec)
I0716 05:42:35.150030 139795883440000 basic_session_run_hooks.py:260] lr = 6.915296e-05 (26.396 sec)
I0716 05:43:00.936875 139795883440000 basic_session_run_hooks.py:692] global_step/sec: 3.87648
I0716 05:43:00.940275 139795883440000 basic_session_run_hooks.py:260] loss = 0.014199471, step = 14028 (25.795 sec)
I0716 05:43:00.944169 139795883440000 basic_session_run_hooks.py:260] lr = 6.842818e-05 (25.794 sec)
I0716 05:43:03.559301 139795883440000 basic_session_run_hooks.py:606] Saving checkpoints for 14040 into ../model/elmo_bigru/model.ckpt.
I0716 05:43:05.664705 139795883440000 estimator.py:1145] Calling model_fn.
I0716 05:43:07.298515 139795883440000 saver.py:1499] Saver not created because there are no variables in the graph to restore
I0716 05:43:07.874066 139795883440000 estimator.py:1147] Done calling model_fn.
I0716 05:43:07.904097 139795883440000 evaluation.py:255] Starting evaluation at 2019-07-16T05:43:07Z
I0716 05:43:08.125681 139795883440000 monitored_session.py:240] Graph was finalized.
I0716 05:43:08.143678 139795883440000 saver.py:1280] Restoring parameters from ../model/elmo_bigru/model.ckpt-14040
I0716 05:43:09.050469 139795883440000 session_manager.py:500] Running local_init_op.
I0716 05:43:09.107095 139795883440000 session_manager.py:502] Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
I0716 05:44:08.006983 139795883440000 evaluation.py:275] Finished evaluation at 2019-07-16-05:44:08
I0716 05:44:08.008811 139795883440000 estimator.py:2039] Saving dict for global step 14040: global_step = 14040, loss = 0.40883175
I0716 05:44:08.016630 139795883440000 estimator.py:2099] Saving 'checkpoint_path' summary for global step 14040: ../model/elmo_bigru/model.ckpt-14040
I0716 05:44:08.101225 139795883440000 estimator.py:368] Loss for final step: 0.0031542224.
Reading ../data/atis.test.w-intent.iob
I0716 05:44:08.701605 139795883440000 estimator.py:1145] Calling model_fn.
I0716 05:44:10.260324 139795883440000 saver.py:1499] Saver not created because there are no variables in the graph to restore
I0716 05:44:10.775668 139795883440000 estimator.py:1147] Done calling model_fn.
I0716 05:44:11.004148 139795883440000 monitored_session.py:240] Graph was finalized.
I0716 05:44:11.021503 139795883440000 saver.py:1280] Restoring parameters from ../model/elmo_bigru/model.ckpt-14040
I0716 05:44:11.922996 139795883440000 session_manager.py:500] Running local_init_op.
I0716 05:44:11.984531 139795883440000 session_manager.py:502] Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.
  'precision', 'predicted', average, warn_for)
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1439: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples.
  'recall', 'true', average, warn_for)
I0716 05:45:10.302753 139795883440000 interactiveshell.py:2882] 
                                          precision    recall  f1-score   support

                             atis_flight      0.983     0.992     0.987       632
                            atis_airfare      0.941     1.000     0.970        48
                     atis_ground_service      1.000     1.000     1.000        36
                            atis_airline      1.000     1.000     1.000        38
                       atis_abbreviation      0.943     1.000     0.971        33
                           atis_aircraft      1.000     0.889     0.941         9
                        atis_flight_time      0.500     1.000     0.667         1
                           atis_quantity      0.429     1.000     0.600         3
                atis_flight#atis_airfare      0.833     0.417     0.556        12
                            atis_airport      1.000     1.000     1.000        18
                           atis_distance      1.000     1.000     1.000        10
                               atis_city      1.000     0.500     0.667         6
                        atis_ground_fare      1.000     0.857     0.923         7
                           atis_capacity      0.952     0.952     0.952        21
                          atis_flight_no      0.889     1.000     0.941         8
                               atis_meal      1.000     0.833     0.909         6
                        atis_restriction      0.000     0.000     0.000         0
             atis_airline#atis_flight_no      0.000     0.000     0.000         0
    atis_ground_service#atis_ground_fare      0.000     0.000     0.000         0
           atis_airfare#atis_flight_time      0.000     0.000     0.000         0
                           atis_cheapest      0.000     0.000     0.000         0
atis_aircraft#atis_flight#atis_flight_no      0.000     0.000     0.000         0

                               micro avg      0.973     0.979     0.976       888
                               macro avg      0.658     0.656     0.640       888
                            weighted avg      0.976     0.979     0.975       888

I0716 05:45:10.341657 139795883440000 interactiveshell.py:2882] 
                              precision    recall  f1-score   support

                           O      0.000     0.000     0.000       0.0
           B-toloc.city_name      0.975     0.994     0.985     716.0
         B-fromloc.city_name      0.990     0.999     0.994     704.0
           I-toloc.city_name      0.978     0.989     0.983     265.0
      B-depart_date.day_name      0.991     0.991     0.991     212.0
              B-airline_name      1.000     1.000     1.000     101.0
         I-fromloc.city_name      0.978     1.000     0.989     177.0
 B-depart_time.period_of_day      1.000     0.923     0.960     130.0
              I-airline_name      1.000     1.000     1.000      65.0
    B-depart_date.day_number      0.981     0.964     0.972      55.0
    B-depart_date.month_name      0.982     0.964     0.973      56.0
          B-depart_time.time      0.864     1.000     0.927      57.0
                B-round_trip      1.000     1.000     1.000      73.0
             B-cost_relative      0.973     0.973     0.973      37.0
                I-round_trip      1.000     1.000     1.000      71.0
                B-flight_mod      0.828     1.000     0.906      24.0
 B-depart_time.time_relative      0.970     0.985     0.977      65.0
          I-depart_time.time      0.945     1.000     0.972      52.0
         B-stoploc.city_name      0.909     1.000     0.952      20.0
                 B-city_name      0.923     0.632     0.750      57.0
                B-class_type      0.960     1.000     0.980      24.0
          B-arrive_time.time      0.971     0.971     0.971      34.0
 B-arrive_time.time_relative      0.967     0.935     0.951      31.0
                I-class_type      1.000     1.000     1.000      17.0
               B-flight_stop      1.000     1.000     1.000      21.0
          I-arrive_time.time      1.000     0.971     0.986      35.0
              B-airline_code      0.971     1.000     0.986      34.0
    I-depart_date.day_number      1.000     0.933     0.966      15.0
      I-fromloc.airport_name      0.441     1.000     0.612      15.0
      B-fromloc.airport_name      0.462     1.000     0.632      12.0
      B-arrive_date.day_name      0.786     1.000     0.880      11.0
          B-toloc.state_code      1.000     1.000     1.000      18.0
B-depart_date.today_relative      1.000     0.889     0.941       9.0
             B-flight_number      0.733     1.000     0.846      11.0
 B-depart_date.date_relative      0.944     1.000     0.971      17.0
          B-toloc.state_name      0.788     0.929     0.852      28.0
           B-fare_basis_code      0.850     1.000     0.919      17.0
               B-flight_time      1.000     1.000     1.000       1.0
                        B-or      1.000     1.000     1.000       3.0
 B-arrive_time.period_of_day      0.750     1.000     0.857       6.0
          B-meal_description      1.000     0.900     0.947      10.0
             I-cost_relative      1.000     0.667     0.800       3.0
              I-airport_name      0.846     0.379     0.524      29.0
               B-fare_amount      1.000     1.000     1.000       2.0
               I-fare_amount      1.000     1.000     1.000       2.0
                 I-city_name      1.000     0.567     0.723      30.0
        I-toloc.airport_name      1.000     1.000     1.000       3.0
            B-transport_type      1.000     1.000     1.000      10.0
    B-arrive_date.month_name      0.714     0.833     0.769       6.0
    B-arrive_date.day_number      0.714     0.833     0.769       6.0
         I-stoploc.city_name      1.000     1.000     1.000      10.0
                      B-meal      1.000     1.000     1.000      16.0
        B-fromloc.state_code      1.000     1.000     1.000      23.0
    B-depart_time.period_mod      1.000     1.000     1.000       5.0
                   B-connect      1.000     1.000     1.000       6.0
               B-flight_days      1.000     1.000     1.000      10.0
        B-toloc.airport_name      1.000     1.000     1.000       3.0
        B-fromloc.state_name      0.944     1.000     0.971      17.0
              B-airport_name      0.800     0.381     0.516      21.0
                   B-economy      1.000     1.000     1.000       6.0
               I-flight_time      1.000     1.000     1.000       1.0
             B-aircraft_code      1.000     0.879     0.935      33.0
                       B-mod      1.000     0.500     0.667       2.0
              B-airport_code      1.000     0.444     0.615       9.0
    B-depart_time.start_time      1.000     0.667     0.800       3.0
      B-depart_time.end_time      1.000     0.333     0.500       3.0
          B-depart_date.year      1.000     1.000     1.000       3.0
            I-transport_type      0.000     0.000     0.000       1.0
          B-restriction_code      0.800     1.000     0.889       4.0
    B-arrive_time.start_time      0.889     1.000     0.941       8.0
        B-toloc.airport_code      1.000     0.750     0.857       4.0
      B-arrive_time.end_time      0.889     1.000     0.941       8.0
      I-arrive_time.end_time      0.889     1.000     0.941       8.0
      I-depart_time.end_time      1.000     0.333     0.500       3.0
               I-flight_stop      0.000     0.000     0.000       0.0
      B-fromloc.airport_code      0.556     1.000     0.714       5.0
          I-restriction_code      1.000     1.000     1.000       3.0
    I-depart_time.start_time      1.000     1.000     1.000       1.0
          I-toloc.state_name      1.000     1.000     1.000       1.0
I-depart_date.today_relative      0.000     0.000     0.000       0.0
 B-arrive_date.date_relative      1.000     1.000     1.000       2.0
                I-flight_mod      1.000     0.167     0.286       6.0
                   I-economy      0.000     0.000     0.000       0.0
 B-return_date.date_relative      1.000     0.333     0.500       3.0
        I-fromloc.state_name      1.000     1.000     1.000       1.0
                B-state_code      1.000     1.000     1.000       1.0
    I-arrive_time.start_time      1.000     1.000     1.000       1.0
    I-arrive_date.day_number      0.000     0.000     0.000       0.0
                 B-meal_code      0.000     0.000     0.000       1.0
 I-depart_time.period_of_day      1.000     1.000     1.000       1.0
                  B-day_name      1.000     0.500     0.667       2.0
             B-period_of_day      1.000     0.750     0.857       4.0
        B-stoploc.state_code      0.000     0.000     0.000       0.0
    B-return_date.month_name      0.000     0.000     0.000       0.0
    B-return_date.day_number      0.000     0.000     0.000       0.0
    B-arrive_time.period_mod      0.000     0.000     0.000       0.0
                 I-meal_code      0.000     0.000     0.000       0.0
        B-toloc.country_name      1.000     1.000     1.000       1.0
                 B-days_code      0.000     0.000     0.000       1.0
 I-arrive_time.period_of_day      0.000     0.000     0.000       0.0
            I-today_relative      0.000     0.000     0.000       0.0
 B-return_time.period_of_day      0.000     0.000     0.000       0.0
                      B-time      0.000     0.000     0.000       0.0
           I-fare_basis_code      0.000     0.000     0.000       0.0
 I-arrive_time.time_relative      1.000     1.000     1.000       4.0
 I-depart_time.time_relative      0.000     0.000     0.000       1.0
            B-today_relative      0.000     0.000     0.000       0.0
                B-state_name      0.000     0.000     0.000       9.0
B-arrive_date.today_relative      0.000     0.000     0.000       0.0
    B-return_time.period_mod      0.000     0.000     0.000       0.0
                B-month_name      0.000     0.000     0.000       0.0
                B-day_number      0.000     0.000     0.000       0.0
 I-return_date.date_relative      0.667     0.667     0.667       3.0
I-return_date.today_relative      0.000     0.000     0.000       0.0
      B-stoploc.airport_name      0.000     0.000     0.000       0.0
             B-time_relative      0.000     0.000     0.000       0.0
                      I-time      0.000     0.000     0.000       0.0
    I-return_date.day_number      0.000     0.000     0.000       0.0
          I-meal_description      0.000     0.000     0.000       0.0
B-return_date.today_relative      0.000     0.000     0.000       0.0
      B-return_date.day_name      0.000     0.000     0.000       2.0

                   micro avg      0.959     0.961     0.960    3657.0
                   macro avg      0.683     0.652     0.652    3657.0
                weighted avg      0.964     0.961     0.957    3657.0

I0716 05:45:10.364182 139795883440000 interactiveshell.py:2882] Best Slot F1: 0.960
I0716 05:45:10.365823 139795883440000 estimator_training.py:186] Not using Distribute Coordinator.
I0716 05:45:10.367828 139795883440000 training.py:612] Running training and evaluation locally (non-distributed).
I0716 05:45:10.370361 139795883440000 training.py:700] Start train and evaluate loop. The evaluate will happen after every checkpoint. Checkpoint frequency is determined based on RunConfig arguments: save_checkpoints_steps 312 or save_checkpoints_secs None.
I0716 05:45:10.443487 139795883440000 estimator.py:1145] Calling model_fn.
I0716 05:45:12.025765 139795883440000 saver.py:1499] Saver not created because there are no variables in the graph to restore
I0716 05:45:12.837126 139795883440000 <ipython-input-6-d6747b40360b>:55] 
[<tf.Variable 'embedding:0' shape=(750, 300) dtype=float32_ref>,
 <tf.Variable 'module/aggregation/weights:0' shape=(3,) dtype=float32>,
 <tf.Variable 'module/aggregation/scaling:0' shape=() dtype=float32>,
 <tf.Variable 'dense/kernel:0' shape=(1324, 300) dtype=float32_ref>,
 <tf.Variable 'dense/bias:0' shape=(300,) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/gates/kernel:0' shape=(600, 600) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/gates/bias:0' shape=(600,) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/candidate/kernel:0' shape=(600, 300) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/candidate/bias:0' shape=(300,) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/gates/kernel:0' shape=(600, 600) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/gates/bias:0' shape=(600,) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/candidate/kernel:0' shape=(600, 300) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/candidate/bias:0' shape=(300,) dtype=float32_ref>,
 <tf.Variable 'dense_1/kernel:0' shape=(600, 23) dtype=float32_ref>,
 <tf.Variable 'dense_1/bias:0' shape=(23,) dtype=float32_ref>,
 <tf.Variable 'dense_2/kernel:0' shape=(600, 122) dtype=float32_ref>,
 <tf.Variable 'dense_2/bias:0' shape=(122,) dtype=float32_ref>]
I0716 05:45:13.959752 139795883440000 estimator.py:1147] Done calling model_fn.
I0716 05:45:13.963981 139795883440000 basic_session_run_hooks.py:541] Create CheckpointSaverHook.
I0716 05:45:14.293300 139795883440000 monitored_session.py:240] Graph was finalized.
I0716 05:45:14.313684 139795883440000 saver.py:1280] Restoring parameters from ../model/elmo_bigru/model.ckpt-14040
I0716 05:45:15.402346 139795883440000 session_manager.py:500] Running local_init_op.
I0716 05:45:15.498242 139795883440000 session_manager.py:502] Done running local_init_op.
I0716 05:45:18.427085 139795883440000 basic_session_run_hooks.py:606] Saving checkpoints for 14040 into ../model/elmo_bigru/model.ckpt.
Reading ../data/atis.train.w-intent.iob
I0716 05:45:23.187589 139795883440000 basic_session_run_hooks.py:262] loss = 0.010440143, step = 14040
I0716 05:45:23.189447 139795883440000 basic_session_run_hooks.py:262] lr = 6.834172e-05
I0716 05:45:51.074040 139795883440000 basic_session_run_hooks.py:692] global_step/sec: 3.58589
I0716 05:45:51.079168 139795883440000 basic_session_run_hooks.py:260] loss = 0.010018, step = 14140 (27.892 sec)
I0716 05:45:51.081705 139795883440000 basic_session_run_hooks.py:260] lr = 6.762545e-05 (27.892 sec)
I0716 05:46:17.054210 139795883440000 basic_session_run_hooks.py:692] global_step/sec: 3.84908
I0716 05:46:17.059381 139795883440000 basic_session_run_hooks.py:260] loss = 0.010949598, step = 14240 (25.980 sec)
I0716 05:46:17.065196 139795883440000 basic_session_run_hooks.py:260] lr = 6.6916684e-05 (25.983 sec)
I0716 05:46:44.069175 139795883440000 basic_session_run_hooks.py:692] global_step/sec: 3.70165
I0716 05:46:44.074118 139795883440000 basic_session_run_hooks.py:260] loss = 0.008432481, step = 14340 (27.015 sec)
I0716 05:46:44.078580 139795883440000 basic_session_run_hooks.py:260] lr = 6.621535e-05 (27.013 sec)
I0716 05:46:46.845710 139795883440000 basic_session_run_hooks.py:606] Saving checkpoints for 14352 into ../model/elmo_bigru/model.ckpt.
I0716 05:46:49.039951 139795883440000 estimator.py:1145] Calling model_fn.
I0716 05:46:50.422496 139795883440000 saver.py:1499] Saver not created because there are no variables in the graph to restore
I0716 05:46:51.007021 139795883440000 estimator.py:1147] Done calling model_fn.
I0716 05:46:51.037232 139795883440000 evaluation.py:255] Starting evaluation at 2019-07-16T05:46:51Z
I0716 05:46:51.265338 139795883440000 monitored_session.py:240] Graph was finalized.
I0716 05:46:51.284544 139795883440000 saver.py:1280] Restoring parameters from ../model/elmo_bigru/model.ckpt-14352
I0716 05:46:52.211852 139795883440000 session_manager.py:500] Running local_init_op.
I0716 05:46:52.275646 139795883440000 session_manager.py:502] Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
I0716 05:47:51.278919 139795883440000 evaluation.py:275] Finished evaluation at 2019-07-16-05:47:51
I0716 05:47:51.280749 139795883440000 estimator.py:2039] Saving dict for global step 14352: global_step = 14352, loss = 0.4107694
I0716 05:47:51.289415 139795883440000 estimator.py:2099] Saving 'checkpoint_path' summary for global step 14352: ../model/elmo_bigru/model.ckpt-14352
I0716 05:47:51.372225 139795883440000 estimator.py:368] Loss for final step: 0.0029536623.
Reading ../data/atis.test.w-intent.iob
I0716 05:47:51.982342 139795883440000 estimator.py:1145] Calling model_fn.
I0716 05:47:53.540017 139795883440000 saver.py:1499] Saver not created because there are no variables in the graph to restore
I0716 05:47:54.252370 139795883440000 estimator.py:1147] Done calling model_fn.
I0716 05:47:54.467233 139795883440000 monitored_session.py:240] Graph was finalized.
I0716 05:47:54.485304 139795883440000 saver.py:1280] Restoring parameters from ../model/elmo_bigru/model.ckpt-14352
I0716 05:47:55.402657 139795883440000 session_manager.py:500] Running local_init_op.
I0716 05:47:55.448067 139795883440000 session_manager.py:502] Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.
  'precision', 'predicted', average, warn_for)
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1439: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples.
  'recall', 'true', average, warn_for)
I0716 05:48:53.400110 139795883440000 interactiveshell.py:2882] 
                                          precision    recall  f1-score   support

                             atis_flight      0.984     0.992     0.988       632
                            atis_airfare      0.941     1.000     0.970        48
                     atis_ground_service      1.000     1.000     1.000        36
                            atis_airline      1.000     1.000     1.000        38
                       atis_abbreviation      0.943     1.000     0.971        33
                           atis_aircraft      1.000     0.889     0.941         9
                        atis_flight_time      0.500     1.000     0.667         1
                           atis_quantity      0.429     1.000     0.600         3
                atis_flight#atis_airfare      0.833     0.417     0.556        12
                            atis_airport      1.000     1.000     1.000        18
                           atis_distance      1.000     1.000     1.000        10
                               atis_city      0.750     0.500     0.600         6
                        atis_ground_fare      1.000     0.857     0.923         7
                           atis_capacity      0.952     0.952     0.952        21
                          atis_flight_no      0.889     1.000     0.941         8
                               atis_meal      1.000     0.833     0.909         6
                        atis_restriction      0.000     0.000     0.000         0
             atis_airline#atis_flight_no      0.000     0.000     0.000         0
    atis_ground_service#atis_ground_fare      0.000     0.000     0.000         0
           atis_airfare#atis_flight_time      0.000     0.000     0.000         0
                           atis_cheapest      0.000     0.000     0.000         0
atis_aircraft#atis_flight#atis_flight_no      0.000     0.000     0.000         0

                               micro avg      0.973     0.979     0.976       888
                               macro avg      0.646     0.656     0.637       888
                            weighted avg      0.975     0.979     0.975       888

I0716 05:48:53.439727 139795883440000 interactiveshell.py:2882] 
                              precision    recall  f1-score   support

                           O      0.000     0.000     0.000       0.0
           B-toloc.city_name      0.975     0.994     0.985     716.0
         B-fromloc.city_name      0.990     0.999     0.994     704.0
           I-toloc.city_name      0.978     0.989     0.983     265.0
      B-depart_date.day_name      0.991     0.991     0.991     212.0
              B-airline_name      1.000     1.000     1.000     101.0
         I-fromloc.city_name      0.973     1.000     0.986     177.0
 B-depart_time.period_of_day      1.000     0.923     0.960     130.0
              I-airline_name      1.000     1.000     1.000      65.0
    B-depart_date.day_number      0.981     0.964     0.972      55.0
    B-depart_date.month_name      0.982     0.964     0.973      56.0
          B-depart_time.time      0.864     1.000     0.927      57.0
                B-round_trip      1.000     0.986     0.993      73.0
             B-cost_relative      0.973     0.973     0.973      37.0
                I-round_trip      1.000     1.000     1.000      71.0
                B-flight_mod      0.828     1.000     0.906      24.0
 B-depart_time.time_relative      0.970     0.985     0.977      65.0
          I-depart_time.time      0.945     1.000     0.972      52.0
         B-stoploc.city_name      0.909     1.000     0.952      20.0
                 B-city_name      0.923     0.632     0.750      57.0
                B-class_type      0.960     1.000     0.980      24.0
          B-arrive_time.time      0.971     0.971     0.971      34.0
 B-arrive_time.time_relative      0.967     0.935     0.951      31.0
                I-class_type      1.000     1.000     1.000      17.0
               B-flight_stop      1.000     1.000     1.000      21.0
          I-arrive_time.time      1.000     0.971     0.986      35.0
              B-airline_code      0.971     1.000     0.986      34.0
    I-depart_date.day_number      1.000     0.933     0.966      15.0
      I-fromloc.airport_name      0.441     1.000     0.612      15.0
      B-fromloc.airport_name      0.462     1.000     0.632      12.0
      B-arrive_date.day_name      0.786     1.000     0.880      11.0
          B-toloc.state_code      1.000     1.000     1.000      18.0
B-depart_date.today_relative      1.000     0.889     0.941       9.0
             B-flight_number      0.733     1.000     0.846      11.0
 B-depart_date.date_relative      0.944     1.000     0.971      17.0
          B-toloc.state_name      0.788     0.929     0.852      28.0
           B-fare_basis_code      0.850     1.000     0.919      17.0
               B-flight_time      1.000     1.000     1.000       1.0
                        B-or      1.000     1.000     1.000       3.0
 B-arrive_time.period_of_day      0.750     1.000     0.857       6.0
          B-meal_description      1.000     0.900     0.947      10.0
             I-cost_relative      1.000     0.667     0.800       3.0
              I-airport_name      0.846     0.379     0.524      29.0
               B-fare_amount      1.000     1.000     1.000       2.0
               I-fare_amount      1.000     1.000     1.000       2.0
                 I-city_name      1.000     0.533     0.696      30.0
        I-toloc.airport_name      1.000     1.000     1.000       3.0
            B-transport_type      1.000     1.000     1.000      10.0
    B-arrive_date.month_name      0.714     0.833     0.769       6.0
    B-arrive_date.day_number      0.714     0.833     0.769       6.0
         I-stoploc.city_name      1.000     1.000     1.000      10.0
                      B-meal      1.000     1.000     1.000      16.0
        B-fromloc.state_code      1.000     1.000     1.000      23.0
    B-depart_time.period_mod      1.000     1.000     1.000       5.0
                   B-connect      1.000     1.000     1.000       6.0
               B-flight_days      1.000     1.000     1.000      10.0
        B-toloc.airport_name      1.000     1.000     1.000       3.0
        B-fromloc.state_name      0.944     1.000     0.971      17.0
              B-airport_name      0.800     0.381     0.516      21.0
                   B-economy      1.000     1.000     1.000       6.0
               I-flight_time      1.000     1.000     1.000       1.0
             B-aircraft_code      1.000     0.879     0.935      33.0
                       B-mod      1.000     0.500     0.667       2.0
              B-airport_code      1.000     0.444     0.615       9.0
    B-depart_time.start_time      1.000     0.667     0.800       3.0
      B-depart_time.end_time      1.000     0.333     0.500       3.0
          B-depart_date.year      1.000     1.000     1.000       3.0
            I-transport_type      0.000     0.000     0.000       1.0
          B-restriction_code      0.800     1.000     0.889       4.0
    B-arrive_time.start_time      0.889     1.000     0.941       8.0
        B-toloc.airport_code      1.000     0.750     0.857       4.0
      B-arrive_time.end_time      0.889     1.000     0.941       8.0
      I-arrive_time.end_time      0.889     1.000     0.941       8.0
      I-depart_time.end_time      1.000     0.333     0.500       3.0
               I-flight_stop      0.000     0.000     0.000       0.0
      B-fromloc.airport_code      0.556     1.000     0.714       5.0
          I-restriction_code      1.000     1.000     1.000       3.0
    I-depart_time.start_time      1.000     1.000     1.000       1.0
          I-toloc.state_name      1.000     1.000     1.000       1.0
I-depart_date.today_relative      0.000     0.000     0.000       0.0
 B-arrive_date.date_relative      1.000     1.000     1.000       2.0
                I-flight_mod      1.000     0.167     0.286       6.0
                   I-economy      0.000     0.000     0.000       0.0
 B-return_date.date_relative      1.000     0.333     0.500       3.0
        I-fromloc.state_name      1.000     1.000     1.000       1.0
                B-state_code      1.000     1.000     1.000       1.0
    I-arrive_time.start_time      1.000     1.000     1.000       1.0
    I-arrive_date.day_number      0.000     0.000     0.000       0.0
                 B-meal_code      0.000     0.000     0.000       1.0
 I-depart_time.period_of_day      1.000     1.000     1.000       1.0
                  B-day_name      1.000     0.500     0.667       2.0
             B-period_of_day      1.000     0.750     0.857       4.0
        B-stoploc.state_code      0.000     0.000     0.000       0.0
    B-return_date.month_name      0.000     0.000     0.000       0.0
    B-return_date.day_number      0.000     0.000     0.000       0.0
    B-arrive_time.period_mod      0.000     0.000     0.000       0.0
                 I-meal_code      0.000     0.000     0.000       0.0
        B-toloc.country_name      1.000     1.000     1.000       1.0
                 B-days_code      0.000     0.000     0.000       1.0
 I-arrive_time.period_of_day      0.000     0.000     0.000       0.0
            I-today_relative      0.000     0.000     0.000       0.0
 B-return_time.period_of_day      0.000     0.000     0.000       0.0
                      B-time      0.000     0.000     0.000       0.0
           I-fare_basis_code      0.000     0.000     0.000       0.0
 I-arrive_time.time_relative      1.000     1.000     1.000       4.0
 I-depart_time.time_relative      0.000     0.000     0.000       1.0
            B-today_relative      0.000     0.000     0.000       0.0
                B-state_name      0.000     0.000     0.000       9.0
B-arrive_date.today_relative      0.000     0.000     0.000       0.0
    B-return_time.period_mod      0.000     0.000     0.000       0.0
                B-month_name      0.000     0.000     0.000       0.0
                B-day_number      0.000     0.000     0.000       0.0
 I-return_date.date_relative      0.667     0.667     0.667       3.0
I-return_date.today_relative      0.000     0.000     0.000       0.0
      B-stoploc.airport_name      0.000     0.000     0.000       0.0
             B-time_relative      0.000     0.000     0.000       0.0
                      I-time      0.000     0.000     0.000       0.0
    I-return_date.day_number      0.000     0.000     0.000       0.0
          I-meal_description      0.000     0.000     0.000       0.0
B-return_date.today_relative      0.000     0.000     0.000       0.0
      B-return_date.day_name      0.000     0.000     0.000       2.0

                   micro avg      0.959     0.960     0.959    3657.0
                   macro avg      0.683     0.652     0.652    3657.0
                weighted avg      0.964     0.960     0.957    3657.0

I0716 05:48:53.476049 139795883440000 interactiveshell.py:2882] Best Slot F1: 0.960
I0716 05:48:53.477536 139795883440000 estimator_training.py:186] Not using Distribute Coordinator.
I0716 05:48:53.483242 139795883440000 training.py:612] Running training and evaluation locally (non-distributed).
I0716 05:48:53.487344 139795883440000 training.py:700] Start train and evaluate loop. The evaluate will happen after every checkpoint. Checkpoint frequency is determined based on RunConfig arguments: save_checkpoints_steps 312 or save_checkpoints_secs None.
I0716 05:48:53.553193 139795883440000 estimator.py:1145] Calling model_fn.
I0716 05:48:54.885581 139795883440000 saver.py:1499] Saver not created because there are no variables in the graph to restore
I0716 05:48:55.670599 139795883440000 <ipython-input-6-d6747b40360b>:55] 
[<tf.Variable 'embedding:0' shape=(750, 300) dtype=float32_ref>,
 <tf.Variable 'module/aggregation/weights:0' shape=(3,) dtype=float32>,
 <tf.Variable 'module/aggregation/scaling:0' shape=() dtype=float32>,
 <tf.Variable 'dense/kernel:0' shape=(1324, 300) dtype=float32_ref>,
 <tf.Variable 'dense/bias:0' shape=(300,) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/gates/kernel:0' shape=(600, 600) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/gates/bias:0' shape=(600,) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/candidate/kernel:0' shape=(600, 300) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/candidate/bias:0' shape=(300,) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/gates/kernel:0' shape=(600, 600) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/gates/bias:0' shape=(600,) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/candidate/kernel:0' shape=(600, 300) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/candidate/bias:0' shape=(300,) dtype=float32_ref>,
 <tf.Variable 'dense_1/kernel:0' shape=(600, 23) dtype=float32_ref>,
 <tf.Variable 'dense_1/bias:0' shape=(23,) dtype=float32_ref>,
 <tf.Variable 'dense_2/kernel:0' shape=(600, 122) dtype=float32_ref>,
 <tf.Variable 'dense_2/bias:0' shape=(122,) dtype=float32_ref>]
I0716 05:48:56.756968 139795883440000 estimator.py:1147] Done calling model_fn.
I0716 05:48:56.761645 139795883440000 basic_session_run_hooks.py:541] Create CheckpointSaverHook.
I0716 05:48:57.091637 139795883440000 monitored_session.py:240] Graph was finalized.
I0716 05:48:57.112784 139795883440000 saver.py:1280] Restoring parameters from ../model/elmo_bigru/model.ckpt-14352
I0716 05:48:58.171647 139795883440000 session_manager.py:500] Running local_init_op.
I0716 05:48:58.240034 139795883440000 session_manager.py:502] Done running local_init_op.
I0716 05:49:01.294649 139795883440000 basic_session_run_hooks.py:606] Saving checkpoints for 14352 into ../model/elmo_bigru/model.ckpt.
Reading ../data/atis.train.w-intent.iob
I0716 05:49:06.017440 139795883440000 basic_session_run_hooks.py:262] loss = 0.005521031, step = 14352
I0716 05:49:06.019191 139795883440000 basic_session_run_hooks.py:262] lr = 6.613168e-05
I0716 05:49:33.814277 139795883440000 basic_session_run_hooks.py:692] global_step/sec: 3.59743
I0716 05:49:33.818360 139795883440000 basic_session_run_hooks.py:260] loss = 0.0083391145, step = 14452 (27.801 sec)
I0716 05:49:33.824078 139795883440000 basic_session_run_hooks.py:260] lr = 6.543857e-05 (27.805 sec)
I0716 05:49:59.538691 139795883440000 basic_session_run_hooks.py:692] global_step/sec: 3.88735
I0716 05:49:59.544354 139795883440000 basic_session_run_hooks.py:260] loss = 0.00884945, step = 14552 (25.726 sec)
I0716 05:49:59.547071 139795883440000 basic_session_run_hooks.py:260] lr = 6.4752734e-05 (25.723 sec)
I0716 05:50:26.581260 139795883440000 basic_session_run_hooks.py:692] global_step/sec: 3.69788
I0716 05:50:26.590217 139795883440000 basic_session_run_hooks.py:260] loss = 0.011286377, step = 14652 (27.046 sec)
I0716 05:50:26.592428 139795883440000 basic_session_run_hooks.py:260] lr = 6.407407e-05 (27.045 sec)
I0716 05:50:29.580576 139795883440000 basic_session_run_hooks.py:606] Saving checkpoints for 14664 into ../model/elmo_bigru/model.ckpt.
I0716 05:50:31.657208 139795883440000 estimator.py:1145] Calling model_fn.
I0716 05:50:33.072262 139795883440000 saver.py:1499] Saver not created because there are no variables in the graph to restore
I0716 05:50:33.644398 139795883440000 estimator.py:1147] Done calling model_fn.
I0716 05:50:33.675320 139795883440000 evaluation.py:255] Starting evaluation at 2019-07-16T05:50:33Z
I0716 05:50:33.899181 139795883440000 monitored_session.py:240] Graph was finalized.
I0716 05:50:33.917637 139795883440000 saver.py:1280] Restoring parameters from ../model/elmo_bigru/model.ckpt-14664
I0716 05:50:34.822535 139795883440000 session_manager.py:500] Running local_init_op.
I0716 05:50:34.882165 139795883440000 session_manager.py:502] Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
I0716 05:51:33.744959 139795883440000 evaluation.py:275] Finished evaluation at 2019-07-16-05:51:33
I0716 05:51:33.746666 139795883440000 estimator.py:2039] Saving dict for global step 14664: global_step = 14664, loss = 0.41299483
I0716 05:51:33.756397 139795883440000 estimator.py:2099] Saving 'checkpoint_path' summary for global step 14664: ../model/elmo_bigru/model.ckpt-14664
I0716 05:51:33.845106 139795883440000 estimator.py:368] Loss for final step: 0.0035273344.
Reading ../data/atis.test.w-intent.iob
I0716 05:51:34.460048 139795883440000 estimator.py:1145] Calling model_fn.
I0716 05:51:36.192692 139795883440000 saver.py:1499] Saver not created because there are no variables in the graph to restore
I0716 05:51:36.715398 139795883440000 estimator.py:1147] Done calling model_fn.
I0716 05:51:36.933637 139795883440000 monitored_session.py:240] Graph was finalized.
I0716 05:51:36.958074 139795883440000 saver.py:1280] Restoring parameters from ../model/elmo_bigru/model.ckpt-14664
I0716 05:51:37.884623 139795883440000 session_manager.py:500] Running local_init_op.
I0716 05:51:37.932129 139795883440000 session_manager.py:502] Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.
  'precision', 'predicted', average, warn_for)
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1439: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples.
  'recall', 'true', average, warn_for)
I0716 05:52:35.795669 139795883440000 interactiveshell.py:2882] 
                                          precision    recall  f1-score   support

                             atis_flight      0.984     0.992     0.988       632
                            atis_airfare      0.941     1.000     0.970        48
                     atis_ground_service      1.000     1.000     1.000        36
                            atis_airline      1.000     1.000     1.000        38
                       atis_abbreviation      0.943     1.000     0.971        33
                           atis_aircraft      1.000     0.889     0.941         9
                        atis_flight_time      0.500     1.000     0.667         1
                           atis_quantity      0.429     1.000     0.600         3
                atis_flight#atis_airfare      0.833     0.417     0.556        12
                            atis_airport      1.000     1.000     1.000        18
                           atis_distance      1.000     1.000     1.000        10
                               atis_city      0.750     0.500     0.600         6
                        atis_ground_fare      1.000     0.857     0.923         7
                           atis_capacity      0.952     0.952     0.952        21
                          atis_flight_no      0.889     1.000     0.941         8
                               atis_meal      1.000     0.833     0.909         6
                        atis_restriction      0.000     0.000     0.000         0
             atis_airline#atis_flight_no      0.000     0.000     0.000         0
    atis_ground_service#atis_ground_fare      0.000     0.000     0.000         0
           atis_airfare#atis_flight_time      0.000     0.000     0.000         0
                           atis_cheapest      0.000     0.000     0.000         0
atis_aircraft#atis_flight#atis_flight_no      0.000     0.000     0.000         0

                               micro avg      0.973     0.979     0.976       888
                               macro avg      0.646     0.656     0.637       888
                            weighted avg      0.975     0.979     0.975       888

I0716 05:52:35.841406 139795883440000 interactiveshell.py:2882] 
                              precision    recall  f1-score   support

                           O      0.000     0.000     0.000       0.0
           B-toloc.city_name      0.977     0.994     0.985     716.0
         B-fromloc.city_name      0.990     0.999     0.994     704.0
           I-toloc.city_name      0.978     0.989     0.983     265.0
      B-depart_date.day_name      0.991     0.991     0.991     212.0
              B-airline_name      1.000     1.000     1.000     101.0
         I-fromloc.city_name      0.978     1.000     0.989     177.0
 B-depart_time.period_of_day      1.000     0.923     0.960     130.0
              I-airline_name      1.000     1.000     1.000      65.0
    B-depart_date.day_number      0.981     0.964     0.972      55.0
    B-depart_date.month_name      0.982     0.964     0.973      56.0
          B-depart_time.time      0.864     1.000     0.927      57.0
                B-round_trip      1.000     1.000     1.000      73.0
             B-cost_relative      0.973     0.973     0.973      37.0
                I-round_trip      1.000     1.000     1.000      71.0
                B-flight_mod      0.828     1.000     0.906      24.0
 B-depart_time.time_relative      0.955     0.985     0.970      65.0
          I-depart_time.time      0.945     1.000     0.972      52.0
         B-stoploc.city_name      0.909     1.000     0.952      20.0
                 B-city_name      0.923     0.632     0.750      57.0
                B-class_type      0.960     1.000     0.980      24.0
          B-arrive_time.time      0.971     0.971     0.971      34.0
 B-arrive_time.time_relative      0.967     0.935     0.951      31.0
                I-class_type      1.000     1.000     1.000      17.0
               B-flight_stop      1.000     1.000     1.000      21.0
          I-arrive_time.time      1.000     0.971     0.986      35.0
              B-airline_code      0.971     1.000     0.986      34.0
    I-depart_date.day_number      1.000     0.933     0.966      15.0
      I-fromloc.airport_name      0.455     1.000     0.625      15.0
      B-fromloc.airport_name      0.462     1.000     0.632      12.0
      B-arrive_date.day_name      0.786     1.000     0.880      11.0
          B-toloc.state_code      1.000     1.000     1.000      18.0
B-depart_date.today_relative      1.000     0.889     0.941       9.0
             B-flight_number      0.733     1.000     0.846      11.0
 B-depart_date.date_relative      0.944     1.000     0.971      17.0
          B-toloc.state_name      0.788     0.929     0.852      28.0
           B-fare_basis_code      0.895     1.000     0.944      17.0
               B-flight_time      1.000     1.000     1.000       1.0
                        B-or      1.000     1.000     1.000       3.0
 B-arrive_time.period_of_day      0.750     1.000     0.857       6.0
          B-meal_description      1.000     0.900     0.947      10.0
             I-cost_relative      1.000     0.667     0.800       3.0
              I-airport_name      0.857     0.414     0.558      29.0
               B-fare_amount      1.000     1.000     1.000       2.0
               I-fare_amount      1.000     1.000     1.000       2.0
                 I-city_name      1.000     0.567     0.723      30.0
        I-toloc.airport_name      1.000     1.000     1.000       3.0
            B-transport_type      1.000     1.000     1.000      10.0
    B-arrive_date.month_name      0.714     0.833     0.769       6.0
    B-arrive_date.day_number      0.714     0.833     0.769       6.0
         I-stoploc.city_name      1.000     1.000     1.000      10.0
                      B-meal      1.000     1.000     1.000      16.0
        B-fromloc.state_code      1.000     1.000     1.000      23.0
    B-depart_time.period_mod      1.000     1.000     1.000       5.0
                   B-connect      1.000     1.000     1.000       6.0
               B-flight_days      1.000     1.000     1.000      10.0
        B-toloc.airport_name      1.000     1.000     1.000       3.0
        B-fromloc.state_name      0.944     1.000     0.971      17.0
              B-airport_name      0.800     0.381     0.516      21.0
                   B-economy      1.000     1.000     1.000       6.0
               I-flight_time      1.000     1.000     1.000       1.0
             B-aircraft_code      1.000     0.879     0.935      33.0
                       B-mod      1.000     0.500     0.667       2.0
              B-airport_code      1.000     0.444     0.615       9.0
    B-depart_time.start_time      1.000     0.667     0.800       3.0
      B-depart_time.end_time      1.000     0.333     0.500       3.0
          B-depart_date.year      1.000     1.000     1.000       3.0
            I-transport_type      0.000     0.000     0.000       1.0
          B-restriction_code      0.800     1.000     0.889       4.0
    B-arrive_time.start_time      0.889     1.000     0.941       8.0
        B-toloc.airport_code      1.000     1.000     1.000       4.0
      B-arrive_time.end_time      0.889     1.000     0.941       8.0
      I-arrive_time.end_time      0.889     1.000     0.941       8.0
      I-depart_time.end_time      1.000     0.333     0.500       3.0
               I-flight_stop      0.000     0.000     0.000       0.0
      B-fromloc.airport_code      0.556     1.000     0.714       5.0
          I-restriction_code      1.000     1.000     1.000       3.0
    I-depart_time.start_time      1.000     1.000     1.000       1.0
          I-toloc.state_name      1.000     1.000     1.000       1.0
I-depart_date.today_relative      0.000     0.000     0.000       0.0
 B-arrive_date.date_relative      1.000     1.000     1.000       2.0
                I-flight_mod      1.000     0.167     0.286       6.0
                   I-economy      0.000     0.000     0.000       0.0
 B-return_date.date_relative      1.000     0.333     0.500       3.0
        I-fromloc.state_name      1.000     1.000     1.000       1.0
                B-state_code      1.000     1.000     1.000       1.0
    I-arrive_time.start_time      1.000     1.000     1.000       1.0
    I-arrive_date.day_number      0.000     0.000     0.000       0.0
                 B-meal_code      0.000     0.000     0.000       1.0
 I-depart_time.period_of_day      1.000     1.000     1.000       1.0
                  B-day_name      1.000     0.500     0.667       2.0
             B-period_of_day      1.000     0.750     0.857       4.0
        B-stoploc.state_code      0.000     0.000     0.000       0.0
    B-return_date.month_name      0.000     0.000     0.000       0.0
    B-return_date.day_number      0.000     0.000     0.000       0.0
    B-arrive_time.period_mod      0.000     0.000     0.000       0.0
                 I-meal_code      0.000     0.000     0.000       0.0
        B-toloc.country_name      1.000     1.000     1.000       1.0
                 B-days_code      1.000     1.000     1.000       1.0
 I-arrive_time.period_of_day      0.000     0.000     0.000       0.0
            I-today_relative      0.000     0.000     0.000       0.0
 B-return_time.period_of_day      0.000     0.000     0.000       0.0
                      B-time      0.000     0.000     0.000       0.0
           I-fare_basis_code      0.000     0.000     0.000       0.0
 I-arrive_time.time_relative      1.000     1.000     1.000       4.0
 I-depart_time.time_relative      0.000     0.000     0.000       1.0
            B-today_relative      0.000     0.000     0.000       0.0
                B-state_name      0.000     0.000     0.000       9.0
B-arrive_date.today_relative      0.000     0.000     0.000       0.0
    B-return_time.period_mod      0.000     0.000     0.000       0.0
                B-month_name      0.000     0.000     0.000       0.0
                B-day_number      0.000     0.000     0.000       0.0
 I-return_date.date_relative      0.667     0.667     0.667       3.0
I-return_date.today_relative      0.000     0.000     0.000       0.0
      B-stoploc.airport_name      0.000     0.000     0.000       0.0
             B-time_relative      0.000     0.000     0.000       0.0
                      I-time      0.000     0.000     0.000       0.0
    I-return_date.day_number      0.000     0.000     0.000       0.0
          I-meal_description      0.000     0.000     0.000       0.0
B-return_date.today_relative      0.000     0.000     0.000       0.0
      B-return_date.day_name      0.000     0.000     0.000       2.0

                   micro avg      0.960     0.961     0.961    3657.0
                   macro avg      0.692     0.663     0.662    3657.0
                weighted avg      0.964     0.961     0.958    3657.0

I0716 05:52:35.869594 139795883440000 interactiveshell.py:2882] Best Slot F1: 0.961
I0716 05:52:35.871094 139795883440000 estimator_training.py:186] Not using Distribute Coordinator.
I0716 05:52:35.872925 139795883440000 training.py:612] Running training and evaluation locally (non-distributed).
I0716 05:52:35.876721 139795883440000 training.py:700] Start train and evaluate loop. The evaluate will happen after every checkpoint. Checkpoint frequency is determined based on RunConfig arguments: save_checkpoints_steps 312 or save_checkpoints_secs None.
I0716 05:52:35.945642 139795883440000 estimator.py:1145] Calling model_fn.
I0716 05:52:37.465623 139795883440000 saver.py:1499] Saver not created because there are no variables in the graph to restore
I0716 05:52:38.056463 139795883440000 <ipython-input-6-d6747b40360b>:55] 
[<tf.Variable 'embedding:0' shape=(750, 300) dtype=float32_ref>,
 <tf.Variable 'module/aggregation/weights:0' shape=(3,) dtype=float32>,
 <tf.Variable 'module/aggregation/scaling:0' shape=() dtype=float32>,
 <tf.Variable 'dense/kernel:0' shape=(1324, 300) dtype=float32_ref>,
 <tf.Variable 'dense/bias:0' shape=(300,) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/gates/kernel:0' shape=(600, 600) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/gates/bias:0' shape=(600,) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/candidate/kernel:0' shape=(600, 300) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/candidate/bias:0' shape=(300,) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/gates/kernel:0' shape=(600, 600) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/gates/bias:0' shape=(600,) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/candidate/kernel:0' shape=(600, 300) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/candidate/bias:0' shape=(300,) dtype=float32_ref>,
 <tf.Variable 'dense_1/kernel:0' shape=(600, 23) dtype=float32_ref>,
 <tf.Variable 'dense_1/bias:0' shape=(23,) dtype=float32_ref>,
 <tf.Variable 'dense_2/kernel:0' shape=(600, 122) dtype=float32_ref>,
 <tf.Variable 'dense_2/bias:0' shape=(122,) dtype=float32_ref>]
I0716 05:52:39.361430 139795883440000 estimator.py:1147] Done calling model_fn.
I0716 05:52:39.365440 139795883440000 basic_session_run_hooks.py:541] Create CheckpointSaverHook.
I0716 05:52:39.690876 139795883440000 monitored_session.py:240] Graph was finalized.
I0716 05:52:39.711251 139795883440000 saver.py:1280] Restoring parameters from ../model/elmo_bigru/model.ckpt-14664
I0716 05:52:40.747165 139795883440000 session_manager.py:500] Running local_init_op.
I0716 05:52:40.822594 139795883440000 session_manager.py:502] Done running local_init_op.
I0716 05:52:43.647684 139795883440000 basic_session_run_hooks.py:606] Saving checkpoints for 14664 into ../model/elmo_bigru/model.ckpt.
Reading ../data/atis.train.w-intent.iob
I0716 05:52:48.409260 139795883440000 basic_session_run_hooks.py:262] loss = 0.009573579, step = 14664
I0716 05:52:48.410746 139795883440000 basic_session_run_hooks.py:262] lr = 6.399311e-05
I0716 05:53:17.095580 139795883440000 basic_session_run_hooks.py:692] global_step/sec: 3.48591
I0716 05:53:17.100201 139795883440000 basic_session_run_hooks.py:260] loss = 0.010442889, step = 14764 (28.691 sec)
I0716 05:53:17.104258 139795883440000 basic_session_run_hooks.py:260] lr = 6.332242e-05 (28.693 sec)
I0716 05:53:42.701854 139795883440000 basic_session_run_hooks.py:692] global_step/sec: 3.90529
I0716 05:53:42.711499 139795883440000 basic_session_run_hooks.py:260] loss = 0.009410427, step = 14864 (25.611 sec)
I0716 05:53:42.713646 139795883440000 basic_session_run_hooks.py:260] lr = 6.265875e-05 (25.609 sec)
I0716 05:54:08.258992 139795883440000 basic_session_run_hooks.py:692] global_step/sec: 3.91281
I0716 05:54:08.267603 139795883440000 basic_session_run_hooks.py:260] loss = 0.006956376, step = 14964 (25.556 sec)
I0716 05:54:08.270033 139795883440000 basic_session_run_hooks.py:260] lr = 6.2002044e-05 (25.556 sec)
I0716 05:54:11.027354 139795883440000 basic_session_run_hooks.py:606] Saving checkpoints for 14976 into ../model/elmo_bigru/model.ckpt.
I0716 05:54:13.016072 139795883440000 estimator.py:1145] Calling model_fn.
I0716 05:54:14.394398 139795883440000 saver.py:1499] Saver not created because there are no variables in the graph to restore
I0716 05:54:15.255473 139795883440000 estimator.py:1147] Done calling model_fn.
I0716 05:54:15.287425 139795883440000 evaluation.py:255] Starting evaluation at 2019-07-16T05:54:15Z
I0716 05:54:15.506654 139795883440000 monitored_session.py:240] Graph was finalized.
I0716 05:54:15.527868 139795883440000 saver.py:1280] Restoring parameters from ../model/elmo_bigru/model.ckpt-14976
I0716 05:54:16.437136 139795883440000 session_manager.py:500] Running local_init_op.
I0716 05:54:16.495558 139795883440000 session_manager.py:502] Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
I0716 05:55:15.609049 139795883440000 evaluation.py:275] Finished evaluation at 2019-07-16-05:55:15
I0716 05:55:15.610839 139795883440000 estimator.py:2039] Saving dict for global step 14976: global_step = 14976, loss = 0.41449836
I0716 05:55:15.617541 139795883440000 estimator.py:2099] Saving 'checkpoint_path' summary for global step 14976: ../model/elmo_bigru/model.ckpt-14976
I0716 05:55:15.705172 139795883440000 estimator.py:368] Loss for final step: 0.0009599638.
Reading ../data/atis.test.w-intent.iob
I0716 05:55:16.311159 139795883440000 estimator.py:1145] Calling model_fn.
I0716 05:55:17.373431 139795883440000 saver.py:1499] Saver not created because there are no variables in the graph to restore
I0716 05:55:18.466601 139795883440000 estimator.py:1147] Done calling model_fn.
I0716 05:55:18.704450 139795883440000 monitored_session.py:240] Graph was finalized.
I0716 05:55:18.721217 139795883440000 saver.py:1280] Restoring parameters from ../model/elmo_bigru/model.ckpt-14976
I0716 05:55:19.648371 139795883440000 session_manager.py:500] Running local_init_op.
I0716 05:55:19.720381 139795883440000 session_manager.py:502] Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.
  'precision', 'predicted', average, warn_for)
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1439: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples.
  'recall', 'true', average, warn_for)
I0716 05:56:18.118442 139795883440000 interactiveshell.py:2882] 
                                          precision    recall  f1-score   support

                             atis_flight      0.984     0.992     0.988       632
                            atis_airfare      0.941     1.000     0.970        48
                     atis_ground_service      1.000     1.000     1.000        36
                            atis_airline      1.000     1.000     1.000        38
                       atis_abbreviation      0.943     1.000     0.971        33
                           atis_aircraft      1.000     0.889     0.941         9
                        atis_flight_time      0.500     1.000     0.667         1
                           atis_quantity      0.429     1.000     0.600         3
                atis_flight#atis_airfare      0.833     0.417     0.556        12
                            atis_airport      1.000     1.000     1.000        18
                           atis_distance      1.000     1.000     1.000        10
                               atis_city      0.750     0.500     0.600         6
                        atis_ground_fare      1.000     0.857     0.923         7
                           atis_capacity      0.952     0.952     0.952        21
                          atis_flight_no      0.889     1.000     0.941         8
                               atis_meal      1.000     0.833     0.909         6
                        atis_restriction      0.000     0.000     0.000         0
             atis_airline#atis_flight_no      0.000     0.000     0.000         0
    atis_ground_service#atis_ground_fare      0.000     0.000     0.000         0
           atis_airfare#atis_flight_time      0.000     0.000     0.000         0
                           atis_cheapest      0.000     0.000     0.000         0
atis_aircraft#atis_flight#atis_flight_no      0.000     0.000     0.000         0

                               micro avg      0.973     0.979     0.976       888
                               macro avg      0.646     0.656     0.637       888
                            weighted avg      0.975     0.979     0.975       888

I0716 05:56:18.155181 139795883440000 interactiveshell.py:2882] 
                              precision    recall  f1-score   support

                           O      0.000     0.000     0.000       0.0
           B-toloc.city_name      0.977     0.994     0.985     716.0
         B-fromloc.city_name      0.992     0.999     0.995     704.0
           I-toloc.city_name      0.978     0.989     0.983     265.0
      B-depart_date.day_name      0.991     0.991     0.991     212.0
              B-airline_name      1.000     1.000     1.000     101.0
         I-fromloc.city_name      0.978     1.000     0.989     177.0
 B-depart_time.period_of_day      1.000     0.923     0.960     130.0
              I-airline_name      1.000     1.000     1.000      65.0
    B-depart_date.day_number      0.981     0.964     0.972      55.0
    B-depart_date.month_name      0.982     0.964     0.973      56.0
          B-depart_time.time      0.864     1.000     0.927      57.0
                B-round_trip      1.000     1.000     1.000      73.0
             B-cost_relative      0.973     0.973     0.973      37.0
                I-round_trip      1.000     1.000     1.000      71.0
                B-flight_mod      0.828     1.000     0.906      24.0
 B-depart_time.time_relative      0.955     0.985     0.970      65.0
          I-depart_time.time      0.945     1.000     0.972      52.0
         B-stoploc.city_name      0.870     1.000     0.930      20.0
                 B-city_name      0.925     0.649     0.763      57.0
                B-class_type      0.960     1.000     0.980      24.0
          B-arrive_time.time      0.971     0.971     0.971      34.0
 B-arrive_time.time_relative      0.967     0.935     0.951      31.0
                I-class_type      1.000     1.000     1.000      17.0
               B-flight_stop      1.000     1.000     1.000      21.0
          I-arrive_time.time      1.000     0.971     0.986      35.0
              B-airline_code      0.971     1.000     0.986      34.0
    I-depart_date.day_number      1.000     0.933     0.966      15.0
      I-fromloc.airport_name      0.441     1.000     0.612      15.0
      B-fromloc.airport_name      0.462     1.000     0.632      12.0
      B-arrive_date.day_name      0.786     1.000     0.880      11.0
          B-toloc.state_code      1.000     1.000     1.000      18.0
B-depart_date.today_relative      1.000     0.889     0.941       9.0
             B-flight_number      0.733     1.000     0.846      11.0
 B-depart_date.date_relative      0.944     1.000     0.971      17.0
          B-toloc.state_name      0.833     0.893     0.862      28.0
           B-fare_basis_code      0.850     1.000     0.919      17.0
               B-flight_time      1.000     1.000     1.000       1.0
                        B-or      1.000     1.000     1.000       3.0
 B-arrive_time.period_of_day      0.750     1.000     0.857       6.0
          B-meal_description      1.000     0.900     0.947      10.0
             I-cost_relative      1.000     0.667     0.800       3.0
              I-airport_name      0.846     0.379     0.524      29.0
               B-fare_amount      1.000     1.000     1.000       2.0
               I-fare_amount      1.000     1.000     1.000       2.0
                 I-city_name      1.000     0.600     0.750      30.0
        I-toloc.airport_name      1.000     1.000     1.000       3.0
            B-transport_type      1.000     1.000     1.000      10.0
    B-arrive_date.month_name      0.714     0.833     0.769       6.0
    B-arrive_date.day_number      0.714     0.833     0.769       6.0
         I-stoploc.city_name      1.000     1.000     1.000      10.0
                      B-meal      0.941     1.000     0.970      16.0
        B-fromloc.state_code      1.000     1.000     1.000      23.0
    B-depart_time.period_mod      1.000     1.000     1.000       5.0
                   B-connect      1.000     1.000     1.000       6.0
               B-flight_days      1.000     1.000     1.000      10.0
        B-toloc.airport_name      1.000     1.000     1.000       3.0
        B-fromloc.state_name      0.944     1.000     0.971      17.0
              B-airport_name      0.667     0.381     0.485      21.0
                   B-economy      1.000     1.000     1.000       6.0
               I-flight_time      1.000     1.000     1.000       1.0
             B-aircraft_code      1.000     0.879     0.935      33.0
                       B-mod      1.000     0.500     0.667       2.0
              B-airport_code      1.000     0.444     0.615       9.0
    B-depart_time.start_time      1.000     0.667     0.800       3.0
      B-depart_time.end_time      1.000     0.333     0.500       3.0
          B-depart_date.year      1.000     1.000     1.000       3.0
            I-transport_type      0.000     0.000     0.000       1.0
          B-restriction_code      0.800     1.000     0.889       4.0
    B-arrive_time.start_time      0.889     1.000     0.941       8.0
        B-toloc.airport_code      1.000     1.000     1.000       4.0
      B-arrive_time.end_time      0.889     1.000     0.941       8.0
      I-arrive_time.end_time      0.889     1.000     0.941       8.0
      I-depart_time.end_time      1.000     0.333     0.500       3.0
               I-flight_stop      0.000     0.000     0.000       0.0
      B-fromloc.airport_code      0.556     1.000     0.714       5.0
          I-restriction_code      1.000     1.000     1.000       3.0
    I-depart_time.start_time      1.000     1.000     1.000       1.0
          I-toloc.state_name      1.000     1.000     1.000       1.0
I-depart_date.today_relative      0.000     0.000     0.000       0.0
 B-arrive_date.date_relative      1.000     1.000     1.000       2.0
                I-flight_mod      1.000     0.167     0.286       6.0
                   I-economy      0.000     0.000     0.000       0.0
 B-return_date.date_relative      1.000     0.333     0.500       3.0
        I-fromloc.state_name      1.000     1.000     1.000       1.0
                B-state_code      1.000     1.000     1.000       1.0
    I-arrive_time.start_time      1.000     1.000     1.000       1.0
    I-arrive_date.day_number      0.000     0.000     0.000       0.0
                 B-meal_code      0.000     0.000     0.000       1.0
 I-depart_time.period_of_day      1.000     1.000     1.000       1.0
                  B-day_name      1.000     0.500     0.667       2.0
             B-period_of_day      1.000     0.750     0.857       4.0
        B-stoploc.state_code      0.000     0.000     0.000       0.0
    B-return_date.month_name      0.000     0.000     0.000       0.0
    B-return_date.day_number      0.000     0.000     0.000       0.0
    B-arrive_time.period_mod      0.000     0.000     0.000       0.0
                 I-meal_code      0.000     0.000     0.000       0.0
        B-toloc.country_name      1.000     1.000     1.000       1.0
                 B-days_code      0.000     0.000     0.000       1.0
 I-arrive_time.period_of_day      0.000     0.000     0.000       0.0
            I-today_relative      0.000     0.000     0.000       0.0
 B-return_time.period_of_day      0.000     0.000     0.000       0.0
                      B-time      0.000     0.000     0.000       0.0
           I-fare_basis_code      0.000     0.000     0.000       0.0
 I-arrive_time.time_relative      1.000     1.000     1.000       4.0
 I-depart_time.time_relative      0.000     0.000     0.000       1.0
            B-today_relative      0.000     0.000     0.000       0.0
                B-state_name      0.000     0.000     0.000       9.0
B-arrive_date.today_relative      0.000     0.000     0.000       0.0
    B-return_time.period_mod      0.000     0.000     0.000       0.0
                B-month_name      0.000     0.000     0.000       0.0
                B-day_number      0.000     0.000     0.000       0.0
 I-return_date.date_relative      0.667     0.667     0.667       3.0
I-return_date.today_relative      0.000     0.000     0.000       0.0
      B-stoploc.airport_name      0.000     0.000     0.000       0.0
             B-time_relative      0.000     0.000     0.000       0.0
                      I-time      0.000     0.000     0.000       0.0
    I-return_date.day_number      0.000     0.000     0.000       0.0
          I-meal_description      0.000     0.000     0.000       0.0
B-return_date.today_relative      0.000     0.000     0.000       0.0
      B-return_date.day_name      0.000     0.000     0.000       2.0

                   micro avg      0.960     0.961     0.960    3657.0
                   macro avg      0.681     0.654     0.653    3657.0
                weighted avg      0.963     0.961     0.958    3657.0

I0716 05:56:18.178083 139795883440000 interactiveshell.py:2882] Best Slot F1: 0.961
I0716 05:56:18.179729 139795883440000 estimator_training.py:186] Not using Distribute Coordinator.
I0716 05:56:18.181779 139795883440000 training.py:612] Running training and evaluation locally (non-distributed).
I0716 05:56:18.183770 139795883440000 training.py:700] Start train and evaluate loop. The evaluate will happen after every checkpoint. Checkpoint frequency is determined based on RunConfig arguments: save_checkpoints_steps 312 or save_checkpoints_secs None.
I0716 05:56:18.252672 139795883440000 estimator.py:1145] Calling model_fn.
I0716 05:56:19.590207 139795883440000 saver.py:1499] Saver not created because there are no variables in the graph to restore
I0716 05:56:20.392826 139795883440000 <ipython-input-6-d6747b40360b>:55] 
[<tf.Variable 'embedding:0' shape=(750, 300) dtype=float32_ref>,
 <tf.Variable 'module/aggregation/weights:0' shape=(3,) dtype=float32>,
 <tf.Variable 'module/aggregation/scaling:0' shape=() dtype=float32>,
 <tf.Variable 'dense/kernel:0' shape=(1324, 300) dtype=float32_ref>,
 <tf.Variable 'dense/bias:0' shape=(300,) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/gates/kernel:0' shape=(600, 600) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/gates/bias:0' shape=(600,) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/candidate/kernel:0' shape=(600, 300) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/candidate/bias:0' shape=(300,) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/gates/kernel:0' shape=(600, 600) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/gates/bias:0' shape=(600,) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/candidate/kernel:0' shape=(600, 300) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/candidate/bias:0' shape=(300,) dtype=float32_ref>,
 <tf.Variable 'dense_1/kernel:0' shape=(600, 23) dtype=float32_ref>,
 <tf.Variable 'dense_1/bias:0' shape=(23,) dtype=float32_ref>,
 <tf.Variable 'dense_2/kernel:0' shape=(600, 122) dtype=float32_ref>,
 <tf.Variable 'dense_2/bias:0' shape=(122,) dtype=float32_ref>]
I0716 05:56:21.472462 139795883440000 estimator.py:1147] Done calling model_fn.
I0716 05:56:21.476660 139795883440000 basic_session_run_hooks.py:541] Create CheckpointSaverHook.
I0716 05:56:21.795012 139795883440000 monitored_session.py:240] Graph was finalized.
I0716 05:56:21.818181 139795883440000 saver.py:1280] Restoring parameters from ../model/elmo_bigru/model.ckpt-14976
I0716 05:56:22.884191 139795883440000 session_manager.py:500] Running local_init_op.
I0716 05:56:22.954678 139795883440000 session_manager.py:502] Done running local_init_op.
I0716 05:56:25.807198 139795883440000 basic_session_run_hooks.py:606] Saving checkpoints for 14976 into ../model/elmo_bigru/model.ckpt.
Reading ../data/atis.train.w-intent.iob
I0716 05:56:30.610659 139795883440000 basic_session_run_hooks.py:262] loss = 0.008995927, step = 14976
I0716 05:56:30.612615 139795883440000 basic_session_run_hooks.py:262] lr = 6.19237e-05
I0716 05:56:57.552406 139795883440000 basic_session_run_hooks.py:692] global_step/sec: 3.71164
I0716 05:56:57.555257 139795883440000 basic_session_run_hooks.py:260] loss = 0.013111635, step = 15076 (26.945 sec)
I0716 05:56:57.557056 139795883440000 basic_session_run_hooks.py:260] lr = 6.127469e-05 (26.944 sec)
I0716 05:57:24.568656 139795883440000 basic_session_run_hooks.py:692] global_step/sec: 3.70148
I0716 05:57:24.571431 139795883440000 basic_session_run_hooks.py:260] loss = 0.011711226, step = 15176 (27.016 sec)
I0716 05:57:24.576102 139795883440000 basic_session_run_hooks.py:260] lr = 6.063249e-05 (27.019 sec)
I0716 05:57:51.199509 139795883440000 basic_session_run_hooks.py:692] global_step/sec: 3.75505
I0716 05:57:51.207821 139795883440000 basic_session_run_hooks.py:260] loss = 0.0053373585, step = 15276 (26.636 sec)
I0716 05:57:51.210559 139795883440000 basic_session_run_hooks.py:260] lr = 5.9997008e-05 (26.634 sec)
I0716 05:57:53.953844 139795883440000 basic_session_run_hooks.py:606] Saving checkpoints for 15288 into ../model/elmo_bigru/model.ckpt.
I0716 05:57:55.954508 139795883440000 estimator.py:1145] Calling model_fn.
I0716 05:57:57.582303 139795883440000 saver.py:1499] Saver not created because there are no variables in the graph to restore
I0716 05:57:58.166869 139795883440000 estimator.py:1147] Done calling model_fn.
I0716 05:57:58.198586 139795883440000 evaluation.py:255] Starting evaluation at 2019-07-16T05:57:58Z
I0716 05:57:58.439807 139795883440000 monitored_session.py:240] Graph was finalized.
I0716 05:57:58.462229 139795883440000 saver.py:1280] Restoring parameters from ../model/elmo_bigru/model.ckpt-15288
I0716 05:57:59.353767 139795883440000 session_manager.py:500] Running local_init_op.
I0716 05:57:59.408347 139795883440000 session_manager.py:502] Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
I0716 05:58:58.318442 139795883440000 evaluation.py:275] Finished evaluation at 2019-07-16-05:58:58
I0716 05:58:58.320344 139795883440000 estimator.py:2039] Saving dict for global step 15288: global_step = 15288, loss = 0.42268425
I0716 05:58:58.328033 139795883440000 estimator.py:2099] Saving 'checkpoint_path' summary for global step 15288: ../model/elmo_bigru/model.ckpt-15288
I0716 05:58:58.413577 139795883440000 estimator.py:368] Loss for final step: 0.0028340353.
Reading ../data/atis.test.w-intent.iob
I0716 05:58:59.028976 139795883440000 estimator.py:1145] Calling model_fn.
I0716 05:59:00.581136 139795883440000 saver.py:1499] Saver not created because there are no variables in the graph to restore
I0716 05:59:01.108357 139795883440000 estimator.py:1147] Done calling model_fn.
I0716 05:59:01.517195 139795883440000 monitored_session.py:240] Graph was finalized.
I0716 05:59:01.536671 139795883440000 saver.py:1280] Restoring parameters from ../model/elmo_bigru/model.ckpt-15288
I0716 05:59:02.462764 139795883440000 session_manager.py:500] Running local_init_op.
I0716 05:59:02.510772 139795883440000 session_manager.py:502] Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.
  'precision', 'predicted', average, warn_for)
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1439: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples.
  'recall', 'true', average, warn_for)
I0716 06:00:00.528573 139795883440000 interactiveshell.py:2882] 
                                          precision    recall  f1-score   support

                             atis_flight      0.983     0.992     0.987       632
                            atis_airfare      0.941     1.000     0.970        48
                     atis_ground_service      1.000     1.000     1.000        36
                            atis_airline      1.000     1.000     1.000        38
                       atis_abbreviation      0.943     1.000     0.971        33
                           atis_aircraft      1.000     0.889     0.941         9
                        atis_flight_time      0.500     1.000     0.667         1
                           atis_quantity      0.429     1.000     0.600         3
                atis_flight#atis_airfare      0.833     0.417     0.556        12
                            atis_airport      1.000     1.000     1.000        18
                           atis_distance      1.000     1.000     1.000        10
                               atis_city      1.000     0.500     0.667         6
                        atis_ground_fare      1.000     0.857     0.923         7
                           atis_capacity      0.952     0.952     0.952        21
                          atis_flight_no      0.889     1.000     0.941         8
                               atis_meal      1.000     0.833     0.909         6
                        atis_restriction      0.000     0.000     0.000         0
             atis_airline#atis_flight_no      0.000     0.000     0.000         0
    atis_ground_service#atis_ground_fare      0.000     0.000     0.000         0
           atis_airfare#atis_flight_time      0.000     0.000     0.000         0
                           atis_cheapest      0.000     0.000     0.000         0
atis_aircraft#atis_flight#atis_flight_no      0.000     0.000     0.000         0

                               micro avg      0.973     0.979     0.976       888
                               macro avg      0.658     0.656     0.640       888
                            weighted avg      0.976     0.979     0.975       888

I0716 06:00:00.569924 139795883440000 interactiveshell.py:2882] 
                              precision    recall  f1-score   support

                           O      0.000     0.000     0.000       0.0
           B-toloc.city_name      0.977     0.994     0.985     716.0
         B-fromloc.city_name      0.990     0.999     0.994     704.0
           I-toloc.city_name      0.978     0.989     0.983     265.0
      B-depart_date.day_name      0.991     0.991     0.991     212.0
              B-airline_name      1.000     1.000     1.000     101.0
         I-fromloc.city_name      0.978     1.000     0.989     177.0
 B-depart_time.period_of_day      1.000     0.923     0.960     130.0
              I-airline_name      1.000     1.000     1.000      65.0
    B-depart_date.day_number      0.981     0.964     0.972      55.0
    B-depart_date.month_name      0.982     0.964     0.973      56.0
          B-depart_time.time      0.864     1.000     0.927      57.0
                B-round_trip      1.000     0.986     0.993      73.0
             B-cost_relative      0.973     0.973     0.973      37.0
                I-round_trip      1.000     1.000     1.000      71.0
                B-flight_mod      0.828     1.000     0.906      24.0
 B-depart_time.time_relative      0.955     0.985     0.970      65.0
          I-depart_time.time      0.945     1.000     0.972      52.0
         B-stoploc.city_name      0.909     1.000     0.952      20.0
                 B-city_name      0.923     0.632     0.750      57.0
                B-class_type      0.960     1.000     0.980      24.0
          B-arrive_time.time      0.971     0.971     0.971      34.0
 B-arrive_time.time_relative      0.967     0.935     0.951      31.0
                I-class_type      1.000     1.000     1.000      17.0
               B-flight_stop      1.000     1.000     1.000      21.0
          I-arrive_time.time      1.000     0.971     0.986      35.0
              B-airline_code      0.971     1.000     0.986      34.0
    I-depart_date.day_number      1.000     0.933     0.966      15.0
      I-fromloc.airport_name      0.441     1.000     0.612      15.0
      B-fromloc.airport_name      0.462     1.000     0.632      12.0
      B-arrive_date.day_name      0.786     1.000     0.880      11.0
          B-toloc.state_code      1.000     1.000     1.000      18.0
B-depart_date.today_relative      1.000     1.000     1.000       9.0
             B-flight_number      0.733     1.000     0.846      11.0
 B-depart_date.date_relative      0.944     1.000     0.971      17.0
          B-toloc.state_name      0.788     0.929     0.852      28.0
           B-fare_basis_code      0.850     1.000     0.919      17.0
               B-flight_time      1.000     1.000     1.000       1.0
                        B-or      1.000     1.000     1.000       3.0
 B-arrive_time.period_of_day      0.750     1.000     0.857       6.0
          B-meal_description      1.000     0.900     0.947      10.0
             I-cost_relative      1.000     0.667     0.800       3.0
              I-airport_name      0.846     0.379     0.524      29.0
               B-fare_amount      1.000     1.000     1.000       2.0
               I-fare_amount      1.000     1.000     1.000       2.0
                 I-city_name      1.000     0.567     0.723      30.0
        I-toloc.airport_name      1.000     1.000     1.000       3.0
            B-transport_type      1.000     1.000     1.000      10.0
    B-arrive_date.month_name      0.714     0.833     0.769       6.0
    B-arrive_date.day_number      0.714     0.833     0.769       6.0
         I-stoploc.city_name      1.000     1.000     1.000      10.0
                      B-meal      1.000     1.000     1.000      16.0
        B-fromloc.state_code      1.000     1.000     1.000      23.0
    B-depart_time.period_mod      1.000     1.000     1.000       5.0
                   B-connect      1.000     1.000     1.000       6.0
               B-flight_days      1.000     1.000     1.000      10.0
        B-toloc.airport_name      1.000     1.000     1.000       3.0
        B-fromloc.state_name      0.944     1.000     0.971      17.0
              B-airport_name      0.800     0.381     0.516      21.0
                   B-economy      1.000     1.000     1.000       6.0
               I-flight_time      1.000     1.000     1.000       1.0
             B-aircraft_code      1.000     0.879     0.935      33.0
                       B-mod      1.000     0.500     0.667       2.0
              B-airport_code      1.000     0.444     0.615       9.0
    B-depart_time.start_time      1.000     0.667     0.800       3.0
      B-depart_time.end_time      1.000     0.333     0.500       3.0
          B-depart_date.year      1.000     1.000     1.000       3.0
            I-transport_type      0.000     0.000     0.000       1.0
          B-restriction_code      0.800     1.000     0.889       4.0
    B-arrive_time.start_time      0.889     1.000     0.941       8.0
        B-toloc.airport_code      1.000     1.000     1.000       4.0
      B-arrive_time.end_time      0.889     1.000     0.941       8.0
      I-arrive_time.end_time      0.889     1.000     0.941       8.0
      I-depart_time.end_time      1.000     0.333     0.500       3.0
               I-flight_stop      0.000     0.000     0.000       0.0
      B-fromloc.airport_code      0.556     1.000     0.714       5.0
          I-restriction_code      1.000     1.000     1.000       3.0
    I-depart_time.start_time      1.000     1.000     1.000       1.0
          I-toloc.state_name      1.000     1.000     1.000       1.0
I-depart_date.today_relative      0.000     0.000     0.000       0.0
 B-arrive_date.date_relative      1.000     1.000     1.000       2.0
                I-flight_mod      1.000     0.167     0.286       6.0
                   I-economy      0.000     0.000     0.000       0.0
 B-return_date.date_relative      1.000     0.333     0.500       3.0
        I-fromloc.state_name      1.000     1.000     1.000       1.0
                B-state_code      1.000     1.000     1.000       1.0
    I-arrive_time.start_time      1.000     1.000     1.000       1.0
    I-arrive_date.day_number      0.000     0.000     0.000       0.0
                 B-meal_code      0.000     0.000     0.000       1.0
 I-depart_time.period_of_day      1.000     1.000     1.000       1.0
                  B-day_name      1.000     0.500     0.667       2.0
             B-period_of_day      1.000     0.750     0.857       4.0
        B-stoploc.state_code      0.000     0.000     0.000       0.0
    B-return_date.month_name      0.000     0.000     0.000       0.0
    B-return_date.day_number      0.000     0.000     0.000       0.0
    B-arrive_time.period_mod      0.000     0.000     0.000       0.0
                 I-meal_code      0.000     0.000     0.000       0.0
        B-toloc.country_name      1.000     1.000     1.000       1.0
                 B-days_code      0.000     0.000     0.000       1.0
 I-arrive_time.period_of_day      0.000     0.000     0.000       0.0
            I-today_relative      0.000     0.000     0.000       0.0
 B-return_time.period_of_day      0.000     0.000     0.000       0.0
                      B-time      0.000     0.000     0.000       0.0
           I-fare_basis_code      0.000     0.000     0.000       0.0
 I-arrive_time.time_relative      1.000     1.000     1.000       4.0
 I-depart_time.time_relative      0.000     0.000     0.000       1.0
            B-today_relative      0.000     0.000     0.000       0.0
                B-state_name      0.000     0.000     0.000       9.0
B-arrive_date.today_relative      0.000     0.000     0.000       0.0
    B-return_time.period_mod      0.000     0.000     0.000       0.0
                B-month_name      0.000     0.000     0.000       0.0
                B-day_number      0.000     0.000     0.000       0.0
 I-return_date.date_relative      0.667     0.667     0.667       3.0
I-return_date.today_relative      0.000     0.000     0.000       0.0
      B-stoploc.airport_name      0.000     0.000     0.000       0.0
             B-time_relative      0.000     0.000     0.000       0.0
                      I-time      0.000     0.000     0.000       0.0
    I-return_date.day_number      0.000     0.000     0.000       0.0
          I-meal_description      0.000     0.000     0.000       0.0
B-return_date.today_relative      0.000     0.000     0.000       0.0
      B-return_date.day_name      0.000     0.000     0.000       2.0

                   micro avg      0.959     0.961     0.960    3657.0
                   macro avg      0.683     0.655     0.654    3657.0
                weighted avg      0.964     0.961     0.958    3657.0

I0716 06:00:00.593960 139795883440000 interactiveshell.py:2882] Best Slot F1: 0.961
I0716 06:00:00.595951 139795883440000 estimator_training.py:186] Not using Distribute Coordinator.
I0716 06:00:00.600968 139795883440000 training.py:612] Running training and evaluation locally (non-distributed).
I0716 06:00:00.603511 139795883440000 training.py:700] Start train and evaluate loop. The evaluate will happen after every checkpoint. Checkpoint frequency is determined based on RunConfig arguments: save_checkpoints_steps 312 or save_checkpoints_secs None.
I0716 06:00:00.673551 139795883440000 estimator.py:1145] Calling model_fn.
I0716 06:00:02.042279 139795883440000 saver.py:1499] Saver not created because there are no variables in the graph to restore
I0716 06:00:02.645030 139795883440000 <ipython-input-6-d6747b40360b>:55] 
[<tf.Variable 'embedding:0' shape=(750, 300) dtype=float32_ref>,
 <tf.Variable 'module/aggregation/weights:0' shape=(3,) dtype=float32>,
 <tf.Variable 'module/aggregation/scaling:0' shape=() dtype=float32>,
 <tf.Variable 'dense/kernel:0' shape=(1324, 300) dtype=float32_ref>,
 <tf.Variable 'dense/bias:0' shape=(300,) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/gates/kernel:0' shape=(600, 600) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/gates/bias:0' shape=(600,) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/candidate/kernel:0' shape=(600, 300) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/candidate/bias:0' shape=(300,) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/gates/kernel:0' shape=(600, 600) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/gates/bias:0' shape=(600,) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/candidate/kernel:0' shape=(600, 300) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/candidate/bias:0' shape=(300,) dtype=float32_ref>,
 <tf.Variable 'dense_1/kernel:0' shape=(600, 23) dtype=float32_ref>,
 <tf.Variable 'dense_1/bias:0' shape=(23,) dtype=float32_ref>,
 <tf.Variable 'dense_2/kernel:0' shape=(600, 122) dtype=float32_ref>,
 <tf.Variable 'dense_2/bias:0' shape=(122,) dtype=float32_ref>]
I0716 06:00:03.956224 139795883440000 estimator.py:1147] Done calling model_fn.
I0716 06:00:03.964102 139795883440000 basic_session_run_hooks.py:541] Create CheckpointSaverHook.
I0716 06:00:04.286618 139795883440000 monitored_session.py:240] Graph was finalized.
I0716 06:00:04.307549 139795883440000 saver.py:1280] Restoring parameters from ../model/elmo_bigru/model.ckpt-15288
I0716 06:00:05.404417 139795883440000 session_manager.py:500] Running local_init_op.
I0716 06:00:05.497148 139795883440000 session_manager.py:502] Done running local_init_op.
I0716 06:00:08.350968 139795883440000 basic_session_run_hooks.py:606] Saving checkpoints for 15288 into ../model/elmo_bigru/model.ckpt.
Reading ../data/atis.train.w-intent.iob
I0716 06:00:13.357057 139795883440000 basic_session_run_hooks.py:262] loss = 0.0077921036, step = 15288
I0716 06:00:13.358960 139795883440000 basic_session_run_hooks.py:262] lr = 5.99212e-05
I0716 06:00:40.077207 139795883440000 basic_session_run_hooks.py:692] global_step/sec: 3.74241
I0716 06:00:40.083547 139795883440000 basic_session_run_hooks.py:260] loss = 0.013172938, step = 15388 (26.726 sec)
I0716 06:00:40.087269 139795883440000 basic_session_run_hooks.py:260] lr = 5.9293186e-05 (26.728 sec)
I0716 06:01:07.204914 139795883440000 basic_session_run_hooks.py:692] global_step/sec: 3.68628
I0716 06:01:07.212323 139795883440000 basic_session_run_hooks.py:260] loss = 0.010772801, step = 15488 (27.129 sec)
I0716 06:01:07.215197 139795883440000 basic_session_run_hooks.py:260] lr = 5.8671747e-05 (27.128 sec)
I0716 06:01:32.990794 139795883440000 basic_session_run_hooks.py:692] global_step/sec: 3.87808
I0716 06:01:32.999497 139795883440000 basic_session_run_hooks.py:260] loss = 0.0069304784, step = 15588 (25.787 sec)
I0716 06:01:33.001451 139795883440000 basic_session_run_hooks.py:260] lr = 5.8056834e-05 (25.786 sec)
I0716 06:01:36.349722 139795883440000 basic_session_run_hooks.py:606] Saving checkpoints for 15600 into ../model/elmo_bigru/model.ckpt.
I0716 06:01:38.454736 139795883440000 estimator.py:1145] Calling model_fn.
I0716 06:01:40.095187 139795883440000 saver.py:1499] Saver not created because there are no variables in the graph to restore
I0716 06:01:40.678136 139795883440000 estimator.py:1147] Done calling model_fn.
I0716 06:01:40.711853 139795883440000 evaluation.py:255] Starting evaluation at 2019-07-16T06:01:40Z
I0716 06:01:40.939875 139795883440000 monitored_session.py:240] Graph was finalized.
I0716 06:01:40.958451 139795883440000 saver.py:1280] Restoring parameters from ../model/elmo_bigru/model.ckpt-15600
I0716 06:01:41.885448 139795883440000 session_manager.py:500] Running local_init_op.
I0716 06:01:41.942841 139795883440000 session_manager.py:502] Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
I0716 06:02:40.721067 139795883440000 evaluation.py:275] Finished evaluation at 2019-07-16-06:02:40
I0716 06:02:40.722697 139795883440000 estimator.py:2039] Saving dict for global step 15600: global_step = 15600, loss = 0.42624673
I0716 06:02:40.728740 139795883440000 estimator.py:2099] Saving 'checkpoint_path' summary for global step 15600: ../model/elmo_bigru/model.ckpt-15600
I0716 06:02:40.827952 139795883440000 estimator.py:368] Loss for final step: 0.009893632.
Reading ../data/atis.test.w-intent.iob
I0716 06:02:41.432628 139795883440000 estimator.py:1145] Calling model_fn.
I0716 06:02:42.993689 139795883440000 saver.py:1499] Saver not created because there are no variables in the graph to restore
I0716 06:02:43.514800 139795883440000 estimator.py:1147] Done calling model_fn.
I0716 06:02:43.729723 139795883440000 monitored_session.py:240] Graph was finalized.
I0716 06:02:43.749116 139795883440000 saver.py:1280] Restoring parameters from ../model/elmo_bigru/model.ckpt-15600
I0716 06:02:44.648586 139795883440000 session_manager.py:500] Running local_init_op.
I0716 06:02:44.698976 139795883440000 session_manager.py:502] Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.
  'precision', 'predicted', average, warn_for)
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1439: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples.
  'recall', 'true', average, warn_for)
I0716 06:03:42.711251 139795883440000 interactiveshell.py:2882] 
                                          precision    recall  f1-score   support

                             atis_flight      0.984     0.991     0.987       632
                            atis_airfare      0.960     1.000     0.980        48
                     atis_ground_service      1.000     1.000     1.000        36
                            atis_airline      1.000     1.000     1.000        38
                       atis_abbreviation      0.943     1.000     0.971        33
                           atis_aircraft      1.000     0.889     0.941         9
                        atis_flight_time      0.500     1.000     0.667         1
                           atis_quantity      0.429     1.000     0.600         3
                atis_flight#atis_airfare      0.833     0.417     0.556        12
                            atis_airport      0.947     1.000     0.973        18
                           atis_distance      1.000     1.000     1.000        10
                               atis_city      0.750     0.500     0.600         6
                        atis_ground_fare      1.000     1.000     1.000         7
                           atis_capacity      0.952     0.952     0.952        21
                          atis_flight_no      0.889     1.000     0.941         8
                               atis_meal      1.000     0.833     0.909         6
                        atis_restriction      0.000     0.000     0.000         0
             atis_airline#atis_flight_no      0.000     0.000     0.000         0
    atis_ground_service#atis_ground_fare      0.000     0.000     0.000         0
           atis_airfare#atis_flight_time      0.000     0.000     0.000         0
                           atis_cheapest      0.000     0.000     0.000         0
atis_aircraft#atis_flight#atis_flight_no      0.000     0.000     0.000         0

                               micro avg      0.973     0.979     0.976       888
                               macro avg      0.645     0.663     0.640       888
                            weighted avg      0.975     0.979     0.975       888

I0716 06:03:42.749567 139795883440000 interactiveshell.py:2882] 
                              precision    recall  f1-score   support

                           O      0.000     0.000     0.000       0.0
           B-toloc.city_name      0.977     0.994     0.985     716.0
         B-fromloc.city_name      0.990     1.000     0.995     704.0
           I-toloc.city_name      0.978     0.989     0.983     265.0
      B-depart_date.day_name      0.991     0.991     0.991     212.0
              B-airline_name      1.000     1.000     1.000     101.0
         I-fromloc.city_name      0.978     1.000     0.989     177.0
 B-depart_time.period_of_day      1.000     0.923     0.960     130.0
              I-airline_name      1.000     1.000     1.000      65.0
    B-depart_date.day_number      0.981     0.964     0.972      55.0
    B-depart_date.month_name      0.982     0.964     0.973      56.0
          B-depart_time.time      0.864     1.000     0.927      57.0
                B-round_trip      1.000     0.986     0.993      73.0
             B-cost_relative      0.973     0.973     0.973      37.0
                I-round_trip      1.000     1.000     1.000      71.0
                B-flight_mod      0.828     1.000     0.906      24.0
 B-depart_time.time_relative      0.970     0.985     0.977      65.0
          I-depart_time.time      0.945     1.000     0.972      52.0
         B-stoploc.city_name      0.952     1.000     0.976      20.0
                 B-city_name      0.923     0.632     0.750      57.0
                B-class_type      0.960     1.000     0.980      24.0
          B-arrive_time.time      0.971     0.971     0.971      34.0
 B-arrive_time.time_relative      0.935     0.935     0.935      31.0
                I-class_type      1.000     1.000     1.000      17.0
               B-flight_stop      1.000     1.000     1.000      21.0
          I-arrive_time.time      1.000     0.971     0.986      35.0
              B-airline_code      0.971     1.000     0.986      34.0
    I-depart_date.day_number      1.000     0.933     0.966      15.0
      I-fromloc.airport_name      0.441     1.000     0.612      15.0
      B-fromloc.airport_name      0.462     1.000     0.632      12.0
      B-arrive_date.day_name      0.786     1.000     0.880      11.0
          B-toloc.state_code      1.000     1.000     1.000      18.0
B-depart_date.today_relative      1.000     0.889     0.941       9.0
             B-flight_number      0.733     1.000     0.846      11.0
 B-depart_date.date_relative      0.944     1.000     0.971      17.0
          B-toloc.state_name      0.839     0.929     0.881      28.0
           B-fare_basis_code      0.895     1.000     0.944      17.0
               B-flight_time      1.000     1.000     1.000       1.0
                        B-or      1.000     1.000     1.000       3.0
 B-arrive_time.period_of_day      0.750     1.000     0.857       6.0
          B-meal_description      1.000     0.900     0.947      10.0
             I-cost_relative      1.000     0.667     0.800       3.0
              I-airport_name      0.846     0.379     0.524      29.0
               B-fare_amount      1.000     1.000     1.000       2.0
               I-fare_amount      1.000     1.000     1.000       2.0
                 I-city_name      1.000     0.567     0.723      30.0
        I-toloc.airport_name      1.000     1.000     1.000       3.0
            B-transport_type      1.000     1.000     1.000      10.0
    B-arrive_date.month_name      0.714     0.833     0.769       6.0
    B-arrive_date.day_number      0.714     0.833     0.769       6.0
         I-stoploc.city_name      1.000     1.000     1.000      10.0
                      B-meal      1.000     1.000     1.000      16.0
        B-fromloc.state_code      1.000     1.000     1.000      23.0
    B-depart_time.period_mod      1.000     1.000     1.000       5.0
                   B-connect      1.000     1.000     1.000       6.0
               B-flight_days      1.000     1.000     1.000      10.0
        B-toloc.airport_name      1.000     1.000     1.000       3.0
        B-fromloc.state_name      0.944     1.000     0.971      17.0
              B-airport_name      0.667     0.381     0.485      21.0
                   B-economy      1.000     1.000     1.000       6.0
               I-flight_time      1.000     1.000     1.000       1.0
             B-aircraft_code      1.000     0.879     0.935      33.0
                       B-mod      1.000     0.500     0.667       2.0
              B-airport_code      1.000     0.444     0.615       9.0
    B-depart_time.start_time      1.000     0.667     0.800       3.0
      B-depart_time.end_time      1.000     0.333     0.500       3.0
          B-depart_date.year      1.000     1.000     1.000       3.0
            I-transport_type      0.000     0.000     0.000       1.0
          B-restriction_code      0.800     1.000     0.889       4.0
    B-arrive_time.start_time      0.889     1.000     0.941       8.0
        B-toloc.airport_code      1.000     1.000     1.000       4.0
      B-arrive_time.end_time      0.889     1.000     0.941       8.0
      I-arrive_time.end_time      0.889     1.000     0.941       8.0
      I-depart_time.end_time      1.000     0.333     0.500       3.0
               I-flight_stop      0.000     0.000     0.000       0.0
      B-fromloc.airport_code      0.556     1.000     0.714       5.0
          I-restriction_code      1.000     1.000     1.000       3.0
    I-depart_time.start_time      1.000     1.000     1.000       1.0
          I-toloc.state_name      1.000     1.000     1.000       1.0
I-depart_date.today_relative      0.000     0.000     0.000       0.0
 B-arrive_date.date_relative      1.000     1.000     1.000       2.0
                I-flight_mod      1.000     0.167     0.286       6.0
                   I-economy      0.000     0.000     0.000       0.0
 B-return_date.date_relative      1.000     0.667     0.800       3.0
        I-fromloc.state_name      1.000     1.000     1.000       1.0
                B-state_code      1.000     1.000     1.000       1.0
    I-arrive_time.start_time      1.000     1.000     1.000       1.0
    I-arrive_date.day_number      0.000     0.000     0.000       0.0
                 B-meal_code      0.000     0.000     0.000       1.0
 I-depart_time.period_of_day      1.000     1.000     1.000       1.0
                  B-day_name      1.000     0.500     0.667       2.0
             B-period_of_day      1.000     0.750     0.857       4.0
        B-stoploc.state_code      0.000     0.000     0.000       0.0
    B-return_date.month_name      0.000     0.000     0.000       0.0
    B-return_date.day_number      0.000     0.000     0.000       0.0
    B-arrive_time.period_mod      0.000     0.000     0.000       0.0
                 I-meal_code      0.000     0.000     0.000       0.0
        B-toloc.country_name      1.000     1.000     1.000       1.0
                 B-days_code      1.000     1.000     1.000       1.0
 I-arrive_time.period_of_day      0.000     0.000     0.000       0.0
            I-today_relative      0.000     0.000     0.000       0.0
 B-return_time.period_of_day      0.000     0.000     0.000       0.0
                      B-time      0.000     0.000     0.000       0.0
           I-fare_basis_code      0.000     0.000     0.000       0.0
 I-arrive_time.time_relative      1.000     0.750     0.857       4.0
 I-depart_time.time_relative      0.000     0.000     0.000       1.0
            B-today_relative      0.000     0.000     0.000       0.0
                B-state_name      0.000     0.000     0.000       9.0
B-arrive_date.today_relative      0.000     0.000     0.000       0.0
    B-return_time.period_mod      0.000     0.000     0.000       0.0
                B-month_name      0.000     0.000     0.000       0.0
                B-day_number      0.000     0.000     0.000       0.0
 I-return_date.date_relative      1.000     0.667     0.800       3.0
I-return_date.today_relative      0.000     0.000     0.000       0.0
      B-stoploc.airport_name      0.000     0.000     0.000       0.0
             B-time_relative      0.000     0.000     0.000       0.0
                      I-time      0.000     0.000     0.000       0.0
    I-return_date.day_number      0.000     0.000     0.000       0.0
          I-meal_description      0.000     0.000     0.000       0.0
B-return_date.today_relative      0.000     0.000     0.000       0.0
      B-return_date.day_name      0.000     0.000     0.000       2.0

                   micro avg      0.960     0.961     0.960    3657.0
                   macro avg      0.694     0.663     0.665    3657.0
                weighted avg      0.964     0.961     0.958    3657.0

I0716 06:03:42.772957 139795883440000 interactiveshell.py:2882] Best Slot F1: 0.961
I0716 06:03:42.774310 139795883440000 estimator_training.py:186] Not using Distribute Coordinator.
I0716 06:03:42.776245 139795883440000 training.py:612] Running training and evaluation locally (non-distributed).
I0716 06:03:42.778315 139795883440000 training.py:700] Start train and evaluate loop. The evaluate will happen after every checkpoint. Checkpoint frequency is determined based on RunConfig arguments: save_checkpoints_steps 312 or save_checkpoints_secs None.
I0716 06:03:42.850522 139795883440000 estimator.py:1145] Calling model_fn.
I0716 06:03:44.378764 139795883440000 saver.py:1499] Saver not created because there are no variables in the graph to restore
I0716 06:03:44.961277 139795883440000 <ipython-input-6-d6747b40360b>:55] 
[<tf.Variable 'embedding:0' shape=(750, 300) dtype=float32_ref>,
 <tf.Variable 'module/aggregation/weights:0' shape=(3,) dtype=float32>,
 <tf.Variable 'module/aggregation/scaling:0' shape=() dtype=float32>,
 <tf.Variable 'dense/kernel:0' shape=(1324, 300) dtype=float32_ref>,
 <tf.Variable 'dense/bias:0' shape=(300,) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/gates/kernel:0' shape=(600, 600) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/gates/bias:0' shape=(600,) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/candidate/kernel:0' shape=(600, 300) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/fw/gru_cell/candidate/bias:0' shape=(300,) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/gates/kernel:0' shape=(600, 600) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/gates/bias:0' shape=(600,) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/candidate/kernel:0' shape=(600, 300) dtype=float32_ref>,
 <tf.Variable 'bidirectional_rnn/bw/gru_cell/candidate/bias:0' shape=(300,) dtype=float32_ref>,
 <tf.Variable 'dense_1/kernel:0' shape=(600, 23) dtype=float32_ref>,
 <tf.Variable 'dense_1/bias:0' shape=(23,) dtype=float32_ref>,
 <tf.Variable 'dense_2/kernel:0' shape=(600, 122) dtype=float32_ref>,
 <tf.Variable 'dense_2/bias:0' shape=(122,) dtype=float32_ref>]
I0716 06:03:46.248441 139795883440000 estimator.py:1147] Done calling model_fn.
I0716 06:03:46.252445 139795883440000 basic_session_run_hooks.py:541] Create CheckpointSaverHook.
I0716 06:03:46.582277 139795883440000 monitored_session.py:240] Graph was finalized.
I0716 06:03:46.600440 139795883440000 saver.py:1280] Restoring parameters from ../model/elmo_bigru/model.ckpt-15600
I0716 06:03:47.670186 139795883440000 session_manager.py:500] Running local_init_op.
I0716 06:03:47.740401 139795883440000 session_manager.py:502] Done running local_init_op.
I0716 06:03:50.567635 139795883440000 basic_session_run_hooks.py:606] Saving checkpoints for 15600 into ../model/elmo_bigru/model.ckpt.
Reading ../data/atis.train.w-intent.iob
I0716 06:03:55.228627 139795883440000 basic_session_run_hooks.py:262] loss = 0.007499742, step = 15600
I0716 06:03:55.230428 139795883440000 basic_session_run_hooks.py:262] lr = 5.798347e-05
I0716 06:04:22.553234 139795883440000 basic_session_run_hooks.py:692] global_step/sec: 3.65965
I0716 06:04:22.559311 139795883440000 basic_session_run_hooks.py:260] loss = 0.0067059495, step = 15700 (27.331 sec)
I0716 06:04:22.561770 139795883440000 basic_session_run_hooks.py:260] lr = 5.737576e-05 (27.331 sec)
I0716 06:04:48.927580 139795883440000 basic_session_run_hooks.py:692] global_step/sec: 3.79156
I0716 06:04:48.931572 139795883440000 basic_session_run_hooks.py:260] loss = 0.009300061, step = 15800 (26.372 sec)
I0716 06:04:48.935711 139795883440000 basic_session_run_hooks.py:260] lr = 5.6774414e-05 (26.374 sec)
I0716 06:05:15.645237 139795883440000 basic_session_run_hooks.py:692] global_step/sec: 3.74285
I0716 06:05:15.651256 139795883440000 basic_session_run_hooks.py:260] loss = 0.007555264, step = 15900 (26.720 sec)
I0716 06:05:15.653822 139795883440000 basic_session_run_hooks.py:260] lr = 5.6179386e-05 (26.718 sec)
I0716 06:05:18.507963 139795883440000 basic_session_run_hooks.py:606] Saving checkpoints for 15912 into ../model/elmo_bigru/model.ckpt.
I0716 06:05:20.802643 139795883440000 estimator.py:1145] Calling model_fn.
I0716 06:05:22.489187 139795883440000 saver.py:1499] Saver not created because there are no variables in the graph to restore
I0716 06:05:23.066193 139795883440000 estimator.py:1147] Done calling model_fn.
I0716 06:05:23.095655 139795883440000 evaluation.py:255] Starting evaluation at 2019-07-16T06:05:23Z
I0716 06:05:23.319568 139795883440000 monitored_session.py:240] Graph was finalized.
I0716 06:05:23.338410 139795883440000 saver.py:1280] Restoring parameters from ../model/elmo_bigru/model.ckpt-15912
I0716 06:05:24.347270 139795883440000 session_manager.py:500] Running local_init_op.
I0716 06:05:24.422134 139795883440000 session_manager.py:502] Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
I0716 06:06:23.721860 139795883440000 evaluation.py:275] Finished evaluation at 2019-07-16-06:06:23
I0716 06:06:23.723529 139795883440000 estimator.py:2039] Saving dict for global step 15912: global_step = 15912, loss = 0.42764378
I0716 06:06:23.731847 139795883440000 estimator.py:2099] Saving 'checkpoint_path' summary for global step 15912: ../model/elmo_bigru/model.ckpt-15912
I0716 06:06:23.813684 139795883440000 estimator.py:368] Loss for final step: 0.0038011419.
Reading ../data/atis.test.w-intent.iob
I0716 06:06:24.433973 139795883440000 estimator.py:1145] Calling model_fn.
I0716 06:06:26.021326 139795883440000 saver.py:1499] Saver not created because there are no variables in the graph to restore
I0716 06:06:26.532943 139795883440000 estimator.py:1147] Done calling model_fn.
I0716 06:06:26.747190 139795883440000 monitored_session.py:240] Graph was finalized.
I0716 06:06:26.767136 139795883440000 saver.py:1280] Restoring parameters from ../model/elmo_bigru/model.ckpt-15912
I0716 06:06:27.689567 139795883440000 session_manager.py:500] Running local_init_op.
I0716 06:06:27.740319 139795883440000 session_manager.py:502] Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.
  'precision', 'predicted', average, warn_for)
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1439: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples.
  'recall', 'true', average, warn_for)
I0716 06:07:25.706608 139795883440000 interactiveshell.py:2882] 
                                          precision    recall  f1-score   support

                             atis_flight      0.984     0.991     0.987       632
                            atis_airfare      0.941     1.000     0.970        48
                     atis_ground_service      1.000     1.000     1.000        36
                            atis_airline      0.974     1.000     0.987        38
                       atis_abbreviation      0.943     1.000     0.971        33
                           atis_aircraft      1.000     0.889     0.941         9
                        atis_flight_time      0.500     1.000     0.667         1
                           atis_quantity      0.429     1.000     0.600         3
                atis_flight#atis_airfare      0.833     0.417     0.556        12
                            atis_airport      0.944     0.944     0.944        18
                           atis_distance      1.000     1.000     1.000        10
                               atis_city      0.750     0.500     0.600         6
                        atis_ground_fare      1.000     0.857     0.923         7
                           atis_capacity      0.952     0.952     0.952        21
                          atis_flight_no      0.889     1.000     0.941         8
                               atis_meal      1.000     0.833     0.909         6
                        atis_restriction      0.000     0.000     0.000         0
             atis_airline#atis_flight_no      0.000     0.000     0.000         0
    atis_ground_service#atis_ground_fare      0.000     0.000     0.000         0
           atis_airfare#atis_flight_time      0.000     0.000     0.000         0
                           atis_cheapest      0.000     0.000     0.000         0
atis_aircraft#atis_flight#atis_flight_no      0.000     0.000     0.000         0

                               micro avg      0.971     0.976     0.974       888
                               macro avg      0.643     0.654     0.634       888
                            weighted avg      0.973     0.976     0.973       888

I0716 06:07:25.744853 139795883440000 interactiveshell.py:2882] 
                              precision    recall  f1-score   support

                           O      0.000     0.000     0.000       0.0
           B-toloc.city_name      0.977     0.994     0.985     716.0
         B-fromloc.city_name      0.990     0.999     0.994     704.0
           I-toloc.city_name      0.978     0.989     0.983     265.0
      B-depart_date.day_name      0.991     0.991     0.991     212.0
              B-airline_name      1.000     1.000     1.000     101.0
         I-fromloc.city_name      0.973     1.000     0.986     177.0
 B-depart_time.period_of_day      0.992     0.923     0.956     130.0
              I-airline_name      1.000     1.000     1.000      65.0
    B-depart_date.day_number      0.981     0.964     0.972      55.0
    B-depart_date.month_name      0.982     0.964     0.973      56.0
          B-depart_time.time      0.864     1.000     0.927      57.0
                B-round_trip      1.000     1.000     1.000      73.0
             B-cost_relative      0.973     0.973     0.973      37.0
                I-round_trip      1.000     1.000     1.000      71.0
                B-flight_mod      0.828     1.000     0.906      24.0
 B-depart_time.time_relative      0.955     0.985     0.970      65.0
          I-depart_time.time      0.945     1.000     0.972      52.0
         B-stoploc.city_name      0.909     1.000     0.952      20.0
                 B-city_name      0.923     0.632     0.750      57.0
                B-class_type      0.960     1.000     0.980      24.0
          B-arrive_time.time      0.971     0.971     0.971      34.0
 B-arrive_time.time_relative      0.967     0.935     0.951      31.0
                I-class_type      1.000     1.000     1.000      17.0
               B-flight_stop      1.000     1.000     1.000      21.0
          I-arrive_time.time      1.000     0.971     0.986      35.0
              B-airline_code      0.971     1.000     0.986      34.0
    I-depart_date.day_number      1.000     0.933     0.966      15.0
      I-fromloc.airport_name      0.441     1.000     0.612      15.0
      B-fromloc.airport_name      0.462     1.000     0.632      12.0
      B-arrive_date.day_name      0.786     1.000     0.880      11.0
          B-toloc.state_code      1.000     1.000     1.000      18.0
B-depart_date.today_relative      1.000     0.889     0.941       9.0
             B-flight_number      0.733     1.000     0.846      11.0
 B-depart_date.date_relative      0.944     1.000     0.971      17.0
          B-toloc.state_name      0.839     0.929     0.881      28.0
           B-fare_basis_code      0.850     1.000     0.919      17.0
               B-flight_time      1.000     1.000     1.000       1.0
                        B-or      1.000     1.000     1.000       3.0
 B-arrive_time.period_of_day      0.750     1.000     0.857       6.0
          B-meal_description      1.000     0.900     0.947      10.0
             I-cost_relative      1.000     0.667     0.800       3.0
              I-airport_name      0.846     0.379     0.524      29.0
               B-fare_amount      1.000     1.000     1.000       2.0
               I-fare_amount      1.000     1.000     1.000       2.0
                 I-city_name      1.000     0.533     0.696      30.0
        I-toloc.airport_name      1.000     1.000     1.000       3.0
            B-transport_type      1.000     1.000     1.000      10.0
    B-arrive_date.month_name      0.714     0.833     0.769       6.0
    B-arrive_date.day_number      0.714     0.833     0.769       6.0
         I-stoploc.city_name      1.000     1.000     1.000      10.0
                      B-meal      1.000     1.000     1.000      16.0
        B-fromloc.state_code      1.000     1.000     1.000      23.0
    B-depart_time.period_mod      1.000     1.000     1.000       5.0
                   B-connect      1.000     1.000     1.000       6.0
               B-flight_days      1.000     1.000     1.000      10.0
        B-toloc.airport_name      1.000     1.000     1.000       3.0
        B-fromloc.state_name      0.944     1.000     0.971      17.0
              B-airport_name      0.667     0.381     0.485      21.0
                   B-economy      1.000     1.000     1.000       6.0
               I-flight_time      1.000     1.000     1.000       1.0
             B-aircraft_code      1.000     0.879     0.935      33.0
                       B-mod      1.000     0.500     0.667       2.0
              B-airport_code      1.000     0.444     0.615       9.0
    B-depart_time.start_time      1.000     0.667     0.800       3.0
      B-depart_time.end_time      1.000     0.333     0.500       3.0
          B-depart_date.year      1.000     1.000     1.000       3.0
            I-transport_type      0.000     0.000     0.000       1.0
          B-restriction_code      0.800     1.000     0.889       4.0
    B-arrive_time.start_time      0.889     1.000     0.941       8.0
        B-toloc.airport_code      1.000     1.000     1.000       4.0
      B-arrive_time.end_time      0.889     1.000     0.941       8.0
      I-arrive_time.end_time      0.889     1.000     0.941       8.0
      I-depart_time.end_time      1.000     0.333     0.500       3.0
               I-flight_stop      0.000     0.000     0.000       0.0
      B-fromloc.airport_code      0.556     1.000     0.714       5.0
          I-restriction_code      1.000     1.000     1.000       3.0
    I-depart_time.start_time      1.000     1.000     1.000       1.0
          I-toloc.state_name      1.000     1.000     1.000       1.0
I-depart_date.today_relative      0.000     0.000     0.000       0.0
 B-arrive_date.date_relative      1.000     1.000     1.000       2.0
                I-flight_mod      1.000     0.167     0.286       6.0
                   I-economy      0.000     0.000     0.000       0.0
 B-return_date.date_relative      1.000     0.667     0.800       3.0
        I-fromloc.state_name      1.000     1.000     1.000       1.0
                B-state_code      1.000     1.000     1.000       1.0
    I-arrive_time.start_time      1.000     1.000     1.000       1.0
    I-arrive_date.day_number      0.000     0.000     0.000       0.0
                 B-meal_code      0.000     0.000     0.000       1.0
 I-depart_time.period_of_day      1.000     1.000     1.000       1.0
                  B-day_name      1.000     0.500     0.667       2.0
             B-period_of_day      1.000     0.750     0.857       4.0
        B-stoploc.state_code      0.000     0.000     0.000       0.0
    B-return_date.month_name      0.000     0.000     0.000       0.0
    B-return_date.day_