In [1]:
"""
We use following lines because we are running on Google Colab
If you are running notebook on a local computer, you don't need this cell
"""
from google.colab import drive
drive.mount('/content/gdrive')
import os
os.chdir('/content/gdrive/My Drive/finch/tensorflow2/spoken_language_understanding/atis/main')
Drive already mounted at /content/gdrive; to attempt to forcibly remount, call drive.mount("/content/gdrive", force_remount=True).
In [2]:
!pip install tensorflow-gpu==2.0.0-beta1 
Requirement already satisfied: tensorflow-gpu==2.0.0-beta1 in /usr/local/lib/python3.6/dist-packages (2.0.0b1)
Requirement already satisfied: wrapt>=1.11.1 in /usr/local/lib/python3.6/dist-packages (from tensorflow-gpu==2.0.0-beta1) (1.11.2)
Requirement already satisfied: tf-estimator-nightly<1.14.0.dev2019060502,>=1.14.0.dev2019060501 in /usr/local/lib/python3.6/dist-packages (from tensorflow-gpu==2.0.0-beta1) (1.14.0.dev2019060501)
Requirement already satisfied: termcolor>=1.1.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow-gpu==2.0.0-beta1) (1.1.0)
Requirement already satisfied: keras-preprocessing>=1.0.5 in /usr/local/lib/python3.6/dist-packages (from tensorflow-gpu==2.0.0-beta1) (1.1.0)
Requirement already satisfied: google-pasta>=0.1.6 in /usr/local/lib/python3.6/dist-packages (from tensorflow-gpu==2.0.0-beta1) (0.1.7)
Requirement already satisfied: absl-py>=0.7.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow-gpu==2.0.0-beta1) (0.7.1)
Requirement already satisfied: tb-nightly<1.14.0a20190604,>=1.14.0a20190603 in /usr/local/lib/python3.6/dist-packages (from tensorflow-gpu==2.0.0-beta1) (1.14.0a20190603)
Requirement already satisfied: wheel>=0.26 in /usr/local/lib/python3.6/dist-packages (from tensorflow-gpu==2.0.0-beta1) (0.33.4)
Requirement already satisfied: numpy<2.0,>=1.14.5 in /usr/local/lib/python3.6/dist-packages (from tensorflow-gpu==2.0.0-beta1) (1.16.4)
Requirement already satisfied: protobuf>=3.6.1 in /usr/local/lib/python3.6/dist-packages (from tensorflow-gpu==2.0.0-beta1) (3.7.1)
Requirement already satisfied: gast>=0.2.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow-gpu==2.0.0-beta1) (0.2.2)
Requirement already satisfied: six>=1.10.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow-gpu==2.0.0-beta1) (1.12.0)
Requirement already satisfied: keras-applications>=1.0.6 in /usr/local/lib/python3.6/dist-packages (from tensorflow-gpu==2.0.0-beta1) (1.0.8)
Requirement already satisfied: grpcio>=1.8.6 in /usr/local/lib/python3.6/dist-packages (from tensorflow-gpu==2.0.0-beta1) (1.15.0)
Requirement already satisfied: astor>=0.6.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow-gpu==2.0.0-beta1) (0.8.0)
Requirement already satisfied: werkzeug>=0.11.15 in /usr/local/lib/python3.6/dist-packages (from tb-nightly<1.14.0a20190604,>=1.14.0a20190603->tensorflow-gpu==2.0.0-beta1) (0.15.4)
Requirement already satisfied: markdown>=2.6.8 in /usr/local/lib/python3.6/dist-packages (from tb-nightly<1.14.0a20190604,>=1.14.0a20190603->tensorflow-gpu==2.0.0-beta1) (3.1.1)
Requirement already satisfied: setuptools>=41.0.0 in /usr/local/lib/python3.6/dist-packages (from tb-nightly<1.14.0a20190604,>=1.14.0a20190603->tensorflow-gpu==2.0.0-beta1) (41.0.1)
Requirement already satisfied: h5py in /usr/local/lib/python3.6/dist-packages (from keras-applications>=1.0.6->tensorflow-gpu==2.0.0-beta1) (2.8.0)
In [3]:
import tensorflow as tf
import pprint
import logging
import time
import numpy as np

from sklearn.metrics import classification_report, f1_score

print("TensorFlow Version", tf.__version__)
print('GPU Enabled:', tf.test.is_gpu_available())
TensorFlow Version 2.0.0-beta1
GPU Enabled: True
In [0]:
def get_vocab(vocab_path):
  word2idx = {}
  with open(vocab_path) as f:
    for i, line in enumerate(f):
      line = line.rstrip()
      word2idx[line] = i
  return word2idx
In [0]:
def data_generator(f_path, params):
  print('Reading', f_path)
  with open(f_path) as f:
    for line in f:
      line = line.rstrip()
      text, slot_intent = line.split('\t')
      words = text.split()[1:-1]
      slot_intent = slot_intent.split()
      slots, intent = slot_intent[1:-1], slot_intent[-1]
      assert len(words) == len(slots)
      
      words = [params['word2idx'].get(w, len(params['word2idx'])) for w in words]
      intent = params['intent2idx'].get(intent, len(params['intent2idx']))
      slots = [params['slot2idx'].get(s, len(params['slot2idx'])) for s in slots]
      
      yield (words, (intent, slots))
In [0]:
def dataset(is_training, params):
  _shapes = ([None], ((), [None]))
  _types = (tf.int32, (tf.int32, tf.int32))
  _pads = (0, (-1, 0))
  
  if is_training:
    ds = tf.data.Dataset.from_generator(
      lambda: data_generator(params['train_path'], params),
      output_shapes = _shapes,
      output_types = _types,)
    ds = ds.shuffle(params['num_samples'])
    ds = ds.padded_batch(params['batch_size'], _shapes, _pads)
    ds = ds.prefetch(tf.data.experimental.AUTOTUNE)
  else:
    ds = tf.data.Dataset.from_generator(
      lambda: data_generator(params['test_path'], params),
      output_shapes = _shapes,
      output_types = _types,)
    ds = ds.padded_batch(params['batch_size'], _shapes, _pads)
    ds = ds.prefetch(tf.data.experimental.AUTOTUNE)
  
  return ds
In [0]:
class LayerNorm(tf.keras.layers.Layer):
  def __init__(self, params):
    super().__init__()
    self._epsilon = 1e-6
    self._units = 2 * params['rnn_units']
  
  def build(self, input_shape):
    self.scale = self.add_weight(name='scale',
                                 shape=[self._units],
                                 initializer=tf.ones_initializer(),
                                 trainable=True)
    self.bias = self.add_weight(name='bias',
                                shape=[self._units],
                                initializer=tf.zeros_initializer(),
                                trainable=True)
    super().build(input_shape)
  
  def call(self, inputs):
    mean, variance = tf.nn.moments(inputs, [-1], keepdims=True)
    norm_x = (inputs - mean) * tf.math.rsqrt(variance + self._epsilon)
    return norm_x * self.scale + self.bias
  
  def compute_output_shape(self, input_shape):
    return input_shape


class EncoderBlock(tf.keras.Model):
  def __init__(self, SubModel, params, name):
    super().__init__(name = name)
    self.layer_norm = LayerNorm(params)
    self.sub_model = SubModel(params)
    self.dropout = tf.keras.layers.Dropout(params['dropout_rate'])
  
  def call(self, inputs, training=False):
    inputs, masks = inputs
    x = self.layer_norm(inputs)
    x = self.sub_model((x, masks), training=training)
    x = self.dropout(x, training=training)
    x += inputs
    return x


class MultiheadSelfAttention(tf.keras.Model):
  def __init__(self, params):
    super().__init__()
    self.qkv_linear = tf.keras.layers.Dense(3 * 2 * params['rnn_units'], name='qkv_linear')
    self.dropout = tf.keras.layers.Dropout(params['dropout_rate'])
    self.out_linear = tf.keras.layers.Dense(2 * params['rnn_units'], name='out_linear')
    self._num_heads = params['num_heads']
  
  def call(self, inputs, training=False):
    x, masks = inputs
    timesteps = tf.shape(x)[1]
    
    q_k_v = self.qkv_linear(x)
    q, k, v = tf.split(q_k_v, 3, axis=-1)
    
    if self._num_heads > 1:
      q = tf.concat(tf.split(q, self._num_heads, axis=2), axis=0)                        
      k = tf.concat(tf.split(k, self._num_heads, axis=2), axis=0)                        
      v = tf.concat(tf.split(v, self._num_heads, axis=2), axis=0)
    
    align = tf.matmul(q, k, transpose_b=True)
    align *= tf.math.rsqrt(tf.cast(k.shape[-1], tf.float32))
    
    if (masks is not None) or (not self._is_bidirectional):
      paddings = tf.fill(tf.shape(align), float('-inf'))
    
    if masks is not None:
      c_masks = tf.tile(masks, [params['num_heads'], 1])
      c_masks = tf.tile(tf.expand_dims(c_masks, 1), [1, timesteps, 1])
      align = tf.where(tf.equal(c_masks, 0), paddings, align)
    
    align = tf.nn.softmax(align)
    align = self.dropout(align, training=training)
    
    if masks is not None:
      q_masks = tf.tile(masks, [params['num_heads'], 1])
      q_masks = tf.tile(tf.expand_dims(q_masks, 2), [1, 1, timesteps])
      align *= tf.cast(q_masks, tf.float32)
    
    x = tf.matmul(align, v)
    if self._num_heads > 1:
      x = tf.concat(tf.split(x, self._num_heads, axis=0), axis=2)
    x = self.out_linear(x)
    
    return x
  

class PointwiseFFN(tf.keras.Model):
  def __init__(self, params):
    super().__init__()
    self.dense_1 = tf.keras.layers.Dense(2 * params['multiplier'] * params['rnn_units'], tf.nn.relu, name='fc_relu')
    self.dropout = tf.keras.layers.Dropout(params['dropout_rate'])
    self.dense_2 = tf.keras.layers.Dense(2 * params['rnn_units'], name='linear')
  
  def call(self, inputs, training=False):
    x, masks = inputs
    return self.dense_2(self.dropout(self.dense_1(x), training=training))
In [0]:
class Model(tf.keras.Model):
  def __init__(self, params: dict):
    super().__init__()
    self.embedding = tf.Variable(np.load(params['vocab_path']),
                                 dtype=tf.float32,
                                 name='pretrained_embedding')
    self.input_dropout = tf.keras.layers.Dropout(params['dropout_rate'])
    
    self.bidir_gru = tf.keras.layers.Bidirectional(tf.keras.layers.GRU(
        params['rnn_units'], return_sequences=True))
    self.transformer = [
        EncoderBlock(MultiheadSelfAttention, params, name='self_attention'),
        EncoderBlock(PointwiseFFN, params, name='pointwise_ffn'),
    ]
    
    self.out_linear_intent = tf.keras.layers.Dense(params['intent_size'], name='output_intent')
    self.out_linear_slot = tf.keras.layers.Dense(params['slot_size'], name='output_slot')
  
  
  def call(self, inputs, training=False):
    if inputs.dtype != tf.int32:
      inputs = tf.cast(inputs, tf.int32)
    mask = tf.sign(inputs)
    
    x = tf.nn.embedding_lookup(self.embedding, inputs)
    x = self.input_dropout(x, training=training)
    x = self.bidir_gru(x, mask=None)
    for block in self.transformer:
      x = block((x, mask), training=training)
    
    x_intent = tf.reduce_max(x, 1)
    x_intent = self.out_linear_intent(x_intent)
    x_slot = self.out_linear_slot(x)
    return (x_intent, x_slot)
In [0]:
params = {
  'train_path': '../data/atis.train.w-intent.iob',
  'test_path': '../data/atis.test.w-intent.iob',
  'word_path': '../vocab/word.txt',
  'vocab_path': '../vocab/word.npy',
  'intent_path': '../vocab/intent.txt',
  'slot_path': '../vocab/slot.txt',
  'batch_size': 16,
  'num_samples': 4978,
  'rnn_units': 300,
  'num_heads': 10,
  'multiplier': 2,
  'dropout_rate': 0.2,
  'clip_norm': 5.0,
  'lr': 3e-4,
  'num_patience': 3,
}
In [0]:
params['word2idx'] = get_vocab(params['word_path'])
params['intent2idx'] = get_vocab(params['intent_path'])
params['slot2idx'] = get_vocab(params['slot_path'])

params['word_size'] = len(params['word2idx']) + 1
params['intent_size'] = len(params['intent2idx']) + 1
params['slot_size'] = len(params['slot2idx']) + 1
In [0]:
def is_descending(history: list):
  history = history[-(params['num_patience']+1):]
  for i in range(1, len(history)):
    if history[i-1] <= history[i]:
      return False
  return True    
In [12]:
model = Model(params)
model.build(input_shape=(None, None))
pprint.pprint([(v.name, v.shape) for v in model.trainable_variables])

decay_lr = tf.optimizers.schedules.ExponentialDecay(params['lr'], 1000, 0.9)
optim = tf.optimizers.Adam(params['lr'])
global_step = 0

history_f1 = []
best_f1 = .0

t0 = time.time()
logger = logging.getLogger('tensorflow')
logger.setLevel(logging.INFO)

while True:
  # TRAINING
  for (words, (intent, slots)) in dataset(is_training=True, params=params):
    with tf.GradientTape() as tape:
      y_intent, y_slots = model(words, training=True)
      loss_intent = tf.nn.sparse_softmax_cross_entropy_with_logits(labels=intent, logits=y_intent)
      loss_intent = tf.reduce_mean(loss_intent)
      # weighted cross entropy (weight of 'O' is set to be small)
      loss_slots = tf.nn.sparse_softmax_cross_entropy_with_logits(labels=slots, logits=y_slots)
      weights = tf.cast(tf.sign(slots), tf.float32)
      padding = tf.constant(1e-2, tf.float32, weights.shape)
      weights = tf.where(tf.equal(weights, 0.), padding, weights)
      loss_slots = tf.reduce_mean(loss_slots * weights)
      # joint loss
      loss = loss_intent + loss_slots
    
    optim.lr.assign(decay_lr(global_step))
    grads = tape.gradient(loss, model.trainable_variables)
    grads, _ = tf.clip_by_global_norm(grads, params['clip_norm'])
    optim.apply_gradients(zip(grads, model.trainable_variables))

    if global_step % 50 == 0:
      logger.info("Step {} | Loss: {:.4f} | Loss_intent: {:.4f} | Loss_slots: {:.4f} | Spent: {:.1f} secs | LR: {:.6f}".format(
          global_step, loss.numpy().item(), loss_intent.numpy().item(), loss_slots.numpy().item(), time.time()-t0, optim.lr.numpy().item()))
      t0 = time.time()
    global_step += 1
    
  # EVALUATION
  intent_true = []
  intent_pred = []
  slot_true = []
  slot_pred = []

  for (words, (intent, slots)) in dataset(is_training=False, params=params):
    y_intent, y_slots = model(words, training=False)
    y_intent = tf.argmax(y_intent, -1)
    y_slots = tf.argmax(y_slots, -1)
    
    intent_true += intent.numpy().flatten().tolist()
    intent_pred += y_intent.numpy().flatten().tolist()
    slot_true += slots.numpy().flatten().tolist()
    slot_pred += y_slots.numpy().flatten().tolist()
    
  f1_slots = f1_score(y_true = slot_true,
                      y_pred = slot_pred,
                      labels = list(params['slot2idx'].values()),
                      sample_weight = np.sign(slot_true),
                      average='micro',)
  history_f1.append(f1_slots)
  
  logger.info('\n'+classification_report(y_true = intent_true,
                                         y_pred = intent_pred,
                                         labels = list(params['intent2idx'].values()),
                                         target_names = list(params['intent2idx'].keys()),
                                         digits=3))
  logger.info('\n'+classification_report(y_true = slot_true,
                                         y_pred = slot_pred,
                                         labels = list(params['slot2idx'].values()),
                                         target_names = list(params['slot2idx'].keys()),
                                         sample_weight = np.sign(slot_true),
                                         digits=3))
  
  if f1_slots > best_f1:
    best_f1 = f1_slots
    # you can save model here
  logger.info("Best Slot F1: {:.3f}".format(best_f1))
  
  if len(history_f1) > params['num_patience'] and is_descending(history_f1):
    logger.info("Testing Slot Accuracy not improved over {} epochs, Early Stop".format(params['num_patience']))
    break
WARNING: Logging before flag parsing goes to stderr.
W0715 07:00:13.604537 140369481070464 deprecation.py:323] From /usr/local/lib/python3.6/dist-packages/tensorflow/python/data/ops/dataset_ops.py:505: py_func (from tensorflow.python.ops.script_ops) is deprecated and will be removed in a future version.
Instructions for updating:
tf.py_func is deprecated in TF V2. Instead, there are two
    options available in V2.
    - tf.py_function takes a python function which manipulates tf eager
    tensors instead of numpy arrays. It's easy to convert a tf eager tensor to
    an ndarray (just call tensor.numpy()) but having access to eager tensors
    means `tf.py_function`s can use accelerators such as GPUs as well as
    being differentiable using a gradient tape.
    - tf.numpy_function maintains the semantics of the deprecated tf.py_func
    (it is not differentiable, and manipulates numpy arrays). It drops the
    stateful argument making all functions stateful.
    
[('bidirectional/forward_gru/kernel:0', TensorShape([300, 900])),
 ('bidirectional/forward_gru/recurrent_kernel:0', TensorShape([300, 900])),
 ('bidirectional/forward_gru/bias:0', TensorShape([2, 900])),
 ('bidirectional/backward_gru/kernel:0', TensorShape([300, 900])),
 ('bidirectional/backward_gru/recurrent_kernel:0', TensorShape([300, 900])),
 ('bidirectional/backward_gru/bias:0', TensorShape([2, 900])),
 ('self_attention/layer_norm/scale:0', TensorShape([600])),
 ('self_attention/layer_norm/bias:0', TensorShape([600])),
 ('self_attention/multihead_self_attention/qkv_linear/kernel:0',
  TensorShape([600, 1800])),
 ('self_attention/multihead_self_attention/qkv_linear/bias:0',
  TensorShape([1800])),
 ('self_attention/multihead_self_attention/out_linear/kernel:0',
  TensorShape([600, 600])),
 ('self_attention/multihead_self_attention/out_linear/bias:0',
  TensorShape([600])),
 ('pointwise_ffn/layer_norm_1/scale:0', TensorShape([600])),
 ('pointwise_ffn/layer_norm_1/bias:0', TensorShape([600])),
 ('pointwise_ffn/pointwise_ffn/fc_relu/kernel:0', TensorShape([600, 1200])),
 ('pointwise_ffn/pointwise_ffn/fc_relu/bias:0', TensorShape([1200])),
 ('pointwise_ffn/pointwise_ffn/linear/kernel:0', TensorShape([1200, 600])),
 ('pointwise_ffn/pointwise_ffn/linear/bias:0', TensorShape([600])),
 ('output_intent/kernel:0', TensorShape([600, 23])),
 ('output_intent/bias:0', TensorShape([23])),
 ('output_slot/kernel:0', TensorShape([600, 122])),
 ('output_slot/bias:0', TensorShape([122])),
 ('pretrained_embedding:0', TensorShape([750, 300]))]
Reading ../data/atis.train.w-intent.iob
W0715 07:00:16.189967 140369481070464 deprecation.py:323] From /usr/local/lib/python3.6/dist-packages/tensorflow/python/ops/clip_ops.py:286: add_dispatch_support.<locals>.wrapper (from tensorflow.python.ops.array_ops) is deprecated and will be removed in a future version.
Instructions for updating:
Use tf.where in 2.0, which has the same broadcast rule as np.where
I0715 07:00:16.295569 140369481070464 interactiveshell.py:2882] Step 0 | Loss: 5.3480 | Loss_intent: 4.2718 | Loss_slots: 1.0761 | Spent: 2.7 secs | LR: 0.000300
I0715 07:00:19.956023 140369481070464 interactiveshell.py:2882] Step 50 | Loss: 0.5650 | Loss_intent: 0.0439 | Loss_slots: 0.5211 | Spent: 3.7 secs | LR: 0.000298
I0715 07:00:23.621835 140369481070464 interactiveshell.py:2882] Step 100 | Loss: 1.0245 | Loss_intent: 0.6387 | Loss_slots: 0.3858 | Spent: 3.7 secs | LR: 0.000297
I0715 07:00:27.247672 140369481070464 interactiveshell.py:2882] Step 150 | Loss: 0.3899 | Loss_intent: 0.0175 | Loss_slots: 0.3724 | Spent: 3.6 secs | LR: 0.000295
I0715 07:00:30.894841 140369481070464 interactiveshell.py:2882] Step 200 | Loss: 0.8707 | Loss_intent: 0.6910 | Loss_slots: 0.1797 | Spent: 3.6 secs | LR: 0.000294
I0715 07:00:34.539896 140369481070464 interactiveshell.py:2882] Step 250 | Loss: 0.4553 | Loss_intent: 0.2807 | Loss_slots: 0.1747 | Spent: 3.6 secs | LR: 0.000292
I0715 07:00:38.181005 140369481070464 interactiveshell.py:2882] Step 300 | Loss: 0.1716 | Loss_intent: 0.0353 | Loss_slots: 0.1363 | Spent: 3.6 secs | LR: 0.000291
Reading ../data/atis.test.w-intent.iob
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.
  'precision', 'predicted', average, warn_for)
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1439: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples.
  'recall', 'true', average, warn_for)
I0715 07:00:40.426919 140369481070464 interactiveshell.py:2882] 
                                          precision    recall  f1-score   support

                             atis_flight      0.946     0.994     0.969       632
                            atis_airfare      1.000     0.896     0.945        48
                     atis_ground_service      1.000     0.917     0.957        36
                            atis_airline      1.000     1.000     1.000        38
                       atis_abbreviation      1.000     0.939     0.969        33
                           atis_aircraft      1.000     1.000     1.000         9
                        atis_flight_time      1.000     1.000     1.000         1
                           atis_quantity      0.333     1.000     0.500         3
                atis_flight#atis_airfare      0.000     0.000     0.000        12
                            atis_airport      1.000     0.833     0.909        18
                           atis_distance      1.000     0.400     0.571        10
                               atis_city      1.000     0.167     0.286         6
                        atis_ground_fare      0.700     1.000     0.824         7
                           atis_capacity      1.000     1.000     1.000        21
                          atis_flight_no      0.889     1.000     0.941         8
                               atis_meal      1.000     0.500     0.667         6
                        atis_restriction      0.000     0.000     0.000         0
             atis_airline#atis_flight_no      0.000     0.000     0.000         0
    atis_ground_service#atis_ground_fare      0.000     0.000     0.000         0
           atis_airfare#atis_flight_time      0.000     0.000     0.000         0
                           atis_cheapest      0.000     0.000     0.000         0
atis_aircraft#atis_flight#atis_flight_no      0.000     0.000     0.000         0

                               micro avg      0.946     0.952     0.949       888
                               macro avg      0.630     0.575     0.570       888
                            weighted avg      0.942     0.952     0.941       888

I0715 07:00:40.470576 140369481070464 interactiveshell.py:2882] 
                              precision    recall  f1-score   support

                           O      0.000     0.000     0.000       0.0
           B-toloc.city_name      0.940     0.979     0.959     716.0
         B-fromloc.city_name      0.936     0.991     0.963     704.0
           I-toloc.city_name      0.808     0.985     0.888     265.0
      B-depart_date.day_name      0.940     0.967     0.953     212.0
              B-airline_name      0.847     0.990     0.913     101.0
         I-fromloc.city_name      0.897     0.881     0.889     177.0
 B-depart_time.period_of_day      0.793     0.885     0.836     130.0
              I-airline_name      0.773     0.892     0.829      65.0
    B-depart_date.day_number      0.743     0.945     0.832      55.0
    B-depart_date.month_name      0.864     0.911     0.887      56.0
          B-depart_time.time      0.746     0.825     0.783      57.0
                B-round_trip      0.853     0.877     0.865      73.0
             B-cost_relative      0.895     0.919     0.907      37.0
                I-round_trip      0.950     0.803     0.870      71.0
                B-flight_mod      0.677     0.875     0.764      24.0
 B-depart_time.time_relative      0.963     0.400     0.565      65.0
          I-depart_time.time      0.413     1.000     0.584      52.0
         B-stoploc.city_name      0.833     0.250     0.385      20.0
                 B-city_name      0.500     0.596     0.544      57.0
                B-class_type      0.667     0.750     0.706      24.0
          B-arrive_time.time      0.778     0.824     0.800      34.0
 B-arrive_time.time_relative      0.295     0.903     0.444      31.0
                I-class_type      0.714     0.882     0.789      17.0
               B-flight_stop      1.000     1.000     1.000      21.0
          I-arrive_time.time      0.833     0.143     0.244      35.0
              B-airline_code      0.810     0.500     0.618      34.0
    I-depart_date.day_number      0.750     0.400     0.522      15.0
      I-fromloc.airport_name      0.250     0.467     0.326      15.0
      B-fromloc.airport_name      0.250     0.167     0.200      12.0
      B-arrive_date.day_name      0.562     0.818     0.667      11.0
          B-toloc.state_code      0.462     1.000     0.632      18.0
B-depart_date.today_relative      1.000     0.333     0.500       9.0
             B-flight_number      0.750     0.273     0.400      11.0
 B-depart_date.date_relative      0.800     0.471     0.593      17.0
          B-toloc.state_name      1.000     0.036     0.069      28.0
           B-fare_basis_code      0.682     0.882     0.769      17.0
               B-flight_time      0.250     1.000     0.400       1.0
                        B-or      0.000     0.000     0.000       3.0
 B-arrive_time.period_of_day      0.000     0.000     0.000       6.0
          B-meal_description      0.700     0.700     0.700      10.0
             I-cost_relative      0.000     0.000     0.000       3.0
              I-airport_name      0.714     0.172     0.278      29.0
               B-fare_amount      1.000     0.500     0.667       2.0
               I-fare_amount      1.000     1.000     1.000       2.0
                 I-city_name      0.800     0.267     0.400      30.0
        I-toloc.airport_name      0.000     0.000     0.000       3.0
            B-transport_type      0.600     0.900     0.720      10.0
    B-arrive_date.month_name      0.000     0.000     0.000       6.0
    B-arrive_date.day_number      0.000     0.000     0.000       6.0
         I-stoploc.city_name      0.000     0.000     0.000      10.0
                      B-meal      0.778     0.438     0.560      16.0
        B-fromloc.state_code      1.000     0.087     0.160      23.0
    B-depart_time.period_mod      0.000     0.000     0.000       5.0
                   B-connect      0.000     0.000     0.000       6.0
               B-flight_days      1.000     0.600     0.750      10.0
        B-toloc.airport_name      0.000     0.000     0.000       3.0
        B-fromloc.state_name      0.000     0.000     0.000      17.0
              B-airport_name      0.500     0.048     0.087      21.0
                   B-economy      1.000     0.500     0.667       6.0
               I-flight_time      0.000     0.000     0.000       1.0
             B-aircraft_code      1.000     0.424     0.596      33.0
                       B-mod      0.000     0.000     0.000       2.0
              B-airport_code      0.000     0.000     0.000       9.0
    B-depart_time.start_time      0.000     0.000     0.000       3.0
      B-depart_time.end_time      0.000     0.000     0.000       3.0
          B-depart_date.year      0.000     0.000     0.000       3.0
            I-transport_type      0.000     0.000     0.000       1.0
          B-restriction_code      0.333     1.000     0.500       4.0
    B-arrive_time.start_time      1.000     0.125     0.222       8.0
        B-toloc.airport_code      0.000     0.000     0.000       4.0
      B-arrive_time.end_time      0.000     0.000     0.000       8.0
      I-arrive_time.end_time      0.000     0.000     0.000       8.0
      I-depart_time.end_time      0.000     0.000     0.000       3.0
               I-flight_stop      0.000     0.000     0.000       0.0
      B-fromloc.airport_code      0.000     0.000     0.000       5.0
          I-restriction_code      0.000     0.000     0.000       3.0
    I-depart_time.start_time      0.000     0.000     0.000       1.0
          I-toloc.state_name      0.000     0.000     0.000       1.0
I-depart_date.today_relative      0.000     0.000     0.000       0.0
 B-arrive_date.date_relative      0.000     0.000     0.000       2.0
                I-flight_mod      0.000     0.000     0.000       6.0
                   I-economy      0.000     0.000     0.000       0.0
 B-return_date.date_relative      0.000     0.000     0.000       3.0
        I-fromloc.state_name      0.000     0.000     0.000       1.0
                B-state_code      0.000     0.000     0.000       1.0
    I-arrive_time.start_time      0.000     0.000     0.000       1.0
    I-arrive_date.day_number      0.000     0.000     0.000       0.0
                 B-meal_code      0.000     0.000     0.000       1.0
 I-depart_time.period_of_day      0.000     0.000     0.000       1.0
                  B-day_name      0.000     0.000     0.000       2.0
             B-period_of_day      0.000     0.000     0.000       4.0
        B-stoploc.state_code      0.000     0.000     0.000       0.0
    B-return_date.month_name      0.000     0.000     0.000       0.0
    B-return_date.day_number      0.000     0.000     0.000       0.0
    B-arrive_time.period_mod      0.000     0.000     0.000       0.0
                 I-meal_code      0.000     0.000     0.000       0.0
        B-toloc.country_name      0.000     0.000     0.000       1.0
                 B-days_code      0.000     0.000     0.000       1.0
 I-arrive_time.period_of_day      0.000     0.000     0.000       0.0
            I-today_relative      0.000     0.000     0.000       0.0
 B-return_time.period_of_day      0.000     0.000     0.000       0.0
                      B-time      0.000     0.000     0.000       0.0
           I-fare_basis_code      0.000     0.000     0.000       0.0
 I-arrive_time.time_relative      0.000     0.000     0.000       4.0
 I-depart_time.time_relative      0.000     0.000     0.000       1.0
            B-today_relative      0.000     0.000     0.000       0.0
                B-state_name      0.000     0.000     0.000       9.0
B-arrive_date.today_relative      0.000     0.000     0.000       0.0
    B-return_time.period_mod      0.000     0.000     0.000       0.0
                B-month_name      0.000     0.000     0.000       0.0
                B-day_number      0.000     0.000     0.000       0.0
 I-return_date.date_relative      0.000     0.000     0.000       3.0
I-return_date.today_relative      0.000     0.000     0.000       0.0
      B-stoploc.airport_name      0.000     0.000     0.000       0.0
             B-time_relative      0.000     0.000     0.000       0.0
                      I-time      0.000     0.000     0.000       0.0
    I-return_date.day_number      0.000     0.000     0.000       0.0
          I-meal_description      0.000     0.000     0.000       0.0
B-return_date.today_relative      0.000     0.000     0.000       0.0
      B-return_date.day_name      0.000     0.000     0.000       2.0

                   micro avg      0.822     0.824     0.823    3657.0
                   macro avg      0.319     0.278     0.266    3657.0
                weighted avg      0.819     0.824     0.797    3657.0

I0715 07:00:40.471406 140369481070464 interactiveshell.py:2882] Best Slot F1: 0.823
Reading ../data/atis.train.w-intent.iob
I0715 07:00:44.003034 140369481070464 interactiveshell.py:2882] Step 350 | Loss: 0.1575 | Loss_intent: 0.0305 | Loss_slots: 0.1270 | Spent: 5.8 secs | LR: 0.000289
I0715 07:00:47.697732 140369481070464 interactiveshell.py:2882] Step 400 | Loss: 0.5410 | Loss_intent: 0.4488 | Loss_slots: 0.0922 | Spent: 3.7 secs | LR: 0.000288
I0715 07:00:51.439484 140369481070464 interactiveshell.py:2882] Step 450 | Loss: 0.0583 | Loss_intent: 0.0008 | Loss_slots: 0.0575 | Spent: 3.7 secs | LR: 0.000286
I0715 07:00:55.187888 140369481070464 interactiveshell.py:2882] Step 500 | Loss: 0.1014 | Loss_intent: 0.0286 | Loss_slots: 0.0728 | Spent: 3.7 secs | LR: 0.000285
I0715 07:00:59.065917 140369481070464 interactiveshell.py:2882] Step 550 | Loss: 0.2422 | Loss_intent: 0.1368 | Loss_slots: 0.1054 | Spent: 3.9 secs | LR: 0.000283
I0715 07:01:02.843440 140369481070464 interactiveshell.py:2882] Step 600 | Loss: 0.0738 | Loss_intent: 0.0050 | Loss_slots: 0.0688 | Spent: 3.8 secs | LR: 0.000282
Reading ../data/atis.test.w-intent.iob
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.
  'precision', 'predicted', average, warn_for)
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1439: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples.
  'recall', 'true', average, warn_for)
I0715 07:01:05.974163 140369481070464 interactiveshell.py:2882] 
                                          precision    recall  f1-score   support

                             atis_flight      0.967     0.989     0.978       632
                            atis_airfare      0.923     1.000     0.960        48
                     atis_ground_service      1.000     1.000     1.000        36
                            atis_airline      1.000     1.000     1.000        38
                       atis_abbreviation      1.000     1.000     1.000        33
                           atis_aircraft      1.000     1.000     1.000         9
                        atis_flight_time      1.000     1.000     1.000         1
                           atis_quantity      0.429     1.000     0.600         3
                atis_flight#atis_airfare      0.000     0.000     0.000        12
                            atis_airport      1.000     0.167     0.286        18
                           atis_distance      1.000     0.400     0.571        10
                               atis_city      0.261     1.000     0.414         6
                        atis_ground_fare      1.000     0.857     0.923         7
                           atis_capacity      1.000     1.000     1.000        21
                          atis_flight_no      0.875     0.875     0.875         8
                               atis_meal      1.000     1.000     1.000         6
                        atis_restriction      0.000     0.000     0.000         0
             atis_airline#atis_flight_no      0.000     0.000     0.000         0
    atis_ground_service#atis_ground_fare      0.000     0.000     0.000         0
           atis_airfare#atis_flight_time      0.000     0.000     0.000         0
                           atis_cheapest      0.000     0.000     0.000         0
atis_aircraft#atis_flight#atis_flight_no      0.000     0.000     0.000         0

                               micro avg      0.947     0.953     0.950       888
                               macro avg      0.612     0.604     0.573       888
                            weighted avg      0.951     0.953     0.942       888

I0715 07:01:06.015882 140369481070464 interactiveshell.py:2882] 
                              precision    recall  f1-score   support

                           O      0.000     0.000     0.000       0.0
           B-toloc.city_name      0.959     0.992     0.975     716.0
         B-fromloc.city_name      0.973     0.987     0.980     704.0
           I-toloc.city_name      0.814     0.989     0.893     265.0
      B-depart_date.day_name      0.926     1.000     0.961     212.0
              B-airline_name      0.951     0.970     0.961     101.0
         I-fromloc.city_name      0.981     0.864     0.919     177.0
 B-depart_time.period_of_day      0.918     0.862     0.889     130.0
              I-airline_name      0.838     0.954     0.892      65.0
    B-depart_date.day_number      0.833     1.000     0.909      55.0
    B-depart_date.month_name      0.947     0.964     0.956      56.0
          B-depart_time.time      0.747     0.982     0.848      57.0
                B-round_trip      1.000     0.918     0.957      73.0
             B-cost_relative      0.923     0.973     0.947      37.0
                I-round_trip      1.000     0.972     0.986      71.0
                B-flight_mod      0.786     0.917     0.846      24.0
 B-depart_time.time_relative      0.947     0.831     0.885      65.0
          I-depart_time.time      0.893     0.962     0.926      52.0
         B-stoploc.city_name      0.704     0.950     0.809      20.0
                 B-city_name      0.900     0.474     0.621      57.0
                B-class_type      0.957     0.917     0.936      24.0
          B-arrive_time.time      0.825     0.971     0.892      34.0
 B-arrive_time.time_relative      0.750     0.871     0.806      31.0
                I-class_type      0.895     1.000     0.944      17.0
               B-flight_stop      1.000     1.000     1.000      21.0
          I-arrive_time.time      0.919     0.971     0.944      35.0
              B-airline_code      0.926     0.735     0.820      34.0
    I-depart_date.day_number      0.875     0.933     0.903      15.0
      I-fromloc.airport_name      0.600     0.600     0.600      15.0
      B-fromloc.airport_name      0.308     0.333     0.320      12.0
      B-arrive_date.day_name      1.000     0.182     0.308      11.0
          B-toloc.state_code      0.720     1.000     0.837      18.0
B-depart_date.today_relative      0.889     0.889     0.889       9.0
             B-flight_number      0.391     0.818     0.529      11.0
 B-depart_date.date_relative      0.762     0.941     0.842      17.0
          B-toloc.state_name      0.786     0.393     0.524      28.0
           B-fare_basis_code      0.800     0.941     0.865      17.0
               B-flight_time      0.000     0.000     0.000       1.0
                        B-or      1.000     0.667     0.800       3.0
 B-arrive_time.period_of_day      0.571     0.667     0.615       6.0
          B-meal_description      0.750     0.900     0.818      10.0
             I-cost_relative      1.000     0.667     0.800       3.0
              I-airport_name      0.640     0.552     0.593      29.0
               B-fare_amount      1.000     0.500     0.667       2.0
               I-fare_amount      1.000     1.000     1.000       2.0
                 I-city_name      0.818     0.300     0.439      30.0
        I-toloc.airport_name      0.000     0.000     0.000       3.0
            B-transport_type      0.667     1.000     0.800      10.0
    B-arrive_date.month_name      0.750     0.500     0.600       6.0
    B-arrive_date.day_number      0.000     0.000     0.000       6.0
         I-stoploc.city_name      0.167     0.100     0.125      10.0
                      B-meal      0.867     0.812     0.839      16.0
        B-fromloc.state_code      1.000     0.652     0.789      23.0
    B-depart_time.period_mod      0.714     1.000     0.833       5.0
                   B-connect      0.857     1.000     0.923       6.0
               B-flight_days      1.000     0.900     0.947      10.0
        B-toloc.airport_name      0.250     0.333     0.286       3.0
        B-fromloc.state_name      0.824     0.824     0.824      17.0
              B-airport_name      0.600     0.571     0.585      21.0
                   B-economy      1.000     0.667     0.800       6.0
               I-flight_time      0.500     1.000     0.667       1.0
             B-aircraft_code      0.958     0.697     0.807      33.0
                       B-mod      0.000     0.000     0.000       2.0
              B-airport_code      0.000     0.000     0.000       9.0
    B-depart_time.start_time      0.273     1.000     0.429       3.0
      B-depart_time.end_time      0.375     1.000     0.545       3.0
          B-depart_date.year      1.000     0.333     0.500       3.0
            I-transport_type      0.000     0.000     0.000       1.0
          B-restriction_code      0.667     0.500     0.571       4.0
    B-arrive_time.start_time      0.000     0.000     0.000       8.0
        B-toloc.airport_code      0.000     0.000     0.000       4.0
      B-arrive_time.end_time      0.000     0.000     0.000       8.0
      I-arrive_time.end_time      0.000     0.000     0.000       8.0
      I-depart_time.end_time      0.273     1.000     0.429       3.0
               I-flight_stop      0.000     0.000     0.000       0.0
      B-fromloc.airport_code      1.000     0.200     0.333       5.0
          I-restriction_code      0.600     1.000     0.750       3.0
    I-depart_time.start_time      0.000     0.000     0.000       1.0
          I-toloc.state_name      0.000     0.000     0.000       1.0
I-depart_date.today_relative      0.000     0.000     0.000       0.0
 B-arrive_date.date_relative      0.000     0.000     0.000       2.0
                I-flight_mod      0.000     0.000     0.000       6.0
                   I-economy      0.000     0.000     0.000       0.0
 B-return_date.date_relative      0.000     0.000     0.000       3.0
        I-fromloc.state_name      0.000     0.000     0.000       1.0
                B-state_code      0.200     1.000     0.333       1.0
    I-arrive_time.start_time      0.000     0.000     0.000       1.0
    I-arrive_date.day_number      0.000     0.000     0.000       0.0
                 B-meal_code      0.000     0.000     0.000       1.0
 I-depart_time.period_of_day      0.000     0.000     0.000       1.0
                  B-day_name      0.000     0.000     0.000       2.0
             B-period_of_day      0.000     0.000     0.000       4.0
        B-stoploc.state_code      0.000     0.000     0.000       0.0
    B-return_date.month_name      0.000     0.000     0.000       0.0
    B-return_date.day_number      0.000     0.000     0.000       0.0
    B-arrive_time.period_mod      0.000     0.000     0.000       0.0
                 I-meal_code      0.000     0.000     0.000       0.0
        B-toloc.country_name      0.000     0.000     0.000       1.0
                 B-days_code      0.000     0.000     0.000       1.0
 I-arrive_time.period_of_day      0.000     0.000     0.000       0.0
            I-today_relative      0.000     0.000     0.000       0.0
 B-return_time.period_of_day      0.000     0.000     0.000       0.0
                      B-time      0.000     0.000     0.000       0.0
           I-fare_basis_code      0.000     0.000     0.000       0.0
 I-arrive_time.time_relative      0.000     0.000     0.000       4.0
 I-depart_time.time_relative      0.000     0.000     0.000       1.0
            B-today_relative      0.000     0.000     0.000       0.0
                B-state_name      0.000     0.000     0.000       9.0
B-arrive_date.today_relative      0.000     0.000     0.000       0.0
    B-return_time.period_mod      0.000     0.000     0.000       0.0
                B-month_name      0.000     0.000     0.000       0.0
                B-day_number      0.000     0.000     0.000       0.0
 I-return_date.date_relative      0.000     0.000     0.000       3.0
I-return_date.today_relative      0.000     0.000     0.000       0.0
      B-stoploc.airport_name      0.000     0.000     0.000       0.0
             B-time_relative      0.000     0.000     0.000       0.0
                      I-time      0.000     0.000     0.000       0.0
    I-return_date.day_number      0.000     0.000     0.000       0.0
          I-meal_description      0.000     0.000     0.000       0.0
B-return_date.today_relative      0.000     0.000     0.000       0.0
      B-return_date.day_name      0.000     0.000     0.000       2.0

                   micro avg      0.895     0.896     0.896    3657.0
                   macro avg      0.428     0.433     0.410    3657.0
                weighted avg      0.885     0.896     0.883    3657.0

I0715 07:01:06.016726 140369481070464 interactiveshell.py:2882] Best Slot F1: 0.896
Reading ../data/atis.train.w-intent.iob
I0715 07:01:08.663485 140369481070464 interactiveshell.py:2882] Step 650 | Loss: 0.1160 | Loss_intent: 0.0017 | Loss_slots: 0.1143 | Spent: 5.8 secs | LR: 0.000280
I0715 07:01:12.331354 140369481070464 interactiveshell.py:2882] Step 700 | Loss: 0.4432 | Loss_intent: 0.3640 | Loss_slots: 0.0792 | Spent: 3.7 secs | LR: 0.000279
I0715 07:01:15.978737 140369481070464 interactiveshell.py:2882] Step 750 | Loss: 0.0695 | Loss_intent: 0.0067 | Loss_slots: 0.0627 | Spent: 3.6 secs | LR: 0.000277
I0715 07:01:19.644731 140369481070464 interactiveshell.py:2882] Step 800 | Loss: 0.1522 | Loss_intent: 0.0935 | Loss_slots: 0.0587 | Spent: 3.7 secs | LR: 0.000276
I0715 07:01:23.324471 140369481070464 interactiveshell.py:2882] Step 850 | Loss: 0.0656 | Loss_intent: 0.0128 | Loss_slots: 0.0529 | Spent: 3.7 secs | LR: 0.000274
I0715 07:01:26.968221 140369481070464 interactiveshell.py:2882] Step 900 | Loss: 0.1980 | Loss_intent: 0.0397 | Loss_slots: 0.1582 | Spent: 3.6 secs | LR: 0.000273
Reading ../data/atis.test.w-intent.iob
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.
  'precision', 'predicted', average, warn_for)
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1439: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples.
  'recall', 'true', average, warn_for)
I0715 07:01:30.965253 140369481070464 interactiveshell.py:2882] 
                                          precision    recall  f1-score   support

                             atis_flight      0.992     0.984     0.988       632
                            atis_airfare      0.960     1.000     0.980        48
                     atis_ground_service      1.000     1.000     1.000        36
                            atis_airline      1.000     1.000     1.000        38
                       atis_abbreviation      0.943     1.000     0.971        33
                           atis_aircraft      1.000     0.889     0.941         9
                        atis_flight_time      0.500     1.000     0.667         1
                           atis_quantity      0.429     1.000     0.600         3
                atis_flight#atis_airfare      0.727     0.667     0.696        12
                            atis_airport      1.000     1.000     1.000        18
                           atis_distance      1.000     1.000     1.000        10
                               atis_city      0.571     0.667     0.615         6
                        atis_ground_fare      1.000     0.857     0.923         7
                           atis_capacity      0.913     1.000     0.955        21
                          atis_flight_no      0.889     1.000     0.941         8
                               atis_meal      0.833     0.833     0.833         6
                        atis_restriction      0.000     0.000     0.000         0
             atis_airline#atis_flight_no      0.000     0.000     0.000         0
    atis_ground_service#atis_ground_fare      0.000     0.000     0.000         0
           atis_airfare#atis_flight_time      0.000     0.000     0.000         0
                           atis_cheapest      0.000     0.000     0.000         0
atis_aircraft#atis_flight#atis_flight_no      0.000     0.000     0.000         0

                               micro avg      0.973     0.979     0.976       888
                               macro avg      0.625     0.677     0.641       888
                            weighted avg      0.977     0.979     0.977       888

I0715 07:01:31.008520 140369481070464 interactiveshell.py:2882] 
                              precision    recall  f1-score   support

                           O      0.000     0.000     0.000       0.0
           B-toloc.city_name      0.969     0.994     0.981     716.0
         B-fromloc.city_name      0.978     0.989     0.983     704.0
           I-toloc.city_name      0.922     0.940     0.931     265.0
      B-depart_date.day_name      0.976     0.976     0.976     212.0
              B-airline_name      0.969     0.941     0.955     101.0
         I-fromloc.city_name      0.993     0.842     0.911     177.0
 B-depart_time.period_of_day      0.966     0.862     0.911     130.0
              I-airline_name      0.914     0.985     0.948      65.0
    B-depart_date.day_number      0.963     0.945     0.954      55.0
    B-depart_date.month_name      0.981     0.946     0.964      56.0
          B-depart_time.time      0.800     0.982     0.882      57.0
                B-round_trip      1.000     0.932     0.965      73.0
             B-cost_relative      1.000     0.973     0.986      37.0
                I-round_trip      1.000     1.000     1.000      71.0
                B-flight_mod      0.800     1.000     0.889      24.0
 B-depart_time.time_relative      0.932     0.846     0.887      65.0
          I-depart_time.time      0.911     0.981     0.944      52.0
         B-stoploc.city_name      0.870     1.000     0.930      20.0
                 B-city_name      0.833     0.526     0.645      57.0
                B-class_type      0.923     1.000     0.960      24.0
          B-arrive_time.time      0.846     0.971     0.904      34.0
 B-arrive_time.time_relative      0.703     0.839     0.765      31.0
                I-class_type      1.000     1.000     1.000      17.0
               B-flight_stop      0.840     1.000     0.913      21.0
          I-arrive_time.time      0.944     0.971     0.958      35.0
              B-airline_code      0.795     0.912     0.849      34.0
    I-depart_date.day_number      1.000     1.000     1.000      15.0
      I-fromloc.airport_name      0.367     0.733     0.489      15.0
      B-fromloc.airport_name      0.296     0.667     0.410      12.0
      B-arrive_date.day_name      0.688     1.000     0.815      11.0
          B-toloc.state_code      0.818     1.000     0.900      18.0
B-depart_date.today_relative      1.000     0.889     0.941       9.0
             B-flight_number      0.727     0.727     0.727      11.0
 B-depart_date.date_relative      0.882     0.882     0.882      17.0
          B-toloc.state_name      0.329     0.893     0.481      28.0
           B-fare_basis_code      0.800     0.941     0.865      17.0
               B-flight_time      0.000     0.000     0.000       1.0
                        B-or      0.667     0.667     0.667       3.0
 B-arrive_time.period_of_day      0.667     1.000     0.800       6.0
          B-meal_description      1.000     0.700     0.824      10.0
             I-cost_relative      1.000     0.667     0.800       3.0
              I-airport_name      0.778     0.241     0.368      29.0
               B-fare_amount      0.500     1.000     0.667       2.0
               I-fare_amount      1.000     1.000     1.000       2.0
                 I-city_name      0.850     0.567     0.680      30.0
        I-toloc.airport_name      0.286     0.667     0.400       3.0
            B-transport_type      0.769     1.000     0.870      10.0
    B-arrive_date.month_name      0.556     0.833     0.667       6.0
    B-arrive_date.day_number      0.714     0.833     0.769       6.0
         I-stoploc.city_name      0.857     0.600     0.706      10.0
                      B-meal      0.615     1.000     0.762      16.0
        B-fromloc.state_code      1.000     0.783     0.878      23.0
    B-depart_time.period_mod      0.600     0.600     0.600       5.0
                   B-connect      1.000     0.667     0.800       6.0
               B-flight_days      1.000     1.000     1.000      10.0
        B-toloc.airport_name      0.333     0.333     0.333       3.0
        B-fromloc.state_name      0.929     0.765     0.839      17.0
              B-airport_name      1.000     0.286     0.444      21.0
                   B-economy      1.000     0.500     0.667       6.0
               I-flight_time      0.500     1.000     0.667       1.0
             B-aircraft_code      1.000     0.879     0.935      33.0
                       B-mod      0.000     0.000     0.000       2.0
              B-airport_code      0.000     0.000     0.000       9.0
    B-depart_time.start_time      1.000     0.333     0.500       3.0
      B-depart_time.end_time      0.667     0.667     0.667       3.0
          B-depart_date.year      1.000     0.667     0.800       3.0
            I-transport_type      0.000     0.000     0.000       1.0
          B-restriction_code      1.000     0.500     0.667       4.0
    B-arrive_time.start_time      0.800     1.000     0.889       8.0
        B-toloc.airport_code      1.000     0.750     0.857       4.0
      B-arrive_time.end_time      0.875     0.875     0.875       8.0
      I-arrive_time.end_time      0.800     1.000     0.889       8.0
      I-depart_time.end_time      0.000     0.000     0.000       3.0
               I-flight_stop      0.000     0.000     0.000       0.0
      B-fromloc.airport_code      0.000     0.000     0.000       5.0
          I-restriction_code      1.000     1.000     1.000       3.0
    I-depart_time.start_time      0.333     1.000     0.500       1.0
          I-toloc.state_name      0.000     0.000     0.000       1.0
I-depart_date.today_relative      0.000     0.000     0.000       0.0
 B-arrive_date.date_relative      1.000     0.500     0.667       2.0
                I-flight_mod      0.000     0.000     0.000       6.0
                   I-economy      0.000     0.000     0.000       0.0
 B-return_date.date_relative      0.000     0.000     0.000       3.0
        I-fromloc.state_name      0.000     0.000     0.000       1.0
                B-state_code      0.333     1.000     0.500       1.0
    I-arrive_time.start_time      0.000     0.000     0.000       1.0
    I-arrive_date.day_number      0.000     0.000     0.000       0.0
                 B-meal_code      0.000     0.000     0.000       1.0
 I-depart_time.period_of_day      0.000     0.000     0.000       1.0
                  B-day_name      1.000     0.500     0.667       2.0
             B-period_of_day      0.000     0.000     0.000       4.0
        B-stoploc.state_code      0.000     0.000     0.000       0.0
    B-return_date.month_name      0.000     0.000     0.000       0.0
    B-return_date.day_number      0.000     0.000     0.000       0.0
    B-arrive_time.period_mod      0.000     0.000     0.000       0.0
                 I-meal_code      0.000     0.000     0.000       0.0
        B-toloc.country_name      0.000     0.000     0.000       1.0
                 B-days_code      0.000     0.000     0.000       1.0
 I-arrive_time.period_of_day      0.000     0.000     0.000       0.0
            I-today_relative      0.000     0.000     0.000       0.0
 B-return_time.period_of_day      0.000     0.000     0.000       0.0
                      B-time      0.000     0.000     0.000       0.0
           I-fare_basis_code      0.000     0.000     0.000       0.0
 I-arrive_time.time_relative      0.000     0.000     0.000       4.0
 I-depart_time.time_relative      0.000     0.000     0.000       1.0
            B-today_relative      0.000     0.000     0.000       0.0
                B-state_name      0.000     0.000     0.000       9.0
B-arrive_date.today_relative      0.000     0.000     0.000       0.0
    B-return_time.period_mod      0.000     0.000     0.000       0.0
                B-month_name      0.000     0.000     0.000       0.0
                B-day_number      0.000     0.000     0.000       0.0
 I-return_date.date_relative      0.000     0.000     0.000       3.0
I-return_date.today_relative      0.000     0.000     0.000       0.0
      B-stoploc.airport_name      0.000     0.000     0.000       0.0
             B-time_relative      0.000     0.000     0.000       0.0
                      I-time      0.000     0.000     0.000       0.0
    I-return_date.day_number      0.000     0.000     0.000       0.0
          I-meal_description      0.000     0.000     0.000       0.0
B-return_date.today_relative      0.000     0.000     0.000       0.0
      B-return_date.day_name      0.000     0.000     0.000       2.0

                   micro avg      0.911     0.913     0.912    3657.0
                   macro avg      0.497     0.500     0.480    3657.0
                weighted avg      0.917     0.913     0.908    3657.0

I0715 07:01:31.009503 140369481070464 interactiveshell.py:2882] Best Slot F1: 0.912
Reading ../data/atis.train.w-intent.iob
I0715 07:01:32.794509 140369481070464 interactiveshell.py:2882] Step 950 | Loss: 0.0878 | Loss_intent: 0.0007 | Loss_slots: 0.0871 | Spent: 5.8 secs | LR: 0.000271
I0715 07:01:36.426969 140369481070464 interactiveshell.py:2882] Step 1000 | Loss: 0.0258 | Loss_intent: 0.0001 | Loss_slots: 0.0258 | Spent: 3.6 secs | LR: 0.000270
I0715 07:01:40.103233 140369481070464 interactiveshell.py:2882] Step 1050 | Loss: 0.0395 | Loss_intent: 0.0075 | Loss_slots: 0.0320 | Spent: 3.7 secs | LR: 0.000269
I0715 07:01:43.755517 140369481070464 interactiveshell.py:2882] Step 1100 | Loss: 0.6457 | Loss_intent: 0.5938 | Loss_slots: 0.0519 | Spent: 3.7 secs | LR: 0.000267
I0715 07:01:47.396603 140369481070464 interactiveshell.py:2882] Step 1150 | Loss: 0.0650 | Loss_intent: 0.0005 | Loss_slots: 0.0645 | Spent: 3.6 secs | LR: 0.000266
I0715 07:01:51.057752 140369481070464 interactiveshell.py:2882] Step 1200 | Loss: 0.1033 | Loss_intent: 0.0009 | Loss_slots: 0.1024 | Spent: 3.7 secs | LR: 0.000264
Reading ../data/atis.test.w-intent.iob
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.
  'precision', 'predicted', average, warn_for)
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1439: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples.
  'recall', 'true', average, warn_for)
I0715 07:01:55.917953 140369481070464 interactiveshell.py:2882] 
                                          precision    recall  f1-score   support

                             atis_flight      0.990     0.987     0.989       632
                            atis_airfare      0.787     1.000     0.881        48
                     atis_ground_service      1.000     1.000     1.000        36
                            atis_airline      1.000     1.000     1.000        38
                       atis_abbreviation      0.943     1.000     0.971        33
                           atis_aircraft      1.000     0.889     0.941         9
                        atis_flight_time      1.000     1.000     1.000         1
                           atis_quantity      0.200     1.000     0.333         3
                atis_flight#atis_airfare      0.875     0.583     0.700        12
                            atis_airport      1.000     1.000     1.000        18
                           atis_distance      1.000     1.000     1.000        10
                               atis_city      1.000     0.500     0.667         6
                        atis_ground_fare      1.000     0.714     0.833         7
                           atis_capacity      1.000     0.857     0.923        21
                          atis_flight_no      1.000     0.250     0.400         8
                               atis_meal      1.000     0.833     0.909         6
                        atis_restriction      0.000     0.000     0.000         0
             atis_airline#atis_flight_no      0.000     0.000     0.000         0
    atis_ground_service#atis_ground_fare      0.000     0.000     0.000         0
           atis_airfare#atis_flight_time      0.000     0.000     0.000         0
                           atis_cheapest      0.000     0.000     0.000         0
atis_aircraft#atis_flight#atis_flight_no      0.000     0.000     0.000         0

                               micro avg      0.962     0.967     0.965       888
                               macro avg      0.673     0.619     0.616       888
                            weighted avg      0.975     0.967     0.966       888

I0715 07:01:55.969513 140369481070464 interactiveshell.py:2882] 
                              precision    recall  f1-score   support

                           O      0.000     0.000     0.000       0.0
           B-toloc.city_name      0.974     0.989     0.981     716.0
         B-fromloc.city_name      0.959     0.994     0.976     704.0
           I-toloc.city_name      0.950     0.932     0.941     265.0
      B-depart_date.day_name      0.995     0.943     0.969     212.0
              B-airline_name      0.933     0.970     0.951     101.0
         I-fromloc.city_name      0.907     0.989     0.946     177.0
 B-depart_time.period_of_day      0.991     0.869     0.926     130.0
              I-airline_name      0.875     0.969     0.920      65.0
    B-depart_date.day_number      0.948     1.000     0.973      55.0
    B-depart_date.month_name      0.981     0.929     0.954      56.0
          B-depart_time.time      0.809     0.965     0.880      57.0
                B-round_trip      1.000     0.986     0.993      73.0
             B-cost_relative      0.947     0.973     0.960      37.0
                I-round_trip      0.986     1.000     0.993      71.0
                B-flight_mod      0.742     0.958     0.836      24.0
 B-depart_time.time_relative      0.982     0.862     0.918      65.0
          I-depart_time.time      0.962     0.981     0.971      52.0
         B-stoploc.city_name      0.833     1.000     0.909      20.0
                 B-city_name      0.926     0.439     0.595      57.0
                B-class_type      0.960     1.000     0.980      24.0
          B-arrive_time.time      0.917     0.971     0.943      34.0
 B-arrive_time.time_relative      0.784     0.935     0.853      31.0
                I-class_type      1.000     1.000     1.000      17.0
               B-flight_stop      1.000     1.000     1.000      21.0
          I-arrive_time.time      0.944     0.971     0.958      35.0
              B-airline_code      0.829     0.853     0.841      34.0
    I-depart_date.day_number      1.000     1.000     1.000      15.0
      I-fromloc.airport_name      0.714     0.667     0.690      15.0
      B-fromloc.airport_name      0.312     0.417     0.357      12.0
      B-arrive_date.day_name      0.440     1.000     0.611      11.0
          B-toloc.state_code      1.000     0.944     0.971      18.0
B-depart_date.today_relative      1.000     0.889     0.941       9.0
             B-flight_number      0.500     0.818     0.621      11.0
 B-depart_date.date_relative      0.789     0.882     0.833      17.0
          B-toloc.state_name      0.549     1.000     0.709      28.0
           B-fare_basis_code      0.850     1.000     0.919      17.0
               B-flight_time      0.333     1.000     0.500       1.0
                        B-or      0.750     1.000     0.857       3.0
 B-arrive_time.period_of_day      0.545     1.000     0.706       6.0
          B-meal_description      1.000     0.800     0.889      10.0
             I-cost_relative      1.000     0.667     0.800       3.0
              I-airport_name      0.810     0.586     0.680      29.0
               B-fare_amount      0.333     1.000     0.500       2.0
               I-fare_amount      1.000     1.000     1.000       2.0
                 I-city_name      0.875     0.233     0.368      30.0
        I-toloc.airport_name      0.333     0.667     0.444       3.0
            B-transport_type      0.833     1.000     0.909      10.0
    B-arrive_date.month_name      0.625     0.833     0.714       6.0
    B-arrive_date.day_number      1.000     0.667     0.800       6.0
         I-stoploc.city_name      0.889     0.800     0.842      10.0
                      B-meal      0.889     1.000     0.941      16.0
        B-fromloc.state_code      1.000     0.957     0.978      23.0
    B-depart_time.period_mod      0.714     1.000     0.833       5.0
                   B-connect      1.000     1.000     1.000       6.0
               B-flight_days      1.000     0.900     0.947      10.0
        B-toloc.airport_name      1.000     0.333     0.500       3.0
        B-fromloc.state_name      0.929     0.765     0.839      17.0
              B-airport_name      0.818     0.429     0.562      21.0
                   B-economy      0.750     1.000     0.857       6.0
               I-flight_time      0.000     0.000     0.000       1.0
             B-aircraft_code      1.000     0.636     0.778      33.0
                       B-mod      0.000     0.000     0.000       2.0
              B-airport_code      0.000     0.000     0.000       9.0
    B-depart_time.start_time      1.000     0.667     0.800       3.0
      B-depart_time.end_time      1.000     0.667     0.800       3.0
          B-depart_date.year      1.000     1.000     1.000       3.0
            I-transport_type      0.000     0.000     0.000       1.0
          B-restriction_code      1.000     0.500     0.667       4.0
    B-arrive_time.start_time      0.875     0.875     0.875       8.0
        B-toloc.airport_code      1.000     0.750     0.857       4.0
      B-arrive_time.end_time      0.800     1.000     0.889       8.0
      I-arrive_time.end_time      0.800     1.000     0.889       8.0
      I-depart_time.end_time      0.000     0.000     0.000       3.0
               I-flight_stop      0.000     0.000     0.000       0.0
      B-fromloc.airport_code      0.667     0.400     0.500       5.0
          I-restriction_code      1.000     1.000     1.000       3.0
    I-depart_time.start_time      0.333     1.000     0.500       1.0
          I-toloc.state_name      0.000     0.000     0.000       1.0
I-depart_date.today_relative      0.000     0.000     0.000       0.0
 B-arrive_date.date_relative      0.500     1.000     0.667       2.0
                I-flight_mod      0.000     0.000     0.000       6.0
                   I-economy      0.000     0.000     0.000       0.0
 B-return_date.date_relative      0.500     0.333     0.400       3.0
        I-fromloc.state_name      0.000     0.000     0.000       1.0
                B-state_code      0.200     1.000     0.333       1.0
    I-arrive_time.start_time      0.000     0.000     0.000       1.0
    I-arrive_date.day_number      0.000     0.000     0.000       0.0
                 B-meal_code      0.000     0.000     0.000       1.0
 I-depart_time.period_of_day      0.000     0.000     0.000       1.0
                  B-day_name      1.000     0.500     0.667       2.0
             B-period_of_day      1.000     0.750     0.857       4.0
        B-stoploc.state_code      0.000     0.000     0.000       0.0
    B-return_date.month_name      0.000     0.000     0.000       0.0
    B-return_date.day_number      0.000     0.000     0.000       0.0
    B-arrive_time.period_mod      0.000     0.000     0.000       0.0
                 I-meal_code      0.000     0.000     0.000       0.0
        B-toloc.country_name      0.000     0.000     0.000       1.0
                 B-days_code      0.500     1.000     0.667       1.0
 I-arrive_time.period_of_day      0.000     0.000     0.000       0.0
            I-today_relative      0.000     0.000     0.000       0.0
 B-return_time.period_of_day      0.000     0.000     0.000       0.0
                      B-time      0.000     0.000     0.000       0.0
           I-fare_basis_code      0.000     0.000     0.000       0.0
 I-arrive_time.time_relative      0.000     0.000     0.000       4.0
 I-depart_time.time_relative      0.000     0.000     0.000       1.0
            B-today_relative      0.000     0.000     0.000       0.0
                B-state_name      0.000     0.000     0.000       9.0
B-arrive_date.today_relative      0.000     0.000     0.000       0.0
    B-return_time.period_mod      0.000     0.000     0.000       0.0
                B-month_name      0.000     0.000     0.000       0.0
                B-day_number      0.000     0.000     0.000       0.0
 I-return_date.date_relative      0.000     0.000     0.000       3.0
I-return_date.today_relative      0.000     0.000     0.000       0.0
      B-stoploc.airport_name      0.000     0.000     0.000       0.0
             B-time_relative      0.000     0.000     0.000       0.0
                      I-time      0.000     0.000     0.000       0.0
    I-return_date.day_number      0.000     0.000     0.000       0.0
          I-meal_description      0.000     0.000     0.000       0.0
B-return_date.today_relative      0.000     0.000     0.000       0.0
      B-return_date.day_name      0.000     0.000     0.000       2.0

                   micro avg      0.922     0.923     0.923    3657.0
                   macro avg      0.528     0.544     0.516    3657.0
                weighted avg      0.924     0.923     0.917    3657.0

I0715 07:01:55.970431 140369481070464 interactiveshell.py:2882] Best Slot F1: 0.923
Reading ../data/atis.train.w-intent.iob
I0715 07:01:56.874175 140369481070464 interactiveshell.py:2882] Step 1250 | Loss: 0.1023 | Loss_intent: 0.0600 | Loss_slots: 0.0423 | Spent: 5.8 secs | LR: 0.000263
I0715 07:02:00.546978 140369481070464 interactiveshell.py:2882] Step 1300 | Loss: 0.7375 | Loss_intent: 0.6986 | Loss_slots: 0.0389 | Spent: 3.7 secs | LR: 0.000262
I0715 07:02:04.247767 140369481070464 interactiveshell.py:2882] Step 1350 | Loss: 0.0603 | Loss_intent: 0.0008 | Loss_slots: 0.0595 | Spent: 3.7 secs | LR: 0.000260
I0715 07:02:07.933224 140369481070464 interactiveshell.py:2882] Step 1400 | Loss: 0.0567 | Loss_intent: 0.0000 | Loss_slots: 0.0567 | Spent: 3.7 secs | LR: 0.000259
I0715 07:02:11.675276 140369481070464 interactiveshell.py:2882] Step 1450 | Loss: 0.2889 | Loss_intent: 0.2415 | Loss_slots: 0.0474 | Spent: 3.7 secs | LR: 0.000257
I0715 07:02:15.395027 140369481070464 interactiveshell.py:2882] Step 1500 | Loss: 0.0104 | Loss_intent: 0.0000 | Loss_slots: 0.0104 | Spent: 3.7 secs | LR: 0.000256
I0715 07:02:19.086988 140369481070464 interactiveshell.py:2882] Step 1550 | Loss: 0.0347 | Loss_intent: 0.0004 | Loss_slots: 0.0344 | Spent: 3.7 secs | LR: 0.000255
Reading ../data/atis.test.w-intent.iob
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.
  'precision', 'predicted', average, warn_for)
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1439: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples.
  'recall', 'true', average, warn_for)
I0715 07:02:21.202833 140369481070464 interactiveshell.py:2882] 
                                          precision    recall  f1-score   support

                             atis_flight      0.975     0.992     0.984       632
                            atis_airfare      0.980     1.000     0.990        48
                     atis_ground_service      1.000     1.000     1.000        36
                            atis_airline      1.000     1.000     1.000        38
                       atis_abbreviation      0.941     0.970     0.955        33
                           atis_aircraft      1.000     0.889     0.941         9
                        atis_flight_time      1.000     1.000     1.000         1
                           atis_quantity      0.375     1.000     0.545         3
                atis_flight#atis_airfare      1.000     0.083     0.154        12
                            atis_airport      1.000     0.944     0.971        18
                           atis_distance      1.000     1.000     1.000        10
                               atis_city      0.750     0.500     0.600         6
                        atis_ground_fare      1.000     1.000     1.000         7
                           atis_capacity      0.952     0.952     0.952        21
                          atis_flight_no      0.889     1.000     0.941         8
                               atis_meal      0.857     1.000     0.923         6
                        atis_restriction      0.000     0.000     0.000         0
             atis_airline#atis_flight_no      0.000     0.000     0.000         0
    atis_ground_service#atis_ground_fare      0.000     0.000     0.000         0
           atis_airfare#atis_flight_time      0.000     0.000     0.000         0
                           atis_cheapest      0.000     0.000     0.000         0
atis_aircraft#atis_flight#atis_flight_no      0.000     0.000     0.000         0

                               micro avg      0.969     0.974     0.971       888
                               macro avg      0.669     0.651     0.634       888
                            weighted avg      0.972     0.974     0.967       888

I0715 07:02:21.244265 140369481070464 interactiveshell.py:2882] 
                              precision    recall  f1-score   support

                           O      0.000     0.000     0.000       0.0
           B-toloc.city_name      0.961     0.997     0.979     716.0
         B-fromloc.city_name      0.982     0.994     0.988     704.0
           I-toloc.city_name      0.926     0.992     0.958     265.0
      B-depart_date.day_name      0.986     0.991     0.988     212.0
              B-airline_name      0.943     0.990     0.966     101.0
         I-fromloc.city_name      0.994     0.904     0.947     177.0
 B-depart_time.period_of_day      0.953     0.931     0.942     130.0
              I-airline_name      0.955     0.985     0.970      65.0
    B-depart_date.day_number      0.964     0.982     0.973      55.0
    B-depart_date.month_name      0.982     0.964     0.973      56.0
          B-depart_time.time      0.862     0.982     0.918      57.0
                B-round_trip      1.000     0.986     0.993      73.0
             B-cost_relative      1.000     0.973     0.986      37.0
                I-round_trip      0.986     1.000     0.993      71.0
                B-flight_mod      0.774     1.000     0.873      24.0
 B-depart_time.time_relative      0.913     0.969     0.940      65.0
          I-depart_time.time      0.981     0.981     0.981      52.0
         B-stoploc.city_name      1.000     0.900     0.947      20.0
                 B-city_name      0.931     0.474     0.628      57.0
                B-class_type      0.960     1.000     0.980      24.0
          B-arrive_time.time      0.892     0.971     0.930      34.0
 B-arrive_time.time_relative      0.848     0.903     0.875      31.0
                I-class_type      1.000     1.000     1.000      17.0
               B-flight_stop      1.000     1.000     1.000      21.0
          I-arrive_time.time      0.971     0.971     0.971      35.0
              B-airline_code      0.800     0.824     0.812      34.0
    I-depart_date.day_number      0.882     1.000     0.938      15.0
      I-fromloc.airport_name      0.375     0.800     0.511      15.0
      B-fromloc.airport_name      0.367     0.917     0.524      12.0
      B-arrive_date.day_name      0.786     1.000     0.880      11.0
          B-toloc.state_code      0.857     1.000     0.923      18.0
B-depart_date.today_relative      1.000     0.889     0.941       9.0
             B-flight_number      0.778     0.636     0.700      11.0
 B-depart_date.date_relative      0.850     1.000     0.919      17.0
          B-toloc.state_name      0.632     0.857     0.727      28.0
           B-fare_basis_code      0.850     1.000     0.919      17.0
               B-flight_time      1.000     1.000     1.000       1.0
                        B-or      1.000     1.000     1.000       3.0
 B-arrive_time.period_of_day      0.714     0.833     0.769       6.0
          B-meal_description      1.000     0.900     0.947      10.0
             I-cost_relative      1.000     0.667     0.800       3.0
              I-airport_name      0.667     0.345     0.455      29.0
               B-fare_amount      0.667     1.000     0.800       2.0
               I-fare_amount      1.000     1.000     1.000       2.0
                 I-city_name      0.778     0.467     0.583      30.0
        I-toloc.airport_name      1.000     1.000     1.000       3.0
            B-transport_type      1.000     1.000     1.000      10.0
    B-arrive_date.month_name      0.714     0.833     0.769       6.0
    B-arrive_date.day_number      0.833     0.833     0.833       6.0
         I-stoploc.city_name      0.800     0.800     0.800      10.0
                      B-meal      0.762     1.000     0.865      16.0
        B-fromloc.state_code      1.000     0.870     0.930      23.0
    B-depart_time.period_mod      1.000     0.600     0.750       5.0
                   B-connect      1.000     1.000     1.000       6.0
               B-flight_days      1.000     1.000     1.000      10.0
        B-toloc.airport_name      1.000     0.333     0.500       3.0
        B-fromloc.state_name      0.889     0.941     0.914      17.0
              B-airport_name      0.714     0.238     0.357      21.0
                   B-economy      1.000     1.000     1.000       6.0
               I-flight_time      0.500     1.000     0.667       1.0
             B-aircraft_code      0.966     0.848     0.903      33.0
                       B-mod      0.000     0.000     0.000       2.0
              B-airport_code      1.000     0.111     0.200       9.0
    B-depart_time.start_time      0.500     0.667     0.571       3.0
      B-depart_time.end_time      1.000     0.667     0.800       3.0
          B-depart_date.year      0.000     0.000     0.000       3.0
            I-transport_type      1.000     1.000     1.000       1.0
          B-restriction_code      1.000     0.500     0.667       4.0
    B-arrive_time.start_time      0.889     1.000     0.941       8.0
        B-toloc.airport_code      1.000     0.750     0.857       4.0
      B-arrive_time.end_time      0.875     0.875     0.875       8.0
      I-arrive_time.end_time      0.800     1.000     0.889       8.0
      I-depart_time.end_time      0.667     0.667     0.667       3.0
               I-flight_stop      0.000     0.000     0.000       0.0
      B-fromloc.airport_code      0.750     0.600     0.667       5.0
          I-restriction_code      1.000     1.000     1.000       3.0
    I-depart_time.start_time      0.000     0.000     0.000       1.0
          I-toloc.state_name      1.000     1.000     1.000       1.0
I-depart_date.today_relative      0.000     0.000     0.000       0.0
 B-arrive_date.date_relative      1.000     0.500     0.667       2.0
                I-flight_mod      0.000     0.000     0.000       6.0
                   I-economy      0.000     0.000     0.000       0.0
 B-return_date.date_relative      1.000     0.333     0.500       3.0
        I-fromloc.state_name      0.000     0.000     0.000       1.0
                B-state_code      1.000     1.000     1.000       1.0
    I-arrive_time.start_time      0.000     0.000     0.000       1.0
    I-arrive_date.day_number      0.000     0.000     0.000       0.0
                 B-meal_code      0.000     0.000     0.000       1.0
 I-depart_time.period_of_day      0.000     0.000     0.000       1.0
                  B-day_name      0.000     0.000     0.000       2.0
             B-period_of_day      0.000     0.000     0.000       4.0
        B-stoploc.state_code      0.000     0.000     0.000       0.0
    B-return_date.month_name      0.000     0.000     0.000       0.0
    B-return_date.day_number      0.000     0.000     0.000       0.0
    B-arrive_time.period_mod      0.000     0.000     0.000       0.0
                 I-meal_code      0.000     0.000     0.000       0.0
        B-toloc.country_name      0.000     0.000     0.000       1.0
                 B-days_code      0.500     1.000     0.667       1.0
 I-arrive_time.period_of_day      0.000     0.000     0.000       0.0
            I-today_relative      0.000     0.000     0.000       0.0
 B-return_time.period_of_day      0.000     0.000     0.000       0.0
                      B-time      0.000     0.000     0.000       0.0
           I-fare_basis_code      0.000     0.000     0.000       0.0
 I-arrive_time.time_relative      0.000     0.000     0.000       4.0
 I-depart_time.time_relative      0.000     0.000     0.000       1.0
            B-today_relative      0.000     0.000     0.000       0.0
                B-state_name      0.000     0.000     0.000       9.0
B-arrive_date.today_relative      0.000     0.000     0.000       0.0
    B-return_time.period_mod      0.000     0.000     0.000       0.0
                B-month_name      0.000     0.000     0.000       0.0
                B-day_number      0.000     0.000     0.000       0.0
 I-return_date.date_relative      0.000     0.000     0.000       3.0
I-return_date.today_relative      0.000     0.000     0.000       0.0
      B-stoploc.airport_name      0.000     0.000     0.000       0.0
             B-time_relative      0.000     0.000     0.000       0.0
                      I-time      0.000     0.000     0.000       0.0
    I-return_date.day_number      0.000     0.000     0.000       0.0
          I-meal_description      0.000     0.000     0.000       0.0
B-return_date.today_relative      0.000     0.000     0.000       0.0
      B-return_date.day_name      0.000     0.000     0.000       2.0

                   micro avg      0.934     0.935     0.935    3657.0
                   macro avg      0.570     0.553     0.546    3657.0
                weighted avg      0.931     0.935     0.927    3657.0

I0715 07:02:21.245017 140369481070464 interactiveshell.py:2882] Best Slot F1: 0.935
Reading ../data/atis.train.w-intent.iob
I0715 07:02:24.943487 140369481070464 interactiveshell.py:2882] Step 1600 | Loss: 0.0583 | Loss_intent: 0.0462 | Loss_slots: 0.0121 | Spent: 5.9 secs | LR: 0.000253
I0715 07:02:28.594769 140369481070464 interactiveshell.py:2882] Step 1650 | Loss: 0.0139 | Loss_intent: 0.0000 | Loss_slots: 0.0139 | Spent: 3.6 secs | LR: 0.000252
I0715 07:02:32.277775 140369481070464 interactiveshell.py:2882] Step 1700 | Loss: 0.0728 | Loss_intent: 0.0000 | Loss_slots: 0.0728 | Spent: 3.7 secs | LR: 0.000251
I0715 07:02:35.938055 140369481070464 interactiveshell.py:2882] Step 1750 | Loss: 0.0769 | Loss_intent: 0.0034 | Loss_slots: 0.0735 | Spent: 3.7 secs | LR: 0.000249
I0715 07:02:39.610734 140369481070464 interactiveshell.py:2882] Step 1800 | Loss: 0.0210 | Loss_intent: 0.0001 | Loss_slots: 0.0208 | Spent: 3.7 secs | LR: 0.000248
I0715 07:02:43.276463 140369481070464 interactiveshell.py:2882] Step 1850 | Loss: 0.0606 | Loss_intent: 0.0002 | Loss_slots: 0.0604 | Spent: 3.7 secs | LR: 0.000247
Reading ../data/atis.test.w-intent.iob
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.
  'precision', 'predicted', average, warn_for)
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1439: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples.
  'recall', 'true', average, warn_for)
I0715 07:02:46.247977 140369481070464 interactiveshell.py:2882] 
                                          precision    recall  f1-score   support

                             atis_flight      0.983     0.984     0.983       632
                            atis_airfare      0.923     1.000     0.960        48
                     atis_ground_service      1.000     1.000     1.000        36
                            atis_airline      1.000     1.000     1.000        38
                       atis_abbreviation      0.943     1.000     0.971        33
                           atis_aircraft      1.000     0.889     0.941         9
                        atis_flight_time      1.000     1.000     1.000         1
                           atis_quantity      0.429     1.000     0.600         3
                atis_flight#atis_airfare      0.875     0.583     0.700        12
                            atis_airport      1.000     0.944     0.971        18
                           atis_distance      1.000     0.900     0.947        10
                               atis_city      0.444     0.667     0.533         6
                        atis_ground_fare      1.000     0.714     0.833         7
                           atis_capacity      0.955     1.000     0.977        21
                          atis_flight_no      1.000     0.500     0.667         8
                               atis_meal      1.000     1.000     1.000         6
                        atis_restriction      0.000     0.000     0.000         0
             atis_airline#atis_flight_no      0.000     0.000     0.000         0
    atis_ground_service#atis_ground_fare      0.000     0.000     0.000         0
           atis_airfare#atis_flight_time      0.000     0.000     0.000         0
                           atis_cheapest      0.000     0.000     0.000         0
atis_aircraft#atis_flight#atis_flight_no      0.000     0.000     0.000         0

                               micro avg      0.965     0.971     0.968       888
                               macro avg      0.661     0.645     0.640       888
                            weighted avg      0.973     0.971     0.970       888

I0715 07:02:46.296122 140369481070464 interactiveshell.py:2882] 
                              precision    recall  f1-score   support

                           O      0.000     0.000     0.000       0.0
           B-toloc.city_name      0.970     0.999     0.984     716.0
         B-fromloc.city_name      0.978     0.996     0.987     704.0
           I-toloc.city_name      0.880     0.996     0.935     265.0
      B-depart_date.day_name      0.991     0.991     0.991     212.0
              B-airline_name      0.971     0.980     0.975     101.0
         I-fromloc.city_name      0.994     0.870     0.928     177.0
 B-depart_time.period_of_day      0.966     0.862     0.911     130.0
              I-airline_name      0.941     0.985     0.962      65.0
    B-depart_date.day_number      0.964     0.982     0.973      55.0
    B-depart_date.month_name      0.982     0.964     0.973      56.0
          B-depart_time.time      0.848     0.982     0.911      57.0
                B-round_trip      1.000     0.986     0.993      73.0
             B-cost_relative      1.000     0.973     0.986      37.0
                I-round_trip      0.986     1.000     0.993      71.0
                B-flight_mod      0.750     1.000     0.857      24.0
 B-depart_time.time_relative      0.969     0.954     0.961      65.0
          I-depart_time.time      0.879     0.981     0.927      52.0
         B-stoploc.city_name      0.952     1.000     0.976      20.0
                 B-city_name      0.882     0.526     0.659      57.0
                B-class_type      0.960     1.000     0.980      24.0
          B-arrive_time.time      0.892     0.971     0.930      34.0
 B-arrive_time.time_relative      0.829     0.935     0.879      31.0
                I-class_type      1.000     1.000     1.000      17.0
               B-flight_stop      1.000     1.000     1.000      21.0
          I-arrive_time.time      0.919     0.971     0.944      35.0
              B-airline_code      0.963     0.765     0.852      34.0
    I-depart_date.day_number      1.000     0.933     0.966      15.0
      I-fromloc.airport_name      0.407     0.733     0.524      15.0
      B-fromloc.airport_name      0.316     0.500     0.387      12.0
      B-arrive_date.day_name      0.786     1.000     0.880      11.0
          B-toloc.state_code      1.000     1.000     1.000      18.0
B-depart_date.today_relative      1.000     0.889     0.941       9.0
             B-flight_number      0.562     0.818     0.667      11.0
 B-depart_date.date_relative      0.810     1.000     0.895      17.0
          B-toloc.state_name      0.641     0.893     0.746      28.0
           B-fare_basis_code      0.842     0.941     0.889      17.0
               B-flight_time      1.000     1.000     1.000       1.0
                        B-or      1.000     1.000     1.000       3.0
 B-arrive_time.period_of_day      0.600     1.000     0.750       6.0
          B-meal_description      1.000     0.900     0.947      10.0
             I-cost_relative      1.000     0.667     0.800       3.0
              I-airport_name      0.688     0.379     0.489      29.0
               B-fare_amount      0.400     1.000     0.571       2.0
               I-fare_amount      1.000     1.000     1.000       2.0
                 I-city_name      0.800     0.533     0.640      30.0
        I-toloc.airport_name      1.000     0.667     0.800       3.0
            B-transport_type      0.833     1.000     0.909      10.0
    B-arrive_date.month_name      0.714     0.833     0.769       6.0
    B-arrive_date.day_number      0.714     0.833     0.769       6.0
         I-stoploc.city_name      1.000     0.300     0.462      10.0
                      B-meal      0.941     1.000     0.970      16.0
        B-fromloc.state_code      1.000     0.957     0.978      23.0
    B-depart_time.period_mod      0.750     0.600     0.667       5.0
                   B-connect      1.000     1.000     1.000       6.0
               B-flight_days      1.000     0.800     0.889      10.0
        B-toloc.airport_name      1.000     0.333     0.500       3.0
        B-fromloc.state_name      0.929     0.765     0.839      17.0
              B-airport_name      0.667     0.286     0.400      21.0
                   B-economy      0.857     1.000     0.923       6.0
               I-flight_time      0.500     1.000     0.667       1.0
             B-aircraft_code      0.958     0.697     0.807      33.0
                       B-mod      0.167     0.500     0.250       2.0
              B-airport_code      0.000     0.000     0.000       9.0
    B-depart_time.start_time      0.500     0.667     0.571       3.0
      B-depart_time.end_time      1.000     0.667     0.800       3.0
          B-depart_date.year      1.000     1.000     1.000       3.0
            I-transport_type      0.000     0.000     0.000       1.0
          B-restriction_code      0.444     1.000     0.615       4.0
    B-arrive_time.start_time      0.889     1.000     0.941       8.0
        B-toloc.airport_code      0.750     0.750     0.750       4.0
      B-arrive_time.end_time      0.875     0.875     0.875       8.0
      I-arrive_time.end_time      0.778     0.875     0.824       8.0
      I-depart_time.end_time      0.667     0.667     0.667       3.0
               I-flight_stop      0.000     0.000     0.000       0.0
      B-fromloc.airport_code      0.750     0.600     0.667       5.0
          I-restriction_code      1.000     1.000     1.000       3.0
    I-depart_time.start_time      0.000     0.000     0.000       1.0
          I-toloc.state_name      0.000     0.000     0.000       1.0
I-depart_date.today_relative      0.000     0.000     0.000       0.0
 B-arrive_date.date_relative      1.000     0.500     0.667       2.0
                I-flight_mod      0.000     0.000     0.000       6.0
                   I-economy      0.000     0.000     0.000       0.0
 B-return_date.date_relative      0.500     0.333     0.400       3.0
        I-fromloc.state_name      0.000     0.000     0.000       1.0
                B-state_code      1.000     1.000     1.000       1.0
    I-arrive_time.start_time      0.000     0.000     0.000       1.0
    I-arrive_date.day_number      0.000     0.000     0.000       0.0
                 B-meal_code      0.000     0.000     0.000       1.0
 I-depart_time.period_of_day      0.000     0.000     0.000       1.0
                  B-day_name      1.000     0.500     0.667       2.0
             B-period_of_day      1.000     0.500     0.667       4.0
        B-stoploc.state_code      0.000     0.000     0.000       0.0
    B-return_date.month_name      0.000     0.000     0.000       0.0
    B-return_date.day_number      0.000     0.000     0.000       0.0
    B-arrive_time.period_mod      0.000     0.000     0.000       0.0
                 I-meal_code      0.000     0.000     0.000       0.0
        B-toloc.country_name      0.000     0.000     0.000       1.0
                 B-days_code      0.000     0.000     0.000       1.0
 I-arrive_time.period_of_day      0.000     0.000     0.000       0.0
            I-today_relative      0.000     0.000     0.000       0.0
 B-return_time.period_of_day      0.000     0.000     0.000       0.0
                      B-time      0.000     0.000     0.000       0.0
           I-fare_basis_code      0.000     0.000     0.000       0.0
 I-arrive_time.time_relative      0.000     0.000     0.000       4.0
 I-depart_time.time_relative      0.000     0.000     0.000       1.0
            B-today_relative      0.000     0.000     0.000       0.0
                B-state_name      1.000     0.333     0.500       9.0
B-arrive_date.today_relative      0.000     0.000     0.000       0.0
    B-return_time.period_mod      0.000     0.000     0.000       0.0
                B-month_name      0.000     0.000     0.000       0.0
                B-day_number      0.000     0.000     0.000       0.0
 I-return_date.date_relative      0.000     0.000     0.000       3.0
I-return_date.today_relative      0.000     0.000     0.000       0.0
      B-stoploc.airport_name      0.000     0.000     0.000       0.0
             B-time_relative      0.000     0.000     0.000       0.0
                      I-time      0.000     0.000     0.000       0.0
    I-return_date.day_number      0.000     0.000     0.000       0.0
          I-meal_description      0.000     0.000     0.000       0.0
B-return_date.today_relative      0.000     0.000     0.000       0.0
      B-return_date.day_name      0.000     0.000     0.000       2.0

                   micro avg      0.929     0.931     0.930    3657.0
                   macro avg      0.558     0.543     0.535    3657.0
                weighted avg      0.930     0.931     0.925    3657.0

I0715 07:02:46.296999 140369481070464 interactiveshell.py:2882] Best Slot F1: 0.935
Reading ../data/atis.train.w-intent.iob
I0715 07:02:49.086195 140369481070464 interactiveshell.py:2882] Step 1900 | Loss: 0.0798 | Loss_intent: 0.0669 | Loss_slots: 0.0128 | Spent: 5.8 secs | LR: 0.000246
I0715 07:02:52.774431 140369481070464 interactiveshell.py:2882] Step 1950 | Loss: 0.0235 | Loss_intent: 0.0001 | Loss_slots: 0.0234 | Spent: 3.7 secs | LR: 0.000244
I0715 07:02:56.473109 140369481070464 interactiveshell.py:2882] Step 2000 | Loss: 0.3186 | Loss_intent: 0.2503 | Loss_slots: 0.0683 | Spent: 3.7 secs | LR: 0.000243
I0715 07:03:00.134052 140369481070464 interactiveshell.py:2882] Step 2050 | Loss: 0.0151 | Loss_intent: 0.0000 | Loss_slots: 0.0151 | Spent: 3.7 secs | LR: 0.000242
I0715 07:03:03.810336 140369481070464 interactiveshell.py:2882] Step 2100 | Loss: 0.0322 | Loss_intent: 0.0034 | Loss_slots: 0.0288 | Spent: 3.7 secs | LR: 0.000240
I0715 07:03:07.472593 140369481070464 interactiveshell.py:2882] Step 2150 | Loss: 0.0141 | Loss_intent: 0.0000 | Loss_slots: 0.0141 | Spent: 3.7 secs | LR: 0.000239
Reading ../data/atis.test.w-intent.iob
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.
  'precision', 'predicted', average, warn_for)
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1439: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples.
  'recall', 'true', average, warn_for)
I0715 07:03:11.317543 140369481070464 interactiveshell.py:2882] 
                                          precision    recall  f1-score   support

                             atis_flight      0.971     0.994     0.982       632
                            atis_airfare      0.941     1.000     0.970        48
                     atis_ground_service      1.000     1.000     1.000        36
                            atis_airline      1.000     0.974     0.987        38
                       atis_abbreviation      0.941     0.970     0.955        33
                           atis_aircraft      0.900     1.000     0.947         9
                        atis_flight_time      1.000     1.000     1.000         1
                           atis_quantity      0.429     1.000     0.600         3
                atis_flight#atis_airfare      0.000     0.000     0.000        12
                            atis_airport      1.000     0.889     0.941        18
                           atis_distance      1.000     0.800     0.889        10
                               atis_city      0.800     0.667     0.727         6
                        atis_ground_fare      1.000     0.714     0.833         7
                           atis_capacity      1.000     0.952     0.976        21
                          atis_flight_no      0.889     1.000     0.941         8
                               atis_meal      1.000     1.000     1.000         6
                        atis_restriction      0.000     0.000     0.000         0
             atis_airline#atis_flight_no      0.000     0.000     0.000         0
    atis_ground_service#atis_ground_fare      0.000     0.000     0.000         0
           atis_airfare#atis_flight_time      0.000     0.000     0.000         0
                           atis_cheapest      0.000     0.000     0.000         0
atis_aircraft#atis_flight#atis_flight_no      0.000     0.000     0.000         0

                               micro avg      0.964     0.970     0.967       888
                               macro avg      0.630     0.635     0.625       888
                            weighted avg      0.955     0.970     0.961       888

I0715 07:03:11.358724 140369481070464 interactiveshell.py:2882] 
                              precision    recall  f1-score   support

                           O      0.000     0.000     0.000       0.0
           B-toloc.city_name      0.960     0.999     0.979     716.0
         B-fromloc.city_name      0.982     0.994     0.988     704.0
           I-toloc.city_name      0.970     0.977     0.974     265.0
      B-depart_date.day_name      0.991     0.986     0.988     212.0
              B-airline_name      0.927     1.000     0.962     101.0
         I-fromloc.city_name      0.950     0.960     0.955     177.0
 B-depart_time.period_of_day      0.991     0.838     0.908     130.0
              I-airline_name      1.000     0.969     0.984      65.0
    B-depart_date.day_number      0.900     0.982     0.939      55.0
    B-depart_date.month_name      0.982     0.964     0.973      56.0
          B-depart_time.time      0.824     0.982     0.896      57.0
                B-round_trip      1.000     0.973     0.986      73.0
             B-cost_relative      1.000     0.973     0.986      37.0
                I-round_trip      1.000     1.000     1.000      71.0
                B-flight_mod      0.759     0.917     0.830      24.0
 B-depart_time.time_relative      0.984     0.938     0.961      65.0
          I-depart_time.time      0.895     0.981     0.936      52.0
         B-stoploc.city_name      1.000     1.000     1.000      20.0
                 B-city_name      0.897     0.456     0.605      57.0
                B-class_type      0.960     1.000     0.980      24.0
          B-arrive_time.time      0.892     0.971     0.930      34.0
 B-arrive_time.time_relative      0.829     0.935     0.879      31.0
                I-class_type      1.000     1.000     1.000      17.0
               B-flight_stop      1.000     1.000     1.000      21.0
          I-arrive_time.time      0.895     0.971     0.932      35.0
              B-airline_code      0.903     0.824     0.862      34.0
    I-depart_date.day_number      1.000     0.933     0.966      15.0
      I-fromloc.airport_name      0.452     0.933     0.609      15.0
      B-fromloc.airport_name      0.360     0.750     0.486      12.0
      B-arrive_date.day_name      0.733     1.000     0.846      11.0
          B-toloc.state_code      1.000     1.000     1.000      18.0
B-depart_date.today_relative      1.000     1.000     1.000       9.0
             B-flight_number      0.444     0.727     0.552      11.0
 B-depart_date.date_relative      0.810     1.000     0.895      17.0
          B-toloc.state_name      0.889     0.857     0.873      28.0
           B-fare_basis_code      0.842     0.941     0.889      17.0
               B-flight_time      0.000     0.000     0.000       1.0
                        B-or      1.000     1.000     1.000       3.0
 B-arrive_time.period_of_day      0.462     1.000     0.632       6.0
          B-meal_description      1.000     0.900     0.947      10.0
             I-cost_relative      1.000     0.667     0.800       3.0
              I-airport_name      0.900     0.310     0.462      29.0
               B-fare_amount      0.400     1.000     0.571       2.0
               I-fare_amount      1.000     1.000     1.000       2.0
                 I-city_name      0.882     0.500     0.638      30.0
        I-toloc.airport_name      0.333     0.667     0.444       3.0
            B-transport_type      0.833     1.000     0.909      10.0
    B-arrive_date.month_name      0.714     0.833     0.769       6.0
    B-arrive_date.day_number      0.833     0.833     0.833       6.0
         I-stoploc.city_name      0.833     1.000     0.909      10.0
                      B-meal      0.941     1.000     0.970      16.0
        B-fromloc.state_code      1.000     1.000     1.000      23.0
    B-depart_time.period_mod      0.833     1.000     0.909       5.0
                   B-connect      1.000     1.000     1.000       6.0
               B-flight_days      1.000     0.900     0.947      10.0
        B-toloc.airport_name      1.000     1.000     1.000       3.0
        B-fromloc.state_name      0.640     0.941     0.762      17.0
              B-airport_name      0.778     0.333     0.467      21.0
                   B-economy      1.000     1.000     1.000       6.0
               I-flight_time      0.500     1.000     0.667       1.0
             B-aircraft_code      1.000     0.697     0.821      33.0
                       B-mod      0.500     0.500     0.500       2.0
              B-airport_code      0.000     0.000     0.000       9.0
    B-depart_time.start_time      1.000     0.667     0.800       3.0
      B-depart_time.end_time      1.000     0.667     0.800       3.0
          B-depart_date.year      1.000     0.667     0.800       3.0
            I-transport_type      0.000     0.000     0.000       1.0
          B-restriction_code      0.800     1.000     0.889       4.0
    B-arrive_time.start_time      0.857     0.750     0.800       8.0
        B-toloc.airport_code      1.000     0.750     0.857       4.0
      B-arrive_time.end_time      0.700     0.875     0.778       8.0
      I-arrive_time.end_time      0.778     0.875     0.824       8.0
      I-depart_time.end_time      0.667     0.667     0.667       3.0
               I-flight_stop      0.000     0.000     0.000       0.0
      B-fromloc.airport_code      0.750     0.600     0.667       5.0
          I-restriction_code      1.000     1.000     1.000       3.0
    I-depart_time.start_time      0.000     0.000     0.000       1.0
          I-toloc.state_name      1.000     1.000     1.000       1.0
I-depart_date.today_relative      0.000     0.000     0.000       0.0
 B-arrive_date.date_relative      1.000     1.000     1.000       2.0
                I-flight_mod      0.000     0.000     0.000       6.0
                   I-economy      0.000     0.000     0.000       0.0
 B-return_date.date_relative      0.000     0.000     0.000       3.0
        I-fromloc.state_name      0.000     0.000     0.000       1.0
                B-state_code      0.333     1.000     0.500       1.0
    I-arrive_time.start_time      0.000     0.000     0.000       1.0
    I-arrive_date.day_number      0.000     0.000     0.000       0.0
                 B-meal_code      0.000     0.000     0.000       1.0
 I-depart_time.period_of_day      0.000     0.000     0.000       1.0
                  B-day_name      1.000     0.500     0.667       2.0
             B-period_of_day      1.000     0.750     0.857       4.0
        B-stoploc.state_code      0.000     0.000     0.000       0.0
    B-return_date.month_name      0.000     0.000     0.000       0.0
    B-return_date.day_number      0.000     0.000     0.000       0.0
    B-arrive_time.period_mod      0.000     0.000     0.000       0.0
                 I-meal_code      0.000     0.000     0.000       0.0
        B-toloc.country_name      0.000     0.000     0.000       1.0
                 B-days_code      0.500     1.000     0.667       1.0
 I-arrive_time.period_of_day      0.000     0.000     0.000       0.0
            I-today_relative      0.000     0.000     0.000       0.0
 B-return_time.period_of_day      0.000     0.000     0.000       0.0
                      B-time      0.000     0.000     0.000       0.0
           I-fare_basis_code      0.000     0.000     0.000       0.0
 I-arrive_time.time_relative      0.000     0.000     0.000       4.0
 I-depart_time.time_relative      0.000     0.000     0.000       1.0
            B-today_relative      0.000     0.000     0.000       0.0
                B-state_name      0.000     0.000     0.000       9.0
B-arrive_date.today_relative      0.000     0.000     0.000       0.0
    B-return_time.period_mod      0.000     0.000     0.000       0.0
                B-month_name      0.000     0.000     0.000       0.0
                B-day_number      0.000     0.000     0.000       0.0
 I-return_date.date_relative      1.000     0.333     0.500       3.0
I-return_date.today_relative      0.000     0.000     0.000       0.0
      B-stoploc.airport_name      0.000     0.000     0.000       0.0
             B-time_relative      0.000     0.000     0.000       0.0
                      I-time      0.000     0.000     0.000       0.0
    I-return_date.day_number      0.000     0.000     0.000       0.0
          I-meal_description      0.000     0.000     0.000       0.0
B-return_date.today_relative      0.000     0.000     0.000       0.0
      B-return_date.day_name      0.000     0.000     0.000       2.0

                   micro avg      0.934     0.935     0.935    3657.0
                   macro avg      0.560     0.570     0.550    3657.0
                weighted avg      0.935     0.935     0.929    3657.0

I0715 07:03:11.359705 140369481070464 interactiveshell.py:2882] Best Slot F1: 0.935
Reading ../data/atis.train.w-intent.iob
I0715 07:03:13.291091 140369481070464 interactiveshell.py:2882] Step 2200 | Loss: 0.0342 | Loss_intent: 0.0000 | Loss_slots: 0.0342 | Spent: 5.8 secs | LR: 0.000238
I0715 07:03:16.941483 140369481070464 interactiveshell.py:2882] Step 2250 | Loss: 0.0207 | Loss_intent: 0.0000 | Loss_slots: 0.0207 | Spent: 3.6 secs | LR: 0.000237
I0715 07:03:20.592763 140369481070464 interactiveshell.py:2882] Step 2300 | Loss: 0.0100 | Loss_intent: 0.0005 | Loss_slots: 0.0095 | Spent: 3.6 secs | LR: 0.000235
I0715 07:03:24.249020 140369481070464 interactiveshell.py:2882] Step 2350 | Loss: 0.0043 | Loss_intent: 0.0000 | Loss_slots: 0.0043 | Spent: 3.7 secs | LR: 0.000234
I0715 07:03:27.929012 140369481070464 interactiveshell.py:2882] Step 2400 | Loss: 0.0360 | Loss_intent: 0.0130 | Loss_slots: 0.0230 | Spent: 3.7 secs | LR: 0.000233
I0715 07:03:31.667760 140369481070464 interactiveshell.py:2882] Step 2450 | Loss: 0.0522 | Loss_intent: 0.0000 | Loss_slots: 0.0522 | Spent: 3.7 secs | LR: 0.000232
Reading ../data/atis.test.w-intent.iob
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.
  'precision', 'predicted', average, warn_for)
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1439: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples.
  'recall', 'true', average, warn_for)
I0715 07:03:36.459711 140369481070464 interactiveshell.py:2882] 
                                          precision    recall  f1-score   support

                             atis_flight      0.981     0.992     0.987       632
                            atis_airfare      0.941     1.000     0.970        48
                     atis_ground_service      1.000     1.000     1.000        36
                            atis_airline      1.000     1.000     1.000        38
                       atis_abbreviation      0.941     0.970     0.955        33
                           atis_aircraft      1.000     0.889     0.941         9
                        atis_flight_time      0.500     1.000     0.667         1
                           atis_quantity      0.429     1.000     0.600         3
                atis_flight#atis_airfare      0.833     0.417     0.556        12
                            atis_airport      1.000     0.889     0.941        18
                           atis_distance      1.000     0.900     0.947        10
                               atis_city      1.000     0.667     0.800         6
                        atis_ground_fare      1.000     0.714     0.833         7
                           atis_capacity      0.913     1.000     0.955        21
                          atis_flight_no      0.889     1.000     0.941         8
                               atis_meal      1.000     0.833     0.909         6
                        atis_restriction      0.000     0.000     0.000         0
             atis_airline#atis_flight_no      0.000     0.000     0.000         0
    atis_ground_service#atis_ground_fare      0.000     0.000     0.000         0
           atis_airfare#atis_flight_time      0.000     0.000     0.000         0
                           atis_cheapest      0.000     0.000     0.000         0
atis_aircraft#atis_flight#atis_flight_no      0.000     0.000     0.000         0

                               micro avg      0.970     0.975     0.972       888
                               macro avg      0.656     0.649     0.636       888
                            weighted avg      0.973     0.975     0.972       888

I0715 07:03:36.500771 140369481070464 interactiveshell.py:2882] 
                              precision    recall  f1-score   support

                           O      0.000     0.000     0.000       0.0
           B-toloc.city_name      0.962     0.999     0.980     716.0
         B-fromloc.city_name      0.985     0.996     0.990     704.0
           I-toloc.city_name      0.973     0.970     0.972     265.0
      B-depart_date.day_name      0.990     0.981     0.986     212.0
              B-airline_name      0.953     1.000     0.976     101.0
         I-fromloc.city_name      0.994     0.904     0.947     177.0
 B-depart_time.period_of_day      0.992     0.900     0.944     130.0
              I-airline_name      1.000     1.000     1.000      65.0
    B-depart_date.day_number      0.964     0.982     0.973      55.0
    B-depart_date.month_name      0.981     0.946     0.964      56.0
          B-depart_time.time      0.833     0.965     0.894      57.0
                B-round_trip      1.000     0.973     0.986      73.0
             B-cost_relative      1.000     0.973     0.986      37.0
                I-round_trip      0.986     1.000     0.993      71.0
                B-flight_mod      0.821     0.958     0.885      24.0
 B-depart_time.time_relative      0.983     0.877     0.927      65.0
          I-depart_time.time      0.927     0.981     0.953      52.0
         B-stoploc.city_name      0.950     0.950     0.950      20.0
                 B-city_name      0.714     0.614     0.660      57.0
                B-class_type      0.960     1.000     0.980      24.0
          B-arrive_time.time      0.943     0.971     0.957      34.0
 B-arrive_time.time_relative      0.909     0.968     0.937      31.0
                I-class_type      1.000     1.000     1.000      17.0
               B-flight_stop      1.000     1.000     1.000      21.0
          I-arrive_time.time      0.919     0.971     0.944      35.0
              B-airline_code      0.882     0.882     0.882      34.0
    I-depart_date.day_number      1.000     0.933     0.966      15.0
      I-fromloc.airport_name      0.412     0.933     0.571      15.0
      B-fromloc.airport_name      0.346     0.750     0.474      12.0
      B-arrive_date.day_name      0.688     1.000     0.815      11.0
          B-toloc.state_code      1.000     1.000     1.000      18.0
B-depart_date.today_relative      1.000     1.000     1.000       9.0
             B-flight_number      0.476     0.909     0.625      11.0
 B-depart_date.date_relative      0.944     1.000     0.971      17.0
          B-toloc.state_name      0.649     0.857     0.738      28.0
           B-fare_basis_code      0.842     0.941     0.889      17.0
               B-flight_time      1.000     1.000     1.000       1.0
                        B-or      0.750     1.000     0.857       3.0
 B-arrive_time.period_of_day      0.667     1.000     0.800       6.0
          B-meal_description      1.000     0.800     0.889      10.0
             I-cost_relative      1.000     0.667     0.800       3.0
              I-airport_name      0.909     0.345     0.500      29.0
               B-fare_amount      0.500     1.000     0.667       2.0
               I-fare_amount      1.000     1.000     1.000       2.0
                 I-city_name      0.645     0.667     0.656      30.0
        I-toloc.airport_name      0.750     1.000     0.857       3.0
            B-transport_type      0.909     1.000     0.952      10.0
    B-arrive_date.month_name      0.625     0.833     0.714       6.0
    B-arrive_date.day_number      0.833     0.833     0.833       6.0
         I-stoploc.city_name      1.000     0.800     0.889      10.0
                      B-meal      0.842     1.000     0.914      16.0
        B-fromloc.state_code      1.000     1.000     1.000      23.0
    B-depart_time.period_mod      1.000     1.000     1.000       5.0
                   B-connect      1.000     1.000     1.000       6.0
               B-flight_days      1.000     1.000     1.000      10.0
        B-toloc.airport_name      1.000     1.000     1.000       3.0
        B-fromloc.state_name      1.000     0.882     0.938      17.0
              B-airport_name      1.000     0.238     0.385      21.0
                   B-economy      1.000     1.000     1.000       6.0
               I-flight_time      1.000     1.000     1.000       1.0
             B-aircraft_code      1.000     0.727     0.842      33.0
                       B-mod      0.000     0.000     0.000       2.0
              B-airport_code      0.000     0.000     0.000       9.0
    B-depart_time.start_time      0.667     0.667     0.667       3.0
      B-depart_time.end_time      1.000     0.667     0.800       3.0
          B-depart_date.year      1.000     0.667     0.800       3.0
            I-transport_type      0.000     0.000     0.000       1.0
          B-restriction_code      0.800     1.000     0.889       4.0
    B-arrive_time.start_time      0.889     1.000     0.941       8.0
        B-toloc.airport_code      1.000     0.500     0.667       4.0
      B-arrive_time.end_time      0.889     1.000     0.941       8.0
      I-arrive_time.end_time      0.875     0.875     0.875       8.0
      I-depart_time.end_time      0.667     0.667     0.667       3.0
               I-flight_stop      0.000     0.000     0.000       0.0
      B-fromloc.airport_code      0.667     0.400     0.500       5.0
          I-restriction_code      1.000     1.000     1.000       3.0
    I-depart_time.start_time      0.000     0.000     0.000       1.0
          I-toloc.state_name      1.000     1.000     1.000       1.0
I-depart_date.today_relative      0.000     0.000     0.000       0.0
 B-arrive_date.date_relative      1.000     1.000     1.000       2.0
                I-flight_mod      0.000     0.000     0.000       6.0
                   I-economy      0.000     0.000     0.000       0.0
 B-return_date.date_relative      0.667     0.667     0.667       3.0
        I-fromloc.state_name      0.500     1.000     0.667       1.0
                B-state_code      1.000     1.000     1.000       1.0
    I-arrive_time.start_time      1.000     1.000     1.000       1.0
    I-arrive_date.day_number      0.000     0.000     0.000       0.0
                 B-meal_code      0.000     0.000     0.000       1.0
 I-depart_time.period_of_day      0.000     0.000     0.000       1.0
                  B-day_name      1.000     0.500     0.667       2.0
             B-period_of_day      1.000     0.750     0.857       4.0
        B-stoploc.state_code      0.000     0.000     0.000       0.0
    B-return_date.month_name      0.000     0.000     0.000       0.0
    B-return_date.day_number      0.000     0.000     0.000       0.0
    B-arrive_time.period_mod      0.000     0.000     0.000       0.0
                 I-meal_code      0.000     0.000     0.000       0.0
        B-toloc.country_name      1.000     1.000     1.000       1.0
                 B-days_code      0.000     0.000     0.000       1.0
 I-arrive_time.period_of_day      0.000     0.000     0.000       0.0
            I-today_relative      0.000     0.000     0.000       0.0
 B-return_time.period_of_day      0.000     0.000     0.000       0.0
                      B-time      0.000     0.000     0.000       0.0
           I-fare_basis_code      0.000     0.000     0.000       0.0
 I-arrive_time.time_relative      1.000     1.000     1.000       4.0
 I-depart_time.time_relative      0.000     0.000     0.000       1.0
            B-today_relative      0.000     0.000     0.000       0.0
                B-state_name      0.000     0.000     0.000       9.0
B-arrive_date.today_relative      0.000     0.000     0.000       0.0
    B-return_time.period_mod      0.000     0.000     0.000       0.0
                B-month_name      0.000     0.000     0.000       0.0
                B-day_number      0.000     0.000     0.000       0.0
 I-return_date.date_relative      1.000     0.333     0.500       3.0
I-return_date.today_relative      0.000     0.000     0.000       0.0
      B-stoploc.airport_name      0.000     0.000     0.000       0.0
             B-time_relative      0.000     0.000     0.000       0.0
                      I-time      0.000     0.000     0.000       0.0
    I-return_date.day_number      0.000     0.000     0.000       0.0
          I-meal_description      0.000     0.000     0.000       0.0
B-return_date.today_relative      0.000     0.000     0.000       0.0
      B-return_date.day_name      0.000     0.000     0.000       2.0

                   micro avg      0.939     0.940     0.940    3657.0
                   macro avg      0.612     0.608     0.598    3657.0
                weighted avg      0.942     0.940     0.936    3657.0

I0715 07:03:36.501689 140369481070464 interactiveshell.py:2882] Best Slot F1: 0.940
Reading ../data/atis.train.w-intent.iob
I0715 07:03:37.569724 140369481070464 interactiveshell.py:2882] Step 2500 | Loss: 0.0157 | Loss_intent: 0.0000 | Loss_slots: 0.0157 | Spent: 5.9 secs | LR: 0.000231
I0715 07:03:41.248736 140369481070464 interactiveshell.py:2882] Step 2550 | Loss: 0.0299 | Loss_intent: 0.0192 | Loss_slots: 0.0107 | Spent: 3.7 secs | LR: 0.000229
I0715 07:03:45.044468 140369481070464 interactiveshell.py:2882] Step 2600 | Loss: 0.0400 | Loss_intent: 0.0004 | Loss_slots: 0.0396 | Spent: 3.8 secs | LR: 0.000228
I0715 07:03:49.092903 140369481070464 interactiveshell.py:2882] Step 2650 | Loss: 0.0432 | Loss_intent: 0.0014 | Loss_slots: 0.0418 | Spent: 4.0 secs | LR: 0.000227
I0715 07:03:53.162835 140369481070464 interactiveshell.py:2882] Step 2700 | Loss: 0.0467 | Loss_intent: 0.0000 | Loss_slots: 0.0467 | Spent: 4.1 secs | LR: 0.000226
I0715 07:03:56.925638 140369481070464 interactiveshell.py:2882] Step 2750 | Loss: 0.0225 | Loss_intent: 0.0000 | Loss_slots: 0.0225 | Spent: 3.8 secs | LR: 0.000225
I0715 07:04:00.583043 140369481070464 interactiveshell.py:2882] Step 2800 | Loss: 0.0487 | Loss_intent: 0.0050 | Loss_slots: 0.0437 | Spent: 3.7 secs | LR: 0.000223
Reading ../data/atis.test.w-intent.iob
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.
  'precision', 'predicted', average, warn_for)
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1439: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples.
  'recall', 'true', average, warn_for)
I0715 07:04:02.534862 140369481070464 interactiveshell.py:2882] 
                                          precision    recall  f1-score   support

                             atis_flight      0.984     0.991     0.987       632
                            atis_airfare      0.960     1.000     0.980        48
                     atis_ground_service      1.000     1.000     1.000        36
                            atis_airline      1.000     1.000     1.000        38
                       atis_abbreviation      0.943     1.000     0.971        33
                           atis_aircraft      0.889     0.889     0.889         9
                        atis_flight_time      1.000     1.000     1.000         1
                           atis_quantity      0.375     1.000     0.545         3
                atis_flight#atis_airfare      0.857     0.500     0.632        12
                            atis_airport      1.000     0.944     0.971        18
                           atis_distance      1.000     1.000     1.000        10
                               atis_city      1.000     0.500     0.667         6
                        atis_ground_fare      1.000     0.857     0.923         7
                           atis_capacity      0.913     1.000     0.955        21
                          atis_flight_no      1.000     1.000     1.000         8
                               atis_meal      1.000     0.833     0.909         6
                        atis_restriction      0.000     0.000     0.000         0
             atis_airline#atis_flight_no      0.000     0.000     0.000         0
    atis_ground_service#atis_ground_fare      0.000     0.000     0.000         0
           atis_airfare#atis_flight_time      0.000     0.000     0.000         0
                           atis_cheapest      0.000     0.000     0.000         0
atis_aircraft#atis_flight#atis_flight_no      0.000     0.000     0.000         0

                               micro avg      0.973     0.979     0.976       888
                               macro avg      0.678     0.660     0.656       888
                            weighted avg      0.977     0.979     0.976       888

I0715 07:04:02.574965 140369481070464 interactiveshell.py:2882] 
                              precision    recall  f1-score   support

                           O      0.000     0.000     0.000       0.0
           B-toloc.city_name      0.975     0.990     0.983     716.0
         B-fromloc.city_name      0.972     0.999     0.985     704.0
           I-toloc.city_name      0.981     1.000     0.991     265.0
      B-depart_date.day_name      0.991     0.986     0.988     212.0
              B-airline_name      0.980     0.980     0.980     101.0
         I-fromloc.city_name      0.994     0.949     0.971     177.0
 B-depart_time.period_of_day      0.992     0.908     0.948     130.0
              I-airline_name      0.985     1.000     0.992      65.0
    B-depart_date.day_number      0.964     0.982     0.973      55.0
    B-depart_date.month_name      0.982     0.964     0.973      56.0
          B-depart_time.time      0.789     0.982     0.875      57.0
                B-round_trip      1.000     0.959     0.979      73.0
             B-cost_relative      1.000     0.973     0.986      37.0
                I-round_trip      0.973     1.000     0.986      71.0
                B-flight_mod      0.800     1.000     0.889      24.0
 B-depart_time.time_relative      0.982     0.846     0.909      65.0
          I-depart_time.time      0.962     0.981     0.971      52.0
         B-stoploc.city_name      1.000     1.000     1.000      20.0
                 B-city_name      0.829     0.596     0.694      57.0
                B-class_type      0.960     1.000     0.980      24.0
          B-arrive_time.time      0.917     0.971     0.943      34.0
 B-arrive_time.time_relative      0.933     0.903     0.918      31.0
                I-class_type      1.000     1.000     1.000      17.0
               B-flight_stop      1.000     1.000     1.000      21.0
          I-arrive_time.time      0.919     0.971     0.944      35.0
              B-airline_code      1.000     0.882     0.938      34.0
    I-depart_date.day_number      1.000     1.000     1.000      15.0
      I-fromloc.airport_name      0.483     0.933     0.636      15.0
      B-fromloc.airport_name      0.478     0.917     0.629      12.0
      B-arrive_date.day_name      0.733     1.000     0.846      11.0
          B-toloc.state_code      1.000     1.000     1.000      18.0
B-depart_date.today_relative      1.000     1.000     1.000       9.0
             B-flight_number      0.529     0.818     0.643      11.0
 B-depart_date.date_relative      0.895     1.000     0.944      17.0
          B-toloc.state_name      0.893     0.893     0.893      28.0
           B-fare_basis_code      0.810     1.000     0.895      17.0
               B-flight_time      0.500     1.000     0.667       1.0
                        B-or      1.000     1.000     1.000       3.0
 B-arrive_time.period_of_day      0.600     1.000     0.750       6.0
          B-meal_description      1.000     0.800     0.889      10.0
             I-cost_relative      1.000     0.667     0.800       3.0
              I-airport_name      0.875     0.483     0.622      29.0
               B-fare_amount      1.000     1.000     1.000       2.0
               I-fare_amount      1.000     1.000     1.000       2.0
                 I-city_name      0.870     0.667     0.755      30.0
        I-toloc.airport_name      1.000     1.000     1.000       3.0
            B-transport_type      0.875     0.700     0.778      10.0
    B-arrive_date.month_name      0.714     0.833     0.769       6.0
    B-arrive_date.day_number      0.833     0.833     0.833       6.0
         I-stoploc.city_name      1.000     0.900     0.947      10.0
                      B-meal      0.889     1.000     0.941      16.0
        B-fromloc.state_code      1.000     1.000     1.000      23.0
    B-depart_time.period_mod      1.000     1.000     1.000       5.0
                   B-connect      1.000     1.000     1.000       6.0
               B-flight_days      1.000     1.000     1.000      10.0
        B-toloc.airport_name      0.750     1.000     0.857       3.0
        B-fromloc.state_name      1.000     0.882     0.938      17.0
              B-airport_name      0.600     0.286     0.387      21.0
                   B-economy      1.000     1.000     1.000       6.0
               I-flight_time      0.000     0.000     0.000       1.0
             B-aircraft_code      0.931     0.818     0.871      33.0
                       B-mod      0.000     0.000     0.000       2.0
              B-airport_code      0.667     0.444     0.533       9.0
    B-depart_time.start_time      0.600     1.000     0.750       3.0
      B-depart_time.end_time      1.000     0.667     0.800       3.0
          B-depart_date.year      1.000     1.000     1.000       3.0
            I-transport_type      0.000     0.000     0.000       1.0
          B-restriction_code      1.000     1.000     1.000       4.0
    B-arrive_time.start_time      1.000     1.000     1.000       8.0
        B-toloc.airport_code      1.000     0.500     0.667       4.0
      B-arrive_time.end_time      0.778     0.875     0.824       8.0
      I-arrive_time.end_time      0.875     0.875     0.875       8.0
      I-depart_time.end_time      0.667     0.667     0.667       3.0
               I-flight_stop      0.000     0.000     0.000       0.0
      B-fromloc.airport_code      0.800     0.800     0.800       5.0
          I-restriction_code      1.000     1.000     1.000       3.0
    I-depart_time.start_time      0.000     0.000     0.000       1.0
          I-toloc.state_name      1.000     1.000     1.000       1.0
I-depart_date.today_relative      0.000     0.000     0.000       0.0
 B-arrive_date.date_relative      1.000     1.000     1.000       2.0
                I-flight_mod      0.000     0.000     0.000       6.0
                   I-economy      0.000     0.000     0.000       0.0
 B-return_date.date_relative      1.000     0.333     0.500       3.0
        I-fromloc.state_name      0.100     1.000     0.182       1.0
                B-state_code      1.000     1.000     1.000       1.0
    I-arrive_time.start_time      0.000     0.000     0.000       1.0
    I-arrive_date.day_number      0.000     0.000     0.000       0.0
                 B-meal_code      0.000     0.000     0.000       1.0
 I-depart_time.period_of_day      0.000     0.000     0.000       1.0
                  B-day_name      1.000     0.500     0.667       2.0
             B-period_of_day      1.000     0.750     0.857       4.0
        B-stoploc.state_code      0.000     0.000     0.000       0.0
    B-return_date.month_name      0.000     0.000     0.000       0.0
    B-return_date.day_number      0.000     0.000     0.000       0.0
    B-arrive_time.period_mod      0.000     0.000     0.000       0.0
                 I-meal_code      0.000     0.000     0.000       0.0
        B-toloc.country_name      1.000     1.000     1.000       1.0
                 B-days_code      0.000     0.000     0.000       1.0
 I-arrive_time.period_of_day      0.000     0.000     0.000       0.0
            I-today_relative      0.000     0.000     0.000       0.0
 B-return_time.period_of_day      0.000     0.000     0.000       0.0
                      B-time      0.000     0.000     0.000       0.0
           I-fare_basis_code      0.000     0.000     0.000       0.0
 I-arrive_time.time_relative      0.571     1.000     0.727       4.0
 I-depart_time.time_relative      0.000     0.000     0.000       1.0
            B-today_relative      0.000     0.000     0.000       0.0
                B-state_name      0.000     0.000     0.000       9.0
B-arrive_date.today_relative      0.000     0.000     0.000       0.0
    B-return_time.period_mod      0.000     0.000     0.000       0.0
                B-month_name      0.000     0.000     0.000       0.0
                B-day_number      0.000     0.000     0.000       0.0
 I-return_date.date_relative      0.000     0.000     0.000       3.0
I-return_date.today_relative      0.000     0.000     0.000       0.0
      B-stoploc.airport_name      0.000     0.000     0.000       0.0
             B-time_relative      0.000     0.000     0.000       0.0
                      I-time      0.000     0.000     0.000       0.0
    I-return_date.day_number      0.000     0.000     0.000       0.0
          I-meal_description      0.000     0.000     0.000       0.0
B-return_date.today_relative      0.000     0.000     0.000       0.0
      B-return_date.day_name      0.000     0.000     0.000       2.0

                   micro avg      0.945     0.947     0.946    3657.0
                   macro avg      0.597     0.601     0.586    3657.0
                weighted avg      0.948     0.947     0.944    3657.0

I0715 07:04:02.575870 140369481070464 interactiveshell.py:2882] Best Slot F1: 0.946
Reading ../data/atis.train.w-intent.iob
I0715 07:04:06.428316 140369481070464 interactiveshell.py:2882] Step 2850 | Loss: 0.0110 | Loss_intent: 0.0000 | Loss_slots: 0.0110 | Spent: 5.8 secs | LR: 0.000222
I0715 07:04:10.085731 140369481070464 interactiveshell.py:2882] Step 2900 | Loss: 0.0131 | Loss_intent: 0.0003 | Loss_slots: 0.0128 | Spent: 3.7 secs | LR: 0.000221
I0715 07:04:13.741718 140369481070464 interactiveshell.py:2882] Step 2950 | Loss: 0.4866 | Loss_intent: 0.4729 | Loss_slots: 0.0137 | Spent: 3.7 secs | LR: 0.000220
I0715 07:04:17.406678 140369481070464 interactiveshell.py:2882] Step 3000 | Loss: 0.0076 | Loss_intent: 0.0009 | Loss_slots: 0.0068 | Spent: 3.7 secs | LR: 0.000219
I0715 07:04:21.093652 140369481070464 interactiveshell.py:2882] Step 3050 | Loss: 0.0437 | Loss_intent: 0.0000 | Loss_slots: 0.0437 | Spent: 3.7 secs | LR: 0.000218
I0715 07:04:24.758536 140369481070464 interactiveshell.py:2882] Step 3100 | Loss: 0.0061 | Loss_intent: 0.0000 | Loss_slots: 0.0061 | Spent: 3.7 secs | LR: 0.000216
Reading ../data/atis.test.w-intent.iob
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.
  'precision', 'predicted', average, warn_for)
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1439: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples.
  'recall', 'true', average, warn_for)
I0715 07:04:27.608251 140369481070464 interactiveshell.py:2882] 
                                          precision    recall  f1-score   support

                             atis_flight      0.986     0.981     0.983       632
                            atis_airfare      0.941     1.000     0.970        48
                     atis_ground_service      1.000     1.000     1.000        36
                            atis_airline      1.000     1.000     1.000        38
                       atis_abbreviation      0.943     1.000     0.971        33
                           atis_aircraft      1.000     0.889     0.941         9
                        atis_flight_time      0.333     1.000     0.500         1
                           atis_quantity      0.429     1.000     0.600         3
                atis_flight#atis_airfare      0.833     0.417     0.556        12
                            atis_airport      0.944     0.944     0.944        18
                           atis_distance      1.000     1.000     1.000        10
                               atis_city      0.400     0.667     0.500         6
                        atis_ground_fare      1.000     0.714     0.833         7
                           atis_capacity      0.913     1.000     0.955        21
                          atis_flight_no      0.889     1.000     0.941         8
                               atis_meal      1.000     0.833     0.909         6
                        atis_restriction      0.000     0.000     0.000         0
             atis_airline#atis_flight_no      0.000     0.000     0.000         0
    atis_ground_service#atis_ground_fare      0.000     0.000     0.000         0
           atis_airfare#atis_flight_time      0.000     0.000     0.000         0
                           atis_cheapest      0.000     0.000     0.000         0
atis_aircraft#atis_flight#atis_flight_no      0.000     0.000     0.000         0

                               micro avg      0.965     0.971     0.968       888
                               macro avg      0.619     0.657     0.618       888
                            weighted avg      0.971     0.971     0.969       888

I0715 07:04:27.648263 140369481070464 interactiveshell.py:2882] 
                              precision    recall  f1-score   support

                           O      0.000     0.000     0.000       0.0
           B-toloc.city_name      0.975     0.994     0.985     716.0
         B-fromloc.city_name      0.980     0.996     0.988     704.0
           I-toloc.city_name      0.966     0.958     0.962     265.0
      B-depart_date.day_name      0.972     0.981     0.977     212.0
              B-airline_name      0.990     0.990     0.990     101.0
         I-fromloc.city_name      0.994     0.932     0.962     177.0
 B-depart_time.period_of_day      0.957     0.862     0.907     130.0
              I-airline_name      1.000     1.000     1.000      65.0
    B-depart_date.day_number      0.964     0.982     0.973      55.0
    B-depart_date.month_name      0.982     0.982     0.982      56.0
          B-depart_time.time      0.848     0.982     0.911      57.0
                B-round_trip      1.000     0.973     0.986      73.0
             B-cost_relative      1.000     0.973     0.986      37.0
                I-round_trip      0.986     1.000     0.993      71.0
                B-flight_mod      0.793     0.958     0.868      24.0
 B-depart_time.time_relative      0.952     0.923     0.938      65.0
          I-depart_time.time      0.911     0.981     0.944      52.0
         B-stoploc.city_name      1.000     1.000     1.000      20.0
                 B-city_name      0.943     0.579     0.717      57.0
                B-class_type      0.960     1.000     0.980      24.0
          B-arrive_time.time      0.892     0.971     0.930      34.0
 B-arrive_time.time_relative      0.853     0.935     0.892      31.0
                I-class_type      1.000     1.000     1.000      17.0
               B-flight_stop      1.000     1.000     1.000      21.0
          I-arrive_time.time      0.944     0.971     0.958      35.0
              B-airline_code      0.971     0.971     0.971      34.0
    I-depart_date.day_number      1.000     1.000     1.000      15.0
      I-fromloc.airport_name      0.560     0.933     0.700      15.0
      B-fromloc.airport_name      0.364     0.667     0.471      12.0
      B-arrive_date.day_name      0.733     1.000     0.846      11.0
          B-toloc.state_code      1.000     1.000     1.000      18.0
B-depart_date.today_relative      1.000     0.889     0.941       9.0
             B-flight_number      0.526     0.909     0.667      11.0
 B-depart_date.date_relative      0.944     1.000     0.971      17.0
          B-toloc.state_name      0.596     1.000     0.747      28.0
           B-fare_basis_code      0.941     0.941     0.941      17.0
               B-flight_time      1.000     1.000     1.000       1.0
                        B-or      1.000     1.000     1.000       3.0
 B-arrive_time.period_of_day      0.600     1.000     0.750       6.0
          B-meal_description      1.000     0.900     0.947      10.0
             I-cost_relative      1.000     0.667     0.800       3.0
              I-airport_name      0.720     0.621     0.667      29.0
               B-fare_amount      0.667     1.000     0.800       2.0
               I-fare_amount      1.000     1.000     1.000       2.0
                 I-city_name      0.889     0.533     0.667      30.0
        I-toloc.airport_name      1.000     1.000     1.000       3.0
            B-transport_type      0.909     1.000     0.952      10.0
    B-arrive_date.month_name      0.833     0.833     0.833       6.0
    B-arrive_date.day_number      0.833     0.833     0.833       6.0
         I-stoploc.city_name      1.000     1.000     1.000      10.0
                      B-meal      0.941     1.000     0.970      16.0
        B-fromloc.state_code      1.000     1.000     1.000      23.0
    B-depart_time.period_mod      0.833     1.000     0.909       5.0
                   B-connect      1.000     1.000     1.000       6.0
               B-flight_days      1.000     1.000     1.000      10.0
        B-toloc.airport_name      1.000     1.000     1.000       3.0
        B-fromloc.state_name      0.696     0.941     0.800      17.0
              B-airport_name      0.750     0.429     0.545      21.0
                   B-economy      1.000     1.000     1.000       6.0
               I-flight_time      1.000     1.000     1.000       1.0
             B-aircraft_code      1.000     0.818     0.900      33.0
                       B-mod      1.000     0.500     0.667       2.0
              B-airport_code      0.667     0.444     0.533       9.0
    B-depart_time.start_time      0.750     1.000     0.857       3.0
      B-depart_time.end_time      0.750     1.000     0.857       3.0
          B-depart_date.year      1.000     0.667     0.800       3.0
            I-transport_type      0.000     0.000     0.000       1.0
          B-restriction_code      0.800     1.000     0.889       4.0
    B-arrive_time.start_time      1.000     1.000     1.000       8.0
        B-toloc.airport_code      1.000     0.750     0.857       4.0
      B-arrive_time.end_time      1.000     0.750     0.857       8.0
      I-arrive_time.end_time      0.875     0.875     0.875       8.0
      I-depart_time.end_time      1.000     0.667     0.800       3.0
               I-flight_stop      0.000     0.000     0.000       0.0
      B-fromloc.airport_code      0.714     1.000     0.833       5.0
          I-restriction_code      1.000     1.000     1.000       3.0
    I-depart_time.start_time      1.000     1.000     1.000       1.0
          I-toloc.state_name      1.000     1.000     1.000       1.0
I-depart_date.today_relative      0.000     0.000     0.000       0.0
 B-arrive_date.date_relative      1.000     1.000     1.000       2.0
                I-flight_mod      0.000     0.000     0.000       6.0
                   I-economy      0.000     0.000     0.000       0.0
 B-return_date.date_relative      0.500     0.333     0.400       3.0
        I-fromloc.state_name      0.000     0.000     0.000       1.0
                B-state_code      1.000     1.000     1.000       1.0
    I-arrive_time.start_time      1.000     1.000     1.000       1.0
    I-arrive_date.day_number      0.000     0.000     0.000       0.0
                 B-meal_code      0.000     0.000     0.000       1.0
 I-depart_time.period_of_day      0.000     0.000     0.000       1.0
                  B-day_name      1.000     0.500     0.667       2.0
             B-period_of_day      0.000     0.000     0.000       4.0
        B-stoploc.state_code      0.000     0.000     0.000       0.0
    B-return_date.month_name      0.000     0.000     0.000       0.0
    B-return_date.day_number      0.000     0.000     0.000       0.0
    B-arrive_time.period_mod      0.000     0.000     0.000       0.0
                 I-meal_code      0.000     0.000     0.000       0.0
        B-toloc.country_name      1.000     1.000     1.000       1.0
                 B-days_code      0.333     1.000     0.500       1.0
 I-arrive_time.period_of_day      0.000     0.000     0.000       0.0
            I-today_relative      0.000     0.000     0.000       0.0
 B-return_time.period_of_day      0.000     0.000     0.000       0.0
                      B-time      0.000     0.000     0.000       0.0
           I-fare_basis_code      0.000     0.000     0.000       0.0
 I-arrive_time.time_relative      1.000     0.500     0.667       4.0
 I-depart_time.time_relative      0.000     0.000     0.000       1.0
            B-today_relative      0.000     0.000     0.000       0.0
                B-state_name      0.000     0.000     0.000       9.0
B-arrive_date.today_relative      0.000     0.000     0.000       0.0
    B-return_time.period_mod      0.000     0.000     0.000       0.0
                B-month_name      0.000     0.000     0.000       0.0
                B-day_number      0.000     0.000     0.000       0.0
 I-return_date.date_relative      1.000     0.333     0.500       3.0
I-return_date.today_relative      0.000     0.000     0.000       0.0
      B-stoploc.airport_name      0.000     0.000     0.000       0.0
             B-time_relative      0.000     0.000     0.000       0.0
                      I-time      0.000     0.000     0.000       0.0
    I-return_date.day_number      0.000     0.000     0.000       0.0
          I-meal_description      0.000     0.000     0.000       0.0
B-return_date.today_relative      0.000     0.000     0.000       0.0
      B-return_date.day_name      0.000     0.000     0.000       2.0

                   micro avg      0.945     0.946     0.946    3657.0
                   macro avg      0.633     0.629     0.620    3657.0
                weighted avg      0.946     0.946     0.943    3657.0

I0715 07:04:27.649199 140369481070464 interactiveshell.py:2882] Best Slot F1: 0.946
Reading ../data/atis.train.w-intent.iob
I0715 07:04:30.608442 140369481070464 interactiveshell.py:2882] Step 3150 | Loss: 0.0089 | Loss_intent: 0.0002 | Loss_slots: 0.0087 | Spent: 5.8 secs | LR: 0.000215
I0715 07:04:34.285097 140369481070464 interactiveshell.py:2882] Step 3200 | Loss: 0.0076 | Loss_intent: 0.0000 | Loss_slots: 0.0076 | Spent: 3.7 secs | LR: 0.000214
I0715 07:04:37.958983 140369481070464 interactiveshell.py:2882] Step 3250 | Loss: 0.0336 | Loss_intent: 0.0088 | Loss_slots: 0.0249 | Spent: 3.7 secs | LR: 0.000213
I0715 07:04:41.637369 140369481070464 interactiveshell.py:2882] Step 3300 | Loss: 0.0265 | Loss_intent: 0.0000 | Loss_slots: 0.0265 | Spent: 3.7 secs | LR: 0.000212
I0715 07:04:45.300895 140369481070464 interactiveshell.py:2882] Step 3350 | Loss: 0.0057 | Loss_intent: 0.0000 | Loss_slots: 0.0057 | Spent: 3.7 secs | LR: 0.000211
I0715 07:04:49.010888 140369481070464 interactiveshell.py:2882] Step 3400 | Loss: 0.0123 | Loss_intent: 0.0001 | Loss_slots: 0.0122 | Spent: 3.7 secs | LR: 0.000210
Reading ../data/atis.test.w-intent.iob
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.
  'precision', 'predicted', average, warn_for)
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1439: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples.
  'recall', 'true', average, warn_for)
I0715 07:04:52.784018 140369481070464 interactiveshell.py:2882] 
                                          precision    recall  f1-score   support

                             atis_flight      0.969     0.983     0.976       632
                            atis_airfare      0.980     1.000     0.990        48
                     atis_ground_service      0.973     1.000     0.986        36
                            atis_airline      1.000     1.000     1.000        38
                       atis_abbreviation      1.000     1.000     1.000        33
                           atis_aircraft      0.800     0.889     0.842         9
                        atis_flight_time      1.000     1.000     1.000         1
                           atis_quantity      0.333     1.000     0.500         3
                atis_flight#atis_airfare      0.750     0.250     0.375        12
                            atis_airport      0.000     0.000     0.000        18
                           atis_distance      1.000     0.900     0.947        10
                               atis_city      0.214     1.000     0.353         6
                        atis_ground_fare      1.000     0.857     0.923         7
                           atis_capacity      1.000     0.810     0.895        21
                          atis_flight_no      1.000     0.750     0.857         8
                               atis_meal      1.000     0.833     0.909         6
                        atis_restriction      0.000     0.000     0.000         0
             atis_airline#atis_flight_no      0.000     0.000     0.000         0
    atis_ground_service#atis_ground_fare      0.000     0.000     0.000         0
           atis_airfare#atis_flight_time      0.000     0.000     0.000         0
                           atis_cheapest      0.000     0.000     0.000         0
atis_aircraft#atis_flight#atis_flight_no      0.000     0.000     0.000         0

                               micro avg      0.941     0.946     0.943       888
                               macro avg      0.592     0.603     0.571       888
                            weighted avg      0.942     0.946     0.940       888

I0715 07:04:52.827507 140369481070464 interactiveshell.py:2882] 
                              precision    recall  f1-score   support

                           O      0.000     0.000     0.000       0.0
           B-toloc.city_name      0.978     0.993     0.985     716.0
         B-fromloc.city_name      0.976     0.997     0.987     704.0
           I-toloc.city_name      0.960     0.985     0.972     265.0
      B-depart_date.day_name      0.986     0.972     0.979     212.0
              B-airline_name      0.990     1.000     0.995     101.0
         I-fromloc.city_name      0.989     0.994     0.992     177.0
 B-depart_time.period_of_day      0.991     0.869     0.926     130.0
              I-airline_name      1.000     1.000     1.000      65.0
    B-depart_date.day_number      0.964     0.982     0.973      55.0
    B-depart_date.month_name      0.982     0.964     0.973      56.0
          B-depart_time.time      0.846     0.965     0.902      57.0
                B-round_trip      1.000     0.986     0.993      73.0
             B-cost_relative      1.000     0.973     0.986      37.0
                I-round_trip      0.986     1.000     0.993      71.0
                B-flight_mod      0.793     0.958     0.868      24.0
 B-depart_time.time_relative      0.984     0.954     0.969      65.0
          I-depart_time.time      0.881     1.000     0.937      52.0
         B-stoploc.city_name      0.833     1.000     0.909      20.0
                 B-city_name      0.897     0.456     0.605      57.0
                B-class_type      0.960     1.000     0.980      24.0
          B-arrive_time.time      0.868     0.971     0.917      34.0
 B-arrive_time.time_relative      0.811     0.968     0.882      31.0
                I-class_type      1.000     1.000     1.000      17.0
               B-flight_stop      1.000     1.000     1.000      21.0
          I-arrive_time.time      1.000     0.943     0.971      35.0
              B-airline_code      0.943     0.971     0.957      34.0
    I-depart_date.day_number      1.000     1.000     1.000      15.0
      I-fromloc.airport_name      0.389     0.933     0.549      15.0
      B-fromloc.airport_name      0.321     0.750     0.450      12.0
      B-arrive_date.day_name      0.647     1.000     0.786      11.0
          B-toloc.state_code      1.000     1.000     1.000      18.0
B-depart_date.today_relative      1.000     0.889     0.941       9.0
             B-flight_number      0.556     0.909     0.690      11.0
 B-depart_date.date_relative      0.944     1.000     0.971      17.0
          B-toloc.state_name      0.675     0.964     0.794      28.0
           B-fare_basis_code      0.850     1.000     0.919      17.0
               B-flight_time      1.000     1.000     1.000       1.0
                        B-or      1.000     1.000     1.000       3.0
 B-arrive_time.period_of_day      0.600     1.000     0.750       6.0
          B-meal_description      1.000     0.900     0.947      10.0
             I-cost_relative      1.000     0.667     0.800       3.0
              I-airport_name      0.833     0.345     0.488      29.0
               B-fare_amount      0.286     1.000     0.444       2.0
               I-fare_amount      1.000     1.000     1.000       2.0
                 I-city_name      0.929     0.433     0.591      30.0
        I-toloc.airport_name      1.000     1.000     1.000       3.0
            B-transport_type      0.909     1.000     0.952      10.0
    B-arrive_date.month_name      0.714     0.833     0.769       6.0
    B-arrive_date.day_number      0.833     0.833     0.833       6.0
         I-stoploc.city_name      1.000     1.000     1.000      10.0
                      B-meal      0.941     1.000     0.970      16.0
        B-fromloc.state_code      1.000     0.957     0.978      23.0
    B-depart_time.period_mod      0.833     1.000     0.909       5.0
                   B-connect      1.000     1.000     1.000       6.0
               B-flight_days      1.000     1.000     1.000      10.0
        B-toloc.airport_name      1.000     1.000     1.000       3.0
        B-fromloc.state_name      1.000     0.941     0.970      17.0
              B-airport_name      0.857     0.286     0.429      21.0
                   B-economy      1.000     1.000     1.000       6.0
               I-flight_time      1.000     1.000     1.000       1.0
             B-aircraft_code      1.000     0.697     0.821      33.0
                       B-mod      1.000     0.500     0.667       2.0
              B-airport_code      1.000     0.444     0.615       9.0
    B-depart_time.start_time      0.500     0.667     0.571       3.0
      B-depart_time.end_time      1.000     0.667     0.800       3.0
          B-depart_date.year      1.000     0.667     0.800       3.0
            I-transport_type      0.000     0.000     0.000       1.0
          B-restriction_code      1.000     1.000     1.000       4.0
    B-arrive_time.start_time      0.889     1.000     0.941       8.0
        B-toloc.airport_code      1.000     0.750     0.857       4.0
      B-arrive_time.end_time      0.875     0.875     0.875       8.0
      I-arrive_time.end_time      0.889     1.000     0.941       8.0
      I-depart_time.end_time      1.000     0.667     0.800       3.0
               I-flight_stop      0.000     0.000     0.000       0.0
      B-fromloc.airport_code      0.800     0.800     0.800       5.0
          I-restriction_code      1.000     1.000     1.000       3.0
    I-depart_time.start_time      1.000     1.000     1.000       1.0
          I-toloc.state_name      1.000     1.000     1.000       1.0
I-depart_date.today_relative      0.000     0.000     0.000       0.0
 B-arrive_date.date_relative      1.000     0.500     0.667       2.0
                I-flight_mod      0.000     0.000     0.000       6.0
                   I-economy      0.000     0.000     0.000       0.0
 B-return_date.date_relative      0.000     0.000     0.000       3.0
        I-fromloc.state_name      1.000     1.000     1.000       1.0
                B-state_code      1.000     1.000     1.000       1.0
    I-arrive_time.start_time      0.500     1.000     0.667       1.0
    I-arrive_date.day_number      0.000     0.000     0.000       0.0
                 B-meal_code      0.000     0.000     0.000       1.0
 I-depart_time.period_of_day      0.000     0.000     0.000       1.0
                  B-day_name      1.000     0.500     0.667       2.0
             B-period_of_day      1.000     0.750     0.857       4.0
        B-stoploc.state_code      0.000     0.000     0.000       0.0
    B-return_date.month_name      0.000     0.000     0.000       0.0
    B-return_date.day_number      0.000     0.000     0.000       0.0
    B-arrive_time.period_mod      0.000     0.000     0.000       0.0
                 I-meal_code      0.000     0.000     0.000       0.0
        B-toloc.country_name      1.000     1.000     1.000       1.0
                 B-days_code      0.500     1.000     0.667       1.0
 I-arrive_time.period_of_day      0.000     0.000     0.000       0.0
            I-today_relative      0.000     0.000     0.000       0.0
 B-return_time.period_of_day      0.000     0.000     0.000       0.0
                      B-time      0.000     0.000     0.000       0.0
           I-fare_basis_code      0.000     0.000     0.000       0.0
 I-arrive_time.time_relative      0.000     0.000     0.000       4.0
 I-depart_time.time_relative      0.000     0.000     0.000       1.0
            B-today_relative      0.000     0.000     0.000       0.0
                B-state_name      0.000     0.000     0.000       9.0
B-arrive_date.today_relative      0.000     0.000     0.000       0.0
    B-return_time.period_mod      0.000     0.000     0.000       0.0
                B-month_name      0.000     0.000     0.000       0.0
                B-day_number      0.000     0.000     0.000       0.0
 I-return_date.date_relative      0.500     0.333     0.400       3.0
I-return_date.today_relative      0.000     0.000     0.000       0.0
      B-stoploc.airport_name      0.000     0.000     0.000       0.0
             B-time_relative      0.000     0.000     0.000       0.0
                      I-time      0.000     0.000     0.000       0.0
    I-return_date.day_number      0.000     0.000     0.000       0.0
          I-meal_description      0.000     0.000     0.000       0.0
B-return_date.today_relative      0.000     0.000     0.000       0.0
      B-return_date.day_name      0.000     0.000     0.000       2.0

                   micro avg      0.943     0.945     0.944    3657.0
                   macro avg      0.630     0.623     0.612    3657.0
                weighted avg      0.948     0.945     0.940    3657.0

I0715 07:04:52.828333 140369481070464 interactiveshell.py:2882] Best Slot F1: 0.946
Reading ../data/atis.train.w-intent.iob
I0715 07:04:54.944830 140369481070464 interactiveshell.py:2882] Step 3450 | Loss: 0.0037 | Loss_intent: 0.0000 | Loss_slots: 0.0037 | Spent: 5.9 secs | LR: 0.000209
I0715 07:04:58.690174 140369481070464 interactiveshell.py:2882] Step 3500 | Loss: 0.0077 | Loss_intent: 0.0000 | Loss_slots: 0.0077 | Spent: 3.7 secs | LR: 0.000207
I0715 07:05:02.370802 140369481070464 interactiveshell.py:2882] Step 3550 | Loss: 0.0134 | Loss_intent: 0.0000 | Loss_slots: 0.0134 | Spent: 3.7 secs | LR: 0.000206
I0715 07:05:06.042471 140369481070464 interactiveshell.py:2882] Step 3600 | Loss: 0.0148 | Loss_intent: 0.0005 | Loss_slots: 0.0143 | Spent: 3.7 secs | LR: 0.000205
I0715 07:05:09.712712 140369481070464 interactiveshell.py:2882] Step 3650 | Loss: 0.0319 | Loss_intent: 0.0000 | Loss_slots: 0.0319 | Spent: 3.7 secs | LR: 0.000204
I0715 07:05:13.395640 140369481070464 interactiveshell.py:2882] Step 3700 | Loss: 0.0026 | Loss_intent: 0.0000 | Loss_slots: 0.0025 | Spent: 3.7 secs | LR: 0.000203
Reading ../data/atis.test.w-intent.iob
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.
  'precision', 'predicted', average, warn_for)
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1439: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples.
  'recall', 'true', average, warn_for)
I0715 07:05:17.993556 140369481070464 interactiveshell.py:2882] 
                                          precision    recall  f1-score   support

                             atis_flight      0.984     0.989     0.987       632
                            atis_airfare      0.941     1.000     0.970        48
                     atis_ground_service      1.000     1.000     1.000        36
                            atis_airline      1.000     1.000     1.000        38
                       atis_abbreviation      0.943     1.000     0.971        33
                           atis_aircraft      0.889     0.889     0.889         9
                        atis_flight_time      1.000     1.000     1.000         1
                           atis_quantity      0.429     1.000     0.600         3
                atis_flight#atis_airfare      0.833     0.417     0.556        12
                            atis_airport      0.947     1.000     0.973        18
                           atis_distance      1.000     1.000     1.000        10
                               atis_city      1.000     0.667     0.800         6
                        atis_ground_fare      1.000     0.714     0.833         7
                           atis_capacity      0.955     1.000     0.977        21
                          atis_flight_no      1.000     1.000     1.000         8
                               atis_meal      1.000     1.000     1.000         6
                        atis_restriction      0.000     0.000     0.000         0
             atis_airline#atis_flight_no      0.000     0.000     0.000         0
    atis_ground_service#atis_ground_fare      0.000     0.000     0.000         0
           atis_airfare#atis_flight_time      0.000     0.000     0.000         0
                           atis_cheapest      0.000     0.000     0.000         0
atis_aircraft#atis_flight#atis_flight_no      0.000     0.000     0.000         0

                               micro avg      0.973     0.979     0.976       888
                               macro avg      0.678     0.667     0.662       888
                            weighted avg      0.976     0.979     0.975       888

I0715 07:05:18.035223 140369481070464 interactiveshell.py:2882] 
                              precision    recall  f1-score   support

                           O      0.000     0.000     0.000       0.0
           B-toloc.city_name      0.974     0.997     0.986     716.0
         B-fromloc.city_name      0.983     0.997     0.990     704.0
           I-toloc.city_name      0.967     1.000     0.983     265.0
      B-depart_date.day_name      0.959     0.986     0.972     212.0
              B-airline_name      1.000     0.990     0.995     101.0
         I-fromloc.city_name      0.989     0.994     0.992     177.0
 B-depart_time.period_of_day      0.991     0.862     0.922     130.0
              I-airline_name      0.970     1.000     0.985      65.0
    B-depart_date.day_number      0.964     0.964     0.964      55.0
    B-depart_date.month_name      0.982     0.964     0.973      56.0
          B-depart_time.time      0.848     0.982     0.911      57.0
                B-round_trip      1.000     0.973     0.986      73.0
             B-cost_relative      1.000     0.973     0.986      37.0
                I-round_trip      0.986     1.000     0.993      71.0
                B-flight_mod      0.786     0.917     0.846      24.0
 B-depart_time.time_relative      0.984     0.969     0.977      65.0
          I-depart_time.time      0.963     1.000     0.981      52.0
         B-stoploc.city_name      0.952     1.000     0.976      20.0
                 B-city_name      0.895     0.596     0.716      57.0
                B-class_type      0.960     1.000     0.980      24.0
          B-arrive_time.time      0.868     0.971     0.917      34.0
 B-arrive_time.time_relative      0.964     0.871     0.915      31.0
                I-class_type      1.000     1.000     1.000      17.0
               B-flight_stop      1.000     1.000     1.000      21.0
          I-arrive_time.time      0.971     0.971     0.971      35.0
              B-airline_code      0.917     0.971     0.943      34.0
    I-depart_date.day_number      1.000     0.933     0.966      15.0
      I-fromloc.airport_name      0.433     0.867     0.578      15.0
      B-fromloc.airport_name      0.333     0.667     0.444      12.0
      B-arrive_date.day_name      0.786     1.000     0.880      11.0
          B-toloc.state_code      1.000     1.000     1.000      18.0
B-depart_date.today_relative      1.000     1.000     1.000       9.0
             B-flight_number      0.733     1.000     0.846      11.0
 B-depart_date.date_relative      0.944     1.000     0.971      17.0
          B-toloc.state_name      0.743     0.929     0.825      28.0
           B-fare_basis_code      0.850     1.000     0.919      17.0
               B-flight_time      1.000     1.000     1.000       1.0
                        B-or      1.000     1.000     1.000       3.0
 B-arrive_time.period_of_day      0.600     1.000     0.750       6.0
          B-meal_description      1.000     0.800     0.889      10.0
             I-cost_relative      1.000     0.667     0.800       3.0
              I-airport_name      0.867     0.448     0.591      29.0
               B-fare_amount      0.500     1.000     0.667       2.0
               I-fare_amount      1.000     1.000     1.000       2.0
                 I-city_name      0.895     0.567     0.694      30.0
        I-toloc.airport_name      1.000     1.000     1.000       3.0
            B-transport_type      0.909     1.000     0.952      10.0
    B-arrive_date.month_name      0.714     0.833     0.769       6.0
    B-arrive_date.day_number      0.714     0.833     0.769       6.0
         I-stoploc.city_name      0.875     0.700     0.778      10.0
                      B-meal      0.889     1.000     0.941      16.0
        B-fromloc.state_code      1.000     1.000     1.000      23.0
    B-depart_time.period_mod      0.833     1.000     0.909       5.0
                   B-connect      1.000     1.000     1.000       6.0
               B-flight_days      1.000     1.000     1.000      10.0
        B-toloc.airport_name      1.000     0.333     0.500       3.0
        B-fromloc.state_name      1.000     0.941     0.970      17.0
              B-airport_name      0.667     0.286     0.400      21.0
                   B-economy      1.000     1.000     1.000       6.0
               I-flight_time      1.000     1.000     1.000       1.0
             B-aircraft_code      1.000     0.848     0.918      33.0
                       B-mod      1.000     0.500     0.667       2.0
              B-airport_code      0.571     0.444     0.500       9.0
    B-depart_time.start_time      0.750     1.000     0.857       3.0
      B-depart_time.end_time      1.000     0.667     0.800       3.0
          B-depart_date.year      1.000     0.667     0.800       3.0
            I-transport_type      0.000     0.000     0.000       1.0
          B-restriction_code      1.000     0.500     0.667       4.0
    B-arrive_time.start_time      1.000     1.000     1.000       8.0
        B-toloc.airport_code      1.000     0.750     0.857       4.0
      B-arrive_time.end_time      0.875     0.875     0.875       8.0
      I-arrive_time.end_time      0.875     0.875     0.875       8.0
      I-depart_time.end_time      1.000     0.667     0.800       3.0
               I-flight_stop      0.000     0.000     0.000       0.0
      B-fromloc.airport_code      0.667     0.800     0.727       5.0
          I-restriction_code      1.000     1.000     1.000       3.0
    I-depart_time.start_time      1.000     1.000     1.000       1.0
          I-toloc.state_name      0.500     1.000     0.667       1.0
I-depart_date.today_relative      0.000     0.000     0.000       0.0
 B-arrive_date.date_relative      1.000     1.000     1.000       2.0
                I-flight_mod      0.000     0.000     0.000       6.0
                   I-economy      0.000     0.000     0.000       0.0
 B-return_date.date_relative      1.000     0.333     0.500       3.0
        I-fromloc.state_name      0.000     0.000     0.000       1.0
                B-state_code      1.000     1.000     1.000       1.0
    I-arrive_time.start_time      1.000     1.000     1.000       1.0
    I-arrive_date.day_number      0.000     0.000     0.000       0.0
                 B-meal_code      0.000     0.000     0.000       1.0
 I-depart_time.period_of_day      0.000     0.000     0.000       1.0
                  B-day_name      1.000     0.500     0.667       2.0
             B-period_of_day      1.000     0.750     0.857       4.0
        B-stoploc.state_code      0.000     0.000     0.000       0.0
    B-return_date.month_name      0.000     0.000     0.000       0.0
    B-return_date.day_number      0.000     0.000     0.000       0.0
    B-arrive_time.period_mod      0.000     0.000     0.000       0.0
                 I-meal_code      0.000     0.000     0.000       0.0
        B-toloc.country_name      0.000     0.000     0.000       1.0
                 B-days_code      0.000     0.000     0.000       1.0
 I-arrive_time.period_of_day      0.000     0.000     0.000       0.0
            I-today_relative      0.000     0.000     0.000       0.0
 B-return_time.period_of_day      0.000     0.000     0.000       0.0
                      B-time      0.000     0.000     0.000       0.0
           I-fare_basis_code      0.000     0.000     0.000       0.0
 I-arrive_time.time_relative      0.667     1.000     0.800       4.0
 I-depart_time.time_relative      0.000     0.000     0.000       1.0
            B-today_relative      0.000     0.000     0.000       0.0
                B-state_name      0.000     0.000     0.000       9.0
B-arrive_date.today_relative      0.000     0.000     0.000       0.0
    B-return_time.period_mod      0.000     0.000     0.000       0.0
                B-month_name      0.000     0.000     0.000       0.0
                B-day_number      0.000     0.000     0.000       0.0
 I-return_date.date_relative      0.000     0.000     0.000       3.0
I-return_date.today_relative      0.000     0.000     0.000       0.0
      B-stoploc.airport_name      0.000     0.000     0.000       0.0
             B-time_relative      0.000     0.000     0.000       0.0
                      I-time      0.000     0.000     0.000       0.0
    I-return_date.day_number      0.000     0.000     0.000       0.0
          I-meal_description      0.000     0.000     0.000       0.0
B-return_date.today_relative      0.000     0.000     0.000       0.0
      B-return_date.day_name      0.000     0.000     0.000       2.0

                   micro avg      0.948     0.950     0.949    3657.0
                   macro avg      0.620     0.603     0.599    3657.0
                weighted avg      0.949     0.950     0.946    3657.0

I0715 07:05:18.036142 140369481070464 interactiveshell.py:2882] Best Slot F1: 0.949
Reading ../data/atis.train.w-intent.iob
I0715 07:05:19.251488 140369481070464 interactiveshell.py:2882] Step 3750 | Loss: 0.0049 | Loss_intent: 0.0000 | Loss_slots: 0.0049 | Spent: 5.9 secs | LR: 0.000202
I0715 07:05:22.945726 140369481070464 interactiveshell.py:2882] Step 3800 | Loss: 0.0522 | Loss_intent: 0.0000 | Loss_slots: 0.0522 | Spent: 3.7 secs | LR: 0.000201
I0715 07:05:26.622527 140369481070464 interactiveshell.py:2882] Step 3850 | Loss: 0.0133 | Loss_intent: 0.0008 | Loss_slots: 0.0125 | Spent: 3.7 secs | LR: 0.000200
I0715 07:05:30.301542 140369481070464 interactiveshell.py:2882] Step 3900 | Loss: 0.0202 | Loss_intent: 0.0000 | Loss_slots: 0.0202 | Spent: 3.7 secs | LR: 0.000199
I0715 07:05:33.956987 140369481070464 interactiveshell.py:2882] Step 3950 | Loss: 0.0051 | Loss_intent: 0.0008 | Loss_slots: 0.0042 | Spent: 3.7 secs | LR: 0.000198
I0715 07:05:37.628471 140369481070464 interactiveshell.py:2882] Step 4000 | Loss: 0.0021 | Loss_intent: 0.0000 | Loss_slots: 0.0020 | Spent: 3.7 secs | LR: 0.000197
I0715 07:05:41.341006 140369481070464 interactiveshell.py:2882] Step 4050 | Loss: 0.0145 | Loss_intent: 0.0000 | Loss_slots: 0.0145 | Spent: 3.7 secs | LR: 0.000196
Reading ../data/atis.test.w-intent.iob
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.
  'precision', 'predicted', average, warn_for)
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1439: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples.
  'recall', 'true', average, warn_for)
I0715 07:05:43.166983 140369481070464 interactiveshell.py:2882] 
                                          precision    recall  f1-score   support

                             atis_flight      0.981     0.989     0.985       632
                            atis_airfare      0.941     1.000     0.970        48
                     atis_ground_service      1.000     1.000     1.000        36
                            atis_airline      1.000     1.000     1.000        38
                       atis_abbreviation      0.943     1.000     0.971        33
                           atis_aircraft      0.692     1.000     0.818         9
                        atis_flight_time      1.000     1.000     1.000         1
                           atis_quantity      0.429     1.000     0.600         3
                atis_flight#atis_airfare      0.750     0.250     0.375        12
                            atis_airport      1.000     1.000     1.000        18
                           atis_distance      1.000     1.000     1.000        10
                               atis_city      1.000     0.667     0.800         6
                        atis_ground_fare      1.000     0.714     0.833         7
                           atis_capacity      1.000     0.952     0.976        21
                          atis_flight_no      1.000     1.000     1.000         8
                               atis_meal      1.000     0.833     0.909         6
                        atis_restriction      0.000     0.000     0.000         0
             atis_airline#atis_flight_no      0.000     0.000     0.000         0
    atis_ground_service#atis_ground_fare      0.000     0.000     0.000         0
           atis_airfare#atis_flight_time      0.000     0.000     0.000         0
                           atis_cheapest      0.000     0.000     0.000         0
atis_aircraft#atis_flight#atis_flight_no      0.000     0.000     0.000         0

                               micro avg      0.970     0.975     0.972       888
                               macro avg      0.670     0.655     0.647       888
                            weighted avg      0.973     0.975     0.971       888

I0715 07:05:43.206777 140369481070464 interactiveshell.py:2882] 
                              precision    recall  f1-score   support

                           O      0.000     0.000     0.000       0.0
           B-toloc.city_name      0.973     0.992     0.982     716.0
         B-fromloc.city_name      0.985     0.997     0.991     704.0
           I-toloc.city_name      0.974     0.996     0.985     265.0
      B-depart_date.day_name      0.972     0.986     0.979     212.0
              B-airline_name      0.990     0.990     0.990     101.0
         I-fromloc.city_name      0.994     0.994     0.994     177.0
 B-depart_time.period_of_day      0.991     0.877     0.931     130.0
              I-airline_name      1.000     1.000     1.000      65.0
    B-depart_date.day_number      0.964     0.982     0.973      55.0
    B-depart_date.month_name      0.982     0.964     0.973      56.0
          B-depart_time.time      0.875     0.982     0.926      57.0
                B-round_trip      1.000     0.986     0.993      73.0
             B-cost_relative      1.000     0.973     0.986      37.0
                I-round_trip      1.000     1.000     1.000      71.0
                B-flight_mod      0.774     1.000     0.873      24.0
 B-depart_time.time_relative      0.984     0.954     0.969      65.0
          I-depart_time.time      0.963     1.000     0.981      52.0
         B-stoploc.city_name      0.952     1.000     0.976      20.0
                 B-city_name      0.879     0.509     0.644      57.0
                B-class_type      0.960     1.000     0.980      24.0
          B-arrive_time.time      0.892     0.971     0.930      34.0
 B-arrive_time.time_relative      0.909     0.968     0.937      31.0
                I-class_type      1.000     1.000     1.000      17.0
               B-flight_stop      1.000     1.000     1.000      21.0
          I-arrive_time.time      0.971     0.971     0.971      35.0
              B-airline_code      0.944     1.000     0.971      34.0
    I-depart_date.day_number      1.000     1.000     1.000      15.0
      I-fromloc.airport_name      0.378     0.933     0.538      15.0
      B-fromloc.airport_name      0.385     0.833     0.526      12.0
      B-arrive_date.day_name      0.786     1.000     0.880      11.0
          B-toloc.state_code      1.000     1.000     1.000      18.0
B-depart_date.today_relative      1.000     1.000     1.000       9.0
             B-flight_number      0.611     1.000     0.759      11.0
 B-depart_date.date_relative      0.944     1.000     0.971      17.0
          B-toloc.state_name      0.920     0.821     0.868      28.0
           B-fare_basis_code      0.895     1.000     0.944      17.0
               B-flight_time      1.000     1.000     1.000       1.0
                        B-or      0.750     1.000     0.857       3.0
 B-arrive_time.period_of_day      0.667     1.000     0.800       6.0
          B-meal_description      1.000     0.800     0.889      10.0
             I-cost_relative      1.000     0.667     0.800       3.0
              I-airport_name      0.909     0.345     0.500      29.0
               B-fare_amount      0.667     1.000     0.800       2.0
               I-fare_amount      1.000     1.000     1.000       2.0
                 I-city_name      1.000     0.567     0.723      30.0
        I-toloc.airport_name      1.000     1.000     1.000       3.0
            B-transport_type      1.000     0.600     0.750      10.0
    B-arrive_date.month_name      0.714     0.833     0.769       6.0
    B-arrive_date.day_number      0.833     0.833     0.833       6.0
         I-stoploc.city_name      0.833     1.000     0.909      10.0
                      B-meal      0.941     1.000     0.970      16.0
        B-fromloc.state_code      1.000     1.000     1.000      23.0
    B-depart_time.period_mod      0.800     0.800     0.800       5.0
                   B-connect      1.000     1.000     1.000       6.0
               B-flight_days      1.000     1.000     1.000      10.0
        B-toloc.airport_name      1.000     1.000     1.000       3.0
        B-fromloc.state_name      1.000     0.824     0.903      17.0
              B-airport_name      0.700     0.333     0.452      21.0
                   B-economy      0.857     1.000     0.923       6.0
               I-flight_time      1.000     1.000     1.000       1.0
             B-aircraft_code      1.000     0.818     0.900      33.0
                       B-mod      0.000     0.000     0.000       2.0
              B-airport_code      0.800     0.444     0.571       9.0
    B-depart_time.start_time      0.750     1.000     0.857       3.0
      B-depart_time.end_time      1.000     0.667     0.800       3.0
          B-depart_date.year      1.000     0.667     0.800       3.0
            I-transport_type      0.200     1.000     0.333       1.0
          B-restriction_code      1.000     0.500     0.667       4.0
    B-arrive_time.start_time      1.000     1.000     1.000       8.0
        B-toloc.airport_code      1.000     0.750     0.857       4.0
      B-arrive_time.end_time      0.875     0.875     0.875       8.0
      I-arrive_time.end_time      0.875     0.875     0.875       8.0
      I-depart_time.end_time      1.000     0.667     0.800       3.0
               I-flight_stop      0.000     0.000     0.000       0.0
      B-fromloc.airport_code      0.714     1.000     0.833       5.0
          I-restriction_code      1.000     1.000     1.000       3.0
    I-depart_time.start_time      1.000     1.000     1.000       1.0
          I-toloc.state_name      0.500     1.000     0.667       1.0
I-depart_date.today_relative      0.000     0.000     0.000       0.0
 B-arrive_date.date_relative      1.000     1.000     1.000       2.0
                I-flight_mod      0.000     0.000     0.000       6.0
                   I-economy      0.000     0.000     0.000       0.0
 B-return_date.date_relative      1.000     0.333     0.500       3.0
        I-fromloc.state_name      0.333     1.000     0.500       1.0
                B-state_code      1.000     1.000     1.000       1.0
    I-arrive_time.start_time      1.000     1.000     1.000       1.0
    I-arrive_date.day_number      0.000     0.000     0.000       0.0
                 B-meal_code      0.000     0.000     0.000       1.0
 I-depart_time.period_of_day      0.000     0.000     0.000       1.0
                  B-day_name      1.000     0.500     0.667       2.0
             B-period_of_day      1.000     1.000     1.000       4.0
        B-stoploc.state_code      0.000     0.000     0.000       0.0
    B-return_date.month_name      0.000     0.000     0.000       0.0
    B-return_date.day_number      0.000     0.000     0.000       0.0
    B-arrive_time.period_mod      0.000     0.000     0.000       0.0
                 I-meal_code      0.000     0.000     0.000       0.0
        B-toloc.country_name      1.000     1.000     1.000       1.0
                 B-days_code      0.250     1.000     0.400       1.0
 I-arrive_time.period_of_day      0.000     0.000     0.000       0.0
            I-today_relative      0.000     0.000     0.000       0.0
 B-return_time.period_of_day      0.000     0.000     0.000       0.0
                      B-time      0.000     0.000     0.000       0.0
           I-fare_basis_code      0.000     0.000     0.000       0.0
 I-arrive_time.time_relative      1.000     0.500     0.667       4.0
 I-depart_time.time_relative      0.000     0.000     0.000       1.0
            B-today_relative      0.000     0.000     0.000       0.0
                B-state_name      0.833     0.556     0.667       9.0
B-arrive_date.today_relative      0.000     0.000     0.000       0.0
    B-return_time.period_mod      0.000     0.000     0.000       0.0
                B-month_name      0.000     0.000     0.000       0.0
                B-day_number      0.000     0.000     0.000       0.0
 I-return_date.date_relative      1.000     0.333     0.500       3.0
I-return_date.today_relative      0.000     0.000     0.000       0.0
      B-stoploc.airport_name      0.000     0.000     0.000       0.0
             B-time_relative      0.000     0.000     0.000       0.0
                      I-time      0.000     0.000     0.000       0.0
    I-return_date.day_number      0.000     0.000     0.000       0.0
          I-meal_description      0.000     0.000     0.000       0.0
B-return_date.today_relative      0.000     0.000     0.000       0.0
      B-return_date.day_name      0.000     0.000     0.000       2.0

                   micro avg      0.949     0.951     0.950    3657.0
                   macro avg      0.650     0.643     0.627    3657.0
                weighted avg      0.958     0.951     0.949    3657.0

I0715 07:05:43.207766 140369481070464 interactiveshell.py:2882] Best Slot F1: 0.950
Reading ../data/atis.train.w-intent.iob
I0715 07:05:47.258957 140369481070464 interactiveshell.py:2882] Step 4100 | Loss: 0.0044 | Loss_intent: 0.0000 | Loss_slots: 0.0044 | Spent: 5.9 secs | LR: 0.000195
I0715 07:05:50.935283 140369481070464 interactiveshell.py:2882] Step 4150 | Loss: 0.0359 | Loss_intent: 0.0000 | Loss_slots: 0.0359 | Spent: 3.7 secs | LR: 0.000194
I0715 07:05:54.663905 140369481070464 interactiveshell.py:2882] Step 4200 | Loss: 0.0268 | Loss_intent: 0.0001 | Loss_slots: 0.0267 | Spent: 3.7 secs | LR: 0.000193
I0715 07:05:58.362050 140369481070464 interactiveshell.py:2882] Step 4250 | Loss: 0.0033 | Loss_intent: 0.0000 | Loss_slots: 0.0033 | Spent: 3.7 secs | LR: 0.000192
I0715 07:06:02.144828 140369481070464 interactiveshell.py:2882] Step 4300 | Loss: 0.0113 | Loss_intent: 0.0013 | Loss_slots: 0.0101 | Spent: 3.8 secs | LR: 0.000191
I0715 07:06:06.093266 140369481070464 interactiveshell.py:2882] Step 4350 | Loss: 0.0041 | Loss_intent: 0.0000 | Loss_slots: 0.0041 | Spent: 3.9 secs | LR: 0.000190
Reading ../data/atis.test.w-intent.iob
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.
  'precision', 'predicted', average, warn_for)
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1439: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples.
  'recall', 'true', average, warn_for)
I0715 07:06:08.831709 140369481070464 interactiveshell.py:2882] 
                                          precision    recall  f1-score   support

                             atis_flight      0.977     0.992     0.984       632
                            atis_airfare      0.980     1.000     0.990        48
                     atis_ground_service      1.000     1.000     1.000        36
                            atis_airline      1.000     1.000     1.000        38
                       atis_abbreviation      0.917     1.000     0.957        33
                           atis_aircraft      1.000     0.889     0.941         9
                        atis_flight_time      1.000     1.000     1.000         1
                           atis_quantity      0.429     1.000     0.600         3
                atis_flight#atis_airfare      0.750     0.250     0.375        12
                            atis_airport      1.000     0.889     0.941        18
                           atis_distance      1.000     1.000     1.000        10
                               atis_city      1.000     0.667     0.800         6
                        atis_ground_fare      1.000     1.000     1.000         7
                           atis_capacity      0.952     0.952     0.952        21
                          atis_flight_no      0.889     1.000     0.941         8
                               atis_meal      1.000     0.833     0.909         6
                        atis_restriction      0.000     0.000     0.000         0
             atis_airline#atis_flight_no      0.000     0.000     0.000         0
    atis_ground_service#atis_ground_fare      0.000     0.000     0.000         0
           atis_airfare#atis_flight_time      0.000     0.000     0.000         0
                           atis_cheapest      0.000     0.000     0.000         0
atis_aircraft#atis_flight#atis_flight_no      0.000     0.000     0.000         0

                               micro avg      0.971     0.976     0.974       888
                               macro avg      0.677     0.658     0.654       888
                            weighted avg      0.972     0.976     0.971       888

I0715 07:06:08.872613 140369481070464 interactiveshell.py:2882] 
                              precision    recall  f1-score   support

                           O      0.000     0.000     0.000       0.0
           B-toloc.city_name      0.971     0.993     0.982     716.0
         B-fromloc.city_name      0.985     0.997     0.991     704.0
           I-toloc.city_name      0.974     1.000     0.987     265.0
      B-depart_date.day_name      0.963     0.986     0.974     212.0
              B-airline_name      0.990     0.990     0.990     101.0
         I-fromloc.city_name      0.994     0.994     0.994     177.0
 B-depart_time.period_of_day      0.991     0.862     0.922     130.0
              I-airline_name      1.000     1.000     1.000      65.0
    B-depart_date.day_number      0.964     0.964     0.964      55.0
    B-depart_date.month_name      0.982     0.964     0.973      56.0
          B-depart_time.time      0.875     0.982     0.926      57.0
                B-round_trip      1.000     0.986     0.993      73.0
             B-cost_relative      1.000     0.973     0.986      37.0
                I-round_trip      1.000     1.000     1.000      71.0
                B-flight_mod      0.800     1.000     0.889      24.0
 B-depart_time.time_relative      0.984     0.969     0.977      65.0
          I-depart_time.time      0.981     1.000     0.990      52.0
         B-stoploc.city_name      0.952     1.000     0.976      20.0
                 B-city_name      0.933     0.491     0.644      57.0
                B-class_type      0.960     1.000     0.980      24.0
          B-arrive_time.time      0.846     0.971     0.904      34.0
 B-arrive_time.time_relative      0.909     0.968     0.937      31.0
                I-class_type      1.000     1.000     1.000      17.0
               B-flight_stop      1.000     1.000     1.000      21.0
          I-arrive_time.time      0.971     0.971     0.971      35.0
              B-airline_code      0.917     0.971     0.943      34.0
    I-depart_date.day_number      1.000     0.933     0.966      15.0
      I-fromloc.airport_name      0.378     0.933     0.538      15.0
      B-fromloc.airport_name      0.346     0.750     0.474      12.0
      B-arrive_date.day_name      0.733     1.000     0.846      11.0
          B-toloc.state_code      1.000     1.000     1.000      18.0
B-depart_date.today_relative      1.000     0.889     0.941       9.0
             B-flight_number      0.625     0.909     0.741      11.0
 B-depart_date.date_relative      0.944     1.000     0.971      17.0
          B-toloc.state_name      0.862     0.893     0.877      28.0
           B-fare_basis_code      0.909     0.588     0.714      17.0
               B-flight_time      1.000     1.000     1.000       1.0
                        B-or      1.000     1.000     1.000       3.0
 B-arrive_time.period_of_day      0.600     1.000     0.750       6.0
          B-meal_description      1.000     0.800     0.889      10.0
             I-cost_relative      1.000     0.667     0.800       3.0
              I-airport_name      0.846     0.379     0.524      29.0
               B-fare_amount      0.400     1.000     0.571       2.0
               I-fare_amount      1.000     1.000     1.000       2.0
                 I-city_name      1.000     0.567     0.723      30.0
        I-toloc.airport_name      1.000     1.000     1.000       3.0
            B-transport_type      0.889     0.800     0.842      10.0
    B-arrive_date.month_name      0.714     0.833     0.769       6.0
    B-arrive_date.day_number      0.714     0.833     0.769       6.0
         I-stoploc.city_name      0.909     1.000     0.952      10.0
                      B-meal      0.941     1.000     0.970      16.0
        B-fromloc.state_code      1.000     1.000     1.000      23.0
    B-depart_time.period_mod      0.833     1.000     0.909       5.0
                   B-connect      1.000     1.000     1.000       6.0
               B-flight_days      1.000     1.000     1.000      10.0
        B-toloc.airport_name      1.000     1.000     1.000       3.0
        B-fromloc.state_name      0.941     0.941     0.941      17.0
              B-airport_name      0.778     0.333     0.467      21.0
                   B-economy      1.000     1.000     1.000       6.0
               I-flight_time      1.000     1.000     1.000       1.0
             B-aircraft_code      1.000     0.758     0.862      33.0
                       B-mod      1.000     0.500     0.667       2.0
              B-airport_code      0.667     0.444     0.533       9.0
    B-depart_time.start_time      0.500     1.000     0.667       3.0
      B-depart_time.end_time      1.000     0.667     0.800       3.0
          B-depart_date.year      0.667     0.667     0.667       3.0
            I-transport_type      0.000     0.000     0.000       1.0
          B-restriction_code      1.000     0.500     0.667       4.0
    B-arrive_time.start_time      1.000     1.000     1.000       8.0
        B-toloc.airport_code      1.000     0.750     0.857       4.0
      B-arrive_time.end_time      0.875     0.875     0.875       8.0
      I-arrive_time.end_time      0.875     0.875     0.875       8.0
      I-depart_time.end_time      1.000     0.667     0.800       3.0
               I-flight_stop      0.000     0.000     0.000       0.0
      B-fromloc.airport_code      0.714     1.000     0.833       5.0
          I-restriction_code      1.000     1.000     1.000       3.0
    I-depart_time.start_time      1.000     1.000     1.000       1.0
          I-toloc.state_name      1.000     1.000     1.000       1.0
I-depart_date.today_relative      0.000     0.000     0.000       0.0
 B-arrive_date.date_relative      1.000     1.000     1.000       2.0
                I-flight_mod      0.000     0.000     0.000       6.0
                   I-economy      0.000     0.000     0.000       0.0
 B-return_date.date_relative      0.000     0.000     0.000       3.0
        I-fromloc.state_name      1.000     1.000     1.000       1.0
                B-state_code      1.000     1.000     1.000       1.0
    I-arrive_time.start_time      1.000     1.000     1.000       1.0
    I-arrive_date.day_number      0.000     0.000     0.000       0.0
                 B-meal_code      1.000     1.000     1.000       1.0
 I-depart_time.period_of_day      0.000     0.000     0.000       1.0
                  B-day_name      1.000     0.500     0.667       2.0
             B-period_of_day      1.000     0.750     0.857       4.0
        B-stoploc.state_code      0.000     0.000     0.000       0.0
    B-return_date.month_name      0.000     0.000     0.000       0.0
    B-return_date.day_number      0.000     0.000     0.000       0.0
    B-arrive_time.period_mod      0.000     0.000     0.000       0.0
                 I-meal_code      0.000     0.000     0.000       0.0
        B-toloc.country_name      1.000     1.000     1.000       1.0
                 B-days_code      1.000     1.000     1.000       1.0
 I-arrive_time.period_of_day      0.000     0.000     0.000       0.0
            I-today_relative      0.000     0.000     0.000       0.0
 B-return_time.period_of_day      0.000     0.000     0.000       0.0
                      B-time      0.000     0.000     0.000       0.0
           I-fare_basis_code      0.000     0.000     0.000       0.0
 I-arrive_time.time_relative      1.000     0.500     0.667       4.0
 I-depart_time.time_relative      0.000     0.000     0.000       1.0
            B-today_relative      0.000     0.000     0.000       0.0
                B-state_name      1.000     0.444     0.615       9.0
B-arrive_date.today_relative      0.000     0.000     0.000       0.0
    B-return_time.period_mod      0.000     0.000     0.000       0.0
                B-month_name      0.000     0.000     0.000       0.0
                B-day_number      0.000     0.000     0.000       0.0
 I-return_date.date_relative      1.000     0.333     0.500       3.0
I-return_date.today_relative      0.000     0.000     0.000       0.0
      B-stoploc.airport_name      0.000     0.000     0.000       0.0
             B-time_relative      0.000     0.000     0.000       0.0
                      I-time      0.000     0.000     0.000       0.0
    I-return_date.day_number      0.000     0.000     0.000       0.0
          I-meal_description      0.000     0.000     0.000       0.0
B-return_date.today_relative      0.000     0.000     0.000       0.0
      B-return_date.day_name      0.000     0.000     0.000       2.0

                   micro avg      0.947     0.948     0.948    3657.0
                   macro avg      0.666     0.639     0.638    3657.0
                weighted avg      0.957     0.948     0.947    3657.0

I0715 07:06:08.873544 140369481070464 interactiveshell.py:2882] Best Slot F1: 0.950
Reading ../data/atis.train.w-intent.iob
I0715 07:06:12.039464 140369481070464 interactiveshell.py:2882] Step 4400 | Loss: 0.0109 | Loss_intent: 0.0038 | Loss_slots: 0.0071 | Spent: 5.9 secs | LR: 0.000189
I0715 07:06:15.837241 140369481070464 interactiveshell.py:2882] Step 4450 | Loss: 0.0029 | Loss_intent: 0.0001 | Loss_slots: 0.0027 | Spent: 3.8 secs | LR: 0.000188
I0715 07:06:20.120241 140369481070464 interactiveshell.py:2882] Step 4500 | Loss: 0.0117 | Loss_intent: 0.0000 | Loss_slots: 0.0117 | Spent: 4.3 secs | LR: 0.000187
I0715 07:06:24.238496 140369481070464 interactiveshell.py:2882] Step 4550 | Loss: 0.0056 | Loss_intent: 0.0000 | Loss_slots: 0.0056 | Spent: 4.1 secs | LR: 0.000186
I0715 07:06:28.893234 140369481070464 interactiveshell.py:2882] Step 4600 | Loss: 0.0024 | Loss_intent: 0.0000 | Loss_slots: 0.0024 | Spent: 4.7 secs | LR: 0.000185
I0715 07:06:32.598230 140369481070464 interactiveshell.py:2882] Step 4650 | Loss: 0.0065 | Loss_intent: 0.0000 | Loss_slots: 0.0065 | Spent: 3.7 secs | LR: 0.000184
Reading ../data/atis.test.w-intent.iob
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.
  'precision', 'predicted', average, warn_for)
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1439: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples.
  'recall', 'true', average, warn_for)
I0715 07:06:36.212586 140369481070464 interactiveshell.py:2882] 
                                          precision    recall  f1-score   support

                             atis_flight      0.981     0.991     0.986       632
                            atis_airfare      0.980     1.000     0.990        48
                     atis_ground_service      0.947     1.000     0.973        36
                            atis_airline      1.000     1.000     1.000        38
                       atis_abbreviation      0.941     0.970     0.955        33
                           atis_aircraft      1.000     0.889     0.941         9
                        atis_flight_time      1.000     1.000     1.000         1
                           atis_quantity      0.333     1.000     0.500         3
                atis_flight#atis_airfare      0.857     0.500     0.632        12
                            atis_airport      0.944     0.944     0.944        18
                           atis_distance      1.000     1.000     1.000        10
                               atis_city      1.000     0.500     0.667         6
                        atis_ground_fare      1.000     0.714     0.833         7
                           atis_capacity      1.000     0.952     0.976        21
                          atis_flight_no      0.889     1.000     0.941         8
                               atis_meal      1.000     0.833     0.909         6
                        atis_restriction      0.000     0.000     0.000         0
             atis_airline#atis_flight_no      0.000     0.000     0.000         0
    atis_ground_service#atis_ground_fare      0.000     0.000     0.000         0
           atis_airfare#atis_flight_time      0.000     0.000     0.000         0
                           atis_cheapest      0.000     0.000     0.000         0
atis_aircraft#atis_flight#atis_flight_no      0.000     0.000     0.000         0

                               micro avg      0.970     0.975     0.972       888
                               macro avg      0.676     0.650     0.648       888
                            weighted avg      0.975     0.975     0.973       888

I0715 07:06:36.253521 140369481070464 interactiveshell.py:2882] 
                              precision    recall  f1-score   support

                           O      0.000     0.000     0.000       0.0
           B-toloc.city_name      0.967     0.996     0.981     716.0
         B-fromloc.city_name      0.987     0.997     0.992     704.0
           I-toloc.city_name      0.953     1.000     0.976     265.0
      B-depart_date.day_name      0.981     0.986     0.984     212.0
              B-airline_name      0.990     0.990     0.990     101.0
         I-fromloc.city_name      0.994     0.977     0.986     177.0
 B-depart_time.period_of_day      0.991     0.885     0.935     130.0
              I-airline_name      1.000     1.000     1.000      65.0
    B-depart_date.day_number      0.964     0.964     0.964      55.0
    B-depart_date.month_name      0.982     0.964     0.973      56.0
          B-depart_time.time      0.875     0.982     0.926      57.0
                B-round_trip      1.000     0.986     0.993      73.0
             B-cost_relative      1.000     0.973     0.986      37.0
                I-round_trip      1.000     1.000     1.000      71.0
                B-flight_mod      0.821     0.958     0.885      24.0
 B-depart_time.time_relative      0.984     0.954     0.969      65.0
          I-depart_time.time      0.929     1.000     0.963      52.0
         B-stoploc.city_name      1.000     1.000     1.000      20.0
                 B-city_name      0.963     0.456     0.619      57.0
                B-class_type      0.960     1.000     0.980      24.0
          B-arrive_time.time      0.805     0.971     0.880      34.0
 B-arrive_time.time_relative      0.900     0.871     0.885      31.0
                I-class_type      1.000     1.000     1.000      17.0
               B-flight_stop      1.000     1.000     1.000      21.0
          I-arrive_time.time      0.971     0.971     0.971      35.0
              B-airline_code      0.970     0.941     0.955      34.0
    I-depart_date.day_number      1.000     0.933     0.966      15.0
      I-fromloc.airport_name      0.368     0.933     0.528      15.0
      B-fromloc.airport_name      0.409     0.750     0.529      12.0
      B-arrive_date.day_name      0.786     1.000     0.880      11.0
          B-toloc.state_code      1.000     1.000     1.000      18.0
B-depart_date.today_relative      1.000     0.889     0.941       9.0
             B-flight_number      0.625     0.909     0.741      11.0
 B-depart_date.date_relative      0.944     1.000     0.971      17.0
          B-toloc.state_name      0.839     0.929     0.881      28.0
           B-fare_basis_code      0.944     1.000     0.971      17.0
               B-flight_time      1.000     1.000     1.000       1.0
                        B-or      0.750     1.000     0.857       3.0
 B-arrive_time.period_of_day      0.667     1.000     0.800       6.0
          B-meal_description      1.000     0.800     0.889      10.0
             I-cost_relative      1.000     0.667     0.800       3.0
              I-airport_name      0.909     0.345     0.500      29.0
               B-fare_amount      1.000     1.000     1.000       2.0
               I-fare_amount      1.000     1.000     1.000       2.0
                 I-city_name      1.000     0.533     0.696      30.0
        I-toloc.airport_name      1.000     1.000     1.000       3.0
            B-transport_type      1.000     0.600     0.750      10.0
    B-arrive_date.month_name      0.714     0.833     0.769       6.0
    B-arrive_date.day_number      0.714     0.833     0.769       6.0
         I-stoploc.city_name      0.900     0.900     0.900      10.0
                      B-meal      0.941     1.000     0.970      16.0
        B-fromloc.state_code      1.000     1.000     1.000      23.0
    B-depart_time.period_mod      1.000     1.000     1.000       5.0
                   B-connect      0.857     1.000     0.923       6.0
               B-flight_days      1.000     1.000     1.000      10.0
        B-toloc.airport_name      1.000     1.000     1.000       3.0
        B-fromloc.state_name      0.727     0.941     0.821      17.0
              B-airport_name      0.750     0.429     0.545      21.0
                   B-economy      1.000     1.000     1.000       6.0
               I-flight_time      1.000     1.000     1.000       1.0
             B-aircraft_code      0.963     0.788     0.867      33.0
                       B-mod      1.000     0.500     0.667       2.0
              B-airport_code      0.667     0.444     0.533       9.0
    B-depart_time.start_time      0.500     1.000     0.667       3.0
      B-depart_time.end_time      1.000     0.667     0.800       3.0
          B-depart_date.year      0.750     1.000     0.857       3.0
            I-transport_type      0.167     1.000     0.286       1.0
          B-restriction_code      1.000     1.000     1.000       4.0
    B-arrive_time.start_time      1.000     0.875     0.933       8.0
        B-toloc.airport_code      1.000     0.750     0.857       4.0
      B-arrive_time.end_time      0.875     0.875     0.875       8.0
      I-arrive_time.end_time      0.875     0.875     0.875       8.0
      I-depart_time.end_time      1.000     0.667     0.800       3.0
               I-flight_stop      0.000     0.000     0.000       0.0
      B-fromloc.airport_code      0.625     1.000     0.769       5.0
          I-restriction_code      1.000     1.000     1.000       3.0
    I-depart_time.start_time      1.000     1.000     1.000       1.0
          I-toloc.state_name      1.000     1.000     1.000       1.0
I-depart_date.today_relative      0.000     0.000     0.000       0.0
 B-arrive_date.date_relative      1.000     1.000     1.000       2.0
                I-flight_mod      0.000     0.000     0.000       6.0
                   I-economy      0.000     0.000     0.000       0.0
 B-return_date.date_relative      1.000     0.333     0.500       3.0
        I-fromloc.state_name      0.000     0.000     0.000       1.0
                B-state_code      1.000     1.000     1.000       1.0
    I-arrive_time.start_time      1.000     1.000     1.000       1.0
    I-arrive_date.day_number      0.000     0.000     0.000       0.0
                 B-meal_code      1.000     1.000     1.000       1.0
 I-depart_time.period_of_day      1.000     1.000     1.000       1.0
                  B-day_name      1.000     0.500     0.667       2.0
             B-period_of_day      1.000     1.000     1.000       4.0
        B-stoploc.state_code      0.000     0.000     0.000       0.0
    B-return_date.month_name      0.000     0.000     0.000       0.0
    B-return_date.day_number      0.000     0.000     0.000       0.0
    B-arrive_time.period_mod      0.000     0.000     0.000       0.0
                 I-meal_code      0.000     0.000     0.000       0.0
        B-toloc.country_name      0.000     0.000     0.000       1.0
                 B-days_code      1.000     1.000     1.000       1.0
 I-arrive_time.period_of_day      0.000     0.000     0.000       0.0
            I-today_relative      0.000     0.000     0.000       0.0
 B-return_time.period_of_day      0.000     0.000     0.000       0.0
                      B-time      0.000     0.000     0.000       0.0
           I-fare_basis_code      0.000     0.000     0.000       0.0
 I-arrive_time.time_relative      0.667     0.500     0.571       4.0
 I-depart_time.time_relative      0.000     0.000     0.000       1.0
            B-today_relative      0.000     0.000     0.000       0.0
                B-state_name      0.000     0.000     0.000       9.0
B-arrive_date.today_relative      0.000     0.000     0.000       0.0
    B-return_time.period_mod      0.000     0.000     0.000       0.0
                B-month_name      0.000     0.000     0.000       0.0
                B-day_number      0.000     0.000     0.000       0.0
 I-return_date.date_relative      0.000     0.000     0.000       3.0
I-return_date.today_relative      0.000     0.000     0.000       0.0
      B-stoploc.airport_name      0.000     0.000     0.000       0.0
             B-time_relative      0.000     0.000     0.000       0.0
                      I-time      0.000     0.000     0.000       0.0
    I-return_date.day_number      0.000     0.000     0.000       0.0
          I-meal_description      0.000     0.000     0.000       0.0
B-return_date.today_relative      0.000     0.000     0.000       0.0
      B-return_date.day_name      0.000     0.000     0.000       2.0

                   micro avg      0.946     0.948     0.947    3657.0
                   macro avg      0.652     0.643     0.634    3657.0
                weighted avg      0.953     0.948     0.945    3657.0

I0715 07:06:36.254346 140369481070464 interactiveshell.py:2882] Best Slot F1: 0.950
Reading ../data/atis.train.w-intent.iob
I0715 07:06:38.487998 140369481070464 interactiveshell.py:2882] Step 4700 | Loss: 0.0077 | Loss_intent: 0.0000 | Loss_slots: 0.0077 | Spent: 5.9 secs | LR: 0.000183
I0715 07:06:42.201415 140369481070464 interactiveshell.py:2882] Step 4750 | Loss: 0.0020 | Loss_intent: 0.0000 | Loss_slots: 0.0020 | Spent: 3.7 secs | LR: 0.000182
I0715 07:06:46.032975 140369481070464 interactiveshell.py:2882] Step 4800 | Loss: 0.0016 | Loss_intent: 0.0000 | Loss_slots: 0.0016 | Spent: 3.8 secs | LR: 0.000181
I0715 07:06:49.768727 140369481070464 interactiveshell.py:2882] Step 4850 | Loss: 0.1327 | Loss_intent: 0.1146 | Loss_slots: 0.0181 | Spent: 3.7 secs | LR: 0.000180
I0715 07:06:53.486173 140369481070464 interactiveshell.py:2882] Step 4900 | Loss: 0.0022 | Loss_intent: 0.0000 | Loss_slots: 0.0021 | Spent: 3.7 secs | LR: 0.000179
I0715 07:06:57.455725 140369481070464 interactiveshell.py:2882] Step 4950 | Loss: 0.0338 | Loss_intent: 0.0000 | Loss_slots: 0.0338 | Spent: 4.0 secs | LR: 0.000178
Reading ../data/atis.test.w-intent.iob
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.
  'precision', 'predicted', average, warn_for)
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1439: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples.
  'recall', 'true', average, warn_for)
I0715 07:07:02.877921 140369481070464 interactiveshell.py:2882] 
                                          precision    recall  f1-score   support

                             atis_flight      0.981     0.989     0.985       632
                            atis_airfare      0.980     1.000     0.990        48
                     atis_ground_service      0.947     1.000     0.973        36
                            atis_airline      1.000     1.000     1.000        38
                       atis_abbreviation      1.000     0.970     0.985        33
                           atis_aircraft      0.889     0.889     0.889         9
                        atis_flight_time      1.000     1.000     1.000         1
                           atis_quantity      0.375     1.000     0.545         3
                atis_flight#atis_airfare      0.833     0.417     0.556        12
                            atis_airport      0.941     0.889     0.914        18
                           atis_distance      1.000     1.000     1.000        10
                               atis_city      1.000     1.000     1.000         6
                        atis_ground_fare      1.000     0.714     0.833         7
                           atis_capacity      1.000     0.952     0.976        21
                          atis_flight_no      1.000     1.000     1.000         8
                               atis_meal      1.000     1.000     1.000         6
                        atis_restriction      0.000     0.000     0.000         0
             atis_airline#atis_flight_no      0.000     0.000     0.000         0
    atis_ground_service#atis_ground_fare      0.000     0.000     0.000         0
           atis_airfare#atis_flight_time      0.000     0.000     0.000         0
                           atis_cheapest      0.000     0.000     0.000         0
atis_aircraft#atis_flight#atis_flight_no      0.000     0.000     0.000         0

                               micro avg      0.971     0.976     0.974       888
                               macro avg      0.679     0.674     0.666       888
                            weighted avg      0.977     0.976     0.975       888

I0715 07:07:02.921999 140369481070464 interactiveshell.py:2882] 
                              precision    recall  f1-score   support

                           O      0.000     0.000     0.000       0.0
           B-toloc.city_name      0.973     0.993     0.983     716.0
         B-fromloc.city_name      0.985     0.994     0.989     704.0
           I-toloc.city_name      0.978     0.992     0.985     265.0
      B-depart_date.day_name      0.981     0.986     0.984     212.0
              B-airline_name      0.971     0.990     0.980     101.0
         I-fromloc.city_name      0.994     0.994     0.994     177.0
 B-depart_time.period_of_day      1.000     0.885     0.939     130.0
              I-airline_name      1.000     1.000     1.000      65.0
    B-depart_date.day_number      0.981     0.964     0.972      55.0
    B-depart_date.month_name      0.982     0.964     0.973      56.0
          B-depart_time.time      0.862     0.982     0.918      57.0
                B-round_trip      1.000     0.986     0.993      73.0
             B-cost_relative      1.000     0.973     0.986      37.0
                I-round_trip      0.986     1.000     0.993      71.0
                B-flight_mod      0.815     0.917     0.863      24.0
 B-depart_time.time_relative      0.984     0.969     0.977      65.0
          I-depart_time.time      0.927     0.981     0.953      52.0
         B-stoploc.city_name      1.000     1.000     1.000      20.0
                 B-city_name      0.941     0.561     0.703      57.0
                B-class_type      0.960     1.000     0.980      24.0
          B-arrive_time.time      0.868     0.971     0.917      34.0
 B-arrive_time.time_relative      0.875     0.903     0.889      31.0
                I-class_type      1.000     1.000     1.000      17.0
               B-flight_stop      1.000     1.000     1.000      21.0
          I-arrive_time.time      0.944     0.971     0.958      35.0
              B-airline_code      0.970     0.941     0.955      34.0
    I-depart_date.day_number      1.000     1.000     1.000      15.0
      I-fromloc.airport_name      0.385     1.000     0.556      15.0
      B-fromloc.airport_name      0.367     0.917     0.524      12.0
      B-arrive_date.day_name      0.733     1.000     0.846      11.0
          B-toloc.state_code      1.000     1.000     1.000      18.0
B-depart_date.today_relative      1.000     1.000     1.000       9.0
             B-flight_number      0.611     1.000     0.759      11.0
 B-depart_date.date_relative      0.895     1.000     0.944      17.0
          B-toloc.state_name      0.781     0.893     0.833      28.0
           B-fare_basis_code      0.944     1.000     0.971      17.0
               B-flight_time      1.000     1.000     1.000       1.0
                        B-or      1.000     1.000     1.000       3.0
 B-arrive_time.period_of_day      0.667     1.000     0.800       6.0
          B-meal_description      1.000     0.900     0.947      10.0
             I-cost_relative      1.000     0.667     0.800       3.0
              I-airport_name      0.909     0.345     0.500      29.0
               B-fare_amount      1.000     1.000     1.000       2.0
               I-fare_amount      1.000     1.000     1.000       2.0
                 I-city_name      1.000     0.600     0.750      30.0
        I-toloc.airport_name      1.000     1.000     1.000       3.0
            B-transport_type      1.000     1.000     1.000      10.0
    B-arrive_date.month_name      0.714     0.833     0.769       6.0
    B-arrive_date.day_number      0.714     0.833     0.769       6.0
         I-stoploc.city_name      0.909     1.000     0.952      10.0
                      B-meal      1.000     1.000     1.000      16.0
        B-fromloc.state_code      1.000     1.000     1.000      23.0
    B-depart_time.period_mod      1.000     1.000     1.000       5.0
                   B-connect      1.000     1.000     1.000       6.0
               B-flight_days      1.000     1.000     1.000      10.0
        B-toloc.airport_name      1.000     1.000     1.000       3.0
        B-fromloc.state_name      0.727     0.941     0.821      17.0
              B-airport_name      0.778     0.333     0.467      21.0
                   B-economy      1.000     1.000     1.000       6.0
               I-flight_time      0.500     1.000     0.667       1.0
             B-aircraft_code      0.960     0.727     0.828      33.0
                       B-mod      1.000     0.500     0.667       2.0
              B-airport_code      0.800     0.444     0.571       9.0
    B-depart_time.start_time      0.600     1.000     0.750       3.0
      B-depart_time.end_time      1.000     0.667     0.800       3.0
          B-depart_date.year      0.500     1.000     0.667       3.0
            I-transport_type      0.500     1.000     0.667       1.0
          B-restriction_code      1.000     1.000     1.000       4.0
    B-arrive_time.start_time      1.000     1.000     1.000       8.0
        B-toloc.airport_code      1.000     0.750     0.857       4.0
      B-arrive_time.end_time      0.875     0.875     0.875       8.0
      I-arrive_time.end_time      0.875     0.875     0.875       8.0
      I-depart_time.end_time      1.000     0.667     0.800       3.0
               I-flight_stop      0.000     0.000     0.000       0.0
      B-fromloc.airport_code      0.833     1.000     0.909       5.0
          I-restriction_code      1.000     1.000     1.000       3.0
    I-depart_time.start_time      1.000     1.000     1.000       1.0
          I-toloc.state_name      1.000     1.000     1.000       1.0
I-depart_date.today_relative      0.000     0.000     0.000       0.0
 B-arrive_date.date_relative      1.000     0.500     0.667       2.0
                I-flight_mod      0.000     0.000     0.000       6.0
                   I-economy      0.000     0.000     0.000       0.0
 B-return_date.date_relative      1.000     0.333     0.500       3.0
        I-fromloc.state_name      1.000     1.000     1.000       1.0
                B-state_code      1.000     1.000     1.000       1.0
    I-arrive_time.start_time      1.000     1.000     1.000       1.0
    I-arrive_date.day_number      0.000     0.000     0.000       0.0
                 B-meal_code      0.000     0.000     0.000       1.0
 I-depart_time.period_of_day      1.000     1.000     1.000       1.0
                  B-day_name      1.000     0.500     0.667       2.0
             B-period_of_day      1.000     0.750     0.857       4.0
        B-stoploc.state_code      0.000     0.000     0.000       0.0
    B-return_date.month_name      0.000     0.000     0.000       0.0
    B-return_date.day_number      0.000     0.000     0.000       0.0
    B-arrive_time.period_mod      0.000     0.000     0.000       0.0
                 I-meal_code      0.000     0.000     0.000       0.0
        B-toloc.country_name      1.000     1.000     1.000       1.0
                 B-days_code      0.500     1.000     0.667       1.0
 I-arrive_time.period_of_day      0.000     0.000     0.000       0.0
            I-today_relative      0.000     0.000     0.000       0.0
 B-return_time.period_of_day      0.000     0.000     0.000       0.0
                      B-time      0.000     0.000     0.000       0.0
           I-fare_basis_code      0.000     0.000     0.000       0.0
 I-arrive_time.time_relative      1.000     0.500     0.667       4.0
 I-depart_time.time_relative      0.000     0.000     0.000       1.0
            B-today_relative      0.000     0.000     0.000       0.0
                B-state_name      0.000     0.000     0.000       9.0
B-arrive_date.today_relative      0.000     0.000     0.000       0.0
    B-return_time.period_mod      0.000     0.000     0.000       0.0
                B-month_name      0.000     0.000     0.000       0.0
                B-day_number      0.000     0.000     0.000       0.0
 I-return_date.date_relative      1.000     0.333     0.500       3.0
I-return_date.today_relative      0.000     0.000     0.000       0.0
      B-stoploc.airport_name      0.000     0.000     0.000       0.0
             B-time_relative      0.000     0.000     0.000       0.0
                      I-time      0.000     0.000     0.000       0.0
    I-return_date.day_number      0.000     0.000     0.000       0.0
          I-meal_description      0.000     0.000     0.000       0.0
B-return_date.today_relative      0.000     0.000     0.000       0.0
      B-return_date.day_name      0.000     0.000     0.000       2.0

                   micro avg      0.950     0.952     0.951    3657.0
                   macro avg      0.670     0.658     0.648    3657.0
                weighted avg      0.956     0.952     0.948    3657.0

I0715 07:07:02.923020 140369481070464 interactiveshell.py:2882] Best Slot F1: 0.951
Reading ../data/atis.train.w-intent.iob
I0715 07:07:04.268871 140369481070464 interactiveshell.py:2882] Step 5000 | Loss: 0.0022 | Loss_intent: 0.0000 | Loss_slots: 0.0022 | Spent: 6.8 secs | LR: 0.000177
I0715 07:07:07.963718 140369481070464 interactiveshell.py:2882] Step 5050 | Loss: 0.0121 | Loss_intent: 0.0000 | Loss_slots: 0.0121 | Spent: 3.7 secs | LR: 0.000176
I0715 07:07:11.673466 140369481070464 interactiveshell.py:2882] Step 5100 | Loss: 0.0051 | Loss_intent: 0.0000 | Loss_slots: 0.0051 | Spent: 3.7 secs | LR: 0.000175
I0715 07:07:15.347370 140369481070464 interactiveshell.py:2882] Step 5150 | Loss: 0.0071 | Loss_intent: 0.0000 | Loss_slots: 0.0071 | Spent: 3.7 secs | LR: 0.000174
I0715 07:07:19.047977 140369481070464 interactiveshell.py:2882] Step 5200 | Loss: 0.0561 | Loss_intent: 0.0000 | Loss_slots: 0.0561 | Spent: 3.7 secs | LR: 0.000173
I0715 07:07:22.757825 140369481070464 interactiveshell.py:2882] Step 5250 | Loss: 0.0011 | Loss_intent: 0.0002 | Loss_slots: 0.0009 | Spent: 3.7 secs | LR: 0.000173
I0715 07:07:26.436365 140369481070464 interactiveshell.py:2882] Step 5300 | Loss: 0.0107 | Loss_intent: 0.0000 | Loss_slots: 0.0107 | Spent: 3.7 secs | LR: 0.000172
Reading ../data/atis.test.w-intent.iob
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.
  'precision', 'predicted', average, warn_for)
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1439: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples.
  'recall', 'true', average, warn_for)
I0715 07:07:28.120135 140369481070464 interactiveshell.py:2882] 
                                          precision    recall  f1-score   support

                             atis_flight      0.984     0.989     0.987       632
                            atis_airfare      0.941     1.000     0.970        48
                     atis_ground_service      1.000     1.000     1.000        36
                            atis_airline      1.000     1.000     1.000        38
                       atis_abbreviation      0.943     1.000     0.971        33
                           atis_aircraft      1.000     0.889     0.941         9
                        atis_flight_time      0.500     1.000     0.667         1
                           atis_quantity      0.333     1.000     0.500         3
                atis_flight#atis_airfare      0.833     0.417     0.556        12
                            atis_airport      0.944     0.944     0.944        18
                           atis_distance      1.000     1.000     1.000        10
                               atis_city      1.000     0.667     0.800         6
                        atis_ground_fare      1.000     0.714     0.833         7
                           atis_capacity      1.000     1.000     1.000        21
                          atis_flight_no      0.889     1.000     0.941         8
                               atis_meal      1.000     1.000     1.000         6
                        atis_restriction      0.000     0.000     0.000         0
             atis_airline#atis_flight_no      0.000     0.000     0.000         0
    atis_ground_service#atis_ground_fare      0.000     0.000     0.000         0
           atis_airfare#atis_flight_time      0.000     0.000     0.000         0
                           atis_cheapest      0.000     0.000     0.000         0
atis_aircraft#atis_flight#atis_flight_no      0.000     0.000     0.000         0

                               micro avg      0.972     0.977     0.975       888
                               macro avg      0.653     0.665     0.641       888
                            weighted avg      0.976     0.977     0.975       888

I0715 07:07:28.180540 140369481070464 interactiveshell.py:2882] 
                              precision    recall  f1-score   support

                           O      0.000     0.000     0.000       0.0
           B-toloc.city_name      0.971     0.993     0.982     716.0
         B-fromloc.city_name      0.979     0.997     0.988     704.0
           I-toloc.city_name      0.985     0.992     0.989     265.0
      B-depart_date.day_name      0.977     0.986     0.981     212.0
              B-airline_name      0.980     0.990     0.985     101.0
         I-fromloc.city_name      0.927     1.000     0.962     177.0
 B-depart_time.period_of_day      1.000     0.885     0.939     130.0
              I-airline_name      1.000     1.000     1.000      65.0
    B-depart_date.day_number      0.981     0.964     0.972      55.0
    B-depart_date.month_name      0.982     0.964     0.973      56.0
          B-depart_time.time      0.862     0.982     0.918      57.0
                B-round_trip      0.986     0.986     0.986      73.0
             B-cost_relative      1.000     0.973     0.986      37.0
                I-round_trip      0.986     1.000     0.993      71.0
                B-flight_mod      0.828     1.000     0.906      24.0
 B-depart_time.time_relative      0.984     0.954     0.969      65.0
          I-depart_time.time      0.962     0.962     0.962      52.0
         B-stoploc.city_name      0.952     1.000     0.976      20.0
                 B-city_name      0.825     0.579     0.680      57.0
                B-class_type      0.960     1.000     0.980      24.0
          B-arrive_time.time      0.917     0.971     0.943      34.0
 B-arrive_time.time_relative      0.882     0.968     0.923      31.0
                I-class_type      1.000     1.000     1.000      17.0
               B-flight_stop      1.000     1.000     1.000      21.0
          I-arrive_time.time      0.919     0.971     0.944      35.0
              B-airline_code      1.000     0.971     0.985      34.0
    I-depart_date.day_number      1.000     0.933     0.966      15.0
      I-fromloc.airport_name      0.469     1.000     0.638      15.0
      B-fromloc.airport_name      0.421     0.667     0.516      12.0
      B-arrive_date.day_name      0.733     1.000     0.846      11.0
          B-toloc.state_code      1.000     1.000     1.000      18.0
B-depart_date.today_relative      1.000     0.889     0.941       9.0
             B-flight_number      0.500     1.000     0.667      11.0
 B-depart_date.date_relative      0.944     1.000     0.971      17.0
          B-toloc.state_name      0.889     0.857     0.873      28.0
           B-fare_basis_code      0.944     1.000     0.971      17.0
               B-flight_time      1.000     1.000     1.000       1.0
                        B-or      1.000     1.000     1.000       3.0
 B-arrive_time.period_of_day      0.600     1.000     0.750       6.0
          B-meal_description      1.000     0.900     0.947      10.0
             I-cost_relative      1.000     0.667     0.800       3.0
              I-airport_name      0.857     0.414     0.558      29.0
               B-fare_amount      0.333     1.000     0.500       2.0
               I-fare_amount      1.000     1.000     1.000       2.0
                 I-city_name      1.000     0.700     0.824      30.0
        I-toloc.airport_name      1.000     1.000     1.000       3.0
            B-transport_type      1.000     0.900     0.947      10.0
    B-arrive_date.month_name      0.714     0.833     0.769       6.0
    B-arrive_date.day_number      0.714     0.833     0.769       6.0
         I-stoploc.city_name      0.889     0.800     0.842      10.0
                      B-meal      1.000     1.000     1.000      16.0
        B-fromloc.state_code      1.000     0.783     0.878      23.0
    B-depart_time.period_mod      1.000     1.000     1.000       5.0
                   B-connect      1.000     1.000     1.000       6.0
               B-flight_days      1.000     1.000     1.000      10.0
        B-toloc.airport_name      1.000     0.667     0.800       3.0
        B-fromloc.state_name      0.800     0.941     0.865      17.0
              B-airport_name      0.750     0.429     0.545      21.0
                   B-economy      1.000     1.000     1.000       6.0
               I-flight_time      1.000     1.000     1.000       1.0
             B-aircraft_code      1.000     0.727     0.842      33.0
                       B-mod      1.000     0.500     0.667       2.0
              B-airport_code      0.500     0.111     0.182       9.0
    B-depart_time.start_time      0.667     0.667     0.667       3.0
      B-depart_time.end_time      1.000     0.667     0.800       3.0
          B-depart_date.year      1.000     0.667     0.800       3.0
            I-transport_type      0.500     1.000     0.667       1.0
          B-restriction_code      1.000     1.000     1.000       4.0
    B-arrive_time.start_time      0.889     1.000     0.941       8.0
        B-toloc.airport_code      1.000     0.500     0.667       4.0
      B-arrive_time.end_time      0.875     0.875     0.875       8.0
      I-arrive_time.end_time      0.875     0.875     0.875       8.0
      I-depart_time.end_time      1.000     0.667     0.800       3.0
               I-flight_stop      0.000     0.000     0.000       0.0
      B-fromloc.airport_code      0.714     1.000     0.833       5.0
          I-restriction_code      1.000     1.000     1.000       3.0
    I-depart_time.start_time      1.000     1.000     1.000       1.0
          I-toloc.state_name      1.000     1.000     1.000       1.0
I-depart_date.today_relative      0.000     0.000     0.000       0.0
 B-arrive_date.date_relative      1.000     1.000     1.000       2.0
                I-flight_mod      0.000     0.000     0.000       6.0
                   I-economy      0.000     0.000     0.000       0.0
 B-return_date.date_relative      0.000     0.000     0.000       3.0
        I-fromloc.state_name      1.000     1.000     1.000       1.0
                B-state_code      1.000     1.000     1.000       1.0
    I-arrive_time.start_time      1.000     1.000     1.000       1.0
    I-arrive_date.day_number      0.000     0.000     0.000       0.0
                 B-meal_code      1.000     1.000     1.000       1.0
 I-depart_time.period_of_day      1.000     1.000     1.000       1.0
                  B-day_name      1.000     0.500     0.667       2.0
             B-period_of_day      1.000     0.750     0.857       4.0
        B-stoploc.state_code      0.000     0.000     0.000       0.0
    B-return_date.month_name      0.000     0.000     0.000       0.0
    B-return_date.day_number      0.000     0.000     0.000       0.0
    B-arrive_time.period_mod      0.000     0.000     0.000       0.0
                 I-meal_code      0.000     0.000     0.000       0.0
        B-toloc.country_name      1.000     1.000     1.000       1.0
                 B-days_code      1.000     1.000     1.000       1.0
 I-arrive_time.period_of_day      0.000     0.000     0.000       0.0
            I-today_relative      0.000     0.000     0.000       0.0
 B-return_time.period_of_day      0.000     0.000     0.000       0.0
                      B-time      0.000     0.000     0.000       0.0
           I-fare_basis_code      0.000     0.000     0.000       0.0
 I-arrive_time.time_relative      1.000     0.500     0.667       4.0
 I-depart_time.time_relative      0.000     0.000     0.000       1.0
            B-today_relative      0.000     0.000     0.000       0.0
                B-state_name      0.000     0.000     0.000       9.0
B-arrive_date.today_relative      0.000     0.000     0.000       0.0
    B-return_time.period_mod      0.000     0.000     0.000       0.0
                B-month_name      0.000     0.000     0.000       0.0
                B-day_number      0.000     0.000     0.000       0.0
 I-return_date.date_relative      0.000     0.000     0.000       3.0
I-return_date.today_relative      0.000     0.000     0.000       0.0
      B-stoploc.airport_name      0.000     0.000     0.000       0.0
             B-time_relative      0.000     0.000     0.000       0.0
                      I-time      0.000     0.000     0.000       0.0
    I-return_date.day_number      0.000     0.000     0.000       0.0
          I-meal_description      0.000     0.000     0.000       0.0
B-return_date.today_relative      0.000     0.000     0.000       0.0
      B-return_date.day_name      0.000     0.000     0.000       2.0

                   micro avg      0.948     0.950     0.949    3657.0
                   macro avg      0.665     0.647     0.644    3657.0
                weighted avg      0.950     0.950     0.946    3657.0

I0715 07:07:28.182085 140369481070464 interactiveshell.py:2882] Best Slot F1: 0.951
Reading ../data/atis.train.w-intent.iob
I0715 07:07:33.437073 140369481070464 interactiveshell.py:2882] Step 5350 | Loss: 0.0016 | Loss_intent: 0.0002 | Loss_slots: 0.0014 | Spent: 7.0 secs | LR: 0.000171
I0715 07:07:37.364877 140369481070464 interactiveshell.py:2882] Step 5400 | Loss: 0.0030 | Loss_intent: 0.0000 | Loss_slots: 0.0030 | Spent: 3.9 secs | LR: 0.000170
I0715 07:07:42.004003 140369481070464 interactiveshell.py:2882] Step 5450 | Loss: 0.0097 | Loss_intent: 0.0000 | Loss_slots: 0.0097 | Spent: 4.6 secs | LR: 0.000169
I0715 07:07:46.191498 140369481070464 interactiveshell.py:2882] Step 5500 | Loss: 0.0297 | Loss_intent: 0.0000 | Loss_slots: 0.0297 | Spent: 4.2 secs | LR: 0.000168
I0715 07:07:49.919486 140369481070464 interactiveshell.py:2882] Step 5550 | Loss: 0.0062 | Loss_intent: 0.0000 | Loss_slots: 0.0062 | Spent: 3.7 secs | LR: 0.000167
I0715 07:07:54.251113 140369481070464 interactiveshell.py:2882] Step 5600 | Loss: 0.0200 | Loss_intent: 0.0000 | Loss_slots: 0.0200 | Spent: 4.3 secs | LR: 0.000166
Reading ../data/atis.test.w-intent.iob
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.
  'precision', 'predicted', average, warn_for)
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1439: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples.
  'recall', 'true', average, warn_for)
I0715 07:07:57.266492 140369481070464 interactiveshell.py:2882] 
                                          precision    recall  f1-score   support

                             atis_flight      0.974     0.992     0.983       632
                            atis_airfare      0.941     1.000     0.970        48
                     atis_ground_service      1.000     1.000     1.000        36
                            atis_airline      1.000     1.000     1.000        38
                       atis_abbreviation      0.943     1.000     0.971        33
                           atis_aircraft      1.000     0.889     0.941         9
                        atis_flight_time      1.000     1.000     1.000         1
                           atis_quantity      0.429     1.000     0.600         3
                atis_flight#atis_airfare      1.000     0.083     0.154        12
                            atis_airport      0.944     0.944     0.944        18
                           atis_distance      1.000     1.000     1.000        10
                               atis_city      1.000     0.667     0.800         6
                        atis_ground_fare      1.000     0.714     0.833         7
                           atis_capacity      0.952     0.952     0.952        21
                          atis_flight_no      1.000     1.000     1.000         8
                               atis_meal      1.000     1.000     1.000         6
                        atis_restriction      0.000     0.000     0.000         0
             atis_airline#atis_flight_no      0.000     0.000     0.000         0
    atis_ground_service#atis_ground_fare      0.000     0.000     0.000         0
           atis_airfare#atis_flight_time      0.000     0.000     0.000         0
                           atis_cheapest      0.000     0.000     0.000         0
atis_aircraft#atis_flight#atis_flight_no      0.000     0.000     0.000         0

                               micro avg      0.969     0.974     0.971       888
                               macro avg      0.690     0.647     0.643       888
                            weighted avg      0.972     0.974     0.967       888

I0715 07:07:57.309628 140369481070464 interactiveshell.py:2882] 
                              precision    recall  f1-score   support

                           O      0.000     0.000     0.000       0.0
           B-toloc.city_name      0.969     0.994     0.981     716.0
         B-fromloc.city_name      0.983     0.997     0.990     704.0
           I-toloc.city_name      0.985     0.989     0.987     265.0
      B-depart_date.day_name      0.981     0.976     0.979     212.0
              B-airline_name      0.971     0.990     0.980     101.0
         I-fromloc.city_name      0.983     0.994     0.989     177.0
 B-depart_time.period_of_day      1.000     0.885     0.939     130.0
              I-airline_name      1.000     1.000     1.000      65.0
    B-depart_date.day_number      0.981     0.964     0.972      55.0
    B-depart_date.month_name      0.982     0.964     0.973      56.0
          B-depart_time.time      0.862     0.982     0.918      57.0
                B-round_trip      1.000     0.986     0.993      73.0
             B-cost_relative      1.000     0.973     0.986      37.0
                I-round_trip      1.000     1.000     1.000      71.0
                B-flight_mod      0.821     0.958     0.885      24.0
 B-depart_time.time_relative      0.984     0.954     0.969      65.0
          I-depart_time.time      0.944     0.981     0.962      52.0
         B-stoploc.city_name      0.952     1.000     0.976      20.0
                 B-city_name      0.833     0.526     0.645      57.0
                B-class_type      0.960     1.000     0.980      24.0
          B-arrive_time.time      0.917     0.971     0.943      34.0
 B-arrive_time.time_relative      0.909     0.968     0.937      31.0
                I-class_type      1.000     1.000     1.000      17.0
               B-flight_stop      1.000     1.000     1.000      21.0
          I-arrive_time.time      0.944     0.971     0.958      35.0
              B-airline_code      1.000     0.912     0.954      34.0
    I-depart_date.day_number      1.000     1.000     1.000      15.0
      I-fromloc.airport_name      0.405     1.000     0.577      15.0
      B-fromloc.airport_name      0.333     0.750     0.462      12.0
      B-arrive_date.day_name      0.647     1.000     0.786      11.0
          B-toloc.state_code      1.000     1.000     1.000      18.0
B-depart_date.today_relative      1.000     0.889     0.941       9.0
             B-flight_number      0.524     1.000     0.688      11.0
 B-depart_date.date_relative      0.944     1.000     0.971      17.0
          B-toloc.state_name      0.857     0.857     0.857      28.0
           B-fare_basis_code      0.909     0.588     0.714      17.0
               B-flight_time      1.000     1.000     1.000       1.0
                        B-or      1.000     1.000     1.000       3.0
 B-arrive_time.period_of_day      0.600     1.000     0.750       6.0
          B-meal_description      1.000     0.900     0.947      10.0
             I-cost_relative      1.000     0.667     0.800       3.0
              I-airport_name      0.909     0.345     0.500      29.0
               B-fare_amount      0.667     1.000     0.800       2.0
               I-fare_amount      1.000     1.000     1.000       2.0
                 I-city_name      0.957     0.733     0.830      30.0
        I-toloc.airport_name      1.000     1.000     1.000       3.0
            B-transport_type      1.000     0.800     0.889      10.0
    B-arrive_date.month_name      0.714     0.833     0.769       6.0
    B-arrive_date.day_number      0.714     0.833     0.769       6.0
         I-stoploc.city_name      1.000     1.000     1.000      10.0
                      B-meal      1.000     1.000     1.000      16.0
        B-fromloc.state_code      1.000     1.000     1.000      23.0
    B-depart_time.period_mod      1.000     1.000     1.000       5.0
                   B-connect      1.000     1.000     1.000       6.0
               B-flight_days      1.000     1.000     1.000      10.0
        B-toloc.airport_name      1.000     1.000     1.000       3.0
        B-fromloc.state_name      1.000     0.941     0.970      17.0
              B-airport_name      0.667     0.286     0.400      21.0
                   B-economy      1.000     1.000     1.000       6.0
               I-flight_time      1.000     1.000     1.000       1.0
             B-aircraft_code      1.000     0.818     0.900      33.0
                       B-mod      1.000     0.500     0.667       2.0
              B-airport_code      0.667     0.444     0.533       9.0
    B-depart_time.start_time      1.000     1.000     1.000       3.0
      B-depart_time.end_time      1.000     0.667     0.800       3.0
          B-depart_date.year      1.000     1.000     1.000       3.0
            I-transport_type      0.333     1.000     0.500       1.0
          B-restriction_code      0.800     1.000     0.889       4.0
    B-arrive_time.start_time      1.000     1.000     1.000       8.0
        B-toloc.airport_code      1.000     0.750     0.857       4.0
      B-arrive_time.end_time      0.875     0.875     0.875       8.0
      I-arrive_time.end_time      0.875     0.875     0.875       8.0
      I-depart_time.end_time      1.000     0.667     0.800       3.0
               I-flight_stop      0.000     0.000     0.000       0.0
      B-fromloc.airport_code      0.800     0.800     0.800       5.0
          I-restriction_code      0.333     1.000     0.500       3.0
    I-depart_time.start_time      1.000     1.000     1.000       1.0
          I-toloc.state_name      1.000     1.000     1.000       1.0
I-depart_date.today_relative      0.000     0.000     0.000       0.0
 B-arrive_date.date_relative      1.000     1.000     1.000       2.0
                I-flight_mod      1.000     0.167     0.286       6.0
                   I-economy      0.000     0.000     0.000       0.0
 B-return_date.date_relative      0.000     0.000     0.000       3.0
        I-fromloc.state_name      1.000     1.000     1.000       1.0
                B-state_code      1.000     1.000     1.000       1.0
    I-arrive_time.start_time      1.000     1.000     1.000       1.0
    I-arrive_date.day_number      0.000     0.000     0.000       0.0
                 B-meal_code      0.000     0.000     0.000       1.0
 I-depart_time.period_of_day      1.000     1.000     1.000       1.0
                  B-day_name      1.000     0.500     0.667       2.0
             B-period_of_day      1.000     0.500     0.667       4.0
        B-stoploc.state_code      0.000     0.000     0.000       0.0
    B-return_date.month_name      0.000     0.000     0.000       0.0
    B-return_date.day_number      0.000     0.000     0.000       0.0
    B-arrive_time.period_mod      0.000     0.000     0.000       0.0
                 I-meal_code      0.000     0.000     0.000       0.0
        B-toloc.country_name      1.000     1.000     1.000       1.0
                 B-days_code      1.000     1.000     1.000       1.0
 I-arrive_time.period_of_day      0.000     0.000     0.000       0.0
            I-today_relative      0.000     0.000     0.000       0.0
 B-return_time.period_of_day      0.000     0.000     0.000       0.0
                      B-time      0.000     0.000     0.000       0.0
           I-fare_basis_code      0.000     0.000     0.000       0.0
 I-arrive_time.time_relative      1.000     0.750     0.857       4.0
 I-depart_time.time_relative      0.000     0.000     0.000       1.0
            B-today_relative      0.000     0.000     0.000       0.0
                B-state_name      0.800     0.444     0.571       9.0
B-arrive_date.today_relative      0.000     0.000     0.000       0.0
    B-return_time.period_mod      0.000     0.000     0.000       0.0
                B-month_name      0.000     0.000     0.000       0.0
                B-day_number      0.000     0.000     0.000       0.0
 I-return_date.date_relative      0.500     0.667     0.571       3.0
I-return_date.today_relative      0.000     0.000     0.000       0.0
      B-stoploc.airport_name      0.000     0.000     0.000       0.0
             B-time_relative      0.000     0.000     0.000       0.0
                      I-time      0.000     0.000     0.000       0.0
    I-return_date.day_number      0.000     0.000     0.000       0.0
          I-meal_description      0.000     0.000     0.000       0.0
B-return_date.today_relative      0.000     0.000     0.000       0.0
      B-return_date.day_name      0.000     0.000     0.000       2.0

                   micro avg      0.949     0.951     0.950    3657.0
                   macro avg      0.676     0.659     0.652    3657.0
                weighted avg      0.958     0.951     0.949    3657.0

I0715 07:07:57.310608 140369481070464 interactiveshell.py:2882] Best Slot F1: 0.951
Reading ../data/atis.train.w-intent.iob
I0715 07:08:00.726484 140369481070464 interactiveshell.py:2882] Step 5650 | Loss: 0.0063 | Loss_intent: 0.0000 | Loss_slots: 0.0063 | Spent: 6.5 secs | LR: 0.000165
I0715 07:08:05.258903 140369481070464 interactiveshell.py:2882] Step 5700 | Loss: 0.0028 | Loss_intent: 0.0000 | Loss_slots: 0.0028 | Spent: 4.5 secs | LR: 0.000165
I0715 07:08:09.389618 140369481070464 interactiveshell.py:2882] Step 5750 | Loss: 0.0095 | Loss_intent: 0.0000 | Loss_slots: 0.0095 | Spent: 4.1 secs | LR: 0.000164
I0715 07:08:13.486037 140369481070464 interactiveshell.py:2882] Step 5800 | Loss: 0.0046 | Loss_intent: 0.0000 | Loss_slots: 0.0046 | Spent: 4.1 secs | LR: 0.000163
I0715 07:08:17.983521 140369481070464 interactiveshell.py:2882] Step 5850 | Loss: 0.0028 | Loss_intent: 0.0000 | Loss_slots: 0.0028 | Spent: 4.5 secs | LR: 0.000162
I0715 07:08:21.835593 140369481070464 interactiveshell.py:2882] Step 5900 | Loss: 0.0014 | Loss_intent: 0.0000 | Loss_slots: 0.0014 | Spent: 3.9 secs | LR: 0.000161
Reading ../data/atis.test.w-intent.iob
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.
  'precision', 'predicted', average, warn_for)
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1439: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples.
  'recall', 'true', average, warn_for)
I0715 07:08:25.748660 140369481070464 interactiveshell.py:2882] 
                                          precision    recall  f1-score   support

                             atis_flight      0.980     0.992     0.986       632
                            atis_airfare      0.941     1.000     0.970        48
                     atis_ground_service      1.000     1.000     1.000        36
                            atis_airline      1.000     1.000     1.000        38
                       atis_abbreviation      0.943     1.000     0.971        33
                           atis_aircraft      1.000     0.889     0.941         9
                        atis_flight_time      1.000     1.000     1.000         1
                           atis_quantity      0.429     1.000     0.600         3
                atis_flight#atis_airfare      0.833     0.417     0.556        12
                            atis_airport      1.000     0.944     0.971        18
                           atis_distance      1.000     1.000     1.000        10
                               atis_city      1.000     0.667     0.800         6
                        atis_ground_fare      1.000     0.714     0.833         7
                           atis_capacity      0.955     1.000     0.977        21
                          atis_flight_no      1.000     1.000     1.000         8
                               atis_meal      1.000     0.833     0.909         6
                        atis_restriction      0.000     0.000     0.000         0
             atis_airline#atis_flight_no      0.000     0.000     0.000         0
    atis_ground_service#atis_ground_fare      0.000     0.000     0.000         0
           atis_airfare#atis_flight_time      0.000     0.000     0.000         0
                           atis_cheapest      0.000     0.000     0.000         0
atis_aircraft#atis_flight#atis_flight_no      0.000     0.000     0.000         0

                               micro avg      0.973     0.979     0.976       888
                               macro avg      0.685     0.657     0.660       888
                            weighted avg      0.975     0.979     0.975       888

I0715 07:08:25.790196 140369481070464 interactiveshell.py:2882] 
                              precision    recall  f1-score   support

                           O      0.000     0.000     0.000       0.0
           B-toloc.city_name      0.981     0.992     0.986     716.0
         B-fromloc.city_name      0.979     0.999     0.989     704.0
           I-toloc.city_name      0.981     0.992     0.987     265.0
      B-depart_date.day_name      0.990     0.981     0.986     212.0
              B-airline_name      0.990     0.980     0.985     101.0
         I-fromloc.city_name      0.983     0.983     0.983     177.0
 B-depart_time.period_of_day      0.992     0.900     0.944     130.0
              I-airline_name      1.000     1.000     1.000      65.0
    B-depart_date.day_number      0.964     0.964     0.964      55.0
    B-depart_date.month_name      0.982     0.964     0.973      56.0
          B-depart_time.time      0.859     0.965     0.909      57.0
                B-round_trip      1.000     0.973     0.986      73.0
             B-cost_relative      1.000     0.973     0.986      37.0
                I-round_trip      1.000     1.000     1.000      71.0
                B-flight_mod      0.815     0.917     0.863      24.0
 B-depart_time.time_relative      0.984     0.969     0.977      65.0
          I-depart_time.time      0.945     1.000     0.972      52.0
         B-stoploc.city_name      1.000     0.950     0.974      20.0
                 B-city_name      0.921     0.614     0.737      57.0
                B-class_type      0.960     1.000     0.980      24.0
          B-arrive_time.time      0.892     0.971     0.930      34.0
 B-arrive_time.time_relative      0.857     0.968     0.909      31.0
                I-class_type      1.000     1.000     1.000      17.0
               B-flight_stop      1.000     1.000     1.000      21.0
          I-arrive_time.time      0.971     0.971     0.971      35.0
              B-airline_code      1.000     0.912     0.954      34.0
    I-depart_date.day_number      1.000     0.933     0.966      15.0
      I-fromloc.airport_name      0.389     0.933     0.549      15.0
      B-fromloc.airport_name      0.300     0.750     0.429      12.0
      B-arrive_date.day_name      0.688     1.000     0.815      11.0
          B-toloc.state_code      1.000     1.000     1.000      18.0
B-depart_date.today_relative      1.000     1.000     1.000       9.0
             B-flight_number      0.733     1.000     0.846      11.0
 B-depart_date.date_relative      0.895     1.000     0.944      17.0
          B-toloc.state_name      0.903     1.000     0.949      28.0
           B-fare_basis_code      0.944     1.000     0.971      17.0
               B-flight_time      1.000     1.000     1.000       1.0
                        B-or      1.000     1.000     1.000       3.0
 B-arrive_time.period_of_day      0.600     1.000     0.750       6.0
          B-meal_description      1.000     0.900     0.947      10.0
             I-cost_relative      1.000     0.667     0.800       3.0
              I-airport_name      0.909     0.345     0.500      29.0
               B-fare_amount      1.000     1.000     1.000       2.0
               I-fare_amount      1.000     1.000     1.000       2.0
                 I-city_name      1.000     0.667     0.800      30.0
        I-toloc.airport_name      1.000     0.667     0.800       3.0
            B-transport_type      0.857     0.600     0.706      10.0
    B-arrive_date.month_name      0.714     0.833     0.769       6.0
    B-arrive_date.day_number      0.714     0.833     0.769       6.0
         I-stoploc.city_name      1.000     0.900     0.947      10.0
                      B-meal      1.000     1.000     1.000      16.0
        B-fromloc.state_code      1.000     1.000     1.000      23.0
    B-depart_time.period_mod      1.000     1.000     1.000       5.0
                   B-connect      1.000     1.000     1.000       6.0
               B-flight_days      1.000     1.000     1.000      10.0
        B-toloc.airport_name      1.000     1.000     1.000       3.0
        B-fromloc.state_name      1.000     0.941     0.970      17.0
              B-airport_name      0.714     0.238     0.357      21.0
                   B-economy      1.000     1.000     1.000       6.0
               I-flight_time      1.000     1.000     1.000       1.0
             B-aircraft_code      0.912     0.939     0.925      33.0
                       B-mod      1.000     0.500     0.667       2.0
              B-airport_code      0.833     0.556     0.667       9.0
    B-depart_time.start_time      0.600     1.000     0.750       3.0
      B-depart_time.end_time      1.000     0.667     0.800       3.0
          B-depart_date.year      0.750     1.000     0.857       3.0
            I-transport_type      0.000     0.000     0.000       1.0
          B-restriction_code      1.000     1.000     1.000       4.0
    B-arrive_time.start_time      1.000     1.000     1.000       8.0
        B-toloc.airport_code      1.000     0.750     0.857       4.0
      B-arrive_time.end_time      0.875     0.875     0.875       8.0
      I-arrive_time.end_time      0.875     0.875     0.875       8.0
      I-depart_time.end_time      1.000     0.667     0.800       3.0
               I-flight_stop      0.000     0.000     0.000       0.0
      B-fromloc.airport_code      0.667     0.800     0.727       5.0
          I-restriction_code      1.000     1.000     1.000       3.0
    I-depart_time.start_time      1.000     1.000     1.000       1.0
          I-toloc.state_name      0.500     1.000     0.667       1.0
I-depart_date.today_relative      0.000     0.000     0.000       0.0
 B-arrive_date.date_relative      1.000     1.000     1.000       2.0
                I-flight_mod      1.000     0.167     0.286       6.0
                   I-economy      0.000     0.000     0.000       0.0
 B-return_date.date_relative      1.000     0.333     0.500       3.0
        I-fromloc.state_name      0.500     1.000     0.667       1.0
                B-state_code      1.000     1.000     1.000       1.0
    I-arrive_time.start_time      1.000     1.000     1.000       1.0
    I-arrive_date.day_number      0.000     0.000     0.000       0.0
                 B-meal_code      0.000     0.000     0.000       1.0
 I-depart_time.period_of_day      1.000     1.000     1.000       1.0
                  B-day_name      1.000     0.500     0.667       2.0
             B-period_of_day      1.000     0.500     0.667       4.0
        B-stoploc.state_code      0.000     0.000     0.000       0.0
    B-return_date.month_name      0.000     0.000     0.000       0.0
    B-return_date.day_number      0.000     0.000     0.000       0.0
    B-arrive_time.period_mod      0.000     0.000     0.000       0.0
                 I-meal_code      0.000     0.000     0.000       0.0
        B-toloc.country_name      1.000     1.000     1.000       1.0
                 B-days_code      0.500     1.000     0.667       1.0
 I-arrive_time.period_of_day      0.000     0.000     0.000       0.0
            I-today_relative      0.000     0.000     0.000       0.0
 B-return_time.period_of_day      0.000     0.000     0.000       0.0
                      B-time      0.000     0.000     0.000       0.0
           I-fare_basis_code      0.000     0.000     0.000       0.0
 I-arrive_time.time_relative      0.000     0.000     0.000       4.0
 I-depart_time.time_relative      0.000     0.000     0.000       1.0
            B-today_relative      0.000     0.000     0.000       0.0
                B-state_name      0.875     0.778     0.824       9.0
B-arrive_date.today_relative      0.000     0.000     0.000       0.0
    B-return_time.period_mod      0.000     0.000     0.000       0.0
                B-month_name      0.000     0.000     0.000       0.0
                B-day_number      0.000     0.000     0.000       0.0
 I-return_date.date_relative      1.000     0.667     0.800       3.0
I-return_date.today_relative      0.000     0.000     0.000       0.0
      B-stoploc.airport_name      0.000     0.000     0.000       0.0
             B-time_relative      0.000     0.000     0.000       0.0
                      I-time      0.000     0.000     0.000       0.0
    I-return_date.day_number      0.000     0.000     0.000       0.0
          I-meal_description      0.000     0.000     0.000       0.0
B-return_date.today_relative      0.000     0.000     0.000       0.0
      B-return_date.day_name      0.000     0.000     0.000       2.0

                   micro avg      0.952     0.954     0.953    3657.0
                   macro avg      0.672     0.651     0.645    3657.0
                weighted avg      0.961     0.954     0.953    3657.0

I0715 07:08:25.791122 140369481070464 interactiveshell.py:2882] Best Slot F1: 0.953
Reading ../data/atis.train.w-intent.iob
I0715 07:08:28.592115 140369481070464 interactiveshell.py:2882] Step 5950 | Loss: 0.0111 | Loss_intent: 0.0000 | Loss_slots: 0.0111 | Spent: 6.8 secs | LR: 0.000160
I0715 07:08:32.514010 140369481070464 interactiveshell.py:2882] Step 6000 | Loss: 0.0039 | Loss_intent: 0.0000 | Loss_slots: 0.0039 | Spent: 3.9 secs | LR: 0.000159
I0715 07:08:36.677737 140369481070464 interactiveshell.py:2882] Step 6050 | Loss: 0.0027 | Loss_intent: 0.0000 | Loss_slots: 0.0027 | Spent: 4.2 secs | LR: 0.000159
I0715 07:08:41.138916 140369481070464 interactiveshell.py:2882] Step 6100 | Loss: 0.0031 | Loss_intent: 0.0000 | Loss_slots: 0.0031 | Spent: 4.5 secs | LR: 0.000158
I0715 07:08:45.192874 140369481070464 interactiveshell.py:2882] Step 6150 | Loss: 0.0011 | Loss_intent: 0.0000 | Loss_slots: 0.0011 | Spent: 4.1 secs | LR: 0.000157
I0715 07:08:49.831898 140369481070464 interactiveshell.py:2882] Step 6200 | Loss: 0.0142 | Loss_intent: 0.0000 | Loss_slots: 0.0142 | Spent: 4.6 secs | LR: 0.000156
Reading ../data/atis.test.w-intent.iob
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.
  'precision', 'predicted', average, warn_for)
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1439: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples.
  'recall', 'true', average, warn_for)
I0715 07:08:54.285982 140369481070464 interactiveshell.py:2882] 
                                          precision    recall  f1-score   support

                             atis_flight      0.983     0.992     0.987       632
                            atis_airfare      0.941     1.000     0.970        48
                     atis_ground_service      1.000     1.000     1.000        36
                            atis_airline      1.000     1.000     1.000        38
                       atis_abbreviation      0.943     1.000     0.971        33
                           atis_aircraft      1.000     0.889     0.941         9
                        atis_flight_time      1.000     1.000     1.000         1
                           atis_quantity      0.429     1.000     0.600         3
                atis_flight#atis_airfare      0.833     0.417     0.556        12
                            atis_airport      1.000     1.000     1.000        18
                           atis_distance      1.000     1.000     1.000        10
                               atis_city      1.000     0.667     0.800         6
                        atis_ground_fare      1.000     0.714     0.833         7
                           atis_capacity      0.955     1.000     0.977        21
                          atis_flight_no      0.889     1.000     0.941         8
                               atis_meal      1.000     0.833     0.909         6
                        atis_restriction      0.000     0.000     0.000         0
             atis_airline#atis_flight_no      0.000     0.000     0.000         0
    atis_ground_service#atis_ground_fare      0.000     0.000     0.000         0
           atis_airfare#atis_flight_time      0.000     0.000     0.000         0
                           atis_cheapest      0.000     0.000     0.000         0
atis_aircraft#atis_flight#atis_flight_no      0.000     0.000     0.000         0

                               micro avg      0.974     0.980     0.977       888
                               macro avg      0.681     0.660     0.658       888
                            weighted avg      0.976     0.980     0.976       888

I0715 07:08:54.328018 140369481070464 interactiveshell.py:2882] 
                              precision    recall  f1-score   support

                           O      0.000     0.000     0.000       0.0
           B-toloc.city_name      0.975     0.994     0.985     716.0
         B-fromloc.city_name      0.980     0.997     0.989     704.0
           I-toloc.city_name      0.985     0.977     0.981     265.0
      B-depart_date.day_name      0.990     0.981     0.986     212.0
              B-airline_name      0.980     0.990     0.985     101.0
         I-fromloc.city_name      0.989     0.994     0.992     177.0
 B-depart_time.period_of_day      1.000     0.885     0.939     130.0
              I-airline_name      1.000     1.000     1.000      65.0
    B-depart_date.day_number      0.964     0.964     0.964      55.0
    B-depart_date.month_name      0.982     0.964     0.973      56.0
          B-depart_time.time      0.875     0.982     0.926      57.0
                B-round_trip      1.000     0.986     0.993      73.0
             B-cost_relative      1.000     0.973     0.986      37.0
                I-round_trip      1.000     1.000     1.000      71.0
                B-flight_mod      0.800     1.000     0.889      24.0
 B-depart_time.time_relative      0.984     0.954     0.969      65.0
          I-depart_time.time      0.912     1.000     0.954      52.0
         B-stoploc.city_name      0.952     1.000     0.976      20.0
                 B-city_name      0.800     0.561     0.660      57.0
                B-class_type      0.960     1.000     0.980      24.0
          B-arrive_time.time      0.892     0.971     0.930      34.0
 B-arrive_time.time_relative      0.882     0.968     0.923      31.0
                I-class_type      1.000     1.000     1.000      17.0
               B-flight_stop      1.000     1.000     1.000      21.0
          I-arrive_time.time      0.971     0.971     0.971      35.0
              B-airline_code      1.000     0.912     0.954      34.0
    I-depart_date.day_number      1.000     0.933     0.966      15.0
      I-fromloc.airport_name      0.400     0.933     0.560      15.0
      B-fromloc.airport_name      0.333     0.583     0.424      12.0
      B-arrive_date.day_name      0.688     1.000     0.815      11.0
          B-toloc.state_code      1.000     1.000     1.000      18.0
B-depart_date.today_relative      1.000     1.000     1.000       9.0
             B-flight_number      0.579     1.000     0.733      11.0
 B-depart_date.date_relative      0.944     1.000     0.971      17.0
          B-toloc.state_name      0.844     0.964     0.900      28.0
           B-fare_basis_code      0.944     1.000     0.971      17.0
               B-flight_time      1.000     1.000     1.000       1.0
                        B-or      1.000     1.000     1.000       3.0
 B-arrive_time.period_of_day      0.600     1.000     0.750       6.0
          B-meal_description      1.000     0.900     0.947      10.0
             I-cost_relative      1.000     0.667     0.800       3.0
              I-airport_name      0.923     0.414     0.571      29.0
               B-fare_amount      0.667     1.000     0.800       2.0
               I-fare_amount      1.000     1.000     1.000       2.0
                 I-city_name      0.880     0.733     0.800      30.0
        I-toloc.airport_name      1.000     1.000     1.000       3.0
            B-transport_type      0.909     1.000     0.952      10.0
    B-arrive_date.month_name      0.714     0.833     0.769       6.0
    B-arrive_date.day_number      0.714     0.833     0.769       6.0
         I-stoploc.city_name      1.000     1.000     1.000      10.0
                      B-meal      1.000     1.000     1.000      16.0
        B-fromloc.state_code      1.000     1.000     1.000      23.0
    B-depart_time.period_mod      1.000     1.000     1.000       5.0
                   B-connect      1.000     1.000     1.000       6.0
               B-flight_days      1.000     1.000     1.000      10.0
        B-toloc.airport_name      1.000     1.000     1.000       3.0
        B-fromloc.state_name      1.000     0.941     0.970      17.0
              B-airport_name      0.700     0.333     0.452      21.0
                   B-economy      1.000     1.000     1.000       6.0
               I-flight_time      1.000     1.000     1.000       1.0
             B-aircraft_code      1.000     0.818     0.900      33.0
                       B-mod      1.000     0.500     0.667       2.0
              B-airport_code      0.800     0.444     0.571       9.0
    B-depart_time.start_time      0.600     1.000     0.750       3.0
      B-depart_time.end_time      1.000     0.667     0.800       3.0
          B-depart_date.year      0.500     1.000     0.667       3.0
            I-transport_type      0.000     0.000     0.000       1.0
          B-restriction_code      1.000     1.000     1.000       4.0
    B-arrive_time.start_time      1.000     1.000     1.000       8.0
        B-toloc.airport_code      1.000     0.750     0.857       4.0
      B-arrive_time.end_time      0.875     0.875     0.875       8.0
      I-arrive_time.end_time      0.875     0.875     0.875       8.0
      I-depart_time.end_time      1.000     0.667     0.800       3.0
               I-flight_stop      0.000     0.000     0.000       0.0
      B-fromloc.airport_code      0.600     0.600     0.600       5.0
          I-restriction_code      1.000     1.000     1.000       3.0
    I-depart_time.start_time      1.000     1.000     1.000       1.0
          I-toloc.state_name      1.000     1.000     1.000       1.0
I-depart_date.today_relative      0.000     0.000     0.000       0.0
 B-arrive_date.date_relative      1.000     1.000     1.000       2.0
                I-flight_mod      0.000     0.000     0.000       6.0
                   I-economy      0.000     0.000     0.000       0.0
 B-return_date.date_relative      0.000     0.000     0.000       3.0
        I-fromloc.state_name      1.000     1.000     1.000       1.0
                B-state_code      1.000     1.000     1.000       1.0
    I-arrive_time.start_time      1.000     1.000     1.000       1.0
    I-arrive_date.day_number      0.000     0.000     0.000       0.0
                 B-meal_code      0.000     0.000     0.000       1.0
 I-depart_time.period_of_day      1.000     1.000     1.000       1.0
                  B-day_name      1.000     0.500     0.667       2.0
             B-period_of_day      1.000     0.750     0.857       4.0
        B-stoploc.state_code      0.000     0.000     0.000       0.0
    B-return_date.month_name      0.000     0.000     0.000       0.0
    B-return_date.day_number      0.000     0.000     0.000       0.0
    B-arrive_time.period_mod      0.000     0.000     0.000       0.0
                 I-meal_code      0.000     0.000     0.000       0.0
        B-toloc.country_name      1.000     1.000     1.000       1.0
                 B-days_code      1.000     1.000     1.000       1.0
 I-arrive_time.period_of_day      0.000     0.000     0.000       0.0
            I-today_relative      0.000     0.000     0.000       0.0
 B-return_time.period_of_day      0.000     0.000     0.000       0.0
                      B-time      0.000     0.000     0.000       0.0
           I-fare_basis_code      0.000     0.000     0.000       0.0
 I-arrive_time.time_relative      1.000     0.500     0.667       4.0
 I-depart_time.time_relative      0.000     0.000     0.000       1.0
            B-today_relative      0.000     0.000     0.000       0.0
                B-state_name      0.667     0.222     0.333       9.0
B-arrive_date.today_relative      0.000     0.000     0.000       0.0
    B-return_time.period_mod      0.000     0.000     0.000       0.0
                B-month_name      0.000     0.000     0.000       0.0
                B-day_number      0.000     0.000     0.000       0.0
 I-return_date.date_relative      0.500     0.667     0.571       3.0
I-return_date.today_relative      0.000     0.000     0.000       0.0
      B-stoploc.airport_name      0.000     0.000     0.000       0.0
             B-time_relative      0.000     0.000     0.000       0.0
                      I-time      0.000     0.000     0.000       0.0
    I-return_date.day_number      0.000     0.000     0.000       0.0
          I-meal_description      0.000     0.000     0.000       0.0
B-return_date.today_relative      0.000     0.000     0.000       0.0
      B-return_date.day_name      0.000     0.000     0.000       2.0

                   micro avg      0.952     0.953     0.952    3657.0
                   macro avg      0.662     0.652     0.647    3657.0
                weighted avg      0.956     0.953     0.951    3657.0

I0715 07:08:54.329007 140369481070464 interactiveshell.py:2882] Best Slot F1: 0.953
Reading ../data/atis.train.w-intent.iob
I0715 07:08:56.131608 140369481070464 interactiveshell.py:2882] Step 6250 | Loss: 0.0031 | Loss_intent: 0.0000 | Loss_slots: 0.0031 | Spent: 6.3 secs | LR: 0.000155
I0715 07:09:01.375751 140369481070464 interactiveshell.py:2882] Step 6300 | Loss: 0.0016 | Loss_intent: 0.0000 | Loss_slots: 0.0016 | Spent: 5.2 secs | LR: 0.000154
I0715 07:09:05.801366 140369481070464 interactiveshell.py:2882] Step 6350 | Loss: 0.0017 | Loss_intent: 0.0000 | Loss_slots: 0.0017 | Spent: 4.4 secs | LR: 0.000154
I0715 07:09:10.478785 140369481070464 interactiveshell.py:2882] Step 6400 | Loss: 0.0013 | Loss_intent: 0.0000 | Loss_slots: 0.0013 | Spent: 4.7 secs | LR: 0.000153
I0715 07:09:14.712074 140369481070464 interactiveshell.py:2882] Step 6450 | Loss: 0.0011 | Loss_intent: 0.0000 | Loss_slots: 0.0011 | Spent: 4.2 secs | LR: 0.000152
I0715 07:09:19.688771 140369481070464 interactiveshell.py:2882] Step 6500 | Loss: 0.0104 | Loss_intent: 0.0000 | Loss_slots: 0.0104 | Spent: 5.0 secs | LR: 0.000151
I0715 07:09:24.454877 140369481070464 interactiveshell.py:2882] Step 6550 | Loss: 0.0062 | Loss_intent: 0.0000 | Loss_slots: 0.0062 | Spent: 4.8 secs | LR: 0.000150
Reading ../data/atis.test.w-intent.iob
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.
  'precision', 'predicted', average, warn_for)
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1439: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples.
  'recall', 'true', average, warn_for)
I0715 07:09:26.305477 140369481070464 interactiveshell.py:2882] 
                                          precision    recall  f1-score   support

                             atis_flight      0.984     0.991     0.987       632
                            atis_airfare      0.960     1.000     0.980        48
                     atis_ground_service      1.000     1.000     1.000        36
                            atis_airline      1.000     1.000     1.000        38
                       atis_abbreviation      0.943     1.000     0.971        33
                           atis_aircraft      1.000     0.889     0.941         9
                        atis_flight_time      1.000     1.000     1.000         1
                           atis_quantity      0.429     1.000     0.600         3
                atis_flight#atis_airfare      0.833     0.417     0.556        12
                            atis_airport      0.947     1.000     0.973        18
                           atis_distance      1.000     1.000     1.000        10
                               atis_city      1.000     0.667     0.800         6
                        atis_ground_fare      1.000     0.857     0.923         7
                           atis_capacity      0.955     1.000     0.977        21
                          atis_flight_no      1.000     1.000     1.000         8
                               atis_meal      1.000     1.000     1.000         6
                        atis_restriction      0.000     0.000     0.000         0
             atis_airline#atis_flight_no      0.000     0.000     0.000         0
    atis_ground_service#atis_ground_fare      0.000     0.000     0.000         0
           atis_airfare#atis_flight_time      0.000     0.000     0.000         0
                           atis_cheapest      0.000     0.000     0.000         0
atis_aircraft#atis_flight#atis_flight_no      0.000     0.000     0.000         0

                               micro avg      0.975     0.981     0.978       888
                               macro avg      0.684     0.674     0.669       888
                            weighted avg      0.978     0.981     0.978       888

I0715 07:09:26.350768 140369481070464 interactiveshell.py:2882] 
                              precision    recall  f1-score   support

                           O      0.000     0.000     0.000       0.0
           B-toloc.city_name      0.975     0.994     0.985     716.0
         B-fromloc.city_name      0.982     0.997     0.989     704.0
           I-toloc.city_name      0.985     0.981     0.983     265.0
      B-depart_date.day_name      0.968     0.986     0.977     212.0
              B-airline_name      0.990     0.990     0.990     101.0
         I-fromloc.city_name      0.983     0.983     0.983     177.0
 B-depart_time.period_of_day      1.000     0.885     0.939     130.0
              I-airline_name      1.000     1.000     1.000      65.0
    B-depart_date.day_number      0.981     0.964     0.972      55.0
    B-depart_date.month_name      0.982     0.964     0.973      56.0
          B-depart_time.time      0.862     0.982     0.918      57.0
                B-round_trip      1.000     0.986     0.993      73.0
             B-cost_relative      1.000     0.973     0.986      37.0
                I-round_trip      1.000     1.000     1.000      71.0
                B-flight_mod      0.828     1.000     0.906      24.0
 B-depart_time.time_relative      0.984     0.969     0.977      65.0
          I-depart_time.time      0.962     0.981     0.971      52.0
         B-stoploc.city_name      0.952     1.000     0.976      20.0
                 B-city_name      0.810     0.596     0.687      57.0
                B-class_type      0.960     1.000     0.980      24.0
          B-arrive_time.time      0.943     0.971     0.957      34.0
 B-arrive_time.time_relative      0.857     0.968     0.909      31.0
                I-class_type      1.000     1.000     1.000      17.0
               B-flight_stop      1.000     1.000     1.000      21.0
          I-arrive_time.time      0.944     0.971     0.958      35.0
              B-airline_code      1.000     0.941     0.970      34.0
    I-depart_date.day_number      1.000     0.933     0.966      15.0
      I-fromloc.airport_name      0.441     1.000     0.612      15.0
      B-fromloc.airport_name      0.414     1.000     0.585      12.0
      B-arrive_date.day_name      0.786     1.000     0.880      11.0
          B-toloc.state_code      1.000     1.000     1.000      18.0
B-depart_date.today_relative      1.000     0.889     0.941       9.0
             B-flight_number      0.611     1.000     0.759      11.0
 B-depart_date.date_relative      0.944     1.000     0.971      17.0
          B-toloc.state_name      0.926     0.893     0.909      28.0
           B-fare_basis_code      0.944     1.000     0.971      17.0
               B-flight_time      1.000     1.000     1.000       1.0
                        B-or      1.000     1.000     1.000       3.0
 B-arrive_time.period_of_day      0.600     1.000     0.750       6.0
          B-meal_description      1.000     0.900     0.947      10.0
             I-cost_relative      1.000     0.667     0.800       3.0
              I-airport_name      0.867     0.448     0.591      29.0
               B-fare_amount      0.667     1.000     0.800       2.0
               I-fare_amount      1.000     1.000     1.000       2.0
                 I-city_name      0.759     0.733     0.746      30.0
        I-toloc.airport_name      1.000     1.000     1.000       3.0
            B-transport_type      1.000     0.800     0.889      10.0
    B-arrive_date.month_name      0.714     0.833     0.769       6.0
    B-arrive_date.day_number      0.714     0.833     0.769       6.0
         I-stoploc.city_name      1.000     0.900     0.947      10.0
                      B-meal      1.000     1.000     1.000      16.0
        B-fromloc.state_code      1.000     1.000     1.000      23.0
    B-depart_time.period_mod      1.000     1.000     1.000       5.0
                   B-connect      1.000     1.000     1.000       6.0
               B-flight_days      1.000     1.000     1.000      10.0
        B-toloc.airport_name      1.000     1.000     1.000       3.0
        B-fromloc.state_name      1.000     0.941     0.970      17.0
              B-airport_name      0.667     0.286     0.400      21.0
                   B-economy      1.000     1.000     1.000       6.0
               I-flight_time      1.000     1.000     1.000       1.0
             B-aircraft_code      0.964     0.818     0.885      33.0
                       B-mod      1.000     0.500     0.667       2.0
              B-airport_code      0.800     0.444     0.571       9.0
    B-depart_time.start_time      1.000     1.000     1.000       3.0
      B-depart_time.end_time      1.000     0.667     0.800       3.0
          B-depart_date.year      0.600     1.000     0.750       3.0
            I-transport_type      0.333     1.000     0.500       1.0
          B-restriction_code      1.000     1.000     1.000       4.0
    B-arrive_time.start_time      1.000     1.000     1.000       8.0
        B-toloc.airport_code      1.000     0.750     0.857       4.0
      B-arrive_time.end_time      0.875     0.875     0.875       8.0
      I-arrive_time.end_time      0.875     0.875     0.875       8.0
      I-depart_time.end_time      1.000     0.667     0.800       3.0
               I-flight_stop      0.000     0.000     0.000       0.0
      B-fromloc.airport_code      1.000     0.400     0.571       5.0
          I-restriction_code      1.000     1.000     1.000       3.0
    I-depart_time.start_time      1.000     1.000     1.000       1.0
          I-toloc.state_name      1.000     1.000     1.000       1.0
I-depart_date.today_relative      0.000     0.000     0.000       0.0
 B-arrive_date.date_relative      1.000     1.000     1.000       2.0
                I-flight_mod      1.000     0.167     0.286       6.0
                   I-economy      0.000     0.000     0.000       0.0
 B-return_date.date_relative      0.000     0.000     0.000       3.0
        I-fromloc.state_name      1.000     1.000     1.000       1.0
                B-state_code      1.000     1.000     1.000       1.0
    I-arrive_time.start_time      1.000     1.000     1.000       1.0
    I-arrive_date.day_number      0.000     0.000     0.000       0.0
                 B-meal_code      0.000     0.000     0.000       1.0
 I-depart_time.period_of_day      1.000     1.000     1.000       1.0
                  B-day_name      1.000     0.500     0.667       2.0
             B-period_of_day      1.000     0.500     0.667       4.0
        B-stoploc.state_code      0.000     0.000     0.000       0.0
    B-return_date.month_name      0.000     0.000     0.000       0.0
    B-return_date.day_number      0.000     0.000     0.000       0.0
    B-arrive_time.period_mod      0.000     0.000     0.000       0.0
                 I-meal_code      0.000     0.000     0.000       0.0
        B-toloc.country_name      1.000     1.000     1.000       1.0
                 B-days_code      1.000     1.000     1.000       1.0
 I-arrive_time.period_of_day      0.000     0.000     0.000       0.0
            I-today_relative      0.000     0.000     0.000       0.0
 B-return_time.period_of_day      0.000     0.000     0.000       0.0
                      B-time      0.000     0.000     0.000       0.0
           I-fare_basis_code      0.000     0.000     0.000       0.0
 I-arrive_time.time_relative      0.000     0.000     0.000       4.0
 I-depart_time.time_relative      0.000     0.000     0.000       1.0
            B-today_relative      0.000     0.000     0.000       0.0
                B-state_name      0.667     0.222     0.333       9.0
B-arrive_date.today_relative      0.000     0.000     0.000       0.0
    B-return_time.period_mod      0.000     0.000     0.000       0.0
                B-month_name      0.000     0.000     0.000       0.0
                B-day_number      0.000     0.000     0.000       0.0
 I-return_date.date_relative      0.667     0.667     0.667       3.0
I-return_date.today_relative      0.000     0.000     0.000       0.0
      B-stoploc.airport_name      0.000     0.000     0.000       0.0
             B-time_relative      0.000     0.000     0.000       0.0
                      I-time      0.000     0.000     0.000       0.0
    I-return_date.day_number      0.000     0.000     0.000       0.0
          I-meal_description      0.000     0.000     0.000       0.0
B-return_date.today_relative      0.000     0.000     0.000       0.0
      B-return_date.day_name      0.000     0.000     0.000       2.0

                   micro avg      0.952     0.954     0.953    3657.0
                   macro avg      0.676     0.654     0.651    3657.0
                weighted avg      0.957     0.954     0.951    3657.0

I0715 07:09:26.351599 140369481070464 interactiveshell.py:2882] Best Slot F1: 0.953
Reading ../data/atis.train.w-intent.iob
I0715 07:09:32.222939 140369481070464 interactiveshell.py:2882] Step 6600 | Loss: 0.0150 | Loss_intent: 0.0000 | Loss_slots: 0.0150 | Spent: 7.8 secs | LR: 0.000150
I0715 07:09:36.796802 140369481070464 interactiveshell.py:2882] Step 6650 | Loss: 0.0067 | Loss_intent: 0.0000 | Loss_slots: 0.0067 | Spent: 4.6 secs | LR: 0.000149
I0715 07:09:42.232475 140369481070464 interactiveshell.py:2882] Step 6700 | Loss: 0.0024 | Loss_intent: 0.0000 | Loss_slots: 0.0024 | Spent: 5.4 secs | LR: 0.000148
I0715 07:09:46.394260 140369481070464 interactiveshell.py:2882] Step 6750 | Loss: 0.0029 | Loss_intent: 0.0000 | Loss_slots: 0.0029 | Spent: 4.2 secs | LR: 0.000147
I0715 07:09:50.740635 140369481070464 interactiveshell.py:2882] Step 6800 | Loss: 0.0007 | Loss_intent: 0.0000 | Loss_slots: 0.0007 | Spent: 4.3 secs | LR: 0.000147
I0715 07:09:56.556538 140369481070464 interactiveshell.py:2882] Step 6850 | Loss: 0.0012 | Loss_intent: 0.0000 | Loss_slots: 0.0012 | Spent: 5.8 secs | LR: 0.000146
Reading ../data/atis.test.w-intent.iob
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.
  'precision', 'predicted', average, warn_for)
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1439: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples.
  'recall', 'true', average, warn_for)
I0715 07:09:59.502226 140369481070464 interactiveshell.py:2882] 
                                          precision    recall  f1-score   support

                             atis_flight      0.983     0.992     0.987       632
                            atis_airfare      0.960     1.000     0.980        48
                     atis_ground_service      1.000     1.000     1.000        36
                            atis_airline      1.000     1.000     1.000        38
                       atis_abbreviation      0.943     1.000     0.971        33
                           atis_aircraft      1.000     0.889     0.941         9
                        atis_flight_time      1.000     1.000     1.000         1
                           atis_quantity      0.429     1.000     0.600         3
                atis_flight#atis_airfare      0.833     0.417     0.556        12
                            atis_airport      1.000     1.000     1.000        18
                           atis_distance      1.000     1.000     1.000        10
                               atis_city      1.000     0.667     0.800         6
                        atis_ground_fare      1.000     0.857     0.923         7
                           atis_capacity      0.955     1.000     0.977        21
                          atis_flight_no      1.000     1.000     1.000         8
                               atis_meal      1.000     1.000     1.000         6
                        atis_restriction      0.000     0.000     0.000         0
             atis_airline#atis_flight_no      0.000     0.000     0.000         0
    atis_ground_service#atis_ground_fare      0.000     0.000     0.000         0
           atis_airfare#atis_flight_time      0.000     0.000     0.000         0
                           atis_cheapest      0.000     0.000     0.000         0
atis_aircraft#atis_flight#atis_flight_no      0.000     0.000     0.000         0

                               micro avg      0.976     0.982     0.979       888
                               macro avg      0.686     0.674     0.670       888
                            weighted avg      0.978     0.982     0.978       888

I0715 07:09:59.548545 140369481070464 interactiveshell.py:2882] 
                              precision    recall  f1-score   support

                           O      0.000     0.000     0.000       0.0
           B-toloc.city_name      0.974     0.994     0.984     716.0
         B-fromloc.city_name      0.986     0.997     0.992     704.0
           I-toloc.city_name      0.981     1.000     0.991     265.0
      B-depart_date.day_name      0.986     0.986     0.986     212.0
              B-airline_name      0.990     0.990     0.990     101.0
         I-fromloc.city_name      0.994     0.989     0.992     177.0
 B-depart_time.period_of_day      1.000     0.892     0.943     130.0
              I-airline_name      1.000     1.000     1.000      65.0
    B-depart_date.day_number      0.982     0.982     0.982      55.0
    B-depart_date.month_name      0.982     0.964     0.973      56.0
          B-depart_time.time      0.864     1.000     0.927      57.0
                B-round_trip      1.000     0.986     0.993      73.0
             B-cost_relative      1.000     0.973     0.986      37.0
                I-round_trip      0.986     1.000     0.993      71.0
                B-flight_mod      0.828     1.000     0.906      24.0
 B-depart_time.time_relative      0.984     0.969     0.977      65.0
          I-depart_time.time      0.911     0.981     0.944      52.0
         B-stoploc.city_name      0.952     1.000     0.976      20.0
                 B-city_name      0.773     0.596     0.673      57.0
                B-class_type      0.960     1.000     0.980      24.0
          B-arrive_time.time      0.943     0.971     0.957      34.0
 B-arrive_time.time_relative      0.909     0.968     0.937      31.0
                I-class_type      1.000     1.000     1.000      17.0
               B-flight_stop      1.000     1.000     1.000      21.0
          I-arrive_time.time      0.944     0.971     0.958      35.0
              B-airline_code      1.000     0.912     0.954      34.0
    I-depart_date.day_number      1.000     0.933     0.966      15.0
      I-fromloc.airport_name      0.385     1.000     0.556      15.0
      B-fromloc.airport_name      0.524     0.917     0.667      12.0
      B-arrive_date.day_name      0.786     1.000     0.880      11.0
          B-toloc.state_code      1.000     1.000     1.000      18.0
B-depart_date.today_relative      1.000     0.889     0.941       9.0
             B-flight_number      0.524     1.000     0.688      11.0
 B-depart_date.date_relative      0.944     1.000     0.971      17.0
          B-toloc.state_name      0.889     0.857     0.873      28.0
           B-fare_basis_code      0.944     1.000     0.971      17.0
               B-flight_time      1.000     1.000     1.000       1.0
                        B-or      1.000     1.000     1.000       3.0
 B-arrive_time.period_of_day      0.667     1.000     0.800       6.0
          B-meal_description      1.000     0.900     0.947      10.0
             I-cost_relative      1.000     0.667     0.800       3.0
              I-airport_name      0.917     0.379     0.537      29.0
               B-fare_amount      1.000     1.000     1.000       2.0
               I-fare_amount      1.000     1.000     1.000       2.0
                 I-city_name      1.000     0.700     0.824      30.0
        I-toloc.airport_name      1.000     1.000     1.000       3.0
            B-transport_type      0.833     1.000     0.909      10.0
    B-arrive_date.month_name      0.714     0.833     0.769       6.0
    B-arrive_date.day_number      0.833     0.833     0.833       6.0
         I-stoploc.city_name      1.000     1.000     1.000      10.0
                      B-meal      1.000     1.000     1.000      16.0
        B-fromloc.state_code      1.000     1.000     1.000      23.0
    B-depart_time.period_mod      1.000     1.000     1.000       5.0
                   B-connect      1.000     1.000     1.000       6.0
               B-flight_days      1.000     1.000     1.000      10.0
        B-toloc.airport_name      1.000     1.000     1.000       3.0
        B-fromloc.state_name      1.000     0.941     0.970      17.0
              B-airport_name      0.750     0.429     0.545      21.0
                   B-economy      1.000     1.000     1.000       6.0
               I-flight_time      1.000     1.000     1.000       1.0
             B-aircraft_code      0.962     0.758     0.847      33.0
                       B-mod      1.000     0.500     0.667       2.0
              B-airport_code      0.833     0.556     0.667       9.0
    B-depart_time.start_time      0.750     1.000     0.857       3.0
      B-depart_time.end_time      1.000     0.667     0.800       3.0
          B-depart_date.year      0.500     1.000     0.667       3.0
            I-transport_type      0.000     0.000     0.000       1.0
          B-restriction_code      1.000     1.000     1.000       4.0
    B-arrive_time.start_time      0.889     1.000     0.941       8.0
        B-toloc.airport_code      1.000     0.750     0.857       4.0
      B-arrive_time.end_time      0.875     0.875     0.875       8.0
      I-arrive_time.end_time      0.875     0.875     0.875       8.0
      I-depart_time.end_time      1.000     0.667     0.800       3.0
               I-flight_stop      0.000     0.000     0.000       0.0
      B-fromloc.airport_code      0.750     0.600     0.667       5.0
          I-restriction_code      1.000     1.000     1.000       3.0
    I-depart_time.start_time      1.000     1.000     1.000       1.0
          I-toloc.state_name      1.000     1.000     1.000       1.0
I-depart_date.today_relative      0.000     0.000     0.000       0.0
 B-arrive_date.date_relative      1.000     1.000     1.000       2.0
                I-flight_mod      1.000     0.167     0.286       6.0
                   I-economy      0.000     0.000     0.000       0.0
 B-return_date.date_relative      0.000     0.000     0.000       3.0
        I-fromloc.state_name      1.000     1.000     1.000       1.0
                B-state_code      1.000     1.000     1.000       1.0
    I-arrive_time.start_time      1.000     1.000     1.000       1.0
    I-arrive_date.day_number      0.000     0.000     0.000       0.0
                 B-meal_code      0.000     0.000     0.000       1.0
 I-depart_time.period_of_day      1.000     1.000     1.000       1.0
                  B-day_name      1.000     0.500     0.667       2.0
             B-period_of_day      1.000     0.750     0.857       4.0
        B-stoploc.state_code      0.000     0.000     0.000       0.0
    B-return_date.month_name      0.000     0.000     0.000       0.0
    B-return_date.day_number      0.000     0.000     0.000       0.0
    B-arrive_time.period_mod      0.000     0.000     0.000       0.0
                 I-meal_code      0.000     0.000     0.000       0.0
        B-toloc.country_name      1.000     1.000     1.000       1.0
                 B-days_code      0.500     1.000     0.667       1.0
 I-arrive_time.period_of_day      0.000     0.000     0.000       0.0
            I-today_relative      0.000     0.000     0.000       0.0
 B-return_time.period_of_day      0.000     0.000     0.000       0.0
                      B-time      0.000     0.000     0.000       0.0
           I-fare_basis_code      0.000     0.000     0.000       0.0
 I-arrive_time.time_relative      1.000     0.500     0.667       4.0
 I-depart_time.time_relative      0.000     0.000     0.000       1.0
            B-today_relative      0.000     0.000     0.000       0.0
                B-state_name      0.667     0.222     0.333       9.0
B-arrive_date.today_relative      0.000     0.000     0.000       0.0
    B-return_time.period_mod      0.000     0.000     0.000       0.0
                B-month_name      0.000     0.000     0.000       0.0
                B-day_number      0.000     0.000     0.000       0.0
 I-return_date.date_relative      0.667     0.667     0.667       3.0
I-return_date.today_relative      0.000     0.000     0.000       0.0
      B-stoploc.airport_name      0.000     0.000     0.000       0.0
             B-time_relative      0.000     0.000     0.000       0.0
                      I-time      0.000     0.000     0.000       0.0
    I-return_date.day_number      0.000     0.000     0.000       0.0
          I-meal_description      0.000     0.000     0.000       0.0
B-return_date.today_relative      0.000     0.000     0.000       0.0
      B-return_date.day_name      0.000     0.000     0.000       2.0

                   micro avg      0.955     0.957     0.956    3657.0
                   macro avg      0.677     0.657     0.654    3657.0
                weighted avg      0.961     0.957     0.954    3657.0

I0715 07:09:59.549368 140369481070464 interactiveshell.py:2882] Best Slot F1: 0.956
Reading ../data/atis.train.w-intent.iob
I0715 07:10:03.701361 140369481070464 interactiveshell.py:2882] Step 6900 | Loss: 0.0003 | Loss_intent: 0.0000 | Loss_slots: 0.0003 | Spent: 7.1 secs | LR: 0.000145
I0715 07:10:08.378329 140369481070464 interactiveshell.py:2882] Step 6950 | Loss: 0.0005 | Loss_intent: 0.0000 | Loss_slots: 0.0005 | Spent: 4.7 secs | LR: 0.000144
I0715 07:10:13.703079 140369481070464 interactiveshell.py:2882] Step 7000 | Loss: 0.0175 | Loss_intent: 0.0000 | Loss_slots: 0.0175 | Spent: 5.3 secs | LR: 0.000143
I0715 07:10:17.752017 140369481070464 interactiveshell.py:2882] Step 7050 | Loss: 0.0010 | Loss_intent: 0.0000 | Loss_slots: 0.0010 | Spent: 4.0 secs | LR: 0.000143
I0715 07:10:21.489963 140369481070464 interactiveshell.py:2882] Step 7100 | Loss: 0.0095 | Loss_intent: 0.0000 | Loss_slots: 0.0095 | Spent: 3.7 secs | LR: 0.000142
I0715 07:10:25.201784 140369481070464 interactiveshell.py:2882] Step 7150 | Loss: 0.0019 | Loss_intent: 0.0000 | Loss_slots: 0.0019 | Spent: 3.7 secs | LR: 0.000141
Reading ../data/atis.test.w-intent.iob
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.
  'precision', 'predicted', average, warn_for)
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1439: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples.
  'recall', 'true', average, warn_for)
I0715 07:10:28.527139 140369481070464 interactiveshell.py:2882] 
                                          precision    recall  f1-score   support

                             atis_flight      0.984     0.992     0.988       632
                            atis_airfare      0.960     1.000     0.980        48
                     atis_ground_service      1.000     1.000     1.000        36
                            atis_airline      1.000     1.000     1.000        38
                       atis_abbreviation      0.943     1.000     0.971        33
                           atis_aircraft      1.000     0.889     0.941         9
                        atis_flight_time      1.000     1.000     1.000         1
                           atis_quantity      0.429     1.000     0.600         3
                atis_flight#atis_airfare      0.833     0.417     0.556        12
                            atis_airport      1.000     1.000     1.000        18
                           atis_distance      1.000     1.000     1.000        10
                               atis_city      1.000     0.667     0.800         6
                        atis_ground_fare      1.000     0.857     0.923         7
                           atis_capacity      0.955     1.000     0.977        21
                          atis_flight_no      1.000     1.000     1.000         8
                               atis_meal      1.000     1.000     1.000         6
                        atis_restriction      0.000     0.000     0.000         0
             atis_airline#atis_flight_no      0.000     0.000     0.000         0
    atis_ground_service#atis_ground_fare      0.000     0.000     0.000         0
           atis_airfare#atis_flight_time      0.000     0.000     0.000         0
                           atis_cheapest      0.000     0.000     0.000         0
atis_aircraft#atis_flight#atis_flight_no      0.000     0.000     0.000         0

                               micro avg      0.976     0.982     0.979       888
                               macro avg      0.687     0.674     0.670       888
                            weighted avg      0.979     0.982     0.979       888

I0715 07:10:28.567814 140369481070464 interactiveshell.py:2882] 
                              precision    recall  f1-score   support

                           O      0.000     0.000     0.000       0.0
           B-toloc.city_name      0.975     0.994     0.985     716.0
         B-fromloc.city_name      0.987     0.997     0.992     704.0
           I-toloc.city_name      0.981     0.989     0.985     265.0
      B-depart_date.day_name      0.972     0.981     0.977     212.0
              B-airline_name      1.000     0.990     0.995     101.0
         I-fromloc.city_name      0.989     0.994     0.992     177.0
 B-depart_time.period_of_day      1.000     0.885     0.939     130.0
              I-airline_name      1.000     0.969     0.984      65.0
    B-depart_date.day_number      0.981     0.964     0.972      55.0
    B-depart_date.month_name      0.982     0.964     0.973      56.0
          B-depart_time.time      0.864     1.000     0.927      57.0
                B-round_trip      1.000     0.986     0.993      73.0
             B-cost_relative      1.000     0.973     0.986      37.0
                I-round_trip      1.000     1.000     1.000      71.0
                B-flight_mod      0.828     1.000     0.906      24.0
 B-depart_time.time_relative      0.969     0.969     0.969      65.0
          I-depart_time.time      0.981     1.000     0.990      52.0
         B-stoploc.city_name      0.952     1.000     0.976      20.0
                 B-city_name      0.846     0.579     0.688      57.0
                B-class_type      0.960     1.000     0.980      24.0
          B-arrive_time.time      0.943     0.971     0.957      34.0
 B-arrive_time.time_relative      0.967     0.935     0.951      31.0
                I-class_type      1.000     1.000     1.000      17.0
               B-flight_stop      1.000     1.000     1.000      21.0
          I-arrive_time.time      0.971     0.971     0.971      35.0
              B-airline_code      1.000     1.000     1.000      34.0
    I-depart_date.day_number      1.000     0.933     0.966      15.0
      I-fromloc.airport_name      0.385     1.000     0.556      15.0
      B-fromloc.airport_name      0.407     0.917     0.564      12.0
      B-arrive_date.day_name      0.688     1.000     0.815      11.0
          B-toloc.state_code      1.000     1.000     1.000      18.0
B-depart_date.today_relative      1.000     0.889     0.941       9.0
             B-flight_number      0.688     1.000     0.815      11.0
 B-depart_date.date_relative      0.944     1.000     0.971      17.0
          B-toloc.state_name      0.844     0.964     0.900      28.0
           B-fare_basis_code      0.909     0.588     0.714      17.0
               B-flight_time      1.000     1.000     1.000       1.0
                        B-or      1.000     1.000     1.000       3.0
 B-arrive_time.period_of_day      0.667     1.000     0.800       6.0
          B-meal_description      1.000     0.800     0.889      10.0
             I-cost_relative      1.000     0.667     0.800       3.0
              I-airport_name      0.909     0.345     0.500      29.0
               B-fare_amount      1.000     1.000     1.000       2.0
               I-fare_amount      1.000     1.000     1.000       2.0
                 I-city_name      0.955     0.700     0.808      30.0
        I-toloc.airport_name      1.000     1.000     1.000       3.0
            B-transport_type      0.889     0.800     0.842      10.0
    B-arrive_date.month_name      0.714     0.833     0.769       6.0
    B-arrive_date.day_number      0.714     0.833     0.769       6.0
         I-stoploc.city_name      0.909     1.000     0.952      10.0
                      B-meal      0.941     1.000     0.970      16.0
        B-fromloc.state_code      1.000     1.000     1.000      23.0
    B-depart_time.period_mod      0.714     1.000     0.833       5.0
                   B-connect      1.000     1.000     1.000       6.0
               B-flight_days      1.000     1.000     1.000      10.0
        B-toloc.airport_name      1.000     1.000     1.000       3.0
        B-fromloc.state_name      1.000     0.941     0.970      17.0
              B-airport_name      0.750     0.286     0.414      21.0
                   B-economy      1.000     1.000     1.000       6.0
               I-flight_time      1.000     1.000     1.000       1.0
             B-aircraft_code      0.964     0.818     0.885      33.0
                       B-mod      1.000     0.500     0.667       2.0
              B-airport_code      0.571     0.444     0.500       9.0
    B-depart_time.start_time      0.750     1.000     0.857       3.0
      B-depart_time.end_time      1.000     0.667     0.800       3.0
          B-depart_date.year      0.600     1.000     0.750       3.0
            I-transport_type      0.000     0.000     0.000       1.0
          B-restriction_code      0.800     1.000     0.889       4.0
    B-arrive_time.start_time      0.889     1.000     0.941       8.0
        B-toloc.airport_code      1.000     0.750     0.857       4.0
      B-arrive_time.end_time      0.875     0.875     0.875       8.0
      I-arrive_time.end_time      0.875     0.875     0.875       8.0
      I-depart_time.end_time      1.000     0.667     0.800       3.0
               I-flight_stop      0.000     0.000     0.000       0.0
      B-fromloc.airport_code      0.714     1.000     0.833       5.0
          I-restriction_code      1.000     1.000     1.000       3.0
    I-depart_time.start_time      1.000     1.000     1.000       1.0
          I-toloc.state_name      1.000     1.000     1.000       1.0
I-depart_date.today_relative      0.000     0.000     0.000       0.0
 B-arrive_date.date_relative      1.000     1.000     1.000       2.0
                I-flight_mod      1.000     0.167     0.286       6.0
                   I-economy      0.000     0.000     0.000       0.0
 B-return_date.date_relative      1.000     0.667     0.800       3.0
        I-fromloc.state_name      1.000     1.000     1.000       1.0
                B-state_code      1.000     1.000     1.000       1.0
    I-arrive_time.start_time      1.000     1.000     1.000       1.0
    I-arrive_date.day_number      0.000     0.000     0.000       0.0
                 B-meal_code      0.000     0.000     0.000       1.0
 I-depart_time.period_of_day      1.000     1.000     1.000       1.0
                  B-day_name      1.000     0.500     0.667       2.0
             B-period_of_day      1.000     0.500     0.667       4.0
        B-stoploc.state_code      0.000     0.000     0.000       0.0
    B-return_date.month_name      0.000     0.000     0.000       0.0
    B-return_date.day_number      0.000     0.000     0.000       0.0
    B-arrive_time.period_mod      0.000     0.000     0.000       0.0
                 I-meal_code      0.000     0.000     0.000       0.0
        B-toloc.country_name      1.000     1.000     1.000       1.0
                 B-days_code      0.000     0.000     0.000       1.0
 I-arrive_time.period_of_day      0.000     0.000     0.000       0.0
            I-today_relative      0.000     0.000     0.000       0.0
 B-return_time.period_of_day      0.000     0.000     0.000       0.0
                      B-time      0.000     0.000     0.000       0.0
           I-fare_basis_code      0.000     0.000     0.000       0.0
 I-arrive_time.time_relative      1.000     1.000     1.000       4.0
 I-depart_time.time_relative      0.000     0.000     0.000       1.0
            B-today_relative      0.000     0.000     0.000       0.0
                B-state_name      0.750     0.333     0.462       9.0
B-arrive_date.today_relative      0.000     0.000     0.000       0.0
    B-return_time.period_mod      0.000     0.000     0.000       0.0
                B-month_name      0.000     0.000     0.000       0.0
                B-day_number      0.000     0.000     0.000       0.0
 I-return_date.date_relative      1.000     0.333     0.500       3.0
I-return_date.today_relative      0.000     0.000     0.000       0.0
      B-stoploc.airport_name      0.000     0.000     0.000       0.0
             B-time_relative      0.000     0.000     0.000       0.0
                      I-time      0.000     0.000     0.000       0.0
    I-return_date.day_number      0.000     0.000     0.000       0.0
          I-meal_description      0.000     0.000     0.000       0.0
B-return_date.today_relative      0.000     0.000     0.000       0.0
      B-return_date.day_name      0.000     0.000     0.000       2.0

                   micro avg      0.952     0.954     0.953    3657.0
                   macro avg      0.677     0.650     0.649    3657.0
                weighted avg      0.962     0.954     0.952    3657.0

I0715 07:10:28.570247 140369481070464 interactiveshell.py:2882] Best Slot F1: 0.956
Reading ../data/atis.train.w-intent.iob
I0715 07:10:31.111117 140369481070464 interactiveshell.py:2882] Step 7200 | Loss: 0.0040 | Loss_intent: 0.0000 | Loss_slots: 0.0040 | Spent: 5.9 secs | LR: 0.000140
I0715 07:10:34.838051 140369481070464 interactiveshell.py:2882] Step 7250 | Loss: 0.0068 | Loss_intent: 0.0000 | Loss_slots: 0.0068 | Spent: 3.7 secs | LR: 0.000140
I0715 07:10:38.570105 140369481070464 interactiveshell.py:2882] Step 7300 | Loss: 0.0030 | Loss_intent: 0.0000 | Loss_slots: 0.0030 | Spent: 3.7 secs | LR: 0.000139
I0715 07:10:42.270381 140369481070464 interactiveshell.py:2882] Step 7350 | Loss: 0.0108 | Loss_intent: 0.0000 | Loss_slots: 0.0108 | Spent: 3.7 secs | LR: 0.000138
I0715 07:10:45.957800 140369481070464 interactiveshell.py:2882] Step 7400 | Loss: 0.0009 | Loss_intent: 0.0000 | Loss_slots: 0.0009 | Spent: 3.7 secs | LR: 0.000138
I0715 07:10:49.704638 140369481070464 interactiveshell.py:2882] Step 7450 | Loss: 0.0017 | Loss_intent: 0.0000 | Loss_slots: 0.0017 | Spent: 3.7 secs | LR: 0.000137
Reading ../data/atis.test.w-intent.iob
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.
  'precision', 'predicted', average, warn_for)
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1439: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples.
  'recall', 'true', average, warn_for)
I0715 07:10:53.885288 140369481070464 interactiveshell.py:2882] 
                                          precision    recall  f1-score   support

                             atis_flight      0.984     0.991     0.987       632
                            atis_airfare      0.980     1.000     0.990        48
                     atis_ground_service      1.000     1.000     1.000        36
                            atis_airline      1.000     1.000     1.000        38
                       atis_abbreviation      0.943     1.000     0.971        33
                           atis_aircraft      1.000     0.889     0.941         9
                        atis_flight_time      1.000     1.000     1.000         1
                           atis_quantity      0.429     1.000     0.600         3
                atis_flight#atis_airfare      0.833     0.417     0.556        12
                            atis_airport      0.944     0.944     0.944        18
                           atis_distance      1.000     1.000     1.000        10
                               atis_city      0.800     0.667     0.727         6
                        atis_ground_fare      1.000     1.000     1.000         7
                           atis_capacity      0.913     1.000     0.955        21
                          atis_flight_no      1.000     1.000     1.000         8
                               atis_meal      1.000     0.833     0.909         6
                        atis_restriction      0.000     0.000     0.000         0
             atis_airline#atis_flight_no      0.000     0.000     0.000         0
    atis_ground_service#atis_ground_fare      0.000     0.000     0.000         0
           atis_airfare#atis_flight_time      0.000     0.000     0.000         0
                           atis_cheapest      0.000     0.000     0.000         0
atis_aircraft#atis_flight#atis_flight_no      0.000     0.000     0.000         0

                               micro avg      0.974     0.980     0.977       888
                               macro avg      0.674     0.670     0.663       888
                            weighted avg      0.977     0.980     0.977       888

I0715 07:10:53.925780 140369481070464 interactiveshell.py:2882] 
                              precision    recall  f1-score   support

                           O      0.000     0.000     0.000       0.0
           B-toloc.city_name      0.970     0.993     0.981     716.0
         B-fromloc.city_name      0.978     0.997     0.987     704.0
           I-toloc.city_name      0.985     1.000     0.993     265.0
      B-depart_date.day_name      0.968     0.986     0.977     212.0
              B-airline_name      1.000     1.000     1.000     101.0
         I-fromloc.city_name      0.989     0.994     0.992     177.0
 B-depart_time.period_of_day      1.000     0.877     0.934     130.0
              I-airline_name      1.000     0.985     0.992      65.0
    B-depart_date.day_number      0.981     0.964     0.972      55.0
    B-depart_date.month_name      0.982     0.964     0.973      56.0
          B-depart_time.time      0.851     1.000     0.919      57.0
                B-round_trip      1.000     0.986     0.993      73.0
             B-cost_relative      1.000     0.973     0.986      37.0
                I-round_trip      1.000     1.000     1.000      71.0
                B-flight_mod      0.828     1.000     0.906      24.0
 B-depart_time.time_relative      0.984     0.938     0.961      65.0
          I-depart_time.time      0.963     1.000     0.981      52.0
         B-stoploc.city_name      0.952     1.000     0.976      20.0
                 B-city_name      0.900     0.474     0.621      57.0
                B-class_type      0.960     1.000     0.980      24.0
          B-arrive_time.time      0.943     0.971     0.957      34.0
 B-arrive_time.time_relative      0.882     0.968     0.923      31.0
                I-class_type      1.000     1.000     1.000      17.0
               B-flight_stop      1.000     1.000     1.000      21.0
          I-arrive_time.time      0.971     0.971     0.971      35.0
              B-airline_code      1.000     0.971     0.985      34.0
    I-depart_date.day_number      1.000     0.933     0.966      15.0
      I-fromloc.airport_name      0.395     1.000     0.566      15.0
      B-fromloc.airport_name      0.391     0.750     0.514      12.0
      B-arrive_date.day_name      0.786     1.000     0.880      11.0
          B-toloc.state_code      1.000     1.000     1.000      18.0
B-depart_date.today_relative      1.000     0.889     0.941       9.0
             B-flight_number      0.524     1.000     0.688      11.0
 B-depart_date.date_relative      0.944     1.000     0.971      17.0
          B-toloc.state_name      0.929     0.929     0.929      28.0
           B-fare_basis_code      0.944     1.000     0.971      17.0
               B-flight_time      1.000     1.000     1.000       1.0
                        B-or      1.000     1.000     1.000       3.0
 B-arrive_time.period_of_day      0.600     1.000     0.750       6.0
          B-meal_description      1.000     0.900     0.947      10.0
             I-cost_relative      1.000     0.667     0.800       3.0
              I-airport_name      0.917     0.379     0.537      29.0
               B-fare_amount      1.000     1.000     1.000       2.0
               I-fare_amount      1.000     1.000     1.000       2.0
                 I-city_name      0.957     0.733     0.830      30.0
        I-toloc.airport_name      1.000     1.000     1.000       3.0
            B-transport_type      1.000     0.700     0.824      10.0
    B-arrive_date.month_name      0.714     0.833     0.769       6.0
    B-arrive_date.day_number      0.714     0.833     0.769       6.0
         I-stoploc.city_name      1.000     1.000     1.000      10.0
                      B-meal      1.000     1.000     1.000      16.0
        B-fromloc.state_code      1.000     1.000     1.000      23.0
    B-depart_time.period_mod      1.000     1.000     1.000       5.0
                   B-connect      1.000     1.000     1.000       6.0
               B-flight_days      1.000     1.000     1.000      10.0
        B-toloc.airport_name      1.000     1.000     1.000       3.0
        B-fromloc.state_name      1.000     0.941     0.970      17.0
              B-airport_name      0.667     0.286     0.400      21.0
                   B-economy      1.000     1.000     1.000       6.0
               I-flight_time      1.000     1.000     1.000       1.0
             B-aircraft_code      1.000     0.818     0.900      33.0
                       B-mod      1.000     0.500     0.667       2.0
              B-airport_code      0.800     0.444     0.571       9.0
    B-depart_time.start_time      0.667     0.667     0.667       3.0
      B-depart_time.end_time      1.000     0.667     0.800       3.0
          B-depart_date.year      0.750     1.000     0.857       3.0
            I-transport_type      0.000     0.000     0.000       1.0
          B-restriction_code      1.000     1.000     1.000       4.0
    B-arrive_time.start_time      0.800     1.000     0.889       8.0
        B-toloc.airport_code      1.000     0.500     0.667       4.0
      B-arrive_time.end_time      0.875     0.875     0.875       8.0
      I-arrive_time.end_time      0.875     0.875     0.875       8.0
      I-depart_time.end_time      1.000     0.667     0.800       3.0
               I-flight_stop      0.000     0.000     0.000       0.0
      B-fromloc.airport_code      0.750     0.600     0.667       5.0
          I-restriction_code      1.000     1.000     1.000       3.0
    I-depart_time.start_time      1.000     1.000     1.000       1.0
          I-toloc.state_name      1.000     1.000     1.000       1.0
I-depart_date.today_relative      0.000     0.000     0.000       0.0
 B-arrive_date.date_relative      1.000     1.000     1.000       2.0
                I-flight_mod      1.000     0.167     0.286       6.0
                   I-economy      0.000     0.000     0.000       0.0
 B-return_date.date_relative      0.000     0.000     0.000       3.0
        I-fromloc.state_name      1.000     1.000     1.000       1.0
                B-state_code      1.000     1.000     1.000       1.0
    I-arrive_time.start_time      1.000     1.000     1.000       1.0
    I-arrive_date.day_number      0.000     0.000     0.000       0.0
                 B-meal_code      0.000     0.000     0.000       1.0
 I-depart_time.period_of_day      1.000     1.000     1.000       1.0
                  B-day_name      1.000     0.500     0.667       2.0
             B-period_of_day      1.000     0.750     0.857       4.0
        B-stoploc.state_code      0.000     0.000     0.000       0.0
    B-return_date.month_name      0.000     0.000     0.000       0.0
    B-return_date.day_number      0.000     0.000     0.000       0.0
    B-arrive_time.period_mod      0.000     0.000     0.000       0.0
                 I-meal_code      0.000     0.000     0.000       0.0
        B-toloc.country_name      1.000     1.000     1.000       1.0
                 B-days_code      1.000     1.000     1.000       1.0
 I-arrive_time.period_of_day      0.000     0.000     0.000       0.0
            I-today_relative      0.000     0.000     0.000       0.0
 B-return_time.period_of_day      0.000     0.000     0.000       0.0
                      B-time      0.000     0.000     0.000       0.0
           I-fare_basis_code      0.000     0.000     0.000       0.0
 I-arrive_time.time_relative      1.000     0.500     0.667       4.0
 I-depart_time.time_relative      0.000     0.000     0.000       1.0
            B-today_relative      0.000     0.000     0.000       0.0
                B-state_name      0.857     0.667     0.750       9.0
B-arrive_date.today_relative      0.000     0.000     0.000       0.0
    B-return_time.period_mod      0.000     0.000     0.000       0.0
                B-month_name      0.000     0.000     0.000       0.0
                B-day_number      0.000     0.000     0.000       0.0
 I-return_date.date_relative      0.667     0.667     0.667       3.0
I-return_date.today_relative      0.000     0.000     0.000       0.0
      B-stoploc.airport_name      0.000     0.000     0.000       0.0
             B-time_relative      0.000     0.000     0.000       0.0
                      I-time      0.000     0.000     0.000       0.0
    I-return_date.day_number      0.000     0.000     0.000       0.0
          I-meal_description      0.000     0.000     0.000       0.0
B-return_date.today_relative      0.000     0.000     0.000       0.0
      B-return_date.day_name      0.000     0.000     0.000       2.0

                   micro avg      0.952     0.954     0.953    3657.0
                   macro avg      0.683     0.650     0.654    3657.0
                weighted avg      0.960     0.954     0.952    3657.0

I0715 07:10:53.926659 140369481070464 interactiveshell.py:2882] Best Slot F1: 0.956
Reading ../data/atis.train.w-intent.iob
I0715 07:10:55.590447 140369481070464 interactiveshell.py:2882] Step 7500 | Loss: 0.0046 | Loss_intent: 0.0000 | Loss_slots: 0.0046 | Spent: 5.9 secs | LR: 0.000136
I0715 07:10:59.301963 140369481070464 interactiveshell.py:2882] Step 7550 | Loss: 0.0018 | Loss_intent: 0.0000 | Loss_slots: 0.0017 | Spent: 3.7 secs | LR: 0.000135
I0715 07:11:02.996295 140369481070464 interactiveshell.py:2882] Step 7600 | Loss: 0.0264 | Loss_intent: 0.0158 | Loss_slots: 0.0106 | Spent: 3.7 secs | LR: 0.000135
I0715 07:11:06.726038 140369481070464 interactiveshell.py:2882] Step 7650 | Loss: 0.0031 | Loss_intent: 0.0000 | Loss_slots: 0.0031 | Spent: 3.7 secs | LR: 0.000134
I0715 07:11:10.634706 140369481070464 interactiveshell.py:2882] Step 7700 | Loss: 0.0140 | Loss_intent: 0.0000 | Loss_slots: 0.0140 | Spent: 3.9 secs | LR: 0.000133
I0715 07:11:14.393905 140369481070464 interactiveshell.py:2882] Step 7750 | Loss: 0.0062 | Loss_intent: 0.0000 | Loss_slots: 0.0062 | Spent: 3.8 secs | LR: 0.000133
Reading ../data/atis.test.w-intent.iob
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.
  'precision', 'predicted', average, warn_for)
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1439: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples.
  'recall', 'true', average, warn_for)
I0715 07:11:19.442478 140369481070464 interactiveshell.py:2882] 
                                          precision    recall  f1-score   support

                             atis_flight      0.983     0.992     0.987       632
                            atis_airfare      0.960     1.000     0.980        48
                     atis_ground_service      1.000     1.000     1.000        36
                            atis_airline      1.000     1.000     1.000        38
                       atis_abbreviation      0.943     1.000     0.971        33
                           atis_aircraft      1.000     0.889     0.941         9
                        atis_flight_time      1.000     1.000     1.000         1
                           atis_quantity      0.429     1.000     0.600         3
                atis_flight#atis_airfare      0.833     0.417     0.556        12
                            atis_airport      1.000     0.944     0.971        18
                           atis_distance      1.000     1.000     1.000        10
                               atis_city      1.000     0.667     0.800         6
                        atis_ground_fare      1.000     0.857     0.923         7
                           atis_capacity      0.955     1.000     0.977        21
                          atis_flight_no      1.000     1.000     1.000         8
                               atis_meal      1.000     1.000     1.000         6
                        atis_restriction      0.000     0.000     0.000         0
             atis_airline#atis_flight_no      0.000     0.000     0.000         0
    atis_ground_service#atis_ground_fare      0.000     0.000     0.000         0
           atis_airfare#atis_flight_time      0.000     0.000     0.000         0
                           atis_cheapest      0.000     0.000     0.000         0
atis_aircraft#atis_flight#atis_flight_no      0.000     0.000     0.000         0

                               micro avg      0.975     0.981     0.978       888
                               macro avg      0.686     0.671     0.668       888
                            weighted avg      0.978     0.981     0.978       888

I0715 07:11:19.483247 140369481070464 interactiveshell.py:2882] 
                              precision    recall  f1-score   support

                           O      0.000     0.000     0.000       0.0
           B-toloc.city_name      0.978     0.993     0.985     716.0
         B-fromloc.city_name      0.980     0.999     0.989     704.0
           I-toloc.city_name      0.981     0.974     0.977     265.0
      B-depart_date.day_name      0.968     0.986     0.977     212.0
              B-airline_name      0.980     0.990     0.985     101.0
         I-fromloc.city_name      0.946     0.994     0.970     177.0
 B-depart_time.period_of_day      0.983     0.877     0.927     130.0
              I-airline_name      1.000     0.985     0.992      65.0
    B-depart_date.day_number      0.982     0.982     0.982      55.0
    B-depart_date.month_name      0.982     0.964     0.973      56.0
          B-depart_time.time      0.864     1.000     0.927      57.0
                B-round_trip      1.000     0.986     0.993      73.0
             B-cost_relative      1.000     0.973     0.986      37.0
                I-round_trip      1.000     1.000     1.000      71.0
                B-flight_mod      0.821     0.958     0.885      24.0
 B-depart_time.time_relative      0.968     0.938     0.953      65.0
          I-depart_time.time      0.962     0.962     0.962      52.0
         B-stoploc.city_name      1.000     1.000     1.000      20.0
                 B-city_name      0.816     0.544     0.653      57.0
                B-class_type      0.960     1.000     0.980      24.0
          B-arrive_time.time      0.943     0.971     0.957      34.0
 B-arrive_time.time_relative      0.853     0.935     0.892      31.0
                I-class_type      1.000     1.000     1.000      17.0
               B-flight_stop      1.000     1.000     1.000      21.0
          I-arrive_time.time      0.919     0.971     0.944      35.0
              B-airline_code      1.000     0.882     0.938      34.0
    I-depart_date.day_number      1.000     0.933     0.966      15.0
      I-fromloc.airport_name      0.375     1.000     0.545      15.0
      B-fromloc.airport_name      0.407     0.917     0.564      12.0
      B-arrive_date.day_name      0.786     1.000     0.880      11.0
          B-toloc.state_code      1.000     1.000     1.000      18.0
B-depart_date.today_relative      1.000     0.889     0.941       9.0
             B-flight_number      0.550     1.000     0.710      11.0
 B-depart_date.date_relative      0.895     1.000     0.944      17.0
          B-toloc.state_name      0.931     0.964     0.947      28.0
           B-fare_basis_code      0.917     0.647     0.759      17.0
               B-flight_time      1.000     1.000     1.000       1.0
                        B-or      0.750     1.000     0.857       3.0
 B-arrive_time.period_of_day      0.545     1.000     0.706       6.0
          B-meal_description      1.000     0.900     0.947      10.0
             I-cost_relative      1.000     0.667     0.800       3.0
              I-airport_name      0.833     0.345     0.488      29.0
               B-fare_amount      1.000     1.000     1.000       2.0
               I-fare_amount      1.000     1.000     1.000       2.0
                 I-city_name      1.000     0.567     0.723      30.0
        I-toloc.airport_name      1.000     1.000     1.000       3.0
            B-transport_type      1.000     0.900     0.947      10.0
    B-arrive_date.month_name      0.714     0.833     0.769       6.0
    B-arrive_date.day_number      0.833     0.833     0.833       6.0
         I-stoploc.city_name      0.909     1.000     0.952      10.0
                      B-meal      1.000     1.000     1.000      16.0
        B-fromloc.state_code      1.000     1.000     1.000      23.0
    B-depart_time.period_mod      1.000     1.000     1.000       5.0
                   B-connect      1.000     1.000     1.000       6.0
               B-flight_days      1.000     1.000     1.000      10.0
        B-toloc.airport_name      1.000     1.000     1.000       3.0
        B-fromloc.state_name      1.000     1.000     1.000      17.0
              B-airport_name      0.857     0.286     0.429      21.0
                   B-economy      1.000     1.000     1.000       6.0
               I-flight_time      1.000     1.000     1.000       1.0
             B-aircraft_code      1.000     0.788     0.881      33.0
                       B-mod      1.000     0.500     0.667       2.0
              B-airport_code      0.800     0.444     0.571       9.0
    B-depart_time.start_time      0.750     1.000     0.857       3.0
      B-depart_time.end_time      1.000     0.667     0.800       3.0
          B-depart_date.year      0.500     1.000     0.667       3.0
            I-transport_type      0.000     0.000     0.000       1.0
          B-restriction_code      1.000     1.000     1.000       4.0
    B-arrive_time.start_time      0.889     1.000     0.941       8.0
        B-toloc.airport_code      1.000     0.750     0.857       4.0
      B-arrive_time.end_time      0.875     0.875     0.875       8.0
      I-arrive_time.end_time      0.875     0.875     0.875       8.0
      I-depart_time.end_time      1.000     0.667     0.800       3.0
               I-flight_stop      0.000     0.000     0.000       0.0
      B-fromloc.airport_code      0.833     1.000     0.909       5.0
          I-restriction_code      1.000     1.000     1.000       3.0
    I-depart_time.start_time      1.000     1.000     1.000       1.0
          I-toloc.state_name      1.000     1.000     1.000       1.0
I-depart_date.today_relative      0.000     0.000     0.000       0.0
 B-arrive_date.date_relative      1.000     1.000     1.000       2.0
                I-flight_mod      1.000     0.167     0.286       6.0
                   I-economy      0.000     0.000     0.000       0.0
 B-return_date.date_relative      1.000     0.333     0.500       3.0
        I-fromloc.state_name      1.000     1.000     1.000       1.0
                B-state_code      1.000     1.000     1.000       1.0
    I-arrive_time.start_time      1.000     1.000     1.000       1.0
    I-arrive_date.day_number      0.000     0.000     0.000       0.0
                 B-meal_code      0.000     0.000     0.000       1.0
 I-depart_time.period_of_day      0.000     0.000     0.000       1.0
                  B-day_name      1.000     0.500     0.667       2.0
             B-period_of_day      1.000     0.500     0.667       4.0
        B-stoploc.state_code      0.000     0.000     0.000       0.0
    B-return_date.month_name      0.000     0.000     0.000       0.0
    B-return_date.day_number      0.000     0.000     0.000       0.0
    B-arrive_time.period_mod      0.000     0.000     0.000       0.0
                 I-meal_code      0.000     0.000     0.000       0.0
        B-toloc.country_name      1.000     1.000     1.000       1.0
                 B-days_code      1.000     1.000     1.000       1.0
 I-arrive_time.period_of_day      0.000     0.000     0.000       0.0
            I-today_relative      0.000     0.000     0.000       0.0
 B-return_time.period_of_day      0.000     0.000     0.000       0.0
                      B-time      0.000     0.000     0.000       0.0
           I-fare_basis_code      0.000     0.000     0.000       0.0
 I-arrive_time.time_relative      0.000     0.000     0.000       4.0
 I-depart_time.time_relative      0.000     0.000     0.000       1.0
            B-today_relative      0.000     0.000     0.000       0.0
                B-state_name      0.800     0.444     0.571       9.0
B-arrive_date.today_relative      0.000     0.000     0.000       0.0
    B-return_time.period_mod      0.000     0.000     0.000       0.0
                B-month_name      0.000     0.000     0.000       0.0
                B-day_number      0.000     0.000     0.000       0.0
 I-return_date.date_relative      0.667     0.667     0.667       3.0
I-return_date.today_relative      0.000     0.000     0.000       0.0
      B-stoploc.airport_name      0.000     0.000     0.000       0.0
             B-time_relative      0.000     0.000     0.000       0.0
                      I-time      0.000     0.000     0.000       0.0
    I-return_date.day_number      0.000     0.000     0.000       0.0
          I-meal_description      0.000     0.000     0.000       0.0
B-return_date.today_relative      0.000     0.000     0.000       0.0
      B-return_date.day_name      0.000     0.000     0.000       2.0

                   micro avg      0.947     0.949     0.948    3657.0
                   macro avg      0.671     0.642     0.641    3657.0
                weighted avg      0.957     0.949     0.947    3657.0

I0715 07:11:19.484075 140369481070464 interactiveshell.py:2882] Best Slot F1: 0.956
Reading ../data/atis.train.w-intent.iob
I0715 07:11:20.251939 140369481070464 interactiveshell.py:2882] Step 7800 | Loss: 0.0011 | Loss_intent: 0.0000 | Loss_slots: 0.0011 | Spent: 5.9 secs | LR: 0.000132
I0715 07:11:23.933047 140369481070464 interactiveshell.py:2882] Step 7850 | Loss: 0.0004 | Loss_intent: 0.0000 | Loss_slots: 0.0004 | Spent: 3.7 secs | LR: 0.000131
I0715 07:11:27.653290 140369481070464 interactiveshell.py:2882] Step 7900 | Loss: 0.0004 | Loss_intent: 0.0000 | Loss_slots: 0.0004 | Spent: 3.7 secs | LR: 0.000131
I0715 07:11:31.378030 140369481070464 interactiveshell.py:2882] Step 7950 | Loss: 0.0031 | Loss_intent: 0.0000 | Loss_slots: 0.0031 | Spent: 3.7 secs | LR: 0.000130
I0715 07:11:35.113211 140369481070464 interactiveshell.py:2882] Step 8000 | Loss: 0.0065 | Loss_intent: 0.0001 | Loss_slots: 0.0064 | Spent: 3.7 secs | LR: 0.000129
I0715 07:11:38.886826 140369481070464 interactiveshell.py:2882] Step 8050 | Loss: 0.0075 | Loss_intent: 0.0000 | Loss_slots: 0.0075 | Spent: 3.8 secs | LR: 0.000128
I0715 07:11:42.624870 140369481070464 interactiveshell.py:2882] Step 8100 | Loss: 0.0125 | Loss_intent: 0.0000 | Loss_slots: 0.0125 | Spent: 3.7 secs | LR: 0.000128
Reading ../data/atis.test.w-intent.iob
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.
  'precision', 'predicted', average, warn_for)
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1439: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples.
  'recall', 'true', average, warn_for)
I0715 07:11:44.883882 140369481070464 interactiveshell.py:2882] 
                                          precision    recall  f1-score   support

                             atis_flight      0.986     0.991     0.988       632
                            atis_airfare      0.960     1.000     0.980        48
                     atis_ground_service      1.000     1.000     1.000        36
                            atis_airline      1.000     1.000     1.000        38
                       atis_abbreviation      0.943     1.000     0.971        33
                           atis_aircraft      1.000     0.889     0.941         9
                        atis_flight_time      1.000     1.000     1.000         1
                           atis_quantity      0.429     1.000     0.600         3
                atis_flight#atis_airfare      0.875     0.583     0.700        12
                            atis_airport      0.944     0.944     0.944        18
                           atis_distance      1.000     1.000     1.000        10
                               atis_city      1.000     0.667     0.800         6
                        atis_ground_fare      1.000     0.857     0.923         7
                           atis_capacity      0.913     1.000     0.955        21
                          atis_flight_no      1.000     1.000     1.000         8
                               atis_meal      1.000     0.833     0.909         6
                        atis_restriction      0.000     0.000     0.000         0
             atis_airline#atis_flight_no      0.000     0.000     0.000         0
    atis_ground_service#atis_ground_fare      0.000     0.000     0.000         0
           atis_airfare#atis_flight_time      0.000     0.000     0.000         0
                           atis_cheapest      0.000     0.000     0.000         0
atis_aircraft#atis_flight#atis_flight_no      0.000     0.000     0.000         0

                               micro avg      0.975     0.981     0.978       888
                               macro avg      0.684     0.671     0.669       888
                            weighted avg      0.979     0.981     0.979       888

I0715 07:11:44.923775 140369481070464 interactiveshell.py:2882] 
                              precision    recall  f1-score   support

                           O      0.000     0.000     0.000       0.0
           B-toloc.city_name      0.975     0.994     0.985     716.0
         B-fromloc.city_name      0.987     0.997     0.992     704.0
           I-toloc.city_name      0.978     0.989     0.983     265.0
      B-depart_date.day_name      0.981     0.986     0.984     212.0
              B-airline_name      0.980     0.990     0.985     101.0
         I-fromloc.city_name      0.967     0.994     0.981     177.0
 B-depart_time.period_of_day      1.000     0.877     0.934     130.0
              I-airline_name      1.000     0.985     0.992      65.0
    B-depart_date.day_number      0.981     0.964     0.972      55.0
    B-depart_date.month_name      0.982     0.964     0.973      56.0
          B-depart_time.time      0.877     1.000     0.934      57.0
                B-round_trip      1.000     0.986     0.993      73.0
             B-cost_relative      0.973     0.973     0.973      37.0
                I-round_trip      0.986     1.000     0.993      71.0
                B-flight_mod      0.828     1.000     0.906      24.0
 B-depart_time.time_relative      0.984     0.938     0.961      65.0
          I-depart_time.time      0.912     1.000     0.954      52.0
         B-stoploc.city_name      1.000     1.000     1.000      20.0
                 B-city_name      0.854     0.614     0.714      57.0
                B-class_type      0.960     1.000     0.980      24.0
          B-arrive_time.time      0.917     0.971     0.943      34.0
 B-arrive_time.time_relative      0.909     0.968     0.937      31.0
                I-class_type      1.000     1.000     1.000      17.0
               B-flight_stop      1.000     1.000     1.000      21.0
          I-arrive_time.time      0.971     0.971     0.971      35.0
              B-airline_code      0.938     0.882     0.909      34.0
    I-depart_date.day_number      1.000     0.933     0.966      15.0
      I-fromloc.airport_name      0.385     1.000     0.556      15.0
      B-fromloc.airport_name      0.440     0.917     0.595      12.0
      B-arrive_date.day_name      0.786     1.000     0.880      11.0
          B-toloc.state_code      1.000     1.000     1.000      18.0
B-depart_date.today_relative      1.000     1.000     1.000       9.0
             B-flight_number      0.579     1.000     0.733      11.0
 B-depart_date.date_relative      0.944     1.000     0.971      17.0
          B-toloc.state_name      0.867     0.929     0.897      28.0
           B-fare_basis_code      0.917     0.647     0.759      17.0
               B-flight_time      1.000     1.000     1.000       1.0
                        B-or      0.750     1.000     0.857       3.0
 B-arrive_time.period_of_day      0.600     1.000     0.750       6.0
          B-meal_description      1.000     0.900     0.947      10.0
             I-cost_relative      1.000     0.667     0.800       3.0
              I-airport_name      0.846     0.379     0.524      29.0
               B-fare_amount      1.000     1.000     1.000       2.0
               I-fare_amount      1.000     1.000     1.000       2.0
                 I-city_name      1.000     0.600     0.750      30.0
        I-toloc.airport_name      1.000     1.000     1.000       3.0
            B-transport_type      1.000     0.700     0.824      10.0
    B-arrive_date.month_name      0.714     0.833     0.769       6.0
    B-arrive_date.day_number      0.714     0.833     0.769       6.0
         I-stoploc.city_name      1.000     1.000     1.000      10.0
                      B-meal      1.000     1.000     1.000      16.0
        B-fromloc.state_code      1.000     1.000     1.000      23.0
    B-depart_time.period_mod      1.000     1.000     1.000       5.0
                   B-connect      1.000     1.000     1.000       6.0
               B-flight_days      1.000     1.000     1.000      10.0
        B-toloc.airport_name      1.000     1.000     1.000       3.0
        B-fromloc.state_name      1.000     0.824     0.903      17.0
              B-airport_name      0.857     0.286     0.429      21.0
                   B-economy      1.000     1.000     1.000       6.0
               I-flight_time      1.000     1.000     1.000       1.0
             B-aircraft_code      1.000     0.818     0.900      33.0
                       B-mod      1.000     0.500     0.667       2.0
              B-airport_code      0.800     0.444     0.571       9.0
    B-depart_time.start_time      0.667     0.667     0.667       3.0
      B-depart_time.end_time      1.000     0.667     0.800       3.0
          B-depart_date.year      0.500     1.000     0.667       3.0
            I-transport_type      0.000     0.000     0.000       1.0
          B-restriction_code      1.000     0.500     0.667       4.0
    B-arrive_time.start_time      0.889     1.000     0.941       8.0
        B-toloc.airport_code      1.000     0.750     0.857       4.0
      B-arrive_time.end_time      0.875     0.875     0.875       8.0
      I-arrive_time.end_time      0.875     0.875     0.875       8.0
      I-depart_time.end_time      1.000     0.667     0.800       3.0
               I-flight_stop      0.000     0.000     0.000       0.0
      B-fromloc.airport_code      0.625     1.000     0.769       5.0
          I-restriction_code      1.000     1.000     1.000       3.0
    I-depart_time.start_time      1.000     1.000     1.000       1.0
          I-toloc.state_name      1.000     1.000     1.000       1.0
I-depart_date.today_relative      0.000     0.000     0.000       0.0
 B-arrive_date.date_relative      1.000     1.000     1.000       2.0
                I-flight_mod      0.500     0.167     0.250       6.0
                   I-economy      0.000     0.000     0.000       0.0
 B-return_date.date_relative      0.000     0.000     0.000       3.0
        I-fromloc.state_name      0.500     1.000     0.667       1.0
                B-state_code      1.000     1.000     1.000       1.0
    I-arrive_time.start_time      1.000     1.000     1.000       1.0
    I-arrive_date.day_number      0.000     0.000     0.000       0.0
                 B-meal_code      0.000     0.000     0.000       1.0
 I-depart_time.period_of_day      1.000     1.000     1.000       1.0
                  B-day_name      1.000     0.500     0.667       2.0
             B-period_of_day      1.000     0.750     0.857       4.0
        B-stoploc.state_code      0.000     0.000     0.000       0.0
    B-return_date.month_name      0.000     0.000     0.000       0.0
    B-return_date.day_number      0.000     0.000     0.000       0.0
    B-arrive_time.period_mod      0.000     0.000     0.000       0.0
                 I-meal_code      0.000     0.000     0.000       0.0
        B-toloc.country_name      1.000     1.000     1.000       1.0
                 B-days_code      0.500     1.000     0.667       1.0
 I-arrive_time.period_of_day      0.000     0.000     0.000       0.0
            I-today_relative      0.000     0.000     0.000       0.0
 B-return_time.period_of_day      0.000     0.000     0.000       0.0
                      B-time      0.000     0.000     0.000       0.0
           I-fare_basis_code      0.000     0.000     0.000       0.0
 I-arrive_time.time_relative      1.000     0.500     0.667       4.0
 I-depart_time.time_relative      0.000     0.000     0.000       1.0
            B-today_relative      0.000     0.000     0.000       0.0
                B-state_name      0.800     0.444     0.571       9.0
B-arrive_date.today_relative      0.000     0.000     0.000       0.0
    B-return_time.period_mod      0.000     0.000     0.000       0.0
                B-month_name      0.000     0.000     0.000       0.0
                B-day_number      0.000     0.000     0.000       0.0
 I-return_date.date_relative      0.667     0.667     0.667       3.0
I-return_date.today_relative      0.000     0.000     0.000       0.0
      B-stoploc.airport_name      0.000     0.000     0.000       0.0
             B-time_relative      0.000     0.000     0.000       0.0
                      I-time      0.000     0.000     0.000       0.0
    I-return_date.day_number      0.000     0.000     0.000       0.0
          I-meal_description      0.000     0.000     0.000       0.0
B-return_date.today_relative      0.000     0.000     0.000       0.0
      B-return_date.day_name      0.000     0.000     0.000       2.0

                   micro avg      0.950     0.951     0.951    3657.0
                   macro avg      0.666     0.647     0.642    3657.0
                weighted avg      0.959     0.951     0.950    3657.0

I0715 07:11:44.924610 140369481070464 interactiveshell.py:2882] Best Slot F1: 0.956
Reading ../data/atis.train.w-intent.iob
I0715 07:11:48.476368 140369481070464 interactiveshell.py:2882] Step 8150 | Loss: 0.0017 | Loss_intent: 0.0000 | Loss_slots: 0.0017 | Spent: 5.9 secs | LR: 0.000127
I0715 07:11:52.175860 140369481070464 interactiveshell.py:2882] Step 8200 | Loss: 0.0124 | Loss_intent: 0.0008 | Loss_slots: 0.0116 | Spent: 3.7 secs | LR: 0.000126
I0715 07:11:55.869318 140369481070464 interactiveshell.py:2882] Step 8250 | Loss: 0.0003 | Loss_intent: 0.0000 | Loss_slots: 0.0003 | Spent: 3.7 secs | LR: 0.000126
I0715 07:11:59.582288 140369481070464 interactiveshell.py:2882] Step 8300 | Loss: 0.0006 | Loss_intent: 0.0000 | Loss_slots: 0.0006 | Spent: 3.7 secs | LR: 0.000125
I0715 07:12:03.295896 140369481070464 interactiveshell.py:2882] Step 8350 | Loss: 0.0026 | Loss_intent: 0.0000 | Loss_slots: 0.0026 | Spent: 3.7 secs | LR: 0.000124
I0715 07:12:06.991579 140369481070464 interactiveshell.py:2882] Step 8400 | Loss: 0.0034 | Loss_intent: 0.0000 | Loss_slots: 0.0034 | Spent: 3.7 secs | LR: 0.000124
Reading ../data/atis.test.w-intent.iob
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.
  'precision', 'predicted', average, warn_for)
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1439: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples.
  'recall', 'true', average, warn_for)
I0715 07:12:10.133359 140369481070464 interactiveshell.py:2882] 
                                          precision    recall  f1-score   support

                             atis_flight      0.981     0.991     0.986       632
                            atis_airfare      0.941     1.000     0.970        48
                     atis_ground_service      1.000     1.000     1.000        36
                            atis_airline      1.000     1.000     1.000        38
                       atis_abbreviation      0.943     1.000     0.971        33
                           atis_aircraft      1.000     0.889     0.941         9
                        atis_flight_time      1.000     1.000     1.000         1
                           atis_quantity      0.429     1.000     0.600         3
                atis_flight#atis_airfare      0.833     0.417     0.556        12
                            atis_airport      1.000     0.944     0.971        18
                           atis_distance      1.000     1.000     1.000        10
                               atis_city      1.000     0.667     0.800         6
                        atis_ground_fare      1.000     0.857     0.923         7
                           atis_capacity      0.955     1.000     0.977        21
                          atis_flight_no      0.889     1.000     0.941         8
                               atis_meal      1.000     0.833     0.909         6
                        atis_restriction      0.000     0.000     0.000         0
             atis_airline#atis_flight_no      0.000     0.000     0.000         0
    atis_ground_service#atis_ground_fare      0.000     0.000     0.000         0
           atis_airfare#atis_flight_time      0.000     0.000     0.000         0
                           atis_cheapest      0.000     0.000     0.000         0
atis_aircraft#atis_flight#atis_flight_no      0.000     0.000     0.000         0

                               micro avg      0.973     0.979     0.976       888
                               macro avg      0.680     0.664     0.661       888
                            weighted avg      0.975     0.979     0.975       888

I0715 07:12:10.173976 140369481070464 interactiveshell.py:2882] 
                              precision    recall  f1-score   support

                           O      0.000     0.000     0.000       0.0
           B-toloc.city_name      0.975     0.994     0.985     716.0
         B-fromloc.city_name      0.989     0.997     0.993     704.0
           I-toloc.city_name      0.977     0.958     0.968     265.0
      B-depart_date.day_name      0.981     0.986     0.984     212.0
              B-airline_name      0.990     0.990     0.990     101.0
         I-fromloc.city_name      0.972     0.994     0.983     177.0
 B-depart_time.period_of_day      0.983     0.900     0.940     130.0
              I-airline_name      1.000     0.985     0.992      65.0
    B-depart_date.day_number      0.981     0.964     0.972      55.0
    B-depart_date.month_name      0.982     0.964     0.973      56.0
          B-depart_time.time      0.877     1.000     0.934      57.0
                B-round_trip      1.000     0.973     0.986      73.0
             B-cost_relative      1.000     0.973     0.986      37.0
                I-round_trip      0.986     1.000     0.993      71.0
                B-flight_mod      0.821     0.958     0.885      24.0
 B-depart_time.time_relative      0.984     0.969     0.977      65.0
          I-depart_time.time      0.963     1.000     0.981      52.0
         B-stoploc.city_name      1.000     1.000     1.000      20.0
                 B-city_name      0.795     0.614     0.693      57.0
                B-class_type      0.960     1.000     0.980      24.0
          B-arrive_time.time      0.917     0.971     0.943      34.0
 B-arrive_time.time_relative      0.909     0.968     0.937      31.0
                I-class_type      1.000     1.000     1.000      17.0
               B-flight_stop      1.000     1.000     1.000      21.0
          I-arrive_time.time      0.971     0.971     0.971      35.0
              B-airline_code      1.000     0.912     0.954      34.0
    I-depart_date.day_number      1.000     0.933     0.966      15.0
      I-fromloc.airport_name      0.395     1.000     0.566      15.0
      B-fromloc.airport_name      0.462     1.000     0.632      12.0
      B-arrive_date.day_name      0.786     1.000     0.880      11.0
          B-toloc.state_code      1.000     1.000     1.000      18.0
B-depart_date.today_relative      1.000     1.000     1.000       9.0
             B-flight_number      0.579     1.000     0.733      11.0
 B-depart_date.date_relative      0.895     1.000     0.944      17.0
          B-toloc.state_name      0.839     0.929     0.881      28.0
           B-fare_basis_code      0.917     0.647     0.759      17.0
               B-flight_time      1.000     1.000     1.000       1.0
                        B-or      1.000     1.000     1.000       3.0
 B-arrive_time.period_of_day      0.667     1.000     0.800       6.0
          B-meal_description      1.000     0.900     0.947      10.0
             I-cost_relative      1.000     0.667     0.800       3.0
              I-airport_name      0.800     0.414     0.545      29.0
               B-fare_amount      0.500     1.000     0.667       2.0
               I-fare_amount      1.000     1.000     1.000       2.0
                 I-city_name      0.739     0.567     0.642      30.0
        I-toloc.airport_name      1.000     1.000     1.000       3.0
            B-transport_type      0.833     1.000     0.909      10.0
    B-arrive_date.month_name      0.714     0.833     0.769       6.0
    B-arrive_date.day_number      0.714     0.833     0.769       6.0
         I-stoploc.city_name      0.909     1.000     0.952      10.0
                      B-meal      1.000     1.000     1.000      16.0
        B-fromloc.state_code      1.000     1.000     1.000      23.0
    B-depart_time.period_mod      1.000     1.000     1.000       5.0
                   B-connect      1.000     1.000     1.000       6.0
               B-flight_days      1.000     1.000     1.000      10.0
        B-toloc.airport_name      1.000     1.000     1.000       3.0
        B-fromloc.state_name      0.850     1.000     0.919      17.0
              B-airport_name      0.857     0.286     0.429      21.0
                   B-economy      1.000     1.000     1.000       6.0
               I-flight_time      1.000     1.000     1.000       1.0
             B-aircraft_code      1.000     0.697     0.821      33.0
                       B-mod      1.000     0.500     0.667       2.0
              B-airport_code      0.714     0.556     0.625       9.0
    B-depart_time.start_time      0.600     1.000     0.750       3.0
      B-depart_time.end_time      1.000     0.667     0.800       3.0
          B-depart_date.year      1.000     1.000     1.000       3.0
            I-transport_type      0.000     0.000     0.000       1.0
          B-restriction_code      1.000     1.000     1.000       4.0
    B-arrive_time.start_time      1.000     1.000     1.000       8.0
        B-toloc.airport_code      1.000     0.750     0.857       4.0
      B-arrive_time.end_time      0.875     0.875     0.875       8.0
      I-arrive_time.end_time      0.875     0.875     0.875       8.0
      I-depart_time.end_time      1.000     0.667     0.800       3.0
               I-flight_stop      0.000     0.000     0.000       0.0
      B-fromloc.airport_code      1.000     1.000     1.000       5.0
          I-restriction_code      1.000     1.000     1.000       3.0
    I-depart_time.start_time      1.000     1.000     1.000       1.0
          I-toloc.state_name      1.000     1.000     1.000       1.0
I-depart_date.today_relative      0.000     0.000     0.000       0.0
 B-arrive_date.date_relative      1.000     1.000     1.000       2.0
                I-flight_mod      0.500     0.167     0.250       6.0
                   I-economy      0.000     0.000     0.000       0.0
 B-return_date.date_relative      0.000     0.000     0.000       3.0
        I-fromloc.state_name      1.000     1.000     1.000       1.0
                B-state_code      1.000     1.000     1.000       1.0
    I-arrive_time.start_time      1.000     1.000     1.000       1.0
    I-arrive_date.day_number      0.000     0.000     0.000       0.0
                 B-meal_code      1.000     1.000     1.000       1.0
 I-depart_time.period_of_day      0.000     0.000     0.000       1.0
                  B-day_name      1.000     0.500     0.667       2.0
             B-period_of_day      1.000     1.000     1.000       4.0
        B-stoploc.state_code      0.000     0.000     0.000       0.0
    B-return_date.month_name      0.000     0.000     0.000       0.0
    B-return_date.day_number      0.000     0.000     0.000       0.0
    B-arrive_time.period_mod      0.000     0.000     0.000       0.0
                 I-meal_code      0.000     0.000     0.000       0.0
        B-toloc.country_name      1.000     1.000     1.000       1.0
                 B-days_code      1.000     1.000     1.000       1.0
 I-arrive_time.period_of_day      0.000     0.000     0.000       0.0
            I-today_relative      0.000     0.000     0.000       0.0
 B-return_time.period_of_day      0.000     0.000     0.000       0.0
                      B-time      0.000     0.000     0.000       0.0
           I-fare_basis_code      0.000     0.000     0.000       0.0
 I-arrive_time.time_relative      1.000     0.500     0.667       4.0
 I-depart_time.time_relative      0.000     0.000     0.000       1.0
            B-today_relative      0.000     0.000     0.000       0.0
                B-state_name      0.000     0.000     0.000       9.0
B-arrive_date.today_relative      0.000     0.000     0.000       0.0
    B-return_time.period_mod      0.000     0.000     0.000       0.0
                B-month_name      0.000     0.000     0.000       0.0
                B-day_number      0.000     0.000     0.000       0.0
 I-return_date.date_relative      0.667     0.667     0.667       3.0
I-return_date.today_relative      0.000     0.000     0.000       0.0
      B-stoploc.airport_name      0.000     0.000     0.000       0.0
             B-time_relative      0.000     0.000     0.000       0.0
                      I-time      0.000     0.000     0.000       0.0
    I-return_date.day_number      0.000     0.000     0.000       0.0
          I-meal_description      0.000     0.000     0.000       0.0
B-return_date.today_relative      0.000     0.000     0.000       0.0
      B-return_date.day_name      0.000     0.000     0.000       2.0

                   micro avg      0.950     0.951     0.951    3657.0
                   macro avg      0.667     0.657     0.651    3657.0
                weighted avg      0.955     0.951     0.949    3657.0

I0715 07:12:10.174953 140369481070464 interactiveshell.py:2882] Best Slot F1: 0.956
Reading ../data/atis.train.w-intent.iob
I0715 07:12:12.850260 140369481070464 interactiveshell.py:2882] Step 8450 | Loss: 0.0005 | Loss_intent: 0.0000 | Loss_slots: 0.0005 | Spent: 5.9 secs | LR: 0.000123
I0715 07:12:16.596926 140369481070464 interactiveshell.py:2882] Step 8500 | Loss: 0.3465 | Loss_intent: 0.3396 | Loss_slots: 0.0069 | Spent: 3.7 secs | LR: 0.000123
I0715 07:12:20.291564 140369481070464 interactiveshell.py:2882] Step 8550 | Loss: 0.0021 | Loss_intent: 0.0000 | Loss_slots: 0.0021 | Spent: 3.7 secs | LR: 0.000122
I0715 07:12:23.983626 140369481070464 interactiveshell.py:2882] Step 8600 | Loss: 0.0007 | Loss_intent: 0.0000 | Loss_slots: 0.0007 | Spent: 3.7 secs | LR: 0.000121
I0715 07:12:27.711224 140369481070464 interactiveshell.py:2882] Step 8650 | Loss: 0.0161 | Loss_intent: 0.0000 | Loss_slots: 0.0161 | Spent: 3.7 secs | LR: 0.000121
I0715 07:12:31.407706 140369481070464 interactiveshell.py:2882] Step 8700 | Loss: 0.0012 | Loss_intent: 0.0000 | Loss_slots: 0.0012 | Spent: 3.7 secs | LR: 0.000120
Reading ../data/atis.test.w-intent.iob
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.
  'precision', 'predicted', average, warn_for)
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1439: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples.
  'recall', 'true', average, warn_for)
I0715 07:12:35.436307 140369481070464 interactiveshell.py:2882] 
                                          precision    recall  f1-score   support

                             atis_flight      0.981     0.992     0.987       632
                            atis_airfare      0.960     1.000     0.980        48
                     atis_ground_service      1.000     1.000     1.000        36
                            atis_airline      1.000     1.000     1.000        38
                       atis_abbreviation      0.943     1.000     0.971        33
                           atis_aircraft      1.000     0.889     0.941         9
                        atis_flight_time      1.000     1.000     1.000         1
                           atis_quantity      0.429     1.000     0.600         3
                atis_flight#atis_airfare      0.833     0.417     0.556        12
                            atis_airport      1.000     1.000     1.000        18
                           atis_distance      1.000     1.000     1.000        10
                               atis_city      1.000     0.667     0.800         6
                        atis_ground_fare      1.000     0.857     0.923         7
                           atis_capacity      0.955     1.000     0.977        21
                          atis_flight_no      1.000     1.000     1.000         8
                               atis_meal      1.000     0.833     0.909         6
                        atis_restriction      0.000     0.000     0.000         0
             atis_airline#atis_flight_no      0.000     0.000     0.000         0
    atis_ground_service#atis_ground_fare      0.000     0.000     0.000         0
           atis_airfare#atis_flight_time      0.000     0.000     0.000         0
                           atis_cheapest      0.000     0.000     0.000         0
atis_aircraft#atis_flight#atis_flight_no      0.000     0.000     0.000         0

                               micro avg      0.975     0.981     0.978       888
                               macro avg      0.686     0.666     0.666       888
                            weighted avg      0.977     0.981     0.977       888

I0715 07:12:35.476153 140369481070464 interactiveshell.py:2882] 
                              precision    recall  f1-score   support

                           O      0.000     0.000     0.000       0.0
           B-toloc.city_name      0.974     0.996     0.985     716.0
         B-fromloc.city_name      0.980     0.999     0.989     704.0
           I-toloc.city_name      0.978     0.996     0.987     265.0
      B-depart_date.day_name      0.977     0.986     0.981     212.0
              B-airline_name      0.990     0.990     0.990     101.0
         I-fromloc.city_name      0.978     0.994     0.986     177.0
 B-depart_time.period_of_day      0.991     0.869     0.926     130.0
              I-airline_name      1.000     0.985     0.992      65.0
    B-depart_date.day_number      0.982     0.982     0.982      55.0
    B-depart_date.month_name      0.982     0.964     0.973      56.0
          B-depart_time.time      0.864     1.000     0.927      57.0
                B-round_trip      1.000     0.986     0.993      73.0
             B-cost_relative      1.000     0.973     0.986      37.0
                I-round_trip      1.000     1.000     1.000      71.0
                B-flight_mod      0.821     0.958     0.885      24.0
 B-depart_time.time_relative      0.984     0.969     0.977      65.0
          I-depart_time.time      0.912     1.000     0.954      52.0
         B-stoploc.city_name      1.000     1.000     1.000      20.0
                 B-city_name      0.889     0.561     0.688      57.0
                B-class_type      0.960     1.000     0.980      24.0
          B-arrive_time.time      0.943     0.971     0.957      34.0
 B-arrive_time.time_relative      0.935     0.935     0.935      31.0
                I-class_type      1.000     1.000     1.000      17.0
               B-flight_stop      1.000     1.000     1.000      21.0
          I-arrive_time.time      0.971     0.971     0.971      35.0
              B-airline_code      1.000     0.912     0.954      34.0
    I-depart_date.day_number      1.000     1.000     1.000      15.0
      I-fromloc.airport_name      0.441     1.000     0.612      15.0
      B-fromloc.airport_name      0.440     0.917     0.595      12.0
      B-arrive_date.day_name      0.786     1.000     0.880      11.0
          B-toloc.state_code      1.000     1.000     1.000      18.0
B-depart_date.today_relative      1.000     1.000     1.000       9.0
             B-flight_number      0.550     1.000     0.710      11.0
 B-depart_date.date_relative      0.850     1.000     0.919      17.0
          B-toloc.state_name      0.929     0.929     0.929      28.0
           B-fare_basis_code      0.917     0.647     0.759      17.0
               B-flight_time      1.000     1.000     1.000       1.0
                        B-or      1.000     1.000     1.000       3.0
 B-arrive_time.period_of_day      0.600     1.000     0.750       6.0
          B-meal_description      1.000     0.900     0.947      10.0
             I-cost_relative      1.000     0.667     0.800       3.0
              I-airport_name      0.846     0.379     0.524      29.0
               B-fare_amount      0.400     1.000     0.571       2.0
               I-fare_amount      1.000     1.000     1.000       2.0
                 I-city_name      1.000     0.567     0.723      30.0
        I-toloc.airport_name      1.000     1.000     1.000       3.0
            B-transport_type      1.000     1.000     1.000      10.0
    B-arrive_date.month_name      0.714     0.833     0.769       6.0
    B-arrive_date.day_number      0.833     0.833     0.833       6.0
         I-stoploc.city_name      1.000     1.000     1.000      10.0
                      B-meal      1.000     1.000     1.000      16.0
        B-fromloc.state_code      1.000     1.000     1.000      23.0
    B-depart_time.period_mod      1.000     1.000     1.000       5.0
                   B-connect      1.000     1.000     1.000       6.0
               B-flight_days      1.000     1.000     1.000      10.0
        B-toloc.airport_name      1.000     1.000     1.000       3.0
        B-fromloc.state_name      1.000     0.941     0.970      17.0
              B-airport_name      0.857     0.286     0.429      21.0
                   B-economy      1.000     1.000     1.000       6.0
               I-flight_time      1.000     1.000     1.000       1.0
             B-aircraft_code      1.000     0.758     0.862      33.0
                       B-mod      1.000     0.500     0.667       2.0
              B-airport_code      0.667     0.444     0.533       9.0
    B-depart_time.start_time      0.750     1.000     0.857       3.0
      B-depart_time.end_time      1.000     0.667     0.800       3.0
          B-depart_date.year      0.600     1.000     0.750       3.0
            I-transport_type      0.000     0.000     0.000       1.0
          B-restriction_code      1.000     1.000     1.000       4.0
    B-arrive_time.start_time      1.000     0.875     0.933       8.0
        B-toloc.airport_code      1.000     0.750     0.857       4.0
      B-arrive_time.end_time      0.875     0.875     0.875       8.0
      I-arrive_time.end_time      0.875     0.875     0.875       8.0
      I-depart_time.end_time      1.000     0.667     0.800       3.0
               I-flight_stop      0.000     0.000     0.000       0.0
      B-fromloc.airport_code      0.833     1.000     0.909       5.0
          I-restriction_code      1.000     1.000     1.000       3.0
    I-depart_time.start_time      1.000     1.000     1.000       1.0
          I-toloc.state_name      1.000     1.000     1.000       1.0
I-depart_date.today_relative      0.000     0.000     0.000       0.0
 B-arrive_date.date_relative      1.000     1.000     1.000       2.0
                I-flight_mod      1.000     0.167     0.286       6.0
                   I-economy      0.000     0.000     0.000       0.0
 B-return_date.date_relative      1.000     0.333     0.500       3.0
        I-fromloc.state_name      1.000     1.000     1.000       1.0
                B-state_code      1.000     1.000     1.000       1.0
    I-arrive_time.start_time      1.000     1.000     1.000       1.0
    I-arrive_date.day_number      0.000     0.000     0.000       0.0
                 B-meal_code      1.000     1.000     1.000       1.0
 I-depart_time.period_of_day      0.000     0.000     0.000       1.0
                  B-day_name      1.000     0.500     0.667       2.0
             B-period_of_day      1.000     0.750     0.857       4.0
        B-stoploc.state_code      0.000     0.000     0.000       0.0
    B-return_date.month_name      0.000     0.000     0.000       0.0
    B-return_date.day_number      0.000     0.000     0.000       0.0
    B-arrive_time.period_mod      0.000     0.000     0.000       0.0
                 I-meal_code      0.000     0.000     0.000       0.0
        B-toloc.country_name      1.000     1.000     1.000       1.0
                 B-days_code      1.000     1.000     1.000       1.0
 I-arrive_time.period_of_day      0.000     0.000     0.000       0.0
            I-today_relative      0.000     0.000     0.000       0.0
 B-return_time.period_of_day      0.000     0.000     0.000       0.0
                      B-time      0.000     0.000     0.000       0.0
           I-fare_basis_code      0.000     0.000     0.000       0.0
 I-arrive_time.time_relative      1.000     1.000     1.000       4.0
 I-depart_time.time_relative      0.000     0.000     0.000       1.0
            B-today_relative      0.000     0.000     0.000       0.0
                B-state_name      0.875     0.778     0.824       9.0
B-arrive_date.today_relative      0.000     0.000     0.000       0.0
    B-return_time.period_mod      0.000     0.000     0.000       0.0
                B-month_name      0.000     0.000     0.000       0.0
                B-day_number      0.000     0.000     0.000       0.0
 I-return_date.date_relative      1.000     0.667     0.800       3.0
I-return_date.today_relative      0.000     0.000     0.000       0.0
      B-stoploc.airport_name      0.000     0.000     0.000       0.0
             B-time_relative      0.000     0.000     0.000       0.0
                      I-time      0.000     0.000     0.000       0.0
    I-return_date.day_number      0.000     0.000     0.000       0.0
          I-meal_description      0.000     0.000     0.000       0.0
B-return_date.today_relative      0.000     0.000     0.000       0.0
      B-return_date.day_name      0.000     0.000     0.000       2.0

                   micro avg      0.953     0.955     0.954    3657.0
                   macro avg      0.692     0.665     0.664    3657.0
                weighted avg      0.962     0.955     0.953    3657.0

I0715 07:12:35.476983 140369481070464 interactiveshell.py:2882] Best Slot F1: 0.956
Reading ../data/atis.train.w-intent.iob
I0715 07:12:37.292228 140369481070464 interactiveshell.py:2882] Step 8750 | Loss: 0.0008 | Loss_intent: 0.0000 | Loss_slots: 0.0008 | Spent: 5.9 secs | LR: 0.000119
I0715 07:12:41.019531 140369481070464 interactiveshell.py:2882] Step 8800 | Loss: 0.0079 | Loss_intent: 0.0000 | Loss_slots: 0.0079 | Spent: 3.7 secs | LR: 0.000119
I0715 07:12:44.703093 140369481070464 interactiveshell.py:2882] Step 8850 | Loss: 0.0010 | Loss_intent: 0.0000 | Loss_slots: 0.0010 | Spent: 3.7 secs | LR: 0.000118
I0715 07:12:48.418054 140369481070464 interactiveshell.py:2882] Step 8900 | Loss: 0.0022 | Loss_intent: 0.0000 | Loss_slots: 0.0022 | Spent: 3.7 secs | LR: 0.000117
I0715 07:12:52.151360 140369481070464 interactiveshell.py:2882] Step 8950 | Loss: 0.0008 | Loss_intent: 0.0000 | Loss_slots: 0.0008 | Spent: 3.7 secs | LR: 0.000117
I0715 07:12:55.940524 140369481070464 interactiveshell.py:2882] Step 9000 | Loss: 0.0017 | Loss_intent: 0.0000 | Loss_slots: 0.0017 | Spent: 3.8 secs | LR: 0.000116
Reading ../data/atis.test.w-intent.iob
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.
  'precision', 'predicted', average, warn_for)
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1439: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples.
  'recall', 'true', average, warn_for)
I0715 07:13:00.954904 140369481070464 interactiveshell.py:2882] 
                                          precision    recall  f1-score   support

                             atis_flight      0.980     0.991     0.985       632
                            atis_airfare      0.941     1.000     0.970        48
                     atis_ground_service      1.000     1.000     1.000        36
                            atis_airline      1.000     1.000     1.000        38
                       atis_abbreviation      0.941     0.970     0.955        33
                           atis_aircraft      1.000     0.889     0.941         9
                        atis_flight_time      1.000     1.000     1.000         1
                           atis_quantity      0.429     1.000     0.600         3
                atis_flight#atis_airfare      0.833     0.417     0.556        12
                            atis_airport      1.000     0.944     0.971        18
                           atis_distance      1.000     1.000     1.000        10
                               atis_city      1.000     0.667     0.800         6
                        atis_ground_fare      1.000     0.857     0.923         7
                           atis_capacity      0.952     0.952     0.952        21
                          atis_flight_no      0.889     1.000     0.941         8
                               atis_meal      1.000     0.833     0.909         6
                        atis_restriction      0.000     0.000     0.000         0
             atis_airline#atis_flight_no      0.000     0.000     0.000         0
    atis_ground_service#atis_ground_fare      0.000     0.000     0.000         0
           atis_airfare#atis_flight_time      0.000     0.000     0.000         0
                           atis_cheapest      0.000     0.000     0.000         0
atis_aircraft#atis_flight#atis_flight_no      0.000     0.000     0.000         0

                               micro avg      0.971     0.976     0.974       888
                               macro avg      0.680     0.660     0.659       888
                            weighted avg      0.974     0.976     0.973       888

I0715 07:13:00.994946 140369481070464 interactiveshell.py:2882] 
                              precision    recall  f1-score   support

                           O      0.000     0.000     0.000       0.0
           B-toloc.city_name      0.971     0.994     0.983     716.0
         B-fromloc.city_name      0.987     0.997     0.992     704.0
           I-toloc.city_name      0.978     0.996     0.987     265.0
      B-depart_date.day_name      0.977     0.986     0.981     212.0
              B-airline_name      0.990     0.990     0.990     101.0
         I-fromloc.city_name      0.989     0.983     0.986     177.0
 B-depart_time.period_of_day      0.991     0.892     0.939     130.0
              I-airline_name      1.000     0.985     0.992      65.0
    B-depart_date.day_number      0.981     0.964     0.972      55.0
    B-depart_date.month_name      0.982     0.964     0.973      56.0
          B-depart_time.time      0.875     0.982     0.926      57.0
                B-round_trip      1.000     0.986     0.993      73.0
             B-cost_relative      1.000     0.973     0.986      37.0
                I-round_trip      1.000     1.000     1.000      71.0
                B-flight_mod      0.821     0.958     0.885      24.0
 B-depart_time.time_relative      0.984     0.954     0.969      65.0
          I-depart_time.time      0.962     0.981     0.971      52.0
         B-stoploc.city_name      1.000     1.000     1.000      20.0
                 B-city_name      0.846     0.579     0.688      57.0
                B-class_type      0.960     1.000     0.980      24.0
          B-arrive_time.time      0.892     0.971     0.930      34.0
 B-arrive_time.time_relative      0.909     0.968     0.937      31.0
                I-class_type      1.000     1.000     1.000      17.0
               B-flight_stop      1.000     1.000     1.000      21.0
          I-arrive_time.time      0.944     0.971     0.958      35.0
              B-airline_code      1.000     0.912     0.954      34.0
    I-depart_date.day_number      1.000     0.933     0.966      15.0
      I-fromloc.airport_name      0.455     1.000     0.625      15.0
      B-fromloc.airport_name      0.423     0.917     0.579      12.0
      B-arrive_date.day_name      0.786     1.000     0.880      11.0
          B-toloc.state_code      1.000     1.000     1.000      18.0
B-depart_date.today_relative      1.000     1.000     1.000       9.0
             B-flight_number      0.500     1.000     0.667      11.0
 B-depart_date.date_relative      0.895     1.000     0.944      17.0
          B-toloc.state_name      0.926     0.893     0.909      28.0
           B-fare_basis_code      0.944     1.000     0.971      17.0
               B-flight_time      1.000     1.000     1.000       1.0
                        B-or      0.750     1.000     0.857       3.0
 B-arrive_time.period_of_day      0.600     1.000     0.750       6.0
          B-meal_description      1.000     0.900     0.947      10.0
             I-cost_relative      1.000     0.667     0.800       3.0
              I-airport_name      0.923     0.414     0.571      29.0
               B-fare_amount      0.500     1.000     0.667       2.0
               I-fare_amount      1.000     1.000     1.000       2.0
                 I-city_name      0.900     0.600     0.720      30.0
        I-toloc.airport_name      0.600     1.000     0.750       3.0
            B-transport_type      0.833     1.000     0.909      10.0
    B-arrive_date.month_name      0.714     0.833     0.769       6.0
    B-arrive_date.day_number      0.714     0.833     0.769       6.0
         I-stoploc.city_name      0.909     1.000     0.952      10.0
                      B-meal      1.000     1.000     1.000      16.0
        B-fromloc.state_code      1.000     1.000     1.000      23.0
    B-depart_time.period_mod      1.000     1.000     1.000       5.0
                   B-connect      1.000     1.000     1.000       6.0
               B-flight_days      1.000     1.000     1.000      10.0
        B-toloc.airport_name      1.000     1.000     1.000       3.0
        B-fromloc.state_name      1.000     0.941     0.970      17.0
              B-airport_name      0.778     0.333     0.467      21.0
                   B-economy      1.000     1.000     1.000       6.0
               I-flight_time      1.000     1.000     1.000       1.0
             B-aircraft_code      1.000     0.667     0.800      33.0
                       B-mod      1.000     0.500     0.667       2.0
              B-airport_code      0.667     0.444     0.533       9.0
    B-depart_time.start_time      0.667     0.667     0.667       3.0
      B-depart_time.end_time      1.000     0.667     0.800       3.0
          B-depart_date.year      0.600     1.000     0.750       3.0
            I-transport_type      0.000     0.000     0.000       1.0
          B-restriction_code      1.000     1.000     1.000       4.0
    B-arrive_time.start_time      0.889     1.000     0.941       8.0
        B-toloc.airport_code      1.000     0.750     0.857       4.0
      B-arrive_time.end_time      0.875     0.875     0.875       8.0
      I-arrive_time.end_time      0.875     0.875     0.875       8.0
      I-depart_time.end_time      1.000     0.667     0.800       3.0
               I-flight_stop      0.000     0.000     0.000       0.0
      B-fromloc.airport_code      0.833     1.000     0.909       5.0
          I-restriction_code      1.000     1.000     1.000       3.0
    I-depart_time.start_time      1.000     1.000     1.000       1.0
          I-toloc.state_name      1.000     1.000     1.000       1.0
I-depart_date.today_relative      0.000     0.000     0.000       0.0
 B-arrive_date.date_relative      1.000     1.000     1.000       2.0
                I-flight_mod      1.000     0.167     0.286       6.0
                   I-economy      0.000     0.000     0.000       0.0
 B-return_date.date_relative      0.000     0.000     0.000       3.0
        I-fromloc.state_name      1.000     1.000     1.000       1.0
                B-state_code      1.000     1.000     1.000       1.0
    I-arrive_time.start_time      1.000     1.000     1.000       1.0
    I-arrive_date.day_number      0.000     0.000     0.000       0.0
                 B-meal_code      1.000     1.000     1.000       1.0
 I-depart_time.period_of_day      0.000     0.000     0.000       1.0
                  B-day_name      1.000     0.500     0.667       2.0
             B-period_of_day      1.000     0.750     0.857       4.0
        B-stoploc.state_code      0.000     0.000     0.000       0.0
    B-return_date.month_name      0.000     0.000     0.000       0.0
    B-return_date.day_number      0.000     0.000     0.000       0.0
    B-arrive_time.period_mod      0.000     0.000     0.000       0.0
                 I-meal_code      0.000     0.000     0.000       0.0
        B-toloc.country_name      1.000     1.000     1.000       1.0
                 B-days_code      1.000     1.000     1.000       1.0
 I-arrive_time.period_of_day      0.000     0.000     0.000       0.0
            I-today_relative      0.000     0.000     0.000       0.0
 B-return_time.period_of_day      0.000     0.000     0.000       0.0
                      B-time      0.000     0.000     0.000       0.0
           I-fare_basis_code      0.000     0.000     0.000       0.0
 I-arrive_time.time_relative      0.667     0.500     0.571       4.0
 I-depart_time.time_relative      0.000     0.000     0.000       1.0
            B-today_relative      0.000     0.000     0.000       0.0
                B-state_name      0.857     0.667     0.750       9.0
B-arrive_date.today_relative      0.000     0.000     0.000       0.0
    B-return_time.period_mod      0.000     0.000     0.000       0.0
                B-month_name      0.000     0.000     0.000       0.0
                B-day_number      0.000     0.000     0.000       0.0
 I-return_date.date_relative      0.667     0.667     0.667       3.0
I-return_date.today_relative      0.000     0.000     0.000       0.0
      B-stoploc.airport_name      0.000     0.000     0.000       0.0
             B-time_relative      0.000     0.000     0.000       0.0
                      I-time      0.000     0.000     0.000       0.0
    I-return_date.day_number      0.000     0.000     0.000       0.0
          I-meal_description      0.000     0.000     0.000       0.0
B-return_date.today_relative      0.000     0.000     0.000       0.0
      B-return_date.day_name      0.000     0.000     0.000       2.0

                   micro avg      0.952     0.954     0.953    3657.0
                   macro avg      0.668     0.658     0.650    3657.0
                weighted avg      0.959     0.954     0.952    3657.0

I0715 07:13:00.995782 140369481070464 interactiveshell.py:2882] Best Slot F1: 0.956
Reading ../data/atis.train.w-intent.iob
I0715 07:13:01.908118 140369481070464 interactiveshell.py:2882] Step 9050 | Loss: 0.0221 | Loss_intent: 0.0000 | Loss_slots: 0.0221 | Spent: 6.0 secs | LR: 0.000116
I0715 07:13:05.620714 140369481070464 interactiveshell.py:2882] Step 9100 | Loss: 0.0008 | Loss_intent: 0.0000 | Loss_slots: 0.0008 | Spent: 3.7 secs | LR: 0.000115
I0715 07:13:09.366204 140369481070464 interactiveshell.py:2882] Step 9150 | Loss: 0.0024 | Loss_intent: 0.0000 | Loss_slots: 0.0024 | Spent: 3.7 secs | LR: 0.000114
I0715 07:13:13.067809 140369481070464 interactiveshell.py:2882] Step 9200 | Loss: 0.0005 | Loss_intent: 0.0000 | Loss_slots: 0.0005 | Spent: 3.7 secs | LR: 0.000114
I0715 07:13:16.769812 140369481070464 interactiveshell.py:2882] Step 9250 | Loss: 0.0037 | Loss_intent: 0.0000 | Loss_slots: 0.0037 | Spent: 3.7 secs | LR: 0.000113
I0715 07:13:20.501927 140369481070464 interactiveshell.py:2882] Step 9300 | Loss: 0.0001 | Loss_intent: 0.0000 | Loss_slots: 0.0001 | Spent: 3.7 secs | LR: 0.000113
I0715 07:13:24.225126 140369481070464 interactiveshell.py:2882] Step 9350 | Loss: 0.0053 | Loss_intent: 0.0000 | Loss_slots: 0.0053 | Spent: 3.7 secs | LR: 0.000112
Reading ../data/atis.test.w-intent.iob
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.
  'precision', 'predicted', average, warn_for)
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1439: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples.
  'recall', 'true', average, warn_for)
I0715 07:13:26.343549 140369481070464 interactiveshell.py:2882] 
                                          precision    recall  f1-score   support

                             atis_flight      0.983     0.991     0.987       632
                            atis_airfare      0.960     1.000     0.980        48
                     atis_ground_service      1.000     1.000     1.000        36
                            atis_airline      1.000     1.000     1.000        38
                       atis_abbreviation      1.000     1.000     1.000        33
                           atis_aircraft      1.000     0.889     0.941         9
                        atis_flight_time      0.500     1.000     0.667         1
                           atis_quantity      0.429     1.000     0.600         3
                atis_flight#atis_airfare      0.833     0.417     0.556        12
                            atis_airport      0.944     0.944     0.944        18
                           atis_distance      1.000     1.000     1.000        10
                               atis_city      1.000     1.000     1.000         6
                        atis_ground_fare      1.000     0.857     0.923         7
                           atis_capacity      0.955     1.000     0.977        21
                          atis_flight_no      0.889     1.000     0.941         8
                               atis_meal      1.000     0.833     0.909         6
                        atis_restriction      0.000     0.000     0.000         0
             atis_airline#atis_flight_no      0.000     0.000     0.000         0
    atis_ground_service#atis_ground_fare      0.000     0.000     0.000         0
           atis_airfare#atis_flight_time      0.000     0.000     0.000         0
                           atis_cheapest      0.000     0.000     0.000         0
atis_aircraft#atis_flight#atis_flight_no      0.000     0.000     0.000         0

                               micro avg      0.975     0.981     0.978       888
                               macro avg      0.659     0.679     0.656       888
                            weighted avg      0.978     0.981     0.978       888

I0715 07:13:26.382138 140369481070464 interactiveshell.py:2882] 
                              precision    recall  f1-score   support

                           O      0.000     0.000     0.000       0.0
           B-toloc.city_name      0.973     0.993     0.983     716.0
         B-fromloc.city_name      0.989     0.997     0.993     704.0
           I-toloc.city_name      0.978     0.996     0.987     265.0
      B-depart_date.day_name      0.977     0.986     0.981     212.0
              B-airline_name      0.980     0.990     0.985     101.0
         I-fromloc.city_name      0.989     0.977     0.983     177.0
 B-depart_time.period_of_day      0.992     0.900     0.944     130.0
              I-airline_name      1.000     0.985     0.992      65.0
    B-depart_date.day_number      0.981     0.964     0.972      55.0
    B-depart_date.month_name      0.982     0.964     0.973      56.0
          B-depart_time.time      0.864     1.000     0.927      57.0
                B-round_trip      1.000     0.986     0.993      73.0
             B-cost_relative      1.000     0.973     0.986      37.0
                I-round_trip      1.000     1.000     1.000      71.0
                B-flight_mod      0.800     1.000     0.889      24.0
 B-depart_time.time_relative      0.969     0.969     0.969      65.0
          I-depart_time.time      0.981     1.000     0.990      52.0
         B-stoploc.city_name      0.870     1.000     0.930      20.0
                 B-city_name      0.906     0.509     0.652      57.0
                B-class_type      0.960     1.000     0.980      24.0
          B-arrive_time.time      0.943     0.971     0.957      34.0
 B-arrive_time.time_relative      0.853     0.935     0.892      31.0
                I-class_type      1.000     1.000     1.000      17.0
               B-flight_stop      1.000     1.000     1.000      21.0
          I-arrive_time.time      0.971     0.971     0.971      35.0
              B-airline_code      0.968     0.882     0.923      34.0
    I-depart_date.day_number      1.000     0.933     0.966      15.0
      I-fromloc.airport_name      0.375     1.000     0.545      15.0
      B-fromloc.airport_name      0.444     1.000     0.615      12.0
      B-arrive_date.day_name      0.733     1.000     0.846      11.0
          B-toloc.state_code      1.000     1.000     1.000      18.0
B-depart_date.today_relative      1.000     1.000     1.000       9.0
             B-flight_number      0.524     1.000     0.688      11.0
 B-depart_date.date_relative      0.944     1.000     0.971      17.0
          B-toloc.state_name      0.926     0.893     0.909      28.0
           B-fare_basis_code      0.909     0.588     0.714      17.0
               B-flight_time      1.000     1.000     1.000       1.0
                        B-or      1.000     1.000     1.000       3.0
 B-arrive_time.period_of_day      0.667     1.000     0.800       6.0
          B-meal_description      1.000     0.900     0.947      10.0
             I-cost_relative      1.000     0.667     0.800       3.0
              I-airport_name      0.917     0.379     0.537      29.0
               B-fare_amount      1.000     1.000     1.000       2.0
               I-fare_amount      1.000     1.000     1.000       2.0
                 I-city_name      1.000     0.633     0.776      30.0
        I-toloc.airport_name      1.000     1.000     1.000       3.0
            B-transport_type      1.000     0.800     0.889      10.0
    B-arrive_date.month_name      0.714     0.833     0.769       6.0
    B-arrive_date.day_number      0.714     0.833     0.769       6.0
         I-stoploc.city_name      0.909     1.000     0.952      10.0
                      B-meal      1.000     1.000     1.000      16.0
        B-fromloc.state_code      1.000     1.000     1.000      23.0
    B-depart_time.period_mod      1.000     1.000     1.000       5.0
                   B-connect      1.000     1.000     1.000       6.0
               B-flight_days      1.000     1.000     1.000      10.0
        B-toloc.airport_name      1.000     1.000     1.000       3.0
        B-fromloc.state_name      1.000     0.941     0.970      17.0
              B-airport_name      0.778     0.333     0.467      21.0
                   B-economy      1.000     1.000     1.000       6.0
               I-flight_time      1.000     1.000     1.000       1.0
             B-aircraft_code      1.000     0.758     0.862      33.0
                       B-mod      1.000     0.500     0.667       2.0
              B-airport_code      0.800     0.444     0.571       9.0
    B-depart_time.start_time      0.750     1.000     0.857       3.0
      B-depart_time.end_time      1.000     0.667     0.800       3.0
          B-depart_date.year      0.429     1.000     0.600       3.0
            I-transport_type      0.000     0.000     0.000       1.0
          B-restriction_code      0.800     1.000     0.889       4.0
    B-arrive_time.start_time      1.000     1.000     1.000       8.0
        B-toloc.airport_code      1.000     0.750     0.857       4.0
      B-arrive_time.end_time      0.875     0.875     0.875       8.0
      I-arrive_time.end_time      0.875     0.875     0.875       8.0
      I-depart_time.end_time      1.000     0.667     0.800       3.0
               I-flight_stop      0.000     0.000     0.000       0.0
      B-fromloc.airport_code      0.833     1.000     0.909       5.0
          I-restriction_code      1.000     1.000     1.000       3.0
    I-depart_time.start_time      1.000     1.000     1.000       1.0
          I-toloc.state_name      1.000     1.000     1.000       1.0
I-depart_date.today_relative      0.000     0.000     0.000       0.0
 B-arrive_date.date_relative      1.000     1.000     1.000       2.0
                I-flight_mod      0.000     0.000     0.000       6.0
                   I-economy      0.000     0.000     0.000       0.0
 B-return_date.date_relative      0.000     0.000     0.000       3.0
        I-fromloc.state_name      0.333     1.000     0.500       1.0
                B-state_code      1.000     1.000     1.000       1.0
    I-arrive_time.start_time      1.000     1.000     1.000       1.0
    I-arrive_date.day_number      0.000     0.000     0.000       0.0
                 B-meal_code      1.000     1.000     1.000       1.0
 I-depart_time.period_of_day      0.000     0.000     0.000       1.0
                  B-day_name      1.000     0.500     0.667       2.0
             B-period_of_day      1.000     0.750     0.857       4.0
        B-stoploc.state_code      0.000     0.000     0.000       0.0
    B-return_date.month_name      0.000     0.000     0.000       0.0
    B-return_date.day_number      0.000     0.000     0.000       0.0
    B-arrive_time.period_mod      0.000     0.000     0.000       0.0
                 I-meal_code      0.000     0.000     0.000       0.0
        B-toloc.country_name      1.000     1.000     1.000       1.0
                 B-days_code      1.000     1.000     1.000       1.0
 I-arrive_time.period_of_day      0.000     0.000     0.000       0.0
            I-today_relative      0.000     0.000     0.000       0.0
 B-return_time.period_of_day      0.000     0.000     0.000       0.0
                      B-time      0.000     0.000     0.000       0.0
           I-fare_basis_code      0.000     0.000     0.000       0.0
 I-arrive_time.time_relative      0.000     0.000     0.000       4.0
 I-depart_time.time_relative      0.000     0.000     0.000       1.0
            B-today_relative      0.000     0.000     0.000       0.0
                B-state_name      0.857     0.667     0.750       9.0
B-arrive_date.today_relative      0.000     0.000     0.000       0.0
    B-return_time.period_mod      0.000     0.000     0.000       0.0
                B-month_name      0.000     0.000     0.000       0.0
                B-day_number      0.000     0.000     0.000       0.0
 I-return_date.date_relative      0.600     1.000     0.750       3.0
I-return_date.today_relative      0.000     0.000     0.000       0.0
      B-stoploc.airport_name      0.000     0.000     0.000       0.0
             B-time_relative      0.000     0.000     0.000       0.0
                      I-time      0.000     0.000     0.000       0.0
    I-return_date.day_number      0.000     0.000     0.000       0.0
          I-meal_description      0.000     0.000     0.000       0.0
B-return_date.today_relative      0.000     0.000     0.000       0.0
      B-return_date.day_name      0.000     0.000     0.000       2.0

                   micro avg      0.950     0.952     0.951    3657.0
                   macro avg      0.658     0.654     0.644    3657.0
                weighted avg      0.959     0.952     0.950    3657.0

I0715 07:13:26.382985 140369481070464 interactiveshell.py:2882] Best Slot F1: 0.956
Reading ../data/atis.train.w-intent.iob
I0715 07:13:30.113051 140369481070464 interactiveshell.py:2882] Step 9400 | Loss: 0.0078 | Loss_intent: 0.0000 | Loss_slots: 0.0078 | Spent: 5.9 secs | LR: 0.000111
I0715 07:13:33.827253 140369481070464 interactiveshell.py:2882] Step 9450 | Loss: 0.0059 | Loss_intent: 0.0000 | Loss_slots: 0.0059 | Spent: 3.7 secs | LR: 0.000111
I0715 07:13:37.538915 140369481070464 interactiveshell.py:2882] Step 9500 | Loss: 0.0007 | Loss_intent: 0.0000 | Loss_slots: 0.0007 | Spent: 3.7 secs | LR: 0.000110
I0715 07:13:41.273530 140369481070464 interactiveshell.py:2882] Step 9550 | Loss: 0.0011 | Loss_intent: 0.0000 | Loss_slots: 0.0011 | Spent: 3.7 secs | LR: 0.000110
I0715 07:13:44.955133 140369481070464 interactiveshell.py:2882] Step 9600 | Loss: 0.0021 | Loss_intent: 0.0000 | Loss_slots: 0.0021 | Spent: 3.7 secs | LR: 0.000109
I0715 07:13:48.652896 140369481070464 interactiveshell.py:2882] Step 9650 | Loss: 0.0138 | Loss_intent: 0.0000 | Loss_slots: 0.0138 | Spent: 3.7 secs | LR: 0.000109
Reading ../data/atis.test.w-intent.iob
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.
  'precision', 'predicted', average, warn_for)
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1439: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples.
  'recall', 'true', average, warn_for)
I0715 07:13:51.686222 140369481070464 interactiveshell.py:2882] 
                                          precision    recall  f1-score   support

                             atis_flight      0.980     0.992     0.986       632
                            atis_airfare      0.960     1.000     0.980        48
                     atis_ground_service      1.000     1.000     1.000        36
                            atis_airline      1.000     1.000     1.000        38
                       atis_abbreviation      1.000     1.000     1.000        33
                           atis_aircraft      1.000     0.889     0.941         9
                        atis_flight_time      1.000     1.000     1.000         1
                           atis_quantity      0.429     1.000     0.600         3
                atis_flight#atis_airfare      0.800     0.333     0.471        12
                            atis_airport      1.000     0.944     0.971        18
                           atis_distance      1.000     1.000     1.000        10
                               atis_city      1.000     1.000     1.000         6
                        atis_ground_fare      1.000     0.857     0.923         7
                           atis_capacity      0.955     1.000     0.977        21
                          atis_flight_no      0.889     1.000     0.941         8
                               atis_meal      1.000     0.833     0.909         6
                        atis_restriction      0.000     0.000     0.000         0
             atis_airline#atis_flight_no      0.000     0.000     0.000         0
    atis_ground_service#atis_ground_fare      0.000     0.000     0.000         0
           atis_airfare#atis_flight_time      0.000     0.000     0.000         0
                           atis_cheapest      0.000     0.000     0.000         0
atis_aircraft#atis_flight#atis_flight_no      0.000     0.000     0.000         0

                               micro avg      0.975     0.981     0.978       888
                               macro avg      0.682     0.675     0.668       888
                            weighted avg      0.977     0.981     0.977       888

I0715 07:13:51.725002 140369481070464 interactiveshell.py:2882] 
                              precision    recall  f1-score   support

                           O      0.000     0.000     0.000       0.0
           B-toloc.city_name      0.975     0.993     0.984     716.0
         B-fromloc.city_name      0.980     0.999     0.989     704.0
           I-toloc.city_name      0.981     0.985     0.983     265.0
      B-depart_date.day_name      0.963     0.986     0.974     212.0
              B-airline_name      0.990     0.990     0.990     101.0
         I-fromloc.city_name      0.989     0.994     0.992     177.0
 B-depart_time.period_of_day      0.991     0.869     0.926     130.0
              I-airline_name      1.000     0.985     0.992      65.0
    B-depart_date.day_number      0.981     0.964     0.972      55.0
    B-depart_date.month_name      0.982     0.964     0.973      56.0
          B-depart_time.time      0.838     1.000     0.912      57.0
                B-round_trip      1.000     0.986     0.993      73.0
             B-cost_relative      1.000     0.973     0.986      37.0
                I-round_trip      1.000     1.000     1.000      71.0
                B-flight_mod      0.828     1.000     0.906      24.0
 B-depart_time.time_relative      0.984     0.938     0.961      65.0
          I-depart_time.time      0.981     1.000     0.990      52.0
         B-stoploc.city_name      1.000     1.000     1.000      20.0
                 B-city_name      0.846     0.579     0.688      57.0
                B-class_type      0.960     1.000     0.980      24.0
          B-arrive_time.time      0.943     0.971     0.957      34.0
 B-arrive_time.time_relative      0.909     0.968     0.937      31.0
                I-class_type      1.000     1.000     1.000      17.0
               B-flight_stop      1.000     1.000     1.000      21.0
          I-arrive_time.time      0.971     0.971     0.971      35.0
              B-airline_code      0.968     0.882     0.923      34.0
    I-depart_date.day_number      1.000     0.933     0.966      15.0
      I-fromloc.airport_name      0.417     1.000     0.588      15.0
      B-fromloc.airport_name      0.429     1.000     0.600      12.0
      B-arrive_date.day_name      0.733     1.000     0.846      11.0
          B-toloc.state_code      1.000     1.000     1.000      18.0
B-depart_date.today_relative      1.000     1.000     1.000       9.0
             B-flight_number      0.688     1.000     0.815      11.0
 B-depart_date.date_relative      0.944     1.000     0.971      17.0
          B-toloc.state_name      0.867     0.929     0.897      28.0
           B-fare_basis_code      0.917     0.647     0.759      17.0
               B-flight_time      1.000     1.000     1.000       1.0
                        B-or      0.750     1.000     0.857       3.0
 B-arrive_time.period_of_day      0.600     1.000     0.750       6.0
          B-meal_description      1.000     0.900     0.947      10.0
             I-cost_relative      1.000     0.667     0.800       3.0
              I-airport_name      0.909     0.345     0.500      29.0
               B-fare_amount      1.000     1.000     1.000       2.0
               I-fare_amount      1.000     1.000     1.000       2.0
                 I-city_name      0.909     0.667     0.769      30.0
        I-toloc.airport_name      1.000     1.000     1.000       3.0
            B-transport_type      1.000     1.000     1.000      10.0
    B-arrive_date.month_name      0.714     0.833     0.769       6.0
    B-arrive_date.day_number      0.714     0.833     0.769       6.0
         I-stoploc.city_name      0.833     1.000     0.909      10.0
                      B-meal      0.941     1.000     0.970      16.0
        B-fromloc.state_code      1.000     1.000     1.000      23.0
    B-depart_time.period_mod      1.000     1.000     1.000       5.0
                   B-connect      1.000     1.000     1.000       6.0
               B-flight_days      1.000     1.000     1.000      10.0
        B-toloc.airport_name      1.000     1.000     1.000       3.0
        B-fromloc.state_name      1.000     0.941     0.970      17.0
              B-airport_name      0.750     0.286     0.414      21.0
                   B-economy      1.000     1.000     1.000       6.0
               I-flight_time      1.000     1.000     1.000       1.0
             B-aircraft_code      1.000     0.939     0.969      33.0
                       B-mod      0.000     0.000     0.000       2.0
              B-airport_code      1.000     0.444     0.615       9.0
    B-depart_time.start_time      0.750     1.000     0.857       3.0
      B-depart_time.end_time      1.000     0.667     0.800       3.0
          B-depart_date.year      0.600     1.000     0.750       3.0
            I-transport_type      0.000     0.000     0.000       1.0
          B-restriction_code      1.000     1.000     1.000       4.0
    B-arrive_time.start_time      1.000     1.000     1.000       8.0
        B-toloc.airport_code      1.000     0.750     0.857       4.0
      B-arrive_time.end_time      0.875     0.875     0.875       8.0
      I-arrive_time.end_time      0.875     0.875     0.875       8.0
      I-depart_time.end_time      1.000     0.667     0.800       3.0
               I-flight_stop      0.000     0.000     0.000       0.0
      B-fromloc.airport_code      1.000     1.000     1.000       5.0
          I-restriction_code      1.000     1.000     1.000       3.0
    I-depart_time.start_time      1.000     1.000     1.000       1.0
          I-toloc.state_name      1.000     1.000     1.000       1.0
I-depart_date.today_relative      0.000     0.000     0.000       0.0
 B-arrive_date.date_relative      1.000     1.000     1.000       2.0
                I-flight_mod      1.000     0.167     0.286       6.0
                   I-economy      0.000     0.000     0.000       0.0
 B-return_date.date_relative      1.000     0.333     0.500       3.0
        I-fromloc.state_name      1.000     1.000     1.000       1.0
                B-state_code      1.000     1.000     1.000       1.0
    I-arrive_time.start_time      1.000     1.000     1.000       1.0
    I-arrive_date.day_number      0.000     0.000     0.000       0.0
                 B-meal_code      0.000     0.000     0.000       1.0
 I-depart_time.period_of_day      0.000     0.000     0.000       1.0
                  B-day_name      1.000     0.500     0.667       2.0
             B-period_of_day      1.000     0.750     0.857       4.0
        B-stoploc.state_code      0.000     0.000     0.000       0.0
    B-return_date.month_name      0.000     0.000     0.000       0.0
    B-return_date.day_number      0.000     0.000     0.000       0.0
    B-arrive_time.period_mod      0.000     0.000     0.000       0.0
                 I-meal_code      0.000     0.000     0.000       0.0
        B-toloc.country_name      1.000     1.000     1.000       1.0
                 B-days_code      0.500     1.000     0.667       1.0
 I-arrive_time.period_of_day      0.000     0.000     0.000       0.0
            I-today_relative      0.000     0.000     0.000       0.0
 B-return_time.period_of_day      0.000     0.000     0.000       0.0
                      B-time      0.000     0.000     0.000       0.0
           I-fare_basis_code      0.000     0.000     0.000       0.0
 I-arrive_time.time_relative      1.000     0.500     0.667       4.0
 I-depart_time.time_relative      0.000     0.000     0.000       1.0
            B-today_relative      0.000     0.000     0.000       0.0
                B-state_name      0.667     0.222     0.333       9.0
B-arrive_date.today_relative      0.000     0.000     0.000       0.0
    B-return_time.period_mod      0.000     0.000     0.000       0.0
                B-month_name      0.000     0.000     0.000       0.0
                B-day_number      0.000     0.000     0.000       0.0
 I-return_date.date_relative      0.667     0.667     0.667       3.0
I-return_date.today_relative      0.000     0.000     0.000       0.0
      B-stoploc.airport_name      0.000     0.000     0.000       0.0
             B-time_relative      0.000     0.000     0.000       0.0
                      I-time      0.000     0.000     0.000       0.0
    I-return_date.day_number      0.000     0.000     0.000       0.0
          I-meal_description      0.000     0.000     0.000       0.0
B-return_date.today_relative      0.000     0.000     0.000       0.0
      B-return_date.day_name      0.000     0.000     0.000       2.0

                   micro avg      0.952     0.953     0.952    3657.0
                   macro avg      0.670     0.647     0.644    3657.0
                weighted avg      0.960     0.953     0.951    3657.0

I0715 07:13:51.725848 140369481070464 interactiveshell.py:2882] Best Slot F1: 0.956
Reading ../data/atis.train.w-intent.iob
I0715 07:13:54.564096 140369481070464 interactiveshell.py:2882] Step 9700 | Loss: 0.0047 | Loss_intent: 0.0000 | Loss_slots: 0.0046 | Spent: 5.9 secs | LR: 0.000108
I0715 07:13:58.284226 140369481070464 interactiveshell.py:2882] Step 9750 | Loss: 0.0054 | Loss_intent: 0.0000 | Loss_slots: 0.0054 | Spent: 3.7 secs | LR: 0.000107
I0715 07:14:02.002186 140369481070464 interactiveshell.py:2882] Step 9800 | Loss: 0.0015 | Loss_intent: 0.0000 | Loss_slots: 0.0015 | Spent: 3.7 secs | LR: 0.000107
I0715 07:14:05.783108 140369481070464 interactiveshell.py:2882] Step 9850 | Loss: 0.0017 | Loss_intent: 0.0000 | Loss_slots: 0.0017 | Spent: 3.8 secs | LR: 0.000106
I0715 07:14:09.917178 140369481070464 interactiveshell.py:2882] Step 9900 | Loss: 0.2931 | Loss_intent: 0.2899 | Loss_slots: 0.0032 | Spent: 4.1 secs | LR: 0.000106
I0715 07:14:14.053593 140369481070464 interactiveshell.py:2882] Step 9950 | Loss: 0.0015 | Loss_intent: 0.0000 | Loss_slots: 0.0015 | Spent: 4.1 secs | LR: 0.000105
Reading ../data/atis.test.w-intent.iob
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.
  'precision', 'predicted', average, warn_for)
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1439: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples.
  'recall', 'true', average, warn_for)
I0715 07:14:18.141286 140369481070464 interactiveshell.py:2882] 
                                          precision    recall  f1-score   support

                             atis_flight      0.978     0.992     0.985       632
                            atis_airfare      0.941     1.000     0.970        48
                     atis_ground_service      1.000     1.000     1.000        36
                            atis_airline      1.000     1.000     1.000        38
                       atis_abbreviation      1.000     1.000     1.000        33
                           atis_aircraft      1.000     0.889     0.941         9
                        atis_flight_time      1.000     1.000     1.000         1
                           atis_quantity      0.429     1.000     0.600         3
                atis_flight#atis_airfare      0.833     0.417     0.556        12
                            atis_airport      1.000     0.944     0.971        18
                           atis_distance      1.000     1.000     1.000        10
                               atis_city      1.000     0.667     0.800         6
                        atis_ground_fare      1.000     0.714     0.833         7
                           atis_capacity      0.955     1.000     0.977        21
                          atis_flight_no      0.889     1.000     0.941         8
                               atis_meal      1.000     0.833     0.909         6
                        atis_restriction      0.000     0.000     0.000         0
             atis_airline#atis_flight_no      0.000     0.000     0.000         0
    atis_ground_service#atis_ground_fare      0.000     0.000     0.000         0
           atis_airfare#atis_flight_time      0.000     0.000     0.000         0
                           atis_cheapest      0.000     0.000     0.000         0
atis_aircraft#atis_flight#atis_flight_no      0.000     0.000     0.000         0

                               micro avg      0.973     0.979     0.976       888
                               macro avg      0.683     0.657     0.658       888
                            weighted avg      0.975     0.979     0.975       888

I0715 07:14:18.180572 140369481070464 interactiveshell.py:2882] 
                              precision    recall  f1-score   support

                           O      0.000     0.000     0.000       0.0
           B-toloc.city_name      0.970     0.993     0.981     716.0
         B-fromloc.city_name      0.983     0.999     0.991     704.0
           I-toloc.city_name      0.978     0.992     0.985     265.0
      B-depart_date.day_name      0.968     0.986     0.977     212.0
              B-airline_name      0.990     0.990     0.990     101.0
         I-fromloc.city_name      0.978     0.994     0.986     177.0
 B-depart_time.period_of_day      0.991     0.869     0.926     130.0
              I-airline_name      1.000     0.985     0.992      65.0
    B-depart_date.day_number      0.981     0.964     0.972      55.0
    B-depart_date.month_name      0.982     0.964     0.973      56.0
          B-depart_time.time      0.862     0.982     0.918      57.0
                B-round_trip      1.000     0.986     0.993      73.0
             B-cost_relative      1.000     0.973     0.986      37.0
                I-round_trip      1.000     0.986     0.993      71.0
                B-flight_mod      0.800     1.000     0.889      24.0
 B-depart_time.time_relative      0.984     0.954     0.969      65.0
          I-depart_time.time      0.981     1.000     0.990      52.0
         B-stoploc.city_name      1.000     1.000     1.000      20.0
                 B-city_name      0.811     0.526     0.638      57.0
                B-class_type      0.960     1.000     0.980      24.0
          B-arrive_time.time      0.892     0.971     0.930      34.0
 B-arrive_time.time_relative      0.909     0.968     0.937      31.0
                I-class_type      1.000     1.000     1.000      17.0
               B-flight_stop      1.000     1.000     1.000      21.0
          I-arrive_time.time      0.971     0.971     0.971      35.0
              B-airline_code      1.000     0.912     0.954      34.0
    I-depart_date.day_number      1.000     0.933     0.966      15.0
      I-fromloc.airport_name      0.395     1.000     0.566      15.0
      B-fromloc.airport_name      0.375     0.750     0.500      12.0
      B-arrive_date.day_name      0.733     1.000     0.846      11.0
          B-toloc.state_code      1.000     1.000     1.000      18.0
B-depart_date.today_relative      1.000     0.889     0.941       9.0
             B-flight_number      0.524     1.000     0.688      11.0
 B-depart_date.date_relative      0.944     1.000     0.971      17.0
          B-toloc.state_name      0.862     0.893     0.877      28.0
           B-fare_basis_code      0.944     1.000     0.971      17.0
               B-flight_time      1.000     1.000     1.000       1.0
                        B-or      1.000     1.000     1.000       3.0
 B-arrive_time.period_of_day      0.600     1.000     0.750       6.0
          B-meal_description      1.000     0.900     0.947      10.0
             I-cost_relative      1.000     0.667     0.800       3.0
              I-airport_name      0.833     0.345     0.488      29.0
               B-fare_amount      1.000     1.000     1.000       2.0
               I-fare_amount      1.000     1.000     1.000       2.0
                 I-city_name      1.000     0.667     0.800      30.0
        I-toloc.airport_name      1.000     1.000     1.000       3.0
            B-transport_type      1.000     0.900     0.947      10.0
    B-arrive_date.month_name      0.714     0.833     0.769       6.0
    B-arrive_date.day_number      0.714     0.833     0.769       6.0
         I-stoploc.city_name      0.909     1.000     0.952      10.0
                      B-meal      0.941     1.000     0.970      16.0
        B-fromloc.state_code      1.000     1.000     1.000      23.0
    B-depart_time.period_mod      0.833     1.000     0.909       5.0
                   B-connect      1.000     1.000     1.000       6.0
               B-flight_days      1.000     1.000     1.000      10.0
        B-toloc.airport_name      1.000     1.000     1.000       3.0
        B-fromloc.state_name      1.000     0.941     0.970      17.0
              B-airport_name      0.857     0.286     0.429      21.0
                   B-economy      1.000     1.000     1.000       6.0
               I-flight_time      1.000     1.000     1.000       1.0
             B-aircraft_code      1.000     0.697     0.821      33.0
                       B-mod      1.000     0.500     0.667       2.0
              B-airport_code      0.667     0.444     0.533       9.0
    B-depart_time.start_time      0.750     1.000     0.857       3.0
      B-depart_time.end_time      1.000     0.667     0.800       3.0
          B-depart_date.year      0.500     1.000     0.667       3.0
            I-transport_type      0.000     0.000     0.000       1.0
          B-restriction_code      1.000     1.000     1.000       4.0
    B-arrive_time.start_time      1.000     1.000     1.000       8.0
        B-toloc.airport_code      1.000     0.750     0.857       4.0
      B-arrive_time.end_time      0.875     0.875     0.875       8.0
      I-arrive_time.end_time      0.875     0.875     0.875       8.0
      I-depart_time.end_time      1.000     0.667     0.800       3.0
               I-flight_stop      0.000     0.000     0.000       0.0
      B-fromloc.airport_code      0.714     1.000     0.833       5.0
          I-restriction_code      1.000     1.000     1.000       3.0
    I-depart_time.start_time      1.000     1.000     1.000       1.0
          I-toloc.state_name      1.000     1.000     1.000       1.0
I-depart_date.today_relative      0.000     0.000     0.000       0.0
 B-arrive_date.date_relative      1.000     1.000     1.000       2.0
                I-flight_mod      0.000     0.000     0.000       6.0
                   I-economy      0.000     0.000     0.000       0.0
 B-return_date.date_relative      0.000     0.000     0.000       3.0
        I-fromloc.state_name      1.000     1.000     1.000       1.0
                B-state_code      1.000     1.000     1.000       1.0
    I-arrive_time.start_time      1.000     1.000     1.000       1.0
    I-arrive_date.day_number      0.000     0.000     0.000       0.0
                 B-meal_code      0.000     0.000     0.000       1.0
 I-depart_time.period_of_day      0.000     0.000     0.000       1.0
                  B-day_name      1.000     0.500     0.667       2.0
             B-period_of_day      1.000     0.750     0.857       4.0
        B-stoploc.state_code      0.000     0.000     0.000       0.0
    B-return_date.month_name      0.000     0.000     0.000       0.0
    B-return_date.day_number      0.000     0.000     0.000       0.0
    B-arrive_time.period_mod      0.000     0.000     0.000       0.0
                 I-meal_code      0.000     0.000     0.000       0.0
        B-toloc.country_name      1.000     1.000     1.000       1.0
                 B-days_code      1.000     1.000     1.000       1.0
 I-arrive_time.period_of_day      0.000     0.000     0.000       0.0
            I-today_relative      0.000     0.000     0.000       0.0
 B-return_time.period_of_day      0.000     0.000     0.000       0.0
                      B-time      0.000     0.000     0.000       0.0
           I-fare_basis_code      0.000     0.000     0.000       0.0
 I-arrive_time.time_relative      1.000     0.500     0.667       4.0
 I-depart_time.time_relative      0.000     0.000     0.000       1.0
            B-today_relative      0.000     0.000     0.000       0.0
                B-state_name      0.667     0.222     0.333       9.0
B-arrive_date.today_relative      0.000     0.000     0.000       0.0
    B-return_time.period_mod      0.000     0.000     0.000       0.0
                B-month_name      0.000     0.000     0.000       0.0
                B-day_number      0.000     0.000     0.000       0.0
 I-return_date.date_relative      0.667     0.667     0.667       3.0
I-return_date.today_relative      0.000     0.000     0.000       0.0
      B-stoploc.airport_name      0.000     0.000     0.000       0.0
             B-time_relative      0.000     0.000     0.000       0.0
                      I-time      0.000     0.000     0.000       0.0
    I-return_date.day_number      0.000     0.000     0.000       0.0
          I-meal_description      0.000     0.000     0.000       0.0
B-return_date.today_relative      0.000     0.000     0.000       0.0
      B-return_date.day_name      0.000     0.000     0.000       2.0

                   micro avg      0.949     0.951     0.950    3657.0
                   macro avg      0.660     0.644     0.641    3657.0
                weighted avg      0.956     0.951     0.948    3657.0

I0715 07:14:18.181581 140369481070464 interactiveshell.py:2882] Best Slot F1: 0.956
Reading ../data/atis.train.w-intent.iob
I0715 07:14:20.152320 140369481070464 interactiveshell.py:2882] Step 10000 | Loss: 0.0069 | Loss_intent: 0.0000 | Loss_slots: 0.0069 | Spent: 6.1 secs | LR: 0.000105
I0715 07:14:23.884406 140369481070464 interactiveshell.py:2882] Step 10050 | Loss: 0.0010 | Loss_intent: 0.0000 | Loss_slots: 0.0010 | Spent: 3.7 secs | LR: 0.000104
I0715 07:14:27.564579 140369481070464 interactiveshell.py:2882] Step 10100 | Loss: 0.0018 | Loss_intent: 0.0000 | Loss_slots: 0.0018 | Spent: 3.7 secs | LR: 0.000104
I0715 07:14:31.289702 140369481070464 interactiveshell.py:2882] Step 10150 | Loss: 0.0010 | Loss_intent: 0.0002 | Loss_slots: 0.0008 | Spent: 3.7 secs | LR: 0.000103
I0715 07:14:35.017127 140369481070464 interactiveshell.py:2882] Step 10200 | Loss: 0.0050 | Loss_intent: 0.0000 | Loss_slots: 0.0050 | Spent: 3.7 secs | LR: 0.000102
I0715 07:14:38.698315 140369481070464 interactiveshell.py:2882] Step 10250 | Loss: 0.0014 | Loss_intent: 0.0000 | Loss_slots: 0.0014 | Spent: 3.7 secs | LR: 0.000102
Reading ../data/atis.test.w-intent.iob
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.
  'precision', 'predicted', average, warn_for)
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1439: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples.
  'recall', 'true', average, warn_for)
I0715 07:14:43.473882 140369481070464 interactiveshell.py:2882] 
                                          precision    recall  f1-score   support

                             atis_flight      0.978     0.991     0.984       632
                            atis_airfare      0.923     1.000     0.960        48
                     atis_ground_service      1.000     1.000     1.000        36
                            atis_airline      1.000     1.000     1.000        38
                       atis_abbreviation      1.000     1.000     1.000        33
                           atis_aircraft      1.000     0.889     0.941         9
                        atis_flight_time      1.000     1.000     1.000         1
                           atis_quantity      0.429     1.000     0.600         3
                atis_flight#atis_airfare      0.833     0.417     0.556        12
                            atis_airport      1.000     0.944     0.971        18
                           atis_distance      1.000     1.000     1.000        10
                               atis_city      1.000     0.667     0.800         6
                        atis_ground_fare      1.000     0.714     0.833         7
                           atis_capacity      0.955     1.000     0.977        21
                          atis_flight_no      0.889     1.000     0.941         8
                               atis_meal      1.000     0.833     0.909         6
                        atis_restriction      0.000     0.000     0.000         0
             atis_airline#atis_flight_no      0.000     0.000     0.000         0
    atis_ground_service#atis_ground_fare      0.000     0.000     0.000         0
           atis_airfare#atis_flight_time      0.000     0.000     0.000         0
                           atis_cheapest      0.000     0.000     0.000         0
atis_aircraft#atis_flight#atis_flight_no      0.000     0.000     0.000         0

                               micro avg      0.972     0.977     0.975       888
                               macro avg      0.682     0.657     0.658       888
                            weighted avg      0.974     0.977     0.974       888

I0715 07:14:43.514234 140369481070464 interactiveshell.py:2882] 
                              precision    recall  f1-score   support

                           O      0.000     0.000     0.000       0.0
           B-toloc.city_name      0.974     0.992     0.983     716.0
         B-fromloc.city_name      0.979     0.999     0.989     704.0
           I-toloc.city_name      0.978     0.989     0.983     265.0
      B-depart_date.day_name      0.977     0.986     0.981     212.0
              B-airline_name      0.990     0.990     0.990     101.0
         I-fromloc.city_name      0.989     0.994     0.992     177.0
 B-depart_time.period_of_day      0.991     0.877     0.931     130.0
              I-airline_name      1.000     1.000     1.000      65.0
    B-depart_date.day_number      0.981     0.964     0.972      55.0
    B-depart_date.month_name      0.982     0.964     0.973      56.0
          B-depart_time.time      0.862     0.982     0.918      57.0
                B-round_trip      1.000     0.986     0.993      73.0
             B-cost_relative      1.000     0.973     0.986      37.0
                I-round_trip      1.000     1.000     1.000      71.0
                B-flight_mod      0.800     1.000     0.889      24.0
 B-depart_time.time_relative      0.984     0.954     0.969      65.0
          I-depart_time.time      0.945     1.000     0.972      52.0
         B-stoploc.city_name      1.000     1.000     1.000      20.0
                 B-city_name      0.810     0.596     0.687      57.0
                B-class_type      0.960     1.000     0.980      24.0
          B-arrive_time.time      0.917     0.971     0.943      34.0
 B-arrive_time.time_relative      0.909     0.968     0.937      31.0
                I-class_type      1.000     1.000     1.000      17.0
               B-flight_stop      1.000     1.000     1.000      21.0
          I-arrive_time.time      0.971     0.971     0.971      35.0
              B-airline_code      0.971     0.971     0.971      34.0
    I-depart_date.day_number      1.000     0.933     0.966      15.0
      I-fromloc.airport_name      0.417     1.000     0.588      15.0
      B-fromloc.airport_name      0.400     0.833     0.541      12.0
      B-arrive_date.day_name      0.786     1.000     0.880      11.0
          B-toloc.state_code      1.000     1.000     1.000      18.0
B-depart_date.today_relative      1.000     1.000     1.000       9.0
             B-flight_number      0.550     1.000     0.710      11.0
 B-depart_date.date_relative      0.944     1.000     0.971      17.0
          B-toloc.state_name      0.926     0.893     0.909      28.0
           B-fare_basis_code      0.895     1.000     0.944      17.0
               B-flight_time      1.000     1.000     1.000       1.0
                        B-or      0.750     1.000     0.857       3.0
 B-arrive_time.period_of_day      0.600     1.000     0.750       6.0
          B-meal_description      1.000     0.900     0.947      10.0
             I-cost_relative      1.000     0.667     0.800       3.0
              I-airport_name      0.909     0.345     0.500      29.0
               B-fare_amount      1.000     1.000     1.000       2.0
               I-fare_amount      1.000     1.000     1.000       2.0
                 I-city_name      1.000     0.567     0.723      30.0
        I-toloc.airport_name      1.000     1.000     1.000       3.0
            B-transport_type      1.000     0.900     0.947      10.0
    B-arrive_date.month_name      0.714     0.833     0.769       6.0
    B-arrive_date.day_number      0.714     0.833     0.769       6.0
         I-stoploc.city_name      0.833     1.000     0.909      10.0
                      B-meal      1.000     1.000     1.000      16.0
        B-fromloc.state_code      1.000     1.000     1.000      23.0
    B-depart_time.period_mod      0.833     1.000     0.909       5.0
                   B-connect      1.000     1.000     1.000       6.0
               B-flight_days      1.000     1.000     1.000      10.0
        B-toloc.airport_name      1.000     1.000     1.000       3.0
        B-fromloc.state_name      1.000     0.941     0.970      17.0
              B-airport_name      0.778     0.333     0.467      21.0
                   B-economy      1.000     1.000     1.000       6.0
               I-flight_time      1.000     1.000     1.000       1.0
             B-aircraft_code      0.929     0.788     0.852      33.0
                       B-mod      1.000     0.500     0.667       2.0
              B-airport_code      1.000     0.444     0.615       9.0
    B-depart_time.start_time      0.750     1.000     0.857       3.0
      B-depart_time.end_time      1.000     0.667     0.800       3.0
          B-depart_date.year      1.000     1.000     1.000       3.0
            I-transport_type      0.500     1.000     0.667       1.0
          B-restriction_code      1.000     1.000     1.000       4.0
    B-arrive_time.start_time      0.875     0.875     0.875       8.0
        B-toloc.airport_code      1.000     0.750     0.857       4.0
      B-arrive_time.end_time      0.875     0.875     0.875       8.0
      I-arrive_time.end_time      0.875     0.875     0.875       8.0
      I-depart_time.end_time      1.000     0.667     0.800       3.0
               I-flight_stop      0.000     0.000     0.000       0.0
      B-fromloc.airport_code      1.000     1.000     1.000       5.0
          I-restriction_code      1.000     1.000     1.000       3.0
    I-depart_time.start_time      1.000     1.000     1.000       1.0
          I-toloc.state_name      1.000     1.000     1.000       1.0
I-depart_date.today_relative      0.000     0.000     0.000       0.0
 B-arrive_date.date_relative      1.000     1.000     1.000       2.0
                I-flight_mod      0.000     0.000     0.000       6.0
                   I-economy      0.000     0.000     0.000       0.0
 B-return_date.date_relative      0.000     0.000     0.000       3.0
        I-fromloc.state_name      1.000     1.000     1.000       1.0
                B-state_code      1.000     1.000     1.000       1.0
    I-arrive_time.start_time      1.000     1.000     1.000       1.0
    I-arrive_date.day_number      0.000     0.000     0.000       0.0
                 B-meal_code      0.000     0.000     0.000       1.0
 I-depart_time.period_of_day      0.000     0.000     0.000       1.0
                  B-day_name      1.000     0.500     0.667       2.0
             B-period_of_day      1.000     0.750     0.857       4.0
        B-stoploc.state_code      0.000     0.000     0.000       0.0
    B-return_date.month_name      0.000     0.000     0.000       0.0
    B-return_date.day_number      0.000     0.000     0.000       0.0
    B-arrive_time.period_mod      0.000     0.000     0.000       0.0
                 I-meal_code      0.000     0.000     0.000       0.0
        B-toloc.country_name      1.000     1.000     1.000       1.0
                 B-days_code      0.500     1.000     0.667       1.0
 I-arrive_time.period_of_day      0.000     0.000     0.000       0.0
            I-today_relative      0.000     0.000     0.000       0.0
 B-return_time.period_of_day      0.000     0.000     0.000       0.0
                      B-time      0.000     0.000     0.000       0.0
           I-fare_basis_code      0.000     0.000     0.000       0.0
 I-arrive_time.time_relative      1.000     0.500     0.667       4.0
 I-depart_time.time_relative      0.000     0.000     0.000       1.0
            B-today_relative      0.000     0.000     0.000       0.0
                B-state_name      0.750     0.333     0.462       9.0
B-arrive_date.today_relative      0.000     0.000     0.000       0.0
    B-return_time.period_mod      0.000     0.000     0.000       0.0
                B-month_name      0.000     0.000     0.000       0.0
                B-day_number      0.000     0.000     0.000       0.0
 I-return_date.date_relative      0.667     0.667     0.667       3.0
I-return_date.today_relative      0.000     0.000     0.000       0.0
      B-stoploc.airport_name      0.000     0.000     0.000       0.0
             B-time_relative      0.000     0.000     0.000       0.0
                      I-time      0.000     0.000     0.000       0.0
    I-return_date.day_number      0.000     0.000     0.000       0.0
          I-meal_description      0.000     0.000     0.000       0.0
B-return_date.today_relative      0.000     0.000     0.000       0.0
      B-return_date.day_name      0.000     0.000     0.000       2.0

                   micro avg      0.952     0.954     0.953    3657.0
                   macro avg      0.667     0.655     0.649    3657.0
                weighted avg      0.958     0.954     0.951    3657.0

I0715 07:14:43.515120 140369481070464 interactiveshell.py:2882] Best Slot F1: 0.956
Reading ../data/atis.train.w-intent.iob
I0715 07:14:44.596433 140369481070464 interactiveshell.py:2882] Step 10300 | Loss: 0.0012 | Loss_intent: 0.0000 | Loss_slots: 0.0012 | Spent: 5.9 secs | LR: 0.000101
I0715 07:14:48.324105 140369481070464 interactiveshell.py:2882] Step 10350 | Loss: 0.0006 | Loss_intent: 0.0000 | Loss_slots: 0.0006 | Spent: 3.7 secs | LR: 0.000101
I0715 07:14:52.032074 140369481070464 interactiveshell.py:2882] Step 10400 | Loss: 0.0017 | Loss_intent: 0.0000 | Loss_slots: 0.0017 | Spent: 3.7 secs | LR: 0.000100
I0715 07:14:55.758438 140369481070464 interactiveshell.py:2882] Step 10450 | Loss: 0.0011 | Loss_intent: 0.0000 | Loss_slots: 0.0011 | Spent: 3.7 secs | LR: 0.000100
I0715 07:14:59.463612 140369481070464 interactiveshell.py:2882] Step 10500 | Loss: 0.0006 | Loss_intent: 0.0000 | Loss_slots: 0.0006 | Spent: 3.7 secs | LR: 0.000099
I0715 07:15:03.649049 140369481070464 interactiveshell.py:2882] Step 10550 | Loss: 0.0258 | Loss_intent: 0.0000 | Loss_slots: 0.0258 | Spent: 4.2 secs | LR: 0.000099
I0715 07:15:08.313146 140369481070464 interactiveshell.py:2882] Step 10600 | Loss: 0.0008 | Loss_intent: 0.0000 | Loss_slots: 0.0008 | Spent: 4.7 secs | LR: 0.000098
Reading ../data/atis.test.w-intent.iob
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.
  'precision', 'predicted', average, warn_for)
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1439: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples.
  'recall', 'true', average, warn_for)
I0715 07:15:10.589503 140369481070464 interactiveshell.py:2882] 
                                          precision    recall  f1-score   support

                             atis_flight      0.978     0.992     0.985       632
                            atis_airfare      0.960     1.000     0.980        48
                     atis_ground_service      1.000     1.000     1.000        36
                            atis_airline      1.000     1.000     1.000        38
                       atis_abbreviation      1.000     1.000     1.000        33
                           atis_aircraft      1.000     0.889     0.941         9
                        atis_flight_time      1.000     1.000     1.000         1
                           atis_quantity      0.429     1.000     0.600         3
                atis_flight#atis_airfare      0.833     0.417     0.556        12
                            atis_airport      1.000     0.944     0.971        18
                           atis_distance      1.000     1.000     1.000        10
                               atis_city      1.000     0.667     0.800         6
                        atis_ground_fare      1.000     0.857     0.923         7
                           atis_capacity      0.955     1.000     0.977        21
                          atis_flight_no      0.889     1.000     0.941         8
                               atis_meal      1.000     0.833     0.909         6
                        atis_restriction      0.000     0.000     0.000         0
             atis_airline#atis_flight_no      0.000     0.000     0.000         0
    atis_ground_service#atis_ground_fare      0.000     0.000     0.000         0
           atis_airfare#atis_flight_time      0.000     0.000     0.000         0
                           atis_cheapest      0.000     0.000     0.000         0
atis_aircraft#atis_flight#atis_flight_no      0.000     0.000     0.000         0

                               micro avg      0.974     0.980     0.977       888
                               macro avg      0.684     0.664     0.663       888
                            weighted avg      0.976     0.980     0.976       888

I0715 07:15:10.633497 140369481070464 interactiveshell.py:2882] 
                              precision    recall  f1-score   support

                           O      0.000     0.000     0.000       0.0
           B-toloc.city_name      0.975     0.993     0.984     716.0
         B-fromloc.city_name      0.987     0.997     0.992     704.0
           I-toloc.city_name      0.978     0.992     0.985     265.0
      B-depart_date.day_name      0.968     0.986     0.977     212.0
              B-airline_name      0.990     0.990     0.990     101.0
         I-fromloc.city_name      0.983     0.994     0.989     177.0
 B-depart_time.period_of_day      1.000     0.877     0.934     130.0
              I-airline_name      1.000     0.969     0.984      65.0
    B-depart_date.day_number      0.982     0.982     0.982      55.0
    B-depart_date.month_name      0.982     0.964     0.973      56.0
          B-depart_time.time      0.838     1.000     0.912      57.0
                B-round_trip      1.000     0.986     0.993      73.0
             B-cost_relative      1.000     0.973     0.986      37.0
                I-round_trip      1.000     1.000     1.000      71.0
                B-flight_mod      0.821     0.958     0.885      24.0
 B-depart_time.time_relative      0.983     0.877     0.927      65.0
          I-depart_time.time      0.962     0.981     0.971      52.0
         B-stoploc.city_name      0.952     1.000     0.976      20.0
                 B-city_name      0.850     0.596     0.701      57.0
                B-class_type      0.960     1.000     0.980      24.0
          B-arrive_time.time      0.943     0.971     0.957      34.0
 B-arrive_time.time_relative      0.906     0.935     0.921      31.0
                I-class_type      1.000     1.000     1.000      17.0
               B-flight_stop      1.000     1.000     1.000      21.0
          I-arrive_time.time      0.944     0.971     0.958      35.0
              B-airline_code      1.000     0.912     0.954      34.0
    I-depart_date.day_number      1.000     1.000     1.000      15.0
      I-fromloc.airport_name      0.417     1.000     0.588      15.0
      B-fromloc.airport_name      0.379     0.917     0.537      12.0
      B-arrive_date.day_name      0.786     1.000     0.880      11.0
          B-toloc.state_code      1.000     1.000     1.000      18.0
B-depart_date.today_relative      1.000     1.000     1.000       9.0
             B-flight_number      0.524     1.000     0.688      11.0
 B-depart_date.date_relative      0.739     1.000     0.850      17.0
          B-toloc.state_name      0.867     0.929     0.897      28.0
           B-fare_basis_code      0.895     1.000     0.944      17.0
               B-flight_time      1.000     1.000     1.000       1.0
                        B-or      0.750     1.000     0.857       3.0
 B-arrive_time.period_of_day      0.600     1.000     0.750       6.0
          B-meal_description      1.000     0.900     0.947      10.0
             I-cost_relative      1.000     0.667     0.800       3.0
              I-airport_name      0.909     0.345     0.500      29.0
               B-fare_amount      1.000     1.000     1.000       2.0
               I-fare_amount      1.000     1.000     1.000       2.0
                 I-city_name      1.000     0.633     0.776      30.0
        I-toloc.airport_name      1.000     1.000     1.000       3.0
            B-transport_type      1.000     0.800     0.889      10.0
    B-arrive_date.month_name      0.714     0.833     0.769       6.0
    B-arrive_date.day_number      0.833     0.833     0.833       6.0
         I-stoploc.city_name      0.909     1.000     0.952      10.0
                      B-meal      1.000     1.000     1.000      16.0
        B-fromloc.state_code      1.000     1.000     1.000      23.0
    B-depart_time.period_mod      1.000     1.000     1.000       5.0
                   B-connect      1.000     1.000     1.000       6.0
               B-flight_days      1.000     1.000     1.000      10.0
        B-toloc.airport_name      1.000     1.000     1.000       3.0
        B-fromloc.state_name      1.000     0.941     0.970      17.0
              B-airport_name      0.750     0.286     0.414      21.0
                   B-economy      1.000     1.000     1.000       6.0
               I-flight_time      1.000     1.000     1.000       1.0
             B-aircraft_code      1.000     0.667     0.800      33.0
                       B-mod      1.000     0.500     0.667       2.0
              B-airport_code      0.667     0.444     0.533       9.0
    B-depart_time.start_time      0.750     1.000     0.857       3.0
      B-depart_time.end_time      1.000     0.667     0.800       3.0
          B-depart_date.year      0.429     1.000     0.600       3.0
            I-transport_type      0.000     0.000     0.000       1.0
          B-restriction_code      1.000     1.000     1.000       4.0
    B-arrive_time.start_time      0.889     1.000     0.941       8.0
        B-toloc.airport_code      1.000     0.750     0.857       4.0
      B-arrive_time.end_time      0.875     0.875     0.875       8.0
      I-arrive_time.end_time      0.875     0.875     0.875       8.0
      I-depart_time.end_time      1.000     0.667     0.800       3.0
               I-flight_stop      0.000     0.000     0.000       0.0
      B-fromloc.airport_code      0.714     1.000     0.833       5.0
          I-restriction_code      1.000     1.000     1.000       3.0
    I-depart_time.start_time      1.000     1.000     1.000       1.0
          I-toloc.state_name      1.000     1.000     1.000       1.0
I-depart_date.today_relative      0.000     0.000     0.000       0.0
 B-arrive_date.date_relative      1.000     1.000     1.000       2.0
                I-flight_mod      1.000     0.167     0.286       6.0
                   I-economy      0.000     0.000     0.000       0.0
 B-return_date.date_relative      1.000     0.333     0.500       3.0
        I-fromloc.state_name      1.000     1.000     1.000       1.0
                B-state_code      1.000     1.000     1.000       1.0
    I-arrive_time.start_time      1.000     1.000     1.000       1.0
    I-arrive_date.day_number      0.000     0.000     0.000       0.0
                 B-meal_code      0.000     0.000     0.000       1.0
 I-depart_time.period_of_day      1.000     1.000     1.000       1.0
                  B-day_name      1.000     0.500     0.667       2.0
             B-period_of_day      1.000     0.750     0.857       4.0
        B-stoploc.state_code      0.000     0.000     0.000       0.0
    B-return_date.month_name      0.000     0.000     0.000       0.0
    B-return_date.day_number      0.000     0.000     0.000       0.0
    B-arrive_time.period_mod      0.000     0.000     0.000       0.0
                 I-meal_code      0.000     0.000     0.000       0.0
        B-toloc.country_name      1.000     1.000     1.000       1.0
                 B-days_code      1.000     1.000     1.000       1.0
 I-arrive_time.period_of_day      0.000     0.000     0.000       0.0
            I-today_relative      0.000     0.000     0.000       0.0
 B-return_time.period_of_day      0.000     0.000     0.000       0.0
                      B-time      0.000     0.000     0.000       0.0
           I-fare_basis_code      0.000     0.000     0.000       0.0
 I-arrive_time.time_relative      1.000     0.500     0.667       4.0
 I-depart_time.time_relative      0.000     0.000     0.000       1.0
            B-today_relative      0.000     0.000     0.000       0.0
                B-state_name      0.800     0.444     0.571       9.0
B-arrive_date.today_relative      0.000     0.000     0.000       0.0
    B-return_time.period_mod      0.000     0.000     0.000       0.0
                B-month_name      0.000     0.000     0.000       0.0
                B-day_number      0.000     0.000     0.000       0.0
 I-return_date.date_relative      1.000     0.667     0.800       3.0
I-return_date.today_relative      0.000     0.000     0.000       0.0
      B-stoploc.airport_name      0.000     0.000     0.000       0.0
             B-time_relative      0.000     0.000     0.000       0.0
                      I-time      0.000     0.000     0.000       0.0
    I-return_date.day_number      0.000     0.000     0.000       0.0
          I-meal_description      0.000     0.000     0.000       0.0
B-return_date.today_relative      0.000     0.000     0.000       0.0
      B-return_date.day_name      0.000     0.000     0.000       2.0

                   micro avg      0.950     0.952     0.951    3657.0
                   macro avg      0.687     0.659     0.657    3657.0
                weighted avg      0.961     0.952     0.951    3657.0

I0715 07:15:10.634499 140369481070464 interactiveshell.py:2882] Best Slot F1: 0.956
Reading ../data/atis.train.w-intent.iob
I0715 07:15:14.859971 140369481070464 interactiveshell.py:2882] Step 10650 | Loss: 0.0010 | Loss_intent: 0.0000 | Loss_slots: 0.0010 | Spent: 6.5 secs | LR: 0.000098
I0715 07:15:18.592778 140369481070464 interactiveshell.py:2882] Step 10700 | Loss: 0.0110 | Loss_intent: 0.0000 | Loss_slots: 0.0110 | Spent: 3.7 secs | LR: 0.000097
I0715 07:15:22.284735 140369481070464 interactiveshell.py:2882] Step 10750 | Loss: 0.0003 | Loss_intent: 0.0000 | Loss_slots: 0.0003 | Spent: 3.7 secs | LR: 0.000097
I0715 07:15:25.991411 140369481070464 interactiveshell.py:2882] Step 10800 | Loss: 0.0016 | Loss_intent: 0.0000 | Loss_slots: 0.0016 | Spent: 3.7 secs | LR: 0.000096
I0715 07:15:29.754814 140369481070464 interactiveshell.py:2882] Step 10850 | Loss: 0.0013 | Loss_intent: 0.0000 | Loss_slots: 0.0013 | Spent: 3.8 secs | LR: 0.000096
I0715 07:15:33.606051 140369481070464 interactiveshell.py:2882] Step 10900 | Loss: 0.0062 | Loss_intent: 0.0000 | Loss_slots: 0.0062 | Spent: 3.9 secs | LR: 0.000095
Reading ../data/atis.test.w-intent.iob
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.
  'precision', 'predicted', average, warn_for)
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1439: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples.
  'recall', 'true', average, warn_for)
I0715 07:15:36.503438 140369481070464 interactiveshell.py:2882] 
                                          precision    recall  f1-score   support

                             atis_flight      0.978     0.989     0.983       632
                            atis_airfare      0.941     1.000     0.970        48
                     atis_ground_service      1.000     1.000     1.000        36
                            atis_airline      1.000     1.000     1.000        38
                       atis_abbreviation      1.000     1.000     1.000        33
                           atis_aircraft      0.889     0.889     0.889         9
                        atis_flight_time      1.000     1.000     1.000         1
                           atis_quantity      0.429     1.000     0.600         3
                atis_flight#atis_airfare      0.833     0.417     0.556        12
                            atis_airport      1.000     0.944     0.971        18
                           atis_distance      1.000     1.000     1.000        10
                               atis_city      1.000     0.667     0.800         6
                        atis_ground_fare      1.000     0.857     0.923         7
                           atis_capacity      0.955     1.000     0.977        21
                          atis_flight_no      0.889     1.000     0.941         8
                               atis_meal      1.000     0.833     0.909         6
                        atis_restriction      0.000     0.000     0.000         0
             atis_airline#atis_flight_no      0.000     0.000     0.000         0
    atis_ground_service#atis_ground_fare      0.000     0.000     0.000         0
           atis_airfare#atis_flight_time      0.000     0.000     0.000         0
                           atis_cheapest      0.000     0.000     0.000         0
atis_aircraft#atis_flight#atis_flight_no      0.000     0.000     0.000         0

                               micro avg      0.972     0.977     0.975       888
                               macro avg      0.678     0.663     0.660       888
                            weighted avg      0.974     0.977     0.974       888

I0715 07:15:36.546086 140369481070464 interactiveshell.py:2882] 
                              precision    recall  f1-score   support

                           O      0.000     0.000     0.000       0.0
           B-toloc.city_name      0.973     0.992     0.982     716.0
         B-fromloc.city_name      0.986     0.997     0.992     704.0
           I-toloc.city_name      0.978     0.992     0.985     265.0
      B-depart_date.day_name      0.986     0.986     0.986     212.0
              B-airline_name      1.000     1.000     1.000     101.0
         I-fromloc.city_name      0.989     0.994     0.992     177.0
 B-depart_time.period_of_day      0.992     0.908     0.948     130.0
              I-airline_name      1.000     0.985     0.992      65.0
    B-depart_date.day_number      0.982     0.982     0.982      55.0
    B-depart_date.month_name      0.982     0.964     0.973      56.0
          B-depart_time.time      0.826     1.000     0.905      57.0
                B-round_trip      1.000     0.986     0.993      73.0
             B-cost_relative      1.000     0.973     0.986      37.0
                I-round_trip      1.000     1.000     1.000      71.0
                B-flight_mod      0.821     0.958     0.885      24.0
 B-depart_time.time_relative      0.984     0.938     0.961      65.0
          I-depart_time.time      0.963     1.000     0.981      52.0
         B-stoploc.city_name      0.909     1.000     0.952      20.0
                 B-city_name      0.846     0.579     0.688      57.0
                B-class_type      0.960     1.000     0.980      24.0
          B-arrive_time.time      0.943     0.971     0.957      34.0
 B-arrive_time.time_relative      0.909     0.968     0.937      31.0
                I-class_type      1.000     1.000     1.000      17.0
               B-flight_stop      1.000     1.000     1.000      21.0
          I-arrive_time.time      0.971     0.971     0.971      35.0
              B-airline_code      1.000     0.912     0.954      34.0
    I-depart_date.day_number      1.000     1.000     1.000      15.0
      I-fromloc.airport_name      0.429     1.000     0.600      15.0
      B-fromloc.airport_name      0.379     0.917     0.537      12.0
      B-arrive_date.day_name      0.786     1.000     0.880      11.0
          B-toloc.state_code      1.000     1.000     1.000      18.0
B-depart_date.today_relative      1.000     1.000     1.000       9.0
             B-flight_number      0.524     1.000     0.688      11.0
 B-depart_date.date_relative      0.895     1.000     0.944      17.0
          B-toloc.state_name      0.862     0.893     0.877      28.0
           B-fare_basis_code      0.895     1.000     0.944      17.0
               B-flight_time      1.000     1.000     1.000       1.0
                        B-or      1.000     1.000     1.000       3.0
 B-arrive_time.period_of_day      0.600     1.000     0.750       6.0
          B-meal_description      1.000     0.900     0.947      10.0
             I-cost_relative      1.000     0.667     0.800       3.0
              I-airport_name      0.917     0.379     0.537      29.0
               B-fare_amount      1.000     1.000     1.000       2.0
               I-fare_amount      1.000     1.000     1.000       2.0
                 I-city_name      1.000     0.600     0.750      30.0
        I-toloc.airport_name      1.000     1.000     1.000       3.0
            B-transport_type      1.000     1.000     1.000      10.0
    B-arrive_date.month_name      0.714     0.833     0.769       6.0
    B-arrive_date.day_number      0.833     0.833     0.833       6.0
         I-stoploc.city_name      0.833     1.000     0.909      10.0
                      B-meal      1.000     1.000     1.000      16.0
        B-fromloc.state_code      1.000     1.000     1.000      23.0
    B-depart_time.period_mod      1.000     1.000     1.000       5.0
                   B-connect      1.000     1.000     1.000       6.0
               B-flight_days      1.000     1.000     1.000      10.0
        B-toloc.airport_name      1.000     1.000     1.000       3.0
        B-fromloc.state_name      1.000     0.941     0.970      17.0
              B-airport_name      0.778     0.333     0.467      21.0
                   B-economy      1.000     1.000     1.000       6.0
               I-flight_time      1.000     1.000     1.000       1.0
             B-aircraft_code      1.000     0.727     0.842      33.0
                       B-mod      1.000     0.500     0.667       2.0
              B-airport_code      0.667     0.444     0.533       9.0
    B-depart_time.start_time      0.600     1.000     0.750       3.0
      B-depart_time.end_time      1.000     0.667     0.800       3.0
          B-depart_date.year      0.429     1.000     0.600       3.0
            I-transport_type      1.000     1.000     1.000       1.0
          B-restriction_code      1.000     1.000     1.000       4.0
    B-arrive_time.start_time      1.000     0.875     0.933       8.0
        B-toloc.airport_code      1.000     0.750     0.857       4.0
      B-arrive_time.end_time      0.875     0.875     0.875       8.0
      I-arrive_time.end_time      0.875     0.875     0.875       8.0
      I-depart_time.end_time      1.000     0.667     0.800       3.0
               I-flight_stop      0.000     0.000     0.000       0.0
      B-fromloc.airport_code      0.714     1.000     0.833       5.0
          I-restriction_code      1.000     1.000     1.000       3.0
    I-depart_time.start_time      1.000     1.000     1.000       1.0
          I-toloc.state_name      1.000     1.000     1.000       1.0
I-depart_date.today_relative      0.000     0.000     0.000       0.0
 B-arrive_date.date_relative      1.000     1.000     1.000       2.0
                I-flight_mod      1.000     0.167     0.286       6.0
                   I-economy      0.000     0.000     0.000       0.0
 B-return_date.date_relative      0.000     0.000     0.000       3.0
        I-fromloc.state_name      1.000     1.000     1.000       1.0
                B-state_code      1.000     1.000     1.000       1.0
    I-arrive_time.start_time      1.000     1.000     1.000       1.0
    I-arrive_date.day_number      0.000     0.000     0.000       0.0
                 B-meal_code      0.000     0.000     0.000       1.0
 I-depart_time.period_of_day      0.000     0.000     0.000       1.0
                  B-day_name      1.000     0.500     0.667       2.0
             B-period_of_day      1.000     0.750     0.857       4.0
        B-stoploc.state_code      0.000     0.000     0.000       0.0
    B-return_date.month_name      0.000     0.000     0.000       0.0
    B-return_date.day_number      0.000     0.000     0.000       0.0
    B-arrive_time.period_mod      0.000     0.000     0.000       0.0
                 I-meal_code      0.000     0.000     0.000       0.0
        B-toloc.country_name      1.000     1.000     1.000       1.0
                 B-days_code      1.000     1.000     1.000       1.0
 I-arrive_time.period_of_day      0.000     0.000     0.000       0.0
            I-today_relative      0.000     0.000     0.000       0.0
 B-return_time.period_of_day      0.000     0.000     0.000       0.0
                      B-time      0.000     0.000     0.000       0.0
           I-fare_basis_code      0.000     0.000     0.000       0.0
 I-arrive_time.time_relative      1.000     0.500     0.667       4.0
 I-depart_time.time_relative      0.000     0.000     0.000       1.0
            B-today_relative      0.000     0.000     0.000       0.0
                B-state_name      0.667     0.222     0.333       9.0
B-arrive_date.today_relative      0.000     0.000     0.000       0.0
    B-return_time.period_mod      0.000     0.000     0.000       0.0
                B-month_name      0.000     0.000     0.000       0.0
                B-day_number      0.000     0.000     0.000       0.0
 I-return_date.date_relative      0.667     0.667     0.667       3.0
I-return_date.today_relative      0.000     0.000     0.000       0.0
      B-stoploc.airport_name      0.000     0.000     0.000       0.0
             B-time_relative      0.000     0.000     0.000       0.0
                      I-time      0.000     0.000     0.000       0.0
    I-return_date.day_number      0.000     0.000     0.000       0.0
          I-meal_description      0.000     0.000     0.000       0.0
B-return_date.today_relative      0.000     0.000     0.000       0.0
      B-return_date.day_name      0.000     0.000     0.000       2.0

                   micro avg      0.953     0.955     0.954    3657.0
                   macro avg      0.677     0.657     0.653    3657.0
                weighted avg      0.961     0.955     0.952    3657.0

I0715 07:15:36.547559 140369481070464 interactiveshell.py:2882] Best Slot F1: 0.956
Reading ../data/atis.train.w-intent.iob
I0715 07:15:39.591775 140369481070464 interactiveshell.py:2882] Step 10950 | Loss: 0.0007 | Loss_intent: 0.0000 | Loss_slots: 0.0007 | Spent: 6.0 secs | LR: 0.000095
I0715 07:15:43.332875 140369481070464 interactiveshell.py:2882] Step 11000 | Loss: 0.0025 | Loss_intent: 0.0000 | Loss_slots: 0.0025 | Spent: 3.7 secs | LR: 0.000094
I0715 07:15:47.066341 140369481070464 interactiveshell.py:2882] Step 11050 | Loss: 0.0007 | Loss_intent: 0.0000 | Loss_slots: 0.0007 | Spent: 3.7 secs | LR: 0.000094
I0715 07:15:50.782530 140369481070464 interactiveshell.py:2882] Step 11100 | Loss: 0.0010 | Loss_intent: 0.0000 | Loss_slots: 0.0010 | Spent: 3.7 secs | LR: 0.000093
I0715 07:15:54.511121 140369481070464 interactiveshell.py:2882] Step 11150 | Loss: 0.0002 | Loss_intent: 0.0000 | Loss_slots: 0.0002 | Spent: 3.7 secs | LR: 0.000093
I0715 07:15:58.215727 140369481070464 interactiveshell.py:2882] Step 11200 | Loss: 0.0012 | Loss_intent: 0.0000 | Loss_slots: 0.0012 | Spent: 3.7 secs | LR: 0.000092
Reading ../data/atis.test.w-intent.iob
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.
  'precision', 'predicted', average, warn_for)
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1439: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples.
  'recall', 'true', average, warn_for)
I0715 07:16:01.961346 140369481070464 interactiveshell.py:2882] 
                                          precision    recall  f1-score   support

                             atis_flight      0.984     0.989     0.987       632
                            atis_airfare      0.941     1.000     0.970        48
                     atis_ground_service      1.000     1.000     1.000        36
                            atis_airline      1.000     1.000     1.000        38
                       atis_abbreviation      0.943     1.000     0.971        33
                           atis_aircraft      1.000     0.889     0.941         9
                        atis_flight_time      0.333     1.000     0.500         1
                           atis_quantity      0.429     1.000     0.600         3
                atis_flight#atis_airfare      0.833     0.417     0.556        12
                            atis_airport      0.944     0.944     0.944        18
                           atis_distance      1.000     1.000     1.000        10
                               atis_city      1.000     0.667     0.800         6
                        atis_ground_fare      1.000     0.857     0.923         7
                           atis_capacity      0.955     1.000     0.977        21
                          atis_flight_no      0.889     1.000     0.941         8
                               atis_meal      1.000     0.833     0.909         6
                        atis_restriction      0.000     0.000     0.000         0
             atis_airline#atis_flight_no      0.000     0.000     0.000         0
    atis_ground_service#atis_ground_fare      0.000     0.000     0.000         0
           atis_airfare#atis_flight_time      0.000     0.000     0.000         0
                           atis_cheapest      0.000     0.000     0.000         0
atis_aircraft#atis_flight#atis_flight_no      0.000     0.000     0.000         0

                               micro avg      0.972     0.977     0.975       888
                               macro avg      0.648     0.663     0.637       888
                            weighted avg      0.975     0.977     0.974       888

I0715 07:16:02.001644 140369481070464 interactiveshell.py:2882] 
                              precision    recall  f1-score   support

                           O      0.000     0.000     0.000       0.0
           B-toloc.city_name      0.970     0.993     0.981     716.0
         B-fromloc.city_name      0.986     0.999     0.992     704.0
           I-toloc.city_name      0.978     0.989     0.983     265.0
      B-depart_date.day_name      0.986     0.986     0.986     212.0
              B-airline_name      0.990     1.000     0.995     101.0
         I-fromloc.city_name      0.962     0.994     0.978     177.0
 B-depart_time.period_of_day      0.992     0.938     0.964     130.0
              I-airline_name      0.985     0.985     0.985      65.0
    B-depart_date.day_number      0.981     0.964     0.972      55.0
    B-depart_date.month_name      0.982     0.964     0.973      56.0
          B-depart_time.time      0.838     1.000     0.912      57.0
                B-round_trip      1.000     0.986     0.993      73.0
             B-cost_relative      1.000     0.973     0.986      37.0
                I-round_trip      1.000     1.000     1.000      71.0
                B-flight_mod      0.828     1.000     0.906      24.0
 B-depart_time.time_relative      0.984     0.923     0.952      65.0
          I-depart_time.time      0.981     1.000     0.990      52.0
         B-stoploc.city_name      1.000     1.000     1.000      20.0
                 B-city_name      0.882     0.526     0.659      57.0
                B-class_type      0.960     1.000     0.980      24.0
          B-arrive_time.time      0.943     0.971     0.957      34.0
 B-arrive_time.time_relative      0.909     0.968     0.937      31.0
                I-class_type      1.000     1.000     1.000      17.0
               B-flight_stop      1.000     1.000     1.000      21.0
          I-arrive_time.time      0.971     0.971     0.971      35.0
              B-airline_code      0.969     0.912     0.939      34.0
    I-depart_date.day_number      1.000     1.000     1.000      15.0
      I-fromloc.airport_name      0.405     1.000     0.577      15.0
      B-fromloc.airport_name      0.379     0.917     0.537      12.0
      B-arrive_date.day_name      0.786     1.000     0.880      11.0
          B-toloc.state_code      1.000     1.000     1.000      18.0
B-depart_date.today_relative      1.000     1.000     1.000       9.0
             B-flight_number      0.500     1.000     0.667      11.0
 B-depart_date.date_relative      0.944     1.000     0.971      17.0
          B-toloc.state_name      0.893     0.893     0.893      28.0
           B-fare_basis_code      0.944     1.000     0.971      17.0
               B-flight_time      1.000     1.000     1.000       1.0
                        B-or      0.750     1.000     0.857       3.0
 B-arrive_time.period_of_day      0.857     1.000     0.923       6.0
          B-meal_description      1.000     0.900     0.947      10.0
             I-cost_relative      1.000     0.667     0.800       3.0
              I-airport_name      0.900     0.310     0.462      29.0
               B-fare_amount      1.000     1.000     1.000       2.0
               I-fare_amount      1.000     1.000     1.000       2.0
                 I-city_name      1.000     0.633     0.776      30.0
        I-toloc.airport_name      1.000     1.000     1.000       3.0
            B-transport_type      1.000     0.800     0.889      10.0
    B-arrive_date.month_name      0.714     0.833     0.769       6.0
    B-arrive_date.day_number      0.714     0.833     0.769       6.0
         I-stoploc.city_name      1.000     0.900     0.947      10.0
                      B-meal      1.000     1.000     1.000      16.0
        B-fromloc.state_code      1.000     1.000     1.000      23.0
    B-depart_time.period_mod      1.000     1.000     1.000       5.0
                   B-connect      1.000     1.000     1.000       6.0
               B-flight_days      1.000     1.000     1.000      10.0
        B-toloc.airport_name      1.000     1.000     1.000       3.0
        B-fromloc.state_name      1.000     1.000     1.000      17.0
              B-airport_name      0.750     0.286     0.414      21.0
                   B-economy      1.000     1.000     1.000       6.0
               I-flight_time      1.000     1.000     1.000       1.0
             B-aircraft_code      1.000     0.788     0.881      33.0
                       B-mod      1.000     0.500     0.667       2.0
              B-airport_code      0.667     0.444     0.533       9.0
    B-depart_time.start_time      0.667     0.667     0.667       3.0
      B-depart_time.end_time      1.000     0.667     0.800       3.0
          B-depart_date.year      1.000     1.000     1.000       3.0
            I-transport_type      0.000     0.000     0.000       1.0
          B-restriction_code      1.000     1.000     1.000       4.0
    B-arrive_time.start_time      0.889     1.000     0.941       8.0
        B-toloc.airport_code      1.000     0.750     0.857       4.0
      B-arrive_time.end_time      0.875     0.875     0.875       8.0
      I-arrive_time.end_time      0.875     0.875     0.875       8.0
      I-depart_time.end_time      1.000     0.667     0.800       3.0
               I-flight_stop      0.000     0.000     0.000       0.0
      B-fromloc.airport_code      0.625     1.000     0.769       5.0
          I-restriction_code      1.000     1.000     1.000       3.0
    I-depart_time.start_time      1.000     1.000     1.000       1.0
          I-toloc.state_name      1.000     1.000     1.000       1.0
I-depart_date.today_relative      0.000     0.000     0.000       0.0
 B-arrive_date.date_relative      1.000     1.000     1.000       2.0
                I-flight_mod      1.000     0.167     0.286       6.0
                   I-economy      0.000     0.000     0.000       0.0
 B-return_date.date_relative      1.000     0.333     0.500       3.0
        I-fromloc.state_name      1.000     1.000     1.000       1.0
                B-state_code      1.000     1.000     1.000       1.0
    I-arrive_time.start_time      1.000     1.000     1.000       1.0
    I-arrive_date.day_number      0.000     0.000     0.000       0.0
                 B-meal_code      1.000     1.000     1.000       1.0
 I-depart_time.period_of_day      1.000     1.000     1.000       1.0
                  B-day_name      1.000     0.500     0.667       2.0
             B-period_of_day      1.000     0.500     0.667       4.0
        B-stoploc.state_code      0.000     0.000     0.000       0.0
    B-return_date.month_name      0.000     0.000     0.000       0.0
    B-return_date.day_number      0.000     0.000     0.000       0.0
    B-arrive_time.period_mod      0.000     0.000     0.000       0.0
                 I-meal_code      0.000     0.000     0.000       0.0
        B-toloc.country_name      0.000     0.000     0.000       1.0
                 B-days_code      1.000     1.000     1.000       1.0
 I-arrive_time.period_of_day      0.000     0.000     0.000       0.0
            I-today_relative      0.000     0.000     0.000       0.0
 B-return_time.period_of_day      0.000     0.000     0.000       0.0
                      B-time      0.000     0.000     0.000       0.0
           I-fare_basis_code      0.000     0.000     0.000       0.0
 I-arrive_time.time_relative      1.000     0.500     0.667       4.0
 I-depart_time.time_relative      0.000     0.000     0.000       1.0
            B-today_relative      0.000     0.000     0.000       0.0
                B-state_name      0.800     0.444     0.571       9.0
B-arrive_date.today_relative      0.000     0.000     0.000       0.0
    B-return_time.period_mod      0.000     0.000     0.000       0.0
                B-month_name      0.000     0.000     0.000       0.0
                B-day_number      0.000     0.000     0.000       0.0
 I-return_date.date_relative      0.667     0.667     0.667       3.0
I-return_date.today_relative      0.000     0.000     0.000       0.0
      B-stoploc.airport_name      0.000     0.000     0.000       0.0
             B-time_relative      0.000     0.000     0.000       0.0
                      I-time      0.000     0.000     0.000       0.0
    I-return_date.day_number      0.000     0.000     0.000       0.0
          I-meal_description      0.000     0.000     0.000       0.0
B-return_date.today_relative      0.000     0.000     0.000       0.0
      B-return_date.day_name      0.000     0.000     0.000       2.0

                   micro avg      0.953     0.955     0.954    3657.0
                   macro avg      0.692     0.656     0.659    3657.0
                weighted avg      0.961     0.955     0.952    3657.0

I0715 07:16:02.002463 140369481070464 interactiveshell.py:2882] Best Slot F1: 0.956
Reading ../data/atis.train.w-intent.iob
I0715 07:16:04.122210 140369481070464 interactiveshell.py:2882] Step 11250 | Loss: 0.0016 | Loss_intent: 0.0000 | Loss_slots: 0.0016 | Spent: 5.9 secs | LR: 0.000092
I0715 07:16:07.852497 140369481070464 interactiveshell.py:2882] Step 11300 | Loss: 0.0008 | Loss_intent: 0.0000 | Loss_slots: 0.0008 | Spent: 3.7 secs | LR: 0.000091
I0715 07:16:11.764161 140369481070464 interactiveshell.py:2882] Step 11350 | Loss: 0.0012 | Loss_intent: 0.0000 | Loss_slots: 0.0012 | Spent: 3.9 secs | LR: 0.000091
I0715 07:16:15.723451 140369481070464 interactiveshell.py:2882] Step 11400 | Loss: 0.0007 | Loss_intent: 0.0000 | Loss_slots: 0.0007 | Spent: 4.0 secs | LR: 0.000090
I0715 07:16:19.513111 140369481070464 interactiveshell.py:2882] Step 11450 | Loss: 0.0007 | Loss_intent: 0.0000 | Loss_slots: 0.0007 | Spent: 3.8 secs | LR: 0.000090
I0715 07:16:23.246039 140369481070464 interactiveshell.py:2882] Step 11500 | Loss: 0.0012 | Loss_intent: 0.0000 | Loss_slots: 0.0012 | Spent: 3.7 secs | LR: 0.000089
Reading ../data/atis.test.w-intent.iob
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.
  'precision', 'predicted', average, warn_for)
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1439: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples.
  'recall', 'true', average, warn_for)
I0715 07:16:27.873739 140369481070464 interactiveshell.py:2882] 
                                          precision    recall  f1-score   support

                             atis_flight      0.983     0.989     0.986       632
                            atis_airfare      0.941     1.000     0.970        48
                     atis_ground_service      1.000     1.000     1.000        36
                            atis_airline      1.000     1.000     1.000        38
                       atis_abbreviation      0.943     1.000     0.971        33
                           atis_aircraft      1.000     0.889     0.941         9
                        atis_flight_time      1.000     1.000     1.000         1
                           atis_quantity      0.429     1.000     0.600         3
                atis_flight#atis_airfare      0.833     0.417     0.556        12
                            atis_airport      0.947     1.000     0.973        18
                           atis_distance      1.000     1.000     1.000        10
                               atis_city      1.000     0.667     0.800         6
                        atis_ground_fare      1.000     0.857     0.923         7
                           atis_capacity      0.955     1.000     0.977        21
                          atis_flight_no      0.889     1.000     0.941         8
                               atis_meal      1.000     0.833     0.909         6
                        atis_restriction      0.000     0.000     0.000         0
             atis_airline#atis_flight_no      0.000     0.000     0.000         0
    atis_ground_service#atis_ground_fare      0.000     0.000     0.000         0
           atis_airfare#atis_flight_time      0.000     0.000     0.000         0
                           atis_cheapest      0.000     0.000     0.000         0
atis_aircraft#atis_flight#atis_flight_no      0.000     0.000     0.000         0

                               micro avg      0.973     0.979     0.976       888
                               macro avg      0.678     0.666     0.661       888
                            weighted avg      0.975     0.979     0.975       888

I0715 07:16:27.917484 140369481070464 interactiveshell.py:2882] 
                              precision    recall  f1-score   support

                           O      0.000     0.000     0.000       0.0
           B-toloc.city_name      0.973     0.993     0.983     716.0
         B-fromloc.city_name      0.985     0.999     0.992     704.0
           I-toloc.city_name      0.978     0.996     0.987     265.0
      B-depart_date.day_name      0.986     0.986     0.986     212.0
              B-airline_name      0.980     0.990     0.985     101.0
         I-fromloc.city_name      0.994     0.994     0.994     177.0
 B-depart_time.period_of_day      1.000     0.908     0.952     130.0
              I-airline_name      0.985     1.000     0.992      65.0
    B-depart_date.day_number      0.981     0.964     0.972      55.0
    B-depart_date.month_name      0.982     0.964     0.973      56.0
          B-depart_time.time      0.848     0.982     0.911      57.0
                B-round_trip      1.000     0.986     0.993      73.0
             B-cost_relative      1.000     0.973     0.986      37.0
                I-round_trip      1.000     1.000     1.000      71.0
                B-flight_mod      0.828     1.000     0.906      24.0
 B-depart_time.time_relative      0.969     0.969     0.969      65.0
          I-depart_time.time      0.963     1.000     0.981      52.0
         B-stoploc.city_name      1.000     1.000     1.000      20.0
                 B-city_name      0.886     0.544     0.674      57.0
                B-class_type      0.960     1.000     0.980      24.0
          B-arrive_time.time      0.917     0.971     0.943      34.0
 B-arrive_time.time_relative      0.906     0.935     0.921      31.0
                I-class_type      1.000     1.000     1.000      17.0
               B-flight_stop      1.000     1.000     1.000      21.0
          I-arrive_time.time      0.971     0.971     0.971      35.0
              B-airline_code      1.000     0.853     0.921      34.0
    I-depart_date.day_number      1.000     1.000     1.000      15.0
      I-fromloc.airport_name      0.395     1.000     0.566      15.0
      B-fromloc.airport_name      0.391     0.750     0.514      12.0
      B-arrive_date.day_name      0.786     1.000     0.880      11.0
          B-toloc.state_code      1.000     1.000     1.000      18.0
B-depart_date.today_relative      1.000     1.000     1.000       9.0
             B-flight_number      0.550     1.000     0.710      11.0
 B-depart_date.date_relative      0.944     1.000     0.971      17.0
          B-toloc.state_name      0.844     0.964     0.900      28.0
           B-fare_basis_code      0.944     1.000     0.971      17.0
               B-flight_time      1.000     1.000     1.000       1.0
                        B-or      1.000     1.000     1.000       3.0
 B-arrive_time.period_of_day      0.600     1.000     0.750       6.0
          B-meal_description      1.000     0.900     0.947      10.0
             I-cost_relative      1.000     0.667     0.800       3.0
              I-airport_name      0.833     0.345     0.488      29.0
               B-fare_amount      1.000     1.000     1.000       2.0
               I-fare_amount      1.000     1.000     1.000       2.0
                 I-city_name      1.000     0.633     0.776      30.0
        I-toloc.airport_name      1.000     1.000     1.000       3.0
            B-transport_type      1.000     1.000     1.000      10.0
    B-arrive_date.month_name      0.714     0.833     0.769       6.0
    B-arrive_date.day_number      0.714     0.833     0.769       6.0
         I-stoploc.city_name      0.909     1.000     0.952      10.0
                      B-meal      1.000     1.000     1.000      16.0
        B-fromloc.state_code      1.000     1.000     1.000      23.0
    B-depart_time.period_mod      1.000     1.000     1.000       5.0
                   B-connect      1.000     1.000     1.000       6.0
               B-flight_days      1.000     1.000     1.000      10.0
        B-toloc.airport_name      1.000     1.000     1.000       3.0
        B-fromloc.state_name      1.000     0.941     0.970      17.0
              B-airport_name      0.875     0.333     0.483      21.0
                   B-economy      1.000     1.000     1.000       6.0
               I-flight_time      1.000     1.000     1.000       1.0
             B-aircraft_code      0.929     0.788     0.852      33.0
                       B-mod      1.000     0.500     0.667       2.0
              B-airport_code      0.800     0.444     0.571       9.0
    B-depart_time.start_time      0.500     0.667     0.571       3.0
      B-depart_time.end_time      1.000     0.667     0.800       3.0
          B-depart_date.year      1.000     1.000     1.000       3.0
            I-transport_type      0.000     0.000     0.000       1.0
          B-restriction_code      1.000     1.000     1.000       4.0
    B-arrive_time.start_time      0.889     1.000     0.941       8.0
        B-toloc.airport_code      1.000     0.750     0.857       4.0
      B-arrive_time.end_time      0.875     0.875     0.875       8.0
      I-arrive_time.end_time      0.875     0.875     0.875       8.0
      I-depart_time.end_time      1.000     0.667     0.800       3.0
               I-flight_stop      0.000     0.000     0.000       0.0
      B-fromloc.airport_code      0.625     1.000     0.769       5.0
          I-restriction_code      1.000     1.000     1.000       3.0
    I-depart_time.start_time      1.000     1.000     1.000       1.0
          I-toloc.state_name      1.000     1.000     1.000       1.0
I-depart_date.today_relative      0.000     0.000     0.000       0.0
 B-arrive_date.date_relative      1.000     1.000     1.000       2.0
                I-flight_mod      1.000     0.167     0.286       6.0
                   I-economy      0.000     0.000     0.000       0.0
 B-return_date.date_relative      1.000     0.333     0.500       3.0
        I-fromloc.state_name      1.000     1.000     1.000       1.0
                B-state_code      1.000     1.000     1.000       1.0
    I-arrive_time.start_time      1.000     1.000     1.000       1.0
    I-arrive_date.day_number      0.000     0.000     0.000       0.0
                 B-meal_code      0.000     0.000     0.000       1.0
 I-depart_time.period_of_day      1.000     1.000     1.000       1.0
                  B-day_name      1.000     0.500     0.667       2.0
             B-period_of_day      1.000     0.750     0.857       4.0
        B-stoploc.state_code      0.000     0.000     0.000       0.0
    B-return_date.month_name      0.000     0.000     0.000       0.0
    B-return_date.day_number      0.000     0.000     0.000       0.0
    B-arrive_time.period_mod      0.000     0.000     0.000       0.0
                 I-meal_code      0.000     0.000     0.000       0.0
        B-toloc.country_name      0.000     0.000     0.000       1.0
                 B-days_code      0.500     1.000     0.667       1.0
 I-arrive_time.period_of_day      0.000     0.000     0.000       0.0
            I-today_relative      0.000     0.000     0.000       0.0
 B-return_time.period_of_day      0.000     0.000     0.000       0.0
                      B-time      0.000     0.000     0.000       0.0
           I-fare_basis_code      0.000     0.000     0.000       0.0
 I-arrive_time.time_relative      1.000     0.500     0.667       4.0
 I-depart_time.time_relative      0.000     0.000     0.000       1.0
            B-today_relative      0.000     0.000     0.000       0.0
                B-state_name      0.833     0.556     0.667       9.0
B-arrive_date.today_relative      0.000     0.000     0.000       0.0
    B-return_time.period_mod      0.000     0.000     0.000       0.0
                B-month_name      0.000     0.000     0.000       0.0
                B-day_number      0.000     0.000     0.000       0.0
 I-return_date.date_relative      0.667     0.667     0.667       3.0
I-return_date.today_relative      0.000     0.000     0.000       0.0
      B-stoploc.airport_name      0.000     0.000     0.000       0.0
             B-time_relative      0.000     0.000     0.000       0.0
                      I-time      0.000     0.000     0.000       0.0
    I-return_date.day_number      0.000     0.000     0.000       0.0
          I-meal_description      0.000     0.000     0.000       0.0
B-return_date.today_relative      0.000     0.000     0.000       0.0
      B-return_date.day_name      0.000     0.000     0.000       2.0

                   micro avg      0.954     0.956     0.955    3657.0
                   macro avg      0.678     0.652     0.651    3657.0
                weighted avg      0.962     0.956     0.954    3657.0

I0715 07:16:27.918501 140369481070464 interactiveshell.py:2882] Best Slot F1: 0.956
Reading ../data/atis.train.w-intent.iob
I0715 07:16:29.136217 140369481070464 interactiveshell.py:2882] Step 11550 | Loss: 0.0092 | Loss_intent: 0.0002 | Loss_slots: 0.0089 | Spent: 5.9 secs | LR: 0.000089
I0715 07:16:32.881626 140369481070464 interactiveshell.py:2882] Step 11600 | Loss: 0.0012 | Loss_intent: 0.0000 | Loss_slots: 0.0012 | Spent: 3.7 secs | LR: 0.000088
I0715 07:16:36.581910 140369481070464 interactiveshell.py:2882] Step 11650 | Loss: 0.0005 | Loss_intent: 0.0000 | Loss_slots: 0.0005 | Spent: 3.7 secs | LR: 0.000088
I0715 07:16:40.304172 140369481070464 interactiveshell.py:2882] Step 11700 | Loss: 0.0078 | Loss_intent: 0.0000 | Loss_slots: 0.0078 | Spent: 3.7 secs | LR: 0.000087
I0715 07:16:44.016797 140369481070464 interactiveshell.py:2882] Step 11750 | Loss: 0.0076 | Loss_intent: 0.0000 | Loss_slots: 0.0076 | Spent: 3.7 secs | LR: 0.000087
I0715 07:16:47.754131 140369481070464 interactiveshell.py:2882] Step 11800 | Loss: 0.0013 | Loss_intent: 0.0000 | Loss_slots: 0.0013 | Spent: 3.7 secs | LR: 0.000087
I0715 07:16:51.482635 140369481070464 interactiveshell.py:2882] Step 11850 | Loss: 0.0126 | Loss_intent: 0.0000 | Loss_slots: 0.0126 | Spent: 3.7 secs | LR: 0.000086
Reading ../data/atis.test.w-intent.iob
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.
  'precision', 'predicted', average, warn_for)
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1439: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples.
  'recall', 'true', average, warn_for)
I0715 07:16:53.339893 140369481070464 interactiveshell.py:2882] 
                                          precision    recall  f1-score   support

                             atis_flight      0.975     0.992     0.984       632
                            atis_airfare      0.941     1.000     0.970        48
                     atis_ground_service      1.000     1.000     1.000        36
                            atis_airline      1.000     1.000     1.000        38
                       atis_abbreviation      1.000     1.000     1.000        33
                           atis_aircraft      1.000     0.889     0.941         9
                        atis_flight_time      1.000     1.000     1.000         1
                           atis_quantity      0.429     1.000     0.600         3
                atis_flight#atis_airfare      0.833     0.417     0.556        12
                            atis_airport      1.000     0.889     0.941        18
                           atis_distance      1.000     1.000     1.000        10
                               atis_city      1.000     0.667     0.800         6
                        atis_ground_fare      1.000     0.714     0.833         7
                           atis_capacity      0.952     0.952     0.952        21
                          atis_flight_no      0.889     1.000     0.941         8
                               atis_meal      1.000     0.833     0.909         6
                        atis_restriction      0.000     0.000     0.000         0
             atis_airline#atis_flight_no      0.000     0.000     0.000         0
    atis_ground_service#atis_ground_fare      0.000     0.000     0.000         0
           atis_airfare#atis_flight_time      0.000     0.000     0.000         0
                           atis_cheapest      0.000     0.000     0.000         0
atis_aircraft#atis_flight#atis_flight_no      0.000     0.000     0.000         0

                               micro avg      0.971     0.976     0.974       888
                               macro avg      0.683     0.652     0.656       888
                            weighted avg      0.973     0.976     0.973       888

I0715 07:16:53.380427 140369481070464 interactiveshell.py:2882] 
                              precision    recall  f1-score   support

                           O      0.000     0.000     0.000       0.0
           B-toloc.city_name      0.970     0.993     0.981     716.0
         B-fromloc.city_name      0.983     0.999     0.991     704.0
           I-toloc.city_name      0.981     0.996     0.989     265.0
      B-depart_date.day_name      0.963     0.986     0.974     212.0
              B-airline_name      1.000     0.990     0.995     101.0
         I-fromloc.city_name      0.989     0.994     0.992     177.0
 B-depart_time.period_of_day      0.983     0.877     0.927     130.0
              I-airline_name      1.000     1.000     1.000      65.0
    B-depart_date.day_number      0.982     0.982     0.982      55.0
    B-depart_date.month_name      0.982     0.982     0.982      56.0
          B-depart_time.time      0.864     1.000     0.927      57.0
                B-round_trip      1.000     0.986     0.993      73.0
             B-cost_relative      1.000     0.973     0.986      37.0
                I-round_trip      1.000     1.000     1.000      71.0
                B-flight_mod      0.821     0.958     0.885      24.0
 B-depart_time.time_relative      0.969     0.969     0.969      65.0
          I-depart_time.time      0.981     1.000     0.990      52.0
         B-stoploc.city_name      1.000     1.000     1.000      20.0
                 B-city_name      0.857     0.526     0.652      57.0
                B-class_type      0.960     1.000     0.980      24.0
          B-arrive_time.time      0.917     0.971     0.943      34.0
 B-arrive_time.time_relative      0.906     0.935     0.921      31.0
                I-class_type      1.000     1.000     1.000      17.0
               B-flight_stop      1.000     1.000     1.000      21.0
          I-arrive_time.time      0.971     0.971     0.971      35.0
              B-airline_code      1.000     0.912     0.954      34.0
    I-depart_date.day_number      1.000     1.000     1.000      15.0
      I-fromloc.airport_name      0.417     1.000     0.588      15.0
      B-fromloc.airport_name      0.320     0.667     0.432      12.0
      B-arrive_date.day_name      0.786     1.000     0.880      11.0
          B-toloc.state_code      1.000     1.000     1.000      18.0
B-depart_date.today_relative      1.000     1.000     1.000       9.0
             B-flight_number      0.688     1.000     0.815      11.0
 B-depart_date.date_relative      0.895     1.000     0.944      17.0
          B-toloc.state_name      0.862     0.893     0.877      28.0
           B-fare_basis_code      0.944     1.000     0.971      17.0
               B-flight_time      1.000     1.000     1.000       1.0
                        B-or      1.000     1.000     1.000       3.0
 B-arrive_time.period_of_day      0.600     1.000     0.750       6.0
          B-meal_description      1.000     0.900     0.947      10.0
             I-cost_relative      1.000     0.667     0.800       3.0
              I-airport_name      0.909     0.345     0.500      29.0
               B-fare_amount      1.000     1.000     1.000       2.0
               I-fare_amount      1.000     1.000     1.000       2.0
                 I-city_name      1.000     0.667     0.800      30.0
        I-toloc.airport_name      1.000     1.000     1.000       3.0
            B-transport_type      1.000     0.800     0.889      10.0
    B-arrive_date.month_name      0.833     0.833     0.833       6.0
    B-arrive_date.day_number      0.833     0.833     0.833       6.0
         I-stoploc.city_name      0.909     1.000     0.952      10.0
                      B-meal      0.941     1.000     0.970      16.0
        B-fromloc.state_code      1.000     1.000     1.000      23.0
    B-depart_time.period_mod      1.000     1.000     1.000       5.0
                   B-connect      1.000     1.000     1.000       6.0
               B-flight_days      1.000     1.000     1.000      10.0
        B-toloc.airport_name      1.000     1.000     1.000       3.0
        B-fromloc.state_name      1.000     0.882     0.938      17.0
              B-airport_name      0.778     0.333     0.467      21.0
                   B-economy      1.000     1.000     1.000       6.0
               I-flight_time      1.000     1.000     1.000       1.0
             B-aircraft_code      0.909     0.909     0.909      33.0
                       B-mod      1.000     0.500     0.667       2.0
              B-airport_code      0.667     0.444     0.533       9.0
    B-depart_time.start_time      0.333     0.667     0.444       3.0
      B-depart_time.end_time      1.000     1.000     1.000       3.0
          B-depart_date.year      1.000     1.000     1.000       3.0
            I-transport_type      0.000     0.000     0.000       1.0
          B-restriction_code      1.000     1.000     1.000       4.0
    B-arrive_time.start_time      0.889     1.000     0.941       8.0
        B-toloc.airport_code      1.000     0.750     0.857       4.0
      B-arrive_time.end_time      1.000     0.875     0.933       8.0
      I-arrive_time.end_time      0.875     0.875     0.875       8.0
      I-depart_time.end_time      1.000     0.667     0.800       3.0
               I-flight_stop      0.000     0.000     0.000       0.0
      B-fromloc.airport_code      0.714     1.000     0.833       5.0
          I-restriction_code      1.000     1.000     1.000       3.0
    I-depart_time.start_time      1.000     1.000     1.000       1.0
          I-toloc.state_name      1.000     1.000     1.000       1.0
I-depart_date.today_relative      0.000     0.000     0.000       0.0
 B-arrive_date.date_relative      1.000     1.000     1.000       2.0
                I-flight_mod      1.000     0.167     0.286       6.0
                   I-economy      0.000     0.000     0.000       0.0
 B-return_date.date_relative      1.000     0.333     0.500       3.0
        I-fromloc.state_name      1.000     1.000     1.000       1.0
                B-state_code      1.000     1.000     1.000       1.0
    I-arrive_time.start_time      1.000     1.000     1.000       1.0
    I-arrive_date.day_number      0.000     0.000     0.000       0.0
                 B-meal_code      0.000     0.000     0.000       1.0
 I-depart_time.period_of_day      0.000     0.000     0.000       1.0
                  B-day_name      1.000     0.500     0.667       2.0
             B-period_of_day      1.000     0.500     0.667       4.0
        B-stoploc.state_code      0.000     0.000     0.000       0.0
    B-return_date.month_name      0.000     0.000     0.000       0.0
    B-return_date.day_number      0.000     0.000     0.000       0.0
    B-arrive_time.period_mod      0.000     0.000     0.000       0.0
                 I-meal_code      0.000     0.000     0.000       0.0
        B-toloc.country_name      1.000     1.000     1.000       1.0
                 B-days_code      1.000     1.000     1.000       1.0
 I-arrive_time.period_of_day      0.000     0.000     0.000       0.0
            I-today_relative      0.000     0.000     0.000       0.0
 B-return_time.period_of_day      0.000     0.000     0.000       0.0
                      B-time      0.000     0.000     0.000       0.0
           I-fare_basis_code      0.000     0.000     0.000       0.0
 I-arrive_time.time_relative      1.000     0.500     0.667       4.0
 I-depart_time.time_relative      0.000     0.000     0.000       1.0
            B-today_relative      0.000     0.000     0.000       0.0
                B-state_name      0.800     0.444     0.571       9.0
B-arrive_date.today_relative      0.000     0.000     0.000       0.0
    B-return_time.period_mod      0.000     0.000     0.000       0.0
                B-month_name      0.000     0.000     0.000       0.0
                B-day_number      0.000     0.000     0.000       0.0
 I-return_date.date_relative      0.667     0.667     0.667       3.0
I-return_date.today_relative      0.000     0.000     0.000       0.0
      B-stoploc.airport_name      0.000     0.000     0.000       0.0
             B-time_relative      0.000     0.000     0.000       0.0
                      I-time      0.000     0.000     0.000       0.0
    I-return_date.day_number      0.000     0.000     0.000       0.0
          I-meal_description      0.000     0.000     0.000       0.0
B-return_date.today_relative      0.000     0.000     0.000       0.0
      B-return_date.day_name      0.000     0.000     0.000       2.0

                   micro avg      0.953     0.955     0.954    3657.0
                   macro avg      0.683     0.650     0.653    3657.0
                weighted avg      0.960     0.955     0.953    3657.0

I0715 07:16:53.381277 140369481070464 interactiveshell.py:2882] Best Slot F1: 0.956
Reading ../data/atis.train.w-intent.iob
I0715 07:16:57.487102 140369481070464 interactiveshell.py:2882] Step 11900 | Loss: 0.0005 | Loss_intent: 0.0000 | Loss_slots: 0.0005 | Spent: 6.0 secs | LR: 0.000086
I0715 07:17:01.267490 140369481070464 interactiveshell.py:2882] Step 11950 | Loss: 0.0017 | Loss_intent: 0.0000 | Loss_slots: 0.0017 | Spent: 3.8 secs | LR: 0.000085
I0715 07:17:04.996175 140369481070464 interactiveshell.py:2882] Step 12000 | Loss: 0.0005 | Loss_intent: 0.0000 | Loss_slots: 0.0005 | Spent: 3.7 secs | LR: 0.000085
I0715 07:17:08.706781 140369481070464 interactiveshell.py:2882] Step 12050 | Loss: 0.0007 | Loss_intent: 0.0000 | Loss_slots: 0.0006 | Spent: 3.7 secs | LR: 0.000084
I0715 07:17:12.409727 140369481070464 interactiveshell.py:2882] Step 12100 | Loss: 0.0014 | Loss_intent: 0.0000 | Loss_slots: 0.0014 | Spent: 3.7 secs | LR: 0.000084
I0715 07:17:16.142866 140369481070464 interactiveshell.py:2882] Step 12150 | Loss: 0.0010 | Loss_intent: 0.0001 | Loss_slots: 0.0008 | Spent: 3.7 secs | LR: 0.000083
Reading ../data/atis.test.w-intent.iob
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.
  'precision', 'predicted', average, warn_for)
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1439: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples.
  'recall', 'true', average, warn_for)
I0715 07:17:18.841804 140369481070464 interactiveshell.py:2882] 
                                          precision    recall  f1-score   support

                             atis_flight      0.980     0.991     0.985       632
                            atis_airfare      0.941     1.000     0.970        48
                     atis_ground_service      1.000     1.000     1.000        36
                            atis_airline      1.000     1.000     1.000        38
                       atis_abbreviation      0.943     1.000     0.971        33
                           atis_aircraft      1.000     0.889     0.941         9
                        atis_flight_time      1.000     1.000     1.000         1
                           atis_quantity      0.429     1.000     0.600         3
                atis_flight#atis_airfare      0.750     0.250     0.375        12
                            atis_airport      1.000     1.000     1.000        18
                           atis_distance      1.000     1.000     1.000        10
                               atis_city      1.000     0.667     0.800         6
                        atis_ground_fare      1.000     0.857     0.923         7
                           atis_capacity      0.955     1.000     0.977        21
                          atis_flight_no      0.889     1.000     0.941         8
                               atis_meal      1.000     0.833     0.909         6
                        atis_restriction      0.000     0.000     0.000         0
             atis_airline#atis_flight_no      0.000     0.000     0.000         0
    atis_ground_service#atis_ground_fare      0.000     0.000     0.000         0
           atis_airfare#atis_flight_time      0.000     0.000     0.000         0
                           atis_cheapest      0.000     0.000     0.000         0
atis_aircraft#atis_flight#atis_flight_no      0.000     0.000     0.000         0

                               micro avg      0.972     0.977     0.975       888
                               macro avg      0.677     0.658     0.654       888
                            weighted avg      0.973     0.977     0.973       888

I0715 07:17:18.882311 140369481070464 interactiveshell.py:2882] 
                              precision    recall  f1-score   support

                           O      0.000     0.000     0.000       0.0
           B-toloc.city_name      0.979     0.992     0.985     716.0
         B-fromloc.city_name      0.986     0.999     0.992     704.0
           I-toloc.city_name      0.978     0.996     0.987     265.0
      B-depart_date.day_name      0.963     0.986     0.974     212.0
              B-airline_name      1.000     0.990     0.995     101.0
         I-fromloc.city_name      0.994     0.994     0.994     177.0
 B-depart_time.period_of_day      0.991     0.869     0.926     130.0
              I-airline_name      1.000     1.000     1.000      65.0
    B-depart_date.day_number      0.981     0.964     0.972      55.0
    B-depart_date.month_name      0.982     0.964     0.973      56.0
          B-depart_time.time      0.836     0.982     0.903      57.0
                B-round_trip      1.000     0.986     0.993      73.0
             B-cost_relative      1.000     0.973     0.986      37.0
                I-round_trip      1.000     1.000     1.000      71.0
                B-flight_mod      0.828     1.000     0.906      24.0
 B-depart_time.time_relative      0.984     0.954     0.969      65.0
          I-depart_time.time      0.981     1.000     0.990      52.0
         B-stoploc.city_name      1.000     1.000     1.000      20.0
                 B-city_name      0.875     0.614     0.722      57.0
                B-class_type      0.960     1.000     0.980      24.0
          B-arrive_time.time      0.917     0.971     0.943      34.0
 B-arrive_time.time_relative      0.882     0.968     0.923      31.0
                I-class_type      1.000     1.000     1.000      17.0
               B-flight_stop      1.000     1.000     1.000      21.0
          I-arrive_time.time      0.971     0.971     0.971      35.0
              B-airline_code      1.000     0.912     0.954      34.0
    I-depart_date.day_number      1.000     0.933     0.966      15.0
      I-fromloc.airport_name      0.405     1.000     0.577      15.0
      B-fromloc.airport_name      0.346     0.750     0.474      12.0
      B-arrive_date.day_name      0.733     1.000     0.846      11.0
          B-toloc.state_code      1.000     1.000     1.000      18.0
B-depart_date.today_relative      1.000     1.000     1.000       9.0
             B-flight_number      0.688     1.000     0.815      11.0
 B-depart_date.date_relative      0.944     1.000     0.971      17.0
          B-toloc.state_name      0.933     1.000     0.966      28.0
           B-fare_basis_code      0.944     1.000     0.971      17.0
               B-flight_time      1.000     1.000     1.000       1.0
                        B-or      1.000     1.000     1.000       3.0
 B-arrive_time.period_of_day      0.545     1.000     0.706       6.0
          B-meal_description      1.000     0.900     0.947      10.0
             I-cost_relative      1.000     0.667     0.800       3.0
              I-airport_name      0.833     0.345     0.488      29.0
               B-fare_amount      1.000     1.000     1.000       2.0
               I-fare_amount      1.000     1.000     1.000       2.0
                 I-city_name      1.000     0.633     0.776      30.0
        I-toloc.airport_name      1.000     1.000     1.000       3.0
            B-transport_type      1.000     0.900     0.947      10.0
    B-arrive_date.month_name      0.714     0.833     0.769       6.0
    B-arrive_date.day_number      0.714     0.833     0.769       6.0
         I-stoploc.city_name      1.000     1.000     1.000      10.0
                      B-meal      0.941     1.000     0.970      16.0
        B-fromloc.state_code      1.000     1.000     1.000      23.0
    B-depart_time.period_mod      1.000     1.000     1.000       5.0
                   B-connect      1.000     1.000     1.000       6.0
               B-flight_days      1.000     1.000     1.000      10.0
        B-toloc.airport_name      1.000     1.000     1.000       3.0
        B-fromloc.state_name      1.000     1.000     1.000      17.0
              B-airport_name      0.875     0.333     0.483      21.0
                   B-economy      1.000     1.000     1.000       6.0
               I-flight_time      1.000     1.000     1.000       1.0
             B-aircraft_code      0.909     0.909     0.909      33.0
                       B-mod      1.000     0.500     0.667       2.0
              B-airport_code      0.000     0.000     0.000       9.0
    B-depart_time.start_time      0.500     0.667     0.571       3.0
      B-depart_time.end_time      1.000     0.667     0.800       3.0
          B-depart_date.year      1.000     1.000     1.000       3.0
            I-transport_type      0.000     0.000     0.000       1.0
          B-restriction_code      1.000     1.000     1.000       4.0
    B-arrive_time.start_time      0.889     1.000     0.941       8.0
        B-toloc.airport_code      1.000     0.750     0.857       4.0
      B-arrive_time.end_time      0.875     0.875     0.875       8.0
      I-arrive_time.end_time      0.875     0.875     0.875       8.0
      I-depart_time.end_time      1.000     0.667     0.800       3.0
               I-flight_stop      0.000     0.000     0.000       0.0
      B-fromloc.airport_code      0.556     1.000     0.714       5.0
          I-restriction_code      1.000     1.000     1.000       3.0
    I-depart_time.start_time      1.000     1.000     1.000       1.0
          I-toloc.state_name      1.000     1.000     1.000       1.0
I-depart_date.today_relative      0.000     0.000     0.000       0.0
 B-arrive_date.date_relative      1.000     1.000     1.000       2.0
                I-flight_mod      1.000     0.167     0.286       6.0
                   I-economy      0.000     0.000     0.000       0.0
 B-return_date.date_relative      1.000     0.333     0.500       3.0
        I-fromloc.state_name      1.000     1.000     1.000       1.0
                B-state_code      1.000     1.000     1.000       1.0
    I-arrive_time.start_time      1.000     1.000     1.000       1.0
    I-arrive_date.day_number      0.000     0.000     0.000       0.0
                 B-meal_code      0.000     0.000     0.000       1.0
 I-depart_time.period_of_day      0.000     0.000     0.000       1.0
                  B-day_name      1.000     0.500     0.667       2.0
             B-period_of_day      1.000     0.500     0.667       4.0
        B-stoploc.state_code      0.000     0.000     0.000       0.0
    B-return_date.month_name      0.000     0.000     0.000       0.0
    B-return_date.day_number      0.000     0.000     0.000       0.0
    B-arrive_time.period_mod      0.000     0.000     0.000       0.0
                 I-meal_code      0.000     0.000     0.000       0.0
        B-toloc.country_name      1.000     1.000     1.000       1.0
                 B-days_code      0.167     1.000     0.286       1.0
 I-arrive_time.period_of_day      0.000     0.000     0.000       0.0
            I-today_relative      0.000     0.000     0.000       0.0
 B-return_time.period_of_day      0.000     0.000     0.000       0.0
                      B-time      0.000     0.000     0.000       0.0
           I-fare_basis_code      0.000     0.000     0.000       0.0
 I-arrive_time.time_relative      1.000     0.500     0.667       4.0
 I-depart_time.time_relative      0.000     0.000     0.000       1.0
            B-today_relative      0.000     0.000     0.000       0.0
                B-state_name      0.857     0.667     0.750       9.0
B-arrive_date.today_relative      0.000     0.000     0.000       0.0
    B-return_time.period_mod      0.000     0.000     0.000       0.0
                B-month_name      0.000     0.000     0.000       0.0
                B-day_number      0.000     0.000     0.000       0.0
 I-return_date.date_relative      0.667     0.667     0.667       3.0
I-return_date.today_relative      0.000     0.000     0.000       0.0
      B-stoploc.airport_name      0.000     0.000     0.000       0.0
             B-time_relative      0.000     0.000     0.000       0.0
                      I-time      0.000     0.000     0.000       0.0
    I-return_date.day_number      0.000     0.000     0.000       0.0
          I-meal_description      0.000     0.000     0.000       0.0
B-return_date.today_relative      0.000     0.000     0.000       0.0
      B-return_date.day_name      0.000     0.000     0.000       2.0

                   micro avg      0.954     0.956     0.955    3657.0
                   macro avg      0.669     0.648     0.643    3657.0
                weighted avg      0.961     0.956     0.953    3657.0

I0715 07:17:18.883245 140369481070464 interactiveshell.py:2882] Best Slot F1: 0.956
Reading ../data/atis.train.w-intent.iob
I0715 07:17:22.045781 140369481070464 interactiveshell.py:2882] Step 12200 | Loss: 0.0216 | Loss_intent: 0.0157 | Loss_slots: 0.0059 | Spent: 5.9 secs | LR: 0.000083
I0715 07:17:25.774448 140369481070464 interactiveshell.py:2882] Step 12250 | Loss: 0.0004 | Loss_intent: 0.0000 | Loss_slots: 0.0004 | Spent: 3.7 secs | LR: 0.000083
I0715 07:17:29.481478 140369481070464 interactiveshell.py:2882] Step 12300 | Loss: 0.0078 | Loss_intent: 0.0000 | Loss_slots: 0.0078 | Spent: 3.7 secs | LR: 0.000082
I0715 07:17:33.213567 140369481070464 interactiveshell.py:2882] Step 12350 | Loss: 0.0019 | Loss_intent: 0.0000 | Loss_slots: 0.0019 | Spent: 3.7 secs | LR: 0.000082
I0715 07:17:36.950441 140369481070464 interactiveshell.py:2882] Step 12400 | Loss: 0.0005 | Loss_intent: 0.0000 | Loss_slots: 0.0005 | Spent: 3.7 secs | LR: 0.000081
I0715 07:17:40.690021 140369481070464 interactiveshell.py:2882] Step 12450 | Loss: 0.0009 | Loss_intent: 0.0000 | Loss_slots: 0.0009 | Spent: 3.7 secs | LR: 0.000081
Reading ../data/atis.test.w-intent.iob
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.
  'precision', 'predicted', average, warn_for)
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1439: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples.
  'recall', 'true', average, warn_for)
I0715 07:17:44.310772 140369481070464 interactiveshell.py:2882] 
                                          precision    recall  f1-score   support

                             atis_flight      0.980     0.992     0.986       632
                            atis_airfare      0.941     1.000     0.970        48
                     atis_ground_service      1.000     1.000     1.000        36
                            atis_airline      1.000     1.000     1.000        38
                       atis_abbreviation      0.943     1.000     0.971        33
                           atis_aircraft      1.000     0.889     0.941         9
                        atis_flight_time      1.000     1.000     1.000         1
                           atis_quantity      0.429     1.000     0.600         3
                atis_flight#atis_airfare      0.800     0.333     0.471        12
                            atis_airport      1.000     0.944     0.971        18
                           atis_distance      1.000     1.000     1.000        10
                               atis_city      1.000     0.667     0.800         6
                        atis_ground_fare      1.000     0.714     0.833         7
                           atis_capacity      0.955     1.000     0.977        21
                          atis_flight_no      0.889     1.000     0.941         8
                               atis_meal      1.000     0.833     0.909         6
                        atis_restriction      0.000     0.000     0.000         0
             atis_airline#atis_flight_no      0.000     0.000     0.000         0
    atis_ground_service#atis_ground_fare      0.000     0.000     0.000         0
           atis_airfare#atis_flight_time      0.000     0.000     0.000         0
                           atis_cheapest      0.000     0.000     0.000         0
atis_aircraft#atis_flight#atis_flight_no      0.000     0.000     0.000         0

                               micro avg      0.972     0.977     0.975       888
                               macro avg      0.679     0.653     0.653       888
                            weighted avg      0.974     0.977     0.973       888

I0715 07:17:44.351804 140369481070464 interactiveshell.py:2882] 
                              precision    recall  f1-score   support

                           O      0.000     0.000     0.000       0.0
           B-toloc.city_name      0.971     0.993     0.982     716.0
         B-fromloc.city_name      0.986     0.999     0.992     704.0
           I-toloc.city_name      0.978     0.996     0.987     265.0
      B-depart_date.day_name      0.963     0.986     0.974     212.0
              B-airline_name      1.000     1.000     1.000     101.0
         I-fromloc.city_name      0.989     0.994     0.992     177.0
 B-depart_time.period_of_day      1.000     0.877     0.934     130.0
              I-airline_name      1.000     1.000     1.000      65.0
    B-depart_date.day_number      0.981     0.964     0.972      55.0
    B-depart_date.month_name      0.982     0.964     0.973      56.0
          B-depart_time.time      0.851     1.000     0.919      57.0
                B-round_trip      1.000     0.986     0.993      73.0
             B-cost_relative      1.000     0.973     0.986      37.0
                I-round_trip      1.000     1.000     1.000      71.0
                B-flight_mod      0.828     1.000     0.906      24.0
 B-depart_time.time_relative      0.984     0.954     0.969      65.0
          I-depart_time.time      0.981     1.000     0.990      52.0
         B-stoploc.city_name      1.000     1.000     1.000      20.0
                 B-city_name      0.939     0.544     0.689      57.0
                B-class_type      0.960     1.000     0.980      24.0
          B-arrive_time.time      0.943     0.971     0.957      34.0
 B-arrive_time.time_relative      0.938     0.968     0.952      31.0
                I-class_type      1.000     1.000     1.000      17.0
               B-flight_stop      1.000     1.000     1.000      21.0
          I-arrive_time.time      0.971     0.971     0.971      35.0
              B-airline_code      1.000     0.912     0.954      34.0
    I-depart_date.day_number      1.000     0.933     0.966      15.0
      I-fromloc.airport_name      0.405     1.000     0.577      15.0
      B-fromloc.airport_name      0.333     0.750     0.462      12.0
      B-arrive_date.day_name      0.786     1.000     0.880      11.0
          B-toloc.state_code      1.000     1.000     1.000      18.0
B-depart_date.today_relative      1.000     1.000     1.000       9.0
             B-flight_number      0.440     1.000     0.611      11.0
 B-depart_date.date_relative      0.944     1.000     0.971      17.0
          B-toloc.state_name      0.867     0.929     0.897      28.0
           B-fare_basis_code      0.944     1.000     0.971      17.0
               B-flight_time      1.000     1.000     1.000       1.0
                        B-or      1.000     1.000     1.000       3.0
 B-arrive_time.period_of_day      0.545     1.000     0.706       6.0
          B-meal_description      1.000     0.900     0.947      10.0
             I-cost_relative      1.000     0.667     0.800       3.0
              I-airport_name      0.909     0.345     0.500      29.0
               B-fare_amount      1.000     1.000     1.000       2.0
               I-fare_amount      1.000     1.000     1.000       2.0
                 I-city_name      1.000     0.667     0.800      30.0
        I-toloc.airport_name      1.000     1.000     1.000       3.0
            B-transport_type      1.000     0.900     0.947      10.0
    B-arrive_date.month_name      0.714     0.833     0.769       6.0
    B-arrive_date.day_number      0.714     0.833     0.769       6.0
         I-stoploc.city_name      1.000     0.900     0.947      10.0
                      B-meal      0.941     1.000     0.970      16.0
        B-fromloc.state_code      1.000     1.000     1.000      23.0
    B-depart_time.period_mod      1.000     1.000     1.000       5.0
                   B-connect      1.000     1.000     1.000       6.0
               B-flight_days      1.000     1.000     1.000      10.0
        B-toloc.airport_name      1.000     1.000     1.000       3.0
        B-fromloc.state_name      0.895     1.000     0.944      17.0
              B-airport_name      0.778     0.333     0.467      21.0
                   B-economy      1.000     1.000     1.000       6.0
               I-flight_time      1.000     1.000     1.000       1.0
             B-aircraft_code      1.000     0.758     0.862      33.0
                       B-mod      1.000     0.500     0.667       2.0
              B-airport_code      0.000     0.000     0.000       9.0
    B-depart_time.start_time      0.500     0.667     0.571       3.0
      B-depart_time.end_time      1.000     0.667     0.800       3.0
          B-depart_date.year      0.429     1.000     0.600       3.0
            I-transport_type      0.000     0.000     0.000       1.0
          B-restriction_code      1.000     1.000     1.000       4.0
    B-arrive_time.start_time      0.889     1.000     0.941       8.0
        B-toloc.airport_code      1.000     0.750     0.857       4.0
      B-arrive_time.end_time      0.875     0.875     0.875       8.0
      I-arrive_time.end_time      0.875     0.875     0.875       8.0
      I-depart_time.end_time      1.000     0.667     0.800       3.0
               I-flight_stop      0.000     0.000     0.000       0.0
      B-fromloc.airport_code      0.625     1.000     0.769       5.0
          I-restriction_code      1.000     1.000     1.000       3.0
    I-depart_time.start_time      1.000     1.000     1.000       1.0
          I-toloc.state_name      1.000     1.000     1.000       1.0
I-depart_date.today_relative      0.000     0.000     0.000       0.0
 B-arrive_date.date_relative      1.000     1.000     1.000       2.0
                I-flight_mod      1.000     0.167     0.286       6.0
                   I-economy      0.000     0.000     0.000       0.0
 B-return_date.date_relative      0.000     0.000     0.000       3.0
        I-fromloc.state_name      0.000     0.000     0.000       1.0
                B-state_code      1.000     1.000     1.000       1.0
    I-arrive_time.start_time      1.000     1.000     1.000       1.0
    I-arrive_date.day_number      0.000     0.000     0.000       0.0
                 B-meal_code      0.000     0.000     0.000       1.0
 I-depart_time.period_of_day      1.000     1.000     1.000       1.0
                  B-day_name      1.000     0.500     0.667       2.0
             B-period_of_day      1.000     0.500     0.667       4.0
        B-stoploc.state_code      0.000     0.000     0.000       0.0
    B-return_date.month_name      0.000     0.000     0.000       0.0
    B-return_date.day_number      0.000     0.000     0.000       0.0
    B-arrive_time.period_mod      0.000     0.000     0.000       0.0
                 I-meal_code      0.000     0.000     0.000       0.0
        B-toloc.country_name      1.000     1.000     1.000       1.0
                 B-days_code      1.000     1.000     1.000       1.0
 I-arrive_time.period_of_day      0.000     0.000     0.000       0.0
            I-today_relative      0.000     0.000     0.000       0.0
 B-return_time.period_of_day      0.000     0.000     0.000       0.0
                      B-time      0.000     0.000     0.000       0.0
           I-fare_basis_code      0.000     0.000     0.000       0.0
 I-arrive_time.time_relative      1.000     1.000     1.000       4.0
 I-depart_time.time_relative      0.000     0.000     0.000       1.0
            B-today_relative      0.000     0.000     0.000       0.0
                B-state_name      1.000     0.444     0.615       9.0
B-arrive_date.today_relative      0.000     0.000     0.000       0.0
    B-return_time.period_mod      0.000     0.000     0.000       0.0
                B-month_name      0.000     0.000     0.000       0.0
                B-day_number      0.000     0.000     0.000       0.0
 I-return_date.date_relative      0.600     1.000     0.750       3.0
I-return_date.today_relative      0.000     0.000     0.000       0.0
      B-stoploc.airport_name      0.000     0.000     0.000       0.0
             B-time_relative      0.000     0.000     0.000       0.0
                      I-time      0.000     0.000     0.000       0.0
    I-return_date.day_number      0.000     0.000     0.000       0.0
          I-meal_description      0.000     0.000     0.000       0.0
B-return_date.today_relative      0.000     0.000     0.000       0.0
      B-return_date.day_name      0.000     0.000     0.000       2.0

                   micro avg      0.952     0.954     0.953    3657.0
                   macro avg      0.663     0.648     0.641    3657.0
                weighted avg      0.960     0.954     0.951    3657.0

I0715 07:17:44.352613 140369481070464 interactiveshell.py:2882] Best Slot F1: 0.956
Reading ../data/atis.train.w-intent.iob
I0715 07:17:46.609795 140369481070464 interactiveshell.py:2882] Step 12500 | Loss: 0.0049 | Loss_intent: 0.0000 | Loss_slots: 0.0049 | Spent: 5.9 secs | LR: 0.000080
I0715 07:17:50.320212 140369481070464 interactiveshell.py:2882] Step 12550 | Loss: 0.0005 | Loss_intent: 0.0000 | Loss_slots: 0.0005 | Spent: 3.7 secs | LR: 0.000080
I0715 07:17:54.053362 140369481070464 interactiveshell.py:2882] Step 12600 | Loss: 0.0002 | Loss_intent: 0.0000 | Loss_slots: 0.0002 | Spent: 3.7 secs | LR: 0.000080
I0715 07:17:57.776855 140369481070464 interactiveshell.py:2882] Step 12650 | Loss: 0.0001 | Loss_intent: 0.0000 | Loss_slots: 0.0001 | Spent: 3.7 secs | LR: 0.000079
I0715 07:18:01.518701 140369481070464 interactiveshell.py:2882] Step 12700 | Loss: 0.0002 | Loss_intent: 0.0000 | Loss_slots: 0.0002 | Spent: 3.7 secs | LR: 0.000079
I0715 07:18:05.235546 140369481070464 interactiveshell.py:2882] Step 12750 | Loss: 0.0009 | Loss_intent: 0.0002 | Loss_slots: 0.0007 | Spent: 3.7 secs | LR: 0.000078
Reading ../data/atis.test.w-intent.iob
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.
  'precision', 'predicted', average, warn_for)
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1439: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples.
  'recall', 'true', average, warn_for)
I0715 07:18:09.744220 140369481070464 interactiveshell.py:2882] 
                                          precision    recall  f1-score   support

                             atis_flight      0.978     0.992     0.985       632
                            atis_airfare      0.941     1.000     0.970        48
                     atis_ground_service      1.000     1.000     1.000        36
                            atis_airline      1.000     1.000     1.000        38
                       atis_abbreviation      0.943     1.000     0.971        33
                           atis_aircraft      1.000     0.889     0.941         9
                        atis_flight_time      1.000     1.000     1.000         1
                           atis_quantity      0.429     1.000     0.600         3
                atis_flight#atis_airfare      0.800     0.333     0.471        12
                            atis_airport      1.000     0.889     0.941        18
                           atis_distance      1.000     1.000     1.000        10
                               atis_city      1.000     0.667     0.800         6
                        atis_ground_fare      1.000     0.714     0.833         7
                           atis_capacity      0.955     1.000     0.977        21
                          atis_flight_no      0.889     1.000     0.941         8
                               atis_meal      1.000     0.833     0.909         6
                        atis_restriction      0.000     0.000     0.000         0
             atis_airline#atis_flight_no      0.000     0.000     0.000         0
    atis_ground_service#atis_ground_fare      0.000     0.000     0.000         0
           atis_airfare#atis_flight_time      0.000     0.000     0.000         0
                           atis_cheapest      0.000     0.000     0.000         0
atis_aircraft#atis_flight#atis_flight_no      0.000     0.000     0.000         0

                               micro avg      0.971     0.976     0.974       888
                               macro avg      0.679     0.651     0.652       888
                            weighted avg      0.972     0.976     0.972       888

I0715 07:18:09.783757 140369481070464 interactiveshell.py:2882] 
                              precision    recall  f1-score   support

                           O      0.000     0.000     0.000       0.0
           B-toloc.city_name      0.970     0.993     0.981     716.0
         B-fromloc.city_name      0.986     0.997     0.992     704.0
           I-toloc.city_name      0.981     0.992     0.987     265.0
      B-depart_date.day_name      0.959     0.986     0.972     212.0
              B-airline_name      1.000     1.000     1.000     101.0
         I-fromloc.city_name      0.983     0.994     0.989     177.0
 B-depart_time.period_of_day      1.000     0.869     0.930     130.0
              I-airline_name      1.000     1.000     1.000      65.0
    B-depart_date.day_number      0.981     0.964     0.972      55.0
    B-depart_date.month_name      0.982     0.964     0.973      56.0
          B-depart_time.time      0.862     0.982     0.918      57.0
                B-round_trip      1.000     0.986     0.993      73.0
             B-cost_relative      1.000     0.973     0.986      37.0
                I-round_trip      1.000     1.000     1.000      71.0
                B-flight_mod      0.828     1.000     0.906      24.0
 B-depart_time.time_relative      0.984     0.954     0.969      65.0
          I-depart_time.time      0.981     1.000     0.990      52.0
         B-stoploc.city_name      0.909     1.000     0.952      20.0
                 B-city_name      0.909     0.526     0.667      57.0
                B-class_type      0.960     1.000     0.980      24.0
          B-arrive_time.time      0.892     0.971     0.930      34.0
 B-arrive_time.time_relative      0.967     0.935     0.951      31.0
                I-class_type      1.000     1.000     1.000      17.0
               B-flight_stop      1.000     1.000     1.000      21.0
          I-arrive_time.time      1.000     0.971     0.986      35.0
              B-airline_code      1.000     0.912     0.954      34.0
    I-depart_date.day_number      1.000     0.933     0.966      15.0
      I-fromloc.airport_name      0.395     1.000     0.566      15.0
      B-fromloc.airport_name      0.393     0.917     0.550      12.0
      B-arrive_date.day_name      0.786     1.000     0.880      11.0
          B-toloc.state_code      1.000     1.000     1.000      18.0
B-depart_date.today_relative      1.000     1.000     1.000       9.0
             B-flight_number      0.500     1.000     0.667      11.0
 B-depart_date.date_relative      0.944     1.000     0.971      17.0
          B-toloc.state_name      0.862     0.893     0.877      28.0
           B-fare_basis_code      0.917     0.647     0.759      17.0
               B-flight_time      1.000     1.000     1.000       1.0
                        B-or      0.750     1.000     0.857       3.0
 B-arrive_time.period_of_day      0.600     1.000     0.750       6.0
          B-meal_description      1.000     0.900     0.947      10.0
             I-cost_relative      1.000     0.667     0.800       3.0
              I-airport_name      0.909     0.345     0.500      29.0
               B-fare_amount      1.000     1.000     1.000       2.0
               I-fare_amount      1.000     1.000     1.000       2.0
                 I-city_name      1.000     0.667     0.800      30.0
        I-toloc.airport_name      1.000     1.000     1.000       3.0
            B-transport_type      1.000     0.900     0.947      10.0
    B-arrive_date.month_name      0.714     0.833     0.769       6.0
    B-arrive_date.day_number      0.714     0.833     0.769       6.0
         I-stoploc.city_name      0.909     1.000     0.952      10.0
                      B-meal      1.000     1.000     1.000      16.0
        B-fromloc.state_code      1.000     1.000     1.000      23.0
    B-depart_time.period_mod      1.000     1.000     1.000       5.0
                   B-connect      1.000     1.000     1.000       6.0
               B-flight_days      1.000     1.000     1.000      10.0
        B-toloc.airport_name      1.000     1.000     1.000       3.0
        B-fromloc.state_name      1.000     0.941     0.970      17.0
              B-airport_name      0.778     0.333     0.467      21.0
                   B-economy      1.000     1.000     1.000       6.0
               I-flight_time      1.000     1.000     1.000       1.0
             B-aircraft_code      1.000     0.758     0.862      33.0
                       B-mod      1.000     0.500     0.667       2.0
              B-airport_code      0.667     0.444     0.533       9.0
    B-depart_time.start_time      0.400     0.667     0.500       3.0
      B-depart_time.end_time      1.000     0.667     0.800       3.0
          B-depart_date.year      0.500     1.000     0.667       3.0
            I-transport_type      0.000     0.000     0.000       1.0
          B-restriction_code      1.000     1.000     1.000       4.0
    B-arrive_time.start_time      0.889     1.000     0.941       8.0
        B-toloc.airport_code      1.000     0.750     0.857       4.0
      B-arrive_time.end_time      0.875     0.875     0.875       8.0
      I-arrive_time.end_time      0.889     1.000     0.941       8.0
      I-depart_time.end_time      1.000     0.667     0.800       3.0
               I-flight_stop      0.000     0.000     0.000       0.0
      B-fromloc.airport_code      1.000     1.000     1.000       5.0
          I-restriction_code      1.000     1.000     1.000       3.0
    I-depart_time.start_time      1.000     1.000     1.000       1.0
          I-toloc.state_name      1.000     1.000     1.000       1.0
I-depart_date.today_relative      0.000     0.000     0.000       0.0
 B-arrive_date.date_relative      1.000     1.000     1.000       2.0
                I-flight_mod      1.000     0.167     0.286       6.0
                   I-economy      0.000     0.000     0.000       0.0
 B-return_date.date_relative      1.000     0.333     0.500       3.0
        I-fromloc.state_name      1.000     1.000     1.000       1.0
                B-state_code      1.000     1.000     1.000       1.0
    I-arrive_time.start_time      1.000     1.000     1.000       1.0
    I-arrive_date.day_number      0.000     0.000     0.000       0.0
                 B-meal_code      0.000     0.000     0.000       1.0
 I-depart_time.period_of_day      1.000     1.000     1.000       1.0
                  B-day_name      1.000     0.500     0.667       2.0
             B-period_of_day      1.000     0.750     0.857       4.0
        B-stoploc.state_code      0.000     0.000     0.000       0.0
    B-return_date.month_name      0.000     0.000     0.000       0.0
    B-return_date.day_number      0.000     0.000     0.000       0.0
    B-arrive_time.period_mod      0.000     0.000     0.000       0.0
                 I-meal_code      0.000     0.000     0.000       0.0
        B-toloc.country_name      1.000     1.000     1.000       1.0
                 B-days_code      1.000     1.000     1.000       1.0
 I-arrive_time.period_of_day      0.000     0.000     0.000       0.0
            I-today_relative      0.000     0.000     0.000       0.0
 B-return_time.period_of_day      0.000     0.000     0.000       0.0
                      B-time      0.000     0.000     0.000       0.0
           I-fare_basis_code      0.000     0.000     0.000       0.0
 I-arrive_time.time_relative      1.000     1.000     1.000       4.0
 I-depart_time.time_relative      0.000     0.000     0.000       1.0
            B-today_relative      0.000     0.000     0.000       0.0
                B-state_name      0.800     0.444     0.571       9.0
B-arrive_date.today_relative      0.000     0.000     0.000       0.0
    B-return_time.period_mod      0.000     0.000     0.000       0.0
                B-month_name      0.000     0.000     0.000       0.0
                B-day_number      0.000     0.000     0.000       0.0
 I-return_date.date_relative      0.667     0.667     0.667       3.0
I-return_date.today_relative      0.000     0.000     0.000       0.0
      B-stoploc.airport_name      0.000     0.000     0.000       0.0
             B-time_relative      0.000     0.000     0.000       0.0
                      I-time      0.000     0.000     0.000       0.0
    I-return_date.day_number      0.000     0.000     0.000       0.0
          I-meal_description      0.000     0.000     0.000       0.0
B-return_date.today_relative      0.000     0.000     0.000       0.0
      B-return_date.day_name      0.000     0.000     0.000       2.0

                   micro avg      0.951     0.953     0.952    3657.0
                   macro avg      0.686     0.661     0.659    3657.0
                weighted avg      0.962     0.953     0.952    3657.0

I0715 07:18:09.784782 140369481070464 interactiveshell.py:2882] Best Slot F1: 0.956
Reading ../data/atis.train.w-intent.iob
I0715 07:18:11.161225 140369481070464 interactiveshell.py:2882] Step 12800 | Loss: 0.0005 | Loss_intent: 0.0000 | Loss_slots: 0.0005 | Spent: 5.9 secs | LR: 0.000078
I0715 07:18:14.950560 140369481070464 interactiveshell.py:2882] Step 12850 | Loss: 0.0013 | Loss_intent: 0.0000 | Loss_slots: 0.0013 | Spent: 3.8 secs | LR: 0.000077
I0715 07:18:18.759115 140369481070464 interactiveshell.py:2882] Step 12900 | Loss: 0.0004 | Loss_intent: 0.0000 | Loss_slots: 0.0004 | Spent: 3.8 secs | LR: 0.000077
I0715 07:18:22.524082 140369481070464 interactiveshell.py:2882] Step 12950 | Loss: 0.0002 | Loss_intent: 0.0000 | Loss_slots: 0.0002 | Spent: 3.8 secs | LR: 0.000077
I0715 07:18:26.257027 140369481070464 interactiveshell.py:2882] Step 13000 | Loss: 0.0017 | Loss_intent: 0.0000 | Loss_slots: 0.0017 | Spent: 3.7 secs | LR: 0.000076
I0715 07:18:29.964232 140369481070464 interactiveshell.py:2882] Step 13050 | Loss: 0.0013 | Loss_intent: 0.0000 | Loss_slots: 0.0013 | Spent: 3.7 secs | LR: 0.000076
I0715 07:18:33.713572 140369481070464 interactiveshell.py:2882] Step 13100 | Loss: 0.0008 | Loss_intent: 0.0000 | Loss_slots: 0.0008 | Spent: 3.7 secs | LR: 0.000075
Reading ../data/atis.test.w-intent.iob
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.
  'precision', 'predicted', average, warn_for)
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1439: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples.
  'recall', 'true', average, warn_for)
I0715 07:18:35.387763 140369481070464 interactiveshell.py:2882] 
                                          precision    recall  f1-score   support

                             atis_flight      0.978     0.992     0.985       632
                            atis_airfare      0.941     1.000     0.970        48
                     atis_ground_service      1.000     1.000     1.000        36
                            atis_airline      1.000     1.000     1.000        38
                       atis_abbreviation      0.943     1.000     0.971        33
                           atis_aircraft      1.000     0.889     0.941         9
                        atis_flight_time      1.000     1.000     1.000         1
                           atis_quantity      0.429     1.000     0.600         3
                atis_flight#atis_airfare      0.800     0.333     0.471        12
                            atis_airport      1.000     0.889     0.941        18
                           atis_distance      1.000     1.000     1.000        10
                               atis_city      1.000     0.667     0.800         6
                        atis_ground_fare      1.000     0.714     0.833         7
                           atis_capacity      0.955     1.000     0.977        21
                          atis_flight_no      0.889     1.000     0.941         8
                               atis_meal      1.000     0.833     0.909         6
                        atis_restriction      0.000     0.000     0.000         0
             atis_airline#atis_flight_no      0.000     0.000     0.000         0
    atis_ground_service#atis_ground_fare      0.000     0.000     0.000         0
           atis_airfare#atis_flight_time      0.000     0.000     0.000         0
                           atis_cheapest      0.000     0.000     0.000         0
atis_aircraft#atis_flight#atis_flight_no      0.000     0.000     0.000         0

                               micro avg      0.971     0.976     0.974       888
                               macro avg      0.679     0.651     0.652       888
                            weighted avg      0.972     0.976     0.972       888

I0715 07:18:35.426975 140369481070464 interactiveshell.py:2882] 
                              precision    recall  f1-score   support

                           O      0.000     0.000     0.000       0.0
           B-toloc.city_name      0.973     0.994     0.983     716.0
         B-fromloc.city_name      0.987     0.997     0.992     704.0
           I-toloc.city_name      0.978     0.992     0.985     265.0
      B-depart_date.day_name      0.963     0.986     0.974     212.0
              B-airline_name      1.000     0.990     0.995     101.0
         I-fromloc.city_name      0.972     0.994     0.983     177.0
 B-depart_time.period_of_day      1.000     0.869     0.930     130.0
              I-airline_name      1.000     1.000     1.000      65.0
    B-depart_date.day_number      0.981     0.964     0.972      55.0
    B-depart_date.month_name      0.982     0.964     0.973      56.0
          B-depart_time.time      0.862     0.982     0.918      57.0
                B-round_trip      1.000     0.986     0.993      73.0
             B-cost_relative      1.000     0.973     0.986      37.0
                I-round_trip      1.000     1.000     1.000      71.0
                B-flight_mod      0.828     1.000     0.906      24.0
 B-depart_time.time_relative      0.984     0.954     0.969      65.0
          I-depart_time.time      0.981     1.000     0.990      52.0
         B-stoploc.city_name      0.952     1.000     0.976      20.0
                 B-city_name      0.892     0.579     0.702      57.0
                B-class_type      0.960     1.000     0.980      24.0
          B-arrive_time.time      0.892     0.971     0.930      34.0
 B-arrive_time.time_relative      0.938     0.968     0.952      31.0
                I-class_type      1.000     1.000     1.000      17.0
               B-flight_stop      1.000     1.000     1.000      21.0
          I-arrive_time.time      0.971     0.971     0.971      35.0
              B-airline_code      1.000     0.912     0.954      34.0
    I-depart_date.day_number      1.000     0.933     0.966      15.0
      I-fromloc.airport_name      0.405     1.000     0.577      15.0
      B-fromloc.airport_name      0.400     1.000     0.571      12.0
      B-arrive_date.day_name      0.733     1.000     0.846      11.0
          B-toloc.state_code      1.000     1.000     1.000      18.0
B-depart_date.today_relative      1.000     1.000     1.000       9.0
             B-flight_number      0.611     1.000     0.759      11.0
 B-depart_date.date_relative      0.944     1.000     0.971      17.0
          B-toloc.state_name      0.962     0.893     0.926      28.0
           B-fare_basis_code      0.944     1.000     0.971      17.0
               B-flight_time      1.000     1.000     1.000       1.0
                        B-or      1.000     1.000     1.000       3.0
 B-arrive_time.period_of_day      0.545     1.000     0.706       6.0
          B-meal_description      1.000     0.900     0.947      10.0
             I-cost_relative      1.000     0.667     0.800       3.0
              I-airport_name      0.833     0.345     0.488      29.0
               B-fare_amount      1.000     1.000     1.000       2.0
               I-fare_amount      1.000     1.000     1.000       2.0
                 I-city_name      1.000     0.600     0.750      30.0
        I-toloc.airport_name      1.000     1.000     1.000       3.0
            B-transport_type      1.000     1.000     1.000      10.0
    B-arrive_date.month_name      0.714     0.833     0.769       6.0
    B-arrive_date.day_number      0.714     0.833     0.769       6.0
         I-stoploc.city_name      0.900     0.900     0.900      10.0
                      B-meal      1.000     1.000     1.000      16.0
        B-fromloc.state_code      1.000     1.000     1.000      23.0
    B-depart_time.period_mod      1.000     1.000     1.000       5.0
                   B-connect      1.000     1.000     1.000       6.0
               B-flight_days      1.000     1.000     1.000      10.0
        B-toloc.airport_name      1.000     1.000     1.000       3.0
        B-fromloc.state_name      1.000     1.000     1.000      17.0
              B-airport_name      0.875     0.333     0.483      21.0
                   B-economy      1.000     1.000     1.000       6.0
               I-flight_time      1.000     1.000     1.000       1.0
             B-aircraft_code      0.967     0.879     0.921      33.0
                       B-mod      1.000     0.500     0.667       2.0
              B-airport_code      0.667     0.444     0.533       9.0
    B-depart_time.start_time      0.500     0.667     0.571       3.0
      B-depart_time.end_time      1.000     0.667     0.800       3.0
          B-depart_date.year      0.750     1.000     0.857       3.0
            I-transport_type      0.000     0.000     0.000       1.0
          B-restriction_code      1.000     1.000     1.000       4.0
    B-arrive_time.start_time      0.889     1.000     0.941       8.0
        B-toloc.airport_code      1.000     0.750     0.857       4.0
      B-arrive_time.end_time      0.875     0.875     0.875       8.0
      I-arrive_time.end_time      0.875     0.875     0.875       8.0
      I-depart_time.end_time      1.000     0.667     0.800       3.0
               I-flight_stop      0.000     0.000     0.000       0.0
      B-fromloc.airport_code      0.833     1.000     0.909       5.0
          I-restriction_code      1.000     1.000     1.000       3.0
    I-depart_time.start_time      1.000     1.000     1.000       1.0
          I-toloc.state_name      1.000     1.000     1.000       1.0
I-depart_date.today_relative      0.000     0.000     0.000       0.0
 B-arrive_date.date_relative      1.000     1.000     1.000       2.0
                I-flight_mod      1.000     0.167     0.286       6.0
                   I-economy      0.000     0.000     0.000       0.0
 B-return_date.date_relative      1.000     0.333     0.500       3.0
        I-fromloc.state_name      1.000     1.000     1.000       1.0
                B-state_code      1.000     1.000     1.000       1.0
    I-arrive_time.start_time      1.000     1.000     1.000       1.0
    I-arrive_date.day_number      0.000     0.000     0.000       0.0
                 B-meal_code      0.000     0.000     0.000       1.0
 I-depart_time.period_of_day      1.000     1.000     1.000       1.0
                  B-day_name      1.000     0.500     0.667       2.0
             B-period_of_day      1.000     0.500     0.667       4.0
        B-stoploc.state_code      0.000     0.000     0.000       0.0
    B-return_date.month_name      0.000     0.000     0.000       0.0
    B-return_date.day_number      0.000     0.000     0.000       0.0
    B-arrive_time.period_mod      0.000     0.000     0.000       0.0
                 I-meal_code      0.000     0.000     0.000       0.0
        B-toloc.country_name      1.000     1.000     1.000       1.0
                 B-days_code      1.000     1.000     1.000       1.0
 I-arrive_time.period_of_day      0.000     0.000     0.000       0.0
            I-today_relative      0.000     0.000     0.000       0.0
 B-return_time.period_of_day      0.000     0.000     0.000       0.0
                      B-time      0.000     0.000     0.000       0.0
           I-fare_basis_code      0.000     0.000     0.000       0.0
 I-arrive_time.time_relative      1.000     1.000     1.000       4.0
 I-depart_time.time_relative      0.000     0.000     0.000       1.0
            B-today_relative      0.000     0.000     0.000       0.0
                B-state_name      0.857     0.667     0.750       9.0
B-arrive_date.today_relative      0.000     0.000     0.000       0.0
    B-return_time.period_mod      0.000     0.000     0.000       0.0
                B-month_name      0.000     0.000     0.000       0.0
                B-day_number      0.000     0.000     0.000       0.0
 I-return_date.date_relative      0.667     0.667     0.667       3.0
I-return_date.today_relative      0.000     0.000     0.000       0.0
      B-stoploc.airport_name      0.000     0.000     0.000       0.0
             B-time_relative      0.000     0.000     0.000       0.0
                      I-time      0.000     0.000     0.000       0.0
    I-return_date.day_number      0.000     0.000     0.000       0.0
          I-meal_description      0.000     0.000     0.000       0.0
B-return_date.today_relative      0.000     0.000     0.000       0.0
      B-return_date.day_name      0.000     0.000     0.000       2.0

                   micro avg      0.955     0.957     0.956    3657.0
                   macro avg      0.691     0.665     0.664    3657.0
                weighted avg      0.963     0.957     0.954    3657.0

I0715 07:18:35.427799 140369481070464 interactiveshell.py:2882] Best Slot F1: 0.956
Reading ../data/atis.train.w-intent.iob
I0715 07:18:39.607806 140369481070464 interactiveshell.py:2882] Step 13150 | Loss: 0.0000 | Loss_intent: 0.0000 | Loss_slots: 0.0000 | Spent: 5.9 secs | LR: 0.000075
I0715 07:18:43.319146 140369481070464 interactiveshell.py:2882] Step 13200 | Loss: 0.0006 | Loss_intent: 0.0000 | Loss_slots: 0.0006 | Spent: 3.7 secs | LR: 0.000075
I0715 07:18:47.071787 140369481070464 interactiveshell.py:2882] Step 13250 | Loss: 0.0011 | Loss_intent: 0.0000 | Loss_slots: 0.0011 | Spent: 3.8 secs | LR: 0.000074
I0715 07:18:50.817675 140369481070464 interactiveshell.py:2882] Step 13300 | Loss: 0.0006 | Loss_intent: 0.0000 | Loss_slots: 0.0006 | Spent: 3.7 secs | LR: 0.000074
I0715 07:18:54.542686 140369481070464 interactiveshell.py:2882] Step 13350 | Loss: 0.0006 | Loss_intent: 0.0000 | Loss_slots: 0.0006 | Spent: 3.7 secs | LR: 0.000073
I0715 07:18:58.261465 140369481070464 interactiveshell.py:2882] Step 13400 | Loss: 0.0002 | Loss_intent: 0.0000 | Loss_slots: 0.0002 | Spent: 3.7 secs | LR: 0.000073
Reading ../data/atis.test.w-intent.iob
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.
  'precision', 'predicted', average, warn_for)
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1439: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples.
  'recall', 'true', average, warn_for)
I0715 07:19:00.830423 140369481070464 interactiveshell.py:2882] 
                                          precision    recall  f1-score   support

                             atis_flight      0.981     0.992     0.987       632
                            atis_airfare      0.941     1.000     0.970        48
                     atis_ground_service      1.000     1.000     1.000        36
                            atis_airline      1.000     1.000     1.000        38
                       atis_abbreviation      0.943     1.000     0.971        33
                           atis_aircraft      1.000     0.889     0.941         9
                        atis_flight_time      1.000     1.000     1.000         1
                           atis_quantity      0.429     1.000     0.600         3
                atis_flight#atis_airfare      0.833     0.417     0.556        12
                            atis_airport      1.000     0.944     0.971        18
                           atis_distance      1.000     1.000     1.000        10
                               atis_city      1.000     0.667     0.800         6
                        atis_ground_fare      1.000     0.714     0.833         7
                           atis_capacity      0.955     1.000     0.977        21
                          atis_flight_no      0.889     1.000     0.941         8
                               atis_meal      1.000     0.833     0.909         6
                        atis_restriction      0.000     0.000     0.000         0
             atis_airline#atis_flight_no      0.000     0.000     0.000         0
    atis_ground_service#atis_ground_fare      0.000     0.000     0.000         0
           atis_airfare#atis_flight_time      0.000     0.000     0.000         0
                           atis_cheapest      0.000     0.000     0.000         0
atis_aircraft#atis_flight#atis_flight_no      0.000     0.000     0.000         0

                               micro avg      0.973     0.979     0.976       888
                               macro avg      0.680     0.657     0.657       888
                            weighted avg      0.975     0.979     0.975       888

I0715 07:19:00.872952 140369481070464 interactiveshell.py:2882] 
                              precision    recall  f1-score   support

                           O      0.000     0.000     0.000       0.0
           B-toloc.city_name      0.969     0.997     0.983     716.0
         B-fromloc.city_name      0.990     0.997     0.994     704.0
           I-toloc.city_name      0.974     1.000     0.987     265.0
      B-depart_date.day_name      0.959     0.986     0.972     212.0
              B-airline_name      1.000     0.990     0.995     101.0
         I-fromloc.city_name      0.994     0.994     0.994     177.0
 B-depart_time.period_of_day      1.000     0.869     0.930     130.0
              I-airline_name      1.000     1.000     1.000      65.0
    B-depart_date.day_number      0.981     0.964     0.972      55.0
    B-depart_date.month_name      0.982     0.964     0.973      56.0
          B-depart_time.time      0.862     0.982     0.918      57.0
                B-round_trip      1.000     0.986     0.993      73.0
             B-cost_relative      1.000     0.973     0.986      37.0
                I-round_trip      1.000     1.000     1.000      71.0
                B-flight_mod      0.828     1.000     0.906      24.0
 B-depart_time.time_relative      0.984     0.954     0.969      65.0
          I-depart_time.time      0.981     1.000     0.990      52.0
         B-stoploc.city_name      1.000     1.000     1.000      20.0
                 B-city_name      0.861     0.544     0.667      57.0
                B-class_type      0.960     1.000     0.980      24.0
          B-arrive_time.time      0.892     0.971     0.930      34.0
 B-arrive_time.time_relative      0.938     0.968     0.952      31.0
                I-class_type      1.000     1.000     1.000      17.0
               B-flight_stop      1.000     1.000     1.000      21.0
          I-arrive_time.time      0.971     0.971     0.971      35.0
              B-airline_code      1.000     0.912     0.954      34.0
    I-depart_date.day_number      1.000     0.933     0.966      15.0
      I-fromloc.airport_name      0.405     1.000     0.577      15.0
      B-fromloc.airport_name      0.379     0.917     0.537      12.0
      B-arrive_date.day_name      0.786     1.000     0.880      11.0
          B-toloc.state_code      1.000     1.000     1.000      18.0
B-depart_date.today_relative      1.000     1.000     1.000       9.0
             B-flight_number      0.550     1.000     0.710      11.0
 B-depart_date.date_relative      0.944     1.000     0.971      17.0
          B-toloc.state_name      0.806     0.893     0.847      28.0
           B-fare_basis_code      0.917     0.647     0.759      17.0
               B-flight_time      1.000     1.000     1.000       1.0
                        B-or      1.000     1.000     1.000       3.0
 B-arrive_time.period_of_day      0.600     1.000     0.750       6.0
          B-meal_description      1.000     0.900     0.947      10.0
             I-cost_relative      1.000     0.667     0.800       3.0
              I-airport_name      0.833     0.345     0.488      29.0
               B-fare_amount      1.000     1.000     1.000       2.0
               I-fare_amount      1.000     1.000     1.000       2.0
                 I-city_name      1.000     0.600     0.750      30.0
        I-toloc.airport_name      1.000     1.000     1.000       3.0
            B-transport_type      1.000     1.000     1.000      10.0
    B-arrive_date.month_name      0.714     0.833     0.769       6.0
    B-arrive_date.day_number      0.714     0.833     0.769       6.0
         I-stoploc.city_name      0.909     1.000     0.952      10.0
                      B-meal      1.000     1.000     1.000      16.0
        B-fromloc.state_code      1.000     1.000     1.000      23.0
    B-depart_time.period_mod      1.000     1.000     1.000       5.0
                   B-connect      1.000     1.000     1.000       6.0
               B-flight_days      1.000     1.000     1.000      10.0
        B-toloc.airport_name      1.000     1.000     1.000       3.0
        B-fromloc.state_name      1.000     0.941     0.970      17.0
              B-airport_name      0.857     0.286     0.429      21.0
                   B-economy      1.000     1.000     1.000       6.0
               I-flight_time      1.000     1.000     1.000       1.0
             B-aircraft_code      1.000     0.788     0.881      33.0
                       B-mod      1.000     0.500     0.667       2.0
              B-airport_code      0.000     0.000     0.000       9.0
    B-depart_time.start_time      0.400     0.667     0.500       3.0
      B-depart_time.end_time      1.000     0.667     0.800       3.0
          B-depart_date.year      0.500     1.000     0.667       3.0
            I-transport_type      0.000     0.000     0.000       1.0
          B-restriction_code      1.000     1.000     1.000       4.0
    B-arrive_time.start_time      0.889     1.000     0.941       8.0
        B-toloc.airport_code      1.000     0.750     0.857       4.0
      B-arrive_time.end_time      0.875     0.875     0.875       8.0
      I-arrive_time.end_time      0.875     0.875     0.875       8.0
      I-depart_time.end_time      1.000     0.667     0.800       3.0
               I-flight_stop      0.000     0.000     0.000       0.0
      B-fromloc.airport_code      0.833     1.000     0.909       5.0
          I-restriction_code      0.429     1.000     0.600       3.0
    I-depart_time.start_time      1.000     1.000     1.000       1.0
          I-toloc.state_name      1.000     1.000     1.000       1.0
I-depart_date.today_relative      0.000     0.000     0.000       0.0
 B-arrive_date.date_relative      1.000     1.000     1.000       2.0
                I-flight_mod      1.000     0.167     0.286       6.0
                   I-economy      0.000     0.000     0.000       0.0
 B-return_date.date_relative      0.000     0.000     0.000       3.0
        I-fromloc.state_name      1.000     1.000     1.000       1.0
                B-state_code      1.000     1.000     1.000       1.0
    I-arrive_time.start_time      1.000     1.000     1.000       1.0
    I-arrive_date.day_number      0.000     0.000     0.000       0.0
                 B-meal_code      0.000     0.000     0.000       1.0
 I-depart_time.period_of_day      1.000     1.000     1.000       1.0
                  B-day_name      1.000     0.500     0.667       2.0
             B-period_of_day      1.000     0.500     0.667       4.0
        B-stoploc.state_code      0.000     0.000     0.000       0.0
    B-return_date.month_name      0.000     0.000     0.000       0.0
    B-return_date.day_number      0.000     0.000     0.000       0.0
    B-arrive_time.period_mod      0.000     0.000     0.000       0.0
                 I-meal_code      0.000     0.000     0.000       0.0
        B-toloc.country_name      1.000     1.000     1.000       1.0
                 B-days_code      1.000     1.000     1.000       1.0
 I-arrive_time.period_of_day      0.000     0.000     0.000       0.0
            I-today_relative      0.000     0.000     0.000       0.0
 B-return_time.period_of_day      0.000     0.000     0.000       0.0
                      B-time      0.000     0.000     0.000       0.0
           I-fare_basis_code      0.000     0.000     0.000       0.0
 I-arrive_time.time_relative      1.000     1.000     1.000       4.0
 I-depart_time.time_relative      0.000     0.000     0.000       1.0
            B-today_relative      0.000     0.000     0.000       0.0
                B-state_name      0.667     0.222     0.333       9.0
B-arrive_date.today_relative      0.000     0.000     0.000       0.0
    B-return_time.period_mod      0.000     0.000     0.000       0.0
                B-month_name      0.000     0.000     0.000       0.0
                B-day_number      0.000     0.000     0.000       0.0
 I-return_date.date_relative      0.667     0.667     0.667       3.0
I-return_date.today_relative      0.000     0.000     0.000       0.0
      B-stoploc.airport_name      0.000     0.000     0.000       0.0
             B-time_relative      0.000     0.000     0.000       0.0
                      I-time      0.000     0.000     0.000       0.0
    I-return_date.day_number      0.000     0.000     0.000       0.0
          I-meal_description      0.000     0.000     0.000       0.0
B-return_date.today_relative      0.000     0.000     0.000       0.0
      B-return_date.day_name      0.000     0.000     0.000       2.0

                   micro avg      0.950     0.952     0.951    3657.0
                   macro avg      0.667     0.650     0.644    3657.0
                weighted avg      0.958     0.952     0.949    3657.0

I0715 07:19:00.873931 140369481070464 interactiveshell.py:2882] Best Slot F1: 0.956
Reading ../data/atis.train.w-intent.iob
I0715 07:19:04.165023 140369481070464 interactiveshell.py:2882] Step 13450 | Loss: 0.0006 | Loss_intent: 0.0000 | Loss_slots: 0.0006 | Spent: 5.9 secs | LR: 0.000073
I0715 07:19:07.894057 140369481070464 interactiveshell.py:2882] Step 13500 | Loss: 0.0010 | Loss_intent: 0.0000 | Loss_slots: 0.0010 | Spent: 3.7 secs | LR: 0.000072
I0715 07:19:11.646944 140369481070464 interactiveshell.py:2882] Step 13550 | Loss: 0.0008 | Loss_intent: 0.0000 | Loss_slots: 0.0008 | Spent: 3.8 secs | LR: 0.000072
I0715 07:19:15.366970 140369481070464 interactiveshell.py:2882] Step 13600 | Loss: 0.0001 | Loss_intent: 0.0000 | Loss_slots: 0.0001 | Spent: 3.7 secs | LR: 0.000072
I0715 07:19:19.472249 140369481070464 interactiveshell.py:2882] Step 13650 | Loss: 0.0010 | Loss_intent: 0.0000 | Loss_slots: 0.0010 | Spent: 4.1 secs | LR: 0.000071
I0715 07:19:23.611179 140369481070464 interactiveshell.py:2882] Step 13700 | Loss: 0.0017 | Loss_intent: 0.0000 | Loss_slots: 0.0017 | Spent: 4.1 secs | LR: 0.000071
Reading ../data/atis.test.w-intent.iob
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.
  'precision', 'predicted', average, warn_for)
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1439: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples.
  'recall', 'true', average, warn_for)
I0715 07:19:27.298950 140369481070464 interactiveshell.py:2882] 
                                          precision    recall  f1-score   support

                             atis_flight      0.981     0.991     0.986       632
                            atis_airfare      0.941     1.000     0.970        48
                     atis_ground_service      1.000     1.000     1.000        36
                            atis_airline      1.000     1.000     1.000        38
                       atis_abbreviation      0.943     1.000     0.971        33
                           atis_aircraft      1.000     0.889     0.941         9
                        atis_flight_time      1.000     1.000     1.000         1
                           atis_quantity      0.429     1.000     0.600         3
                atis_flight#atis_airfare      0.833     0.417     0.556        12
                            atis_airport      0.944     0.944     0.944        18
                           atis_distance      1.000     1.000     1.000        10
                               atis_city      1.000     0.667     0.800         6
                        atis_ground_fare      1.000     0.714     0.833         7
                           atis_capacity      0.955     1.000     0.977        21
                          atis_flight_no      0.889     1.000     0.941         8
                               atis_meal      1.000     0.833     0.909         6
                        atis_restriction      0.000     0.000     0.000         0
             atis_airline#atis_flight_no      0.000     0.000     0.000         0
    atis_ground_service#atis_ground_fare      0.000     0.000     0.000         0
           atis_airfare#atis_flight_time      0.000     0.000     0.000         0
                           atis_cheapest      0.000     0.000     0.000         0
atis_aircraft#atis_flight#atis_flight_no      0.000     0.000     0.000         0

                               micro avg      0.972     0.977     0.975       888
                               macro avg      0.678     0.657     0.656       888
                            weighted avg      0.974     0.977     0.974       888

I0715 07:19:27.339954 140369481070464 interactiveshell.py:2882] 
                              precision    recall  f1-score   support

                           O      0.000     0.000     0.000       0.0
           B-toloc.city_name      0.974     0.994     0.984     716.0
         B-fromloc.city_name      0.986     0.999     0.992     704.0
           I-toloc.city_name      0.978     1.000     0.989     265.0
      B-depart_date.day_name      0.959     0.986     0.972     212.0
              B-airline_name      1.000     1.000     1.000     101.0
         I-fromloc.city_name      0.994     0.994     0.994     177.0
 B-depart_time.period_of_day      1.000     0.869     0.930     130.0
              I-airline_name      1.000     1.000     1.000      65.0
    B-depart_date.day_number      0.981     0.964     0.972      55.0
    B-depart_date.month_name      0.982     0.964     0.973      56.0
          B-depart_time.time      0.848     0.982     0.911      57.0
                B-round_trip      1.000     0.986     0.993      73.0
             B-cost_relative      1.000     0.973     0.986      37.0
                I-round_trip      1.000     1.000     1.000      71.0
                B-flight_mod      0.828     1.000     0.906      24.0
 B-depart_time.time_relative      0.984     0.969     0.977      65.0
          I-depart_time.time      0.981     0.981     0.981      52.0
         B-stoploc.city_name      1.000     1.000     1.000      20.0
                 B-city_name      0.912     0.544     0.681      57.0
                B-class_type      0.960     1.000     0.980      24.0
          B-arrive_time.time      0.917     0.971     0.943      34.0
 B-arrive_time.time_relative      0.968     0.968     0.968      31.0
                I-class_type      1.000     1.000     1.000      17.0
               B-flight_stop      1.000     1.000     1.000      21.0
          I-arrive_time.time      0.944     0.971     0.958      35.0
              B-airline_code      1.000     0.912     0.954      34.0
    I-depart_date.day_number      1.000     1.000     1.000      15.0
      I-fromloc.airport_name      0.405     1.000     0.577      15.0
      B-fromloc.airport_name      0.379     0.917     0.537      12.0
      B-arrive_date.day_name      0.786     1.000     0.880      11.0
          B-toloc.state_code      1.000     1.000     1.000      18.0
B-depart_date.today_relative      1.000     1.000     1.000       9.0
             B-flight_number      0.440     1.000     0.611      11.0
 B-depart_date.date_relative      0.944     1.000     0.971      17.0
          B-toloc.state_name      0.800     1.000     0.889      28.0
           B-fare_basis_code      0.944     1.000     0.971      17.0
               B-flight_time      1.000     1.000     1.000       1.0
                        B-or      1.000     1.000     1.000       3.0
 B-arrive_time.period_of_day      0.600     1.000     0.750       6.0
          B-meal_description      1.000     0.900     0.947      10.0
             I-cost_relative      1.000     0.667     0.800       3.0
              I-airport_name      0.909     0.345     0.500      29.0
               B-fare_amount      0.500     1.000     0.667       2.0
               I-fare_amount      1.000     1.000     1.000       2.0
                 I-city_name      1.000     0.600     0.750      30.0
        I-toloc.airport_name      1.000     1.000     1.000       3.0
            B-transport_type      1.000     0.800     0.889      10.0
    B-arrive_date.month_name      0.714     0.833     0.769       6.0
    B-arrive_date.day_number      0.714     0.833     0.769       6.0
         I-stoploc.city_name      1.000     1.000     1.000      10.0
                      B-meal      1.000     1.000     1.000      16.0
        B-fromloc.state_code      1.000     1.000     1.000      23.0
    B-depart_time.period_mod      1.000     1.000     1.000       5.0
                   B-connect      1.000     1.000     1.000       6.0
               B-flight_days      1.000     1.000     1.000      10.0
        B-toloc.airport_name      1.000     1.000     1.000       3.0
        B-fromloc.state_name      1.000     0.941     0.970      17.0
              B-airport_name      0.778     0.333     0.467      21.0
                   B-economy      1.000     1.000     1.000       6.0
               I-flight_time      1.000     1.000     1.000       1.0
             B-aircraft_code      1.000     0.758     0.862      33.0
                       B-mod      1.000     0.500     0.667       2.0
              B-airport_code      0.000     0.000     0.000       9.0
    B-depart_time.start_time      0.400     0.667     0.500       3.0
      B-depart_time.end_time      1.000     0.667     0.800       3.0
          B-depart_date.year      0.600     1.000     0.750       3.0
            I-transport_type      0.000     0.000     0.000       1.0
          B-restriction_code      1.000     1.000     1.000       4.0
    B-arrive_time.start_time      0.889     1.000     0.941       8.0
        B-toloc.airport_code      1.000     0.750     0.857       4.0
      B-arrive_time.end_time      0.875     0.875     0.875       8.0
      I-arrive_time.end_time      0.875     0.875     0.875       8.0
      I-depart_time.end_time      1.000     0.667     0.800       3.0
               I-flight_stop      0.000     0.000     0.000       0.0
      B-fromloc.airport_code      0.800     0.800     0.800       5.0
          I-restriction_code      1.000     1.000     1.000       3.0
    I-depart_time.start_time      1.000     1.000     1.000       1.0
          I-toloc.state_name      1.000     1.000     1.000       1.0
I-depart_date.today_relative      0.000     0.000     0.000       0.0
 B-arrive_date.date_relative      1.000     1.000     1.000       2.0
                I-flight_mod      1.000     0.167     0.286       6.0
                   I-economy      0.000     0.000     0.000       0.0
 B-return_date.date_relative      0.000     0.000     0.000       3.0
        I-fromloc.state_name      1.000     1.000     1.000       1.0
                B-state_code      1.000     1.000     1.000       1.0
    I-arrive_time.start_time      1.000     1.000     1.000       1.0
    I-arrive_date.day_number      0.000     0.000     0.000       0.0
                 B-meal_code      0.000     0.000     0.000       1.0
 I-depart_time.period_of_day      1.000     1.000     1.000       1.0
                  B-day_name      1.000     0.500     0.667       2.0
             B-period_of_day      1.000     0.500     0.667       4.0
        B-stoploc.state_code      0.000     0.000     0.000       0.0
    B-return_date.month_name      0.000     0.000     0.000       0.0
    B-return_date.day_number      0.000     0.000     0.000       0.0
    B-arrive_time.period_mod      0.000     0.000     0.000       0.0
                 I-meal_code      0.000     0.000     0.000       0.0
        B-toloc.country_name      1.000     1.000     1.000       1.0
                 B-days_code      1.000     1.000     1.000       1.0
 I-arrive_time.period_of_day      0.000     0.000     0.000       0.0
            I-today_relative      0.000     0.000     0.000       0.0
 B-return_time.period_of_day      0.000     0.000     0.000       0.0
                      B-time      0.000     0.000     0.000       0.0
           I-fare_basis_code      0.000     0.000     0.000       0.0
 I-arrive_time.time_relative      1.000     1.000     1.000       4.0
 I-depart_time.time_relative      0.000     0.000     0.000       1.0
            B-today_relative      0.000     0.000     0.000       0.0
                B-state_name      0.500     0.111     0.182       9.0
B-arrive_date.today_relative      0.000     0.000     0.000       0.0
    B-return_time.period_mod      0.000     0.000     0.000       0.0
                B-month_name      0.000     0.000     0.000       0.0
                B-day_number      0.000     0.000     0.000       0.0
 I-return_date.date_relative      0.667     0.667     0.667       3.0
I-return_date.today_relative      0.000     0.000     0.000       0.0
      B-stoploc.airport_name      0.000     0.000     0.000       0.0
             B-time_relative      0.000     0.000     0.000       0.0
                      I-time      0.000     0.000     0.000       0.0
    I-return_date.day_number      0.000     0.000     0.000       0.0
          I-meal_description      0.000     0.000     0.000       0.0
B-return_date.today_relative      0.000     0.000     0.000       0.0
      B-return_date.day_name      0.000     0.000     0.000       2.0

                   micro avg      0.952     0.953     0.952    3657.0
                   macro avg      0.667     0.650     0.644    3657.0
                weighted avg      0.959     0.953     0.950    3657.0

I0715 07:19:27.340771 140369481070464 interactiveshell.py:2882] Best Slot F1: 0.956
Reading ../data/atis.train.w-intent.iob
I0715 07:19:29.758219 140369481070464 interactiveshell.py:2882] Step 13750 | Loss: 0.0096 | Loss_intent: 0.0001 | Loss_slots: 0.0095 | Spent: 6.1 secs | LR: 0.000070
I0715 07:19:33.538915 140369481070464 interactiveshell.py:2882] Step 13800 | Loss: 0.0008 | Loss_intent: 0.0000 | Loss_slots: 0.0008 | Spent: 3.8 secs | LR: 0.000070
I0715 07:19:37.328183 140369481070464 interactiveshell.py:2882] Step 13850 | Loss: 0.0026 | Loss_intent: 0.0000 | Loss_slots: 0.0026 | Spent: 3.8 secs | LR: 0.000070
I0715 07:19:41.136522 140369481070464 interactiveshell.py:2882] Step 13900 | Loss: 0.0007 | Loss_intent: 0.0000 | Loss_slots: 0.0007 | Spent: 3.8 secs | LR: 0.000069
I0715 07:19:44.901601 140369481070464 interactiveshell.py:2882] Step 13950 | Loss: 0.0002 | Loss_intent: 0.0000 | Loss_slots: 0.0002 | Spent: 3.8 secs | LR: 0.000069
I0715 07:19:48.642276 140369481070464 interactiveshell.py:2882] Step 14000 | Loss: 0.0008 | Loss_intent: 0.0000 | Loss_slots: 0.0008 | Spent: 3.7 secs | LR: 0.000069
Reading ../data/atis.test.w-intent.iob
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.
  'precision', 'predicted', average, warn_for)
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1439: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples.
  'recall', 'true', average, warn_for)
I0715 07:19:53.000216 140369481070464 interactiveshell.py:2882] 
                                          precision    recall  f1-score   support

                             atis_flight      0.980     0.992     0.986       632
                            atis_airfare      0.960     1.000     0.980        48
                     atis_ground_service      1.000     1.000     1.000        36
                            atis_airline      1.000     1.000     1.000        38
                       atis_abbreviation      0.943     1.000     0.971        33
                           atis_aircraft      1.000     0.889     0.941         9
                        atis_flight_time      1.000     1.000     1.000         1
                           atis_quantity      0.429     1.000     0.600         3
                atis_flight#atis_airfare      0.833     0.417     0.556        12
                            atis_airport      1.000     0.889     0.941        18
                           atis_distance      1.000     1.000     1.000        10
                               atis_city      1.000     0.667     0.800         6
                        atis_ground_fare      1.000     0.714     0.833         7
                           atis_capacity      0.955     1.000     0.977        21
                          atis_flight_no      0.889     1.000     0.941         8
                               atis_meal      1.000     0.833     0.909         6
                        atis_restriction      0.000     0.000     0.000         0
             atis_airline#atis_flight_no      0.000     0.000     0.000         0
    atis_ground_service#atis_ground_fare      0.000     0.000     0.000         0
           atis_airfare#atis_flight_time      0.000     0.000     0.000         0
                           atis_cheapest      0.000     0.000     0.000         0
atis_aircraft#atis_flight#atis_flight_no      0.000     0.000     0.000         0

                               micro avg      0.972     0.977     0.975       888
                               macro avg      0.681     0.655     0.656       888
                            weighted avg      0.975     0.977     0.974       888

I0715 07:19:53.041692 140369481070464 interactiveshell.py:2882] 
                              precision    recall  f1-score   support

                           O      0.000     0.000     0.000       0.0
           B-toloc.city_name      0.970     0.996     0.983     716.0
         B-fromloc.city_name      0.980     0.999     0.989     704.0
           I-toloc.city_name      0.971     1.000     0.985     265.0
      B-depart_date.day_name      0.959     0.986     0.972     212.0
              B-airline_name      0.981     1.000     0.990     101.0
         I-fromloc.city_name      0.994     0.994     0.994     177.0
 B-depart_time.period_of_day      1.000     0.877     0.934     130.0
              I-airline_name      1.000     1.000     1.000      65.0
    B-depart_date.day_number      0.982     0.982     0.982      55.0
    B-depart_date.month_name      0.982     0.964     0.973      56.0
          B-depart_time.time      0.838     1.000     0.912      57.0
                B-round_trip      1.000     0.986     0.993      73.0
             B-cost_relative      1.000     0.973     0.986      37.0
                I-round_trip      1.000     1.000     1.000      71.0
                B-flight_mod      0.828     1.000     0.906      24.0
 B-depart_time.time_relative      0.968     0.938     0.953      65.0
          I-depart_time.time      0.981     1.000     0.990      52.0
         B-stoploc.city_name      0.952     1.000     0.976      20.0
                 B-city_name      0.931     0.474     0.628      57.0
                B-class_type      0.960     1.000     0.980      24.0
          B-arrive_time.time      0.943     0.971     0.957      34.0
 B-arrive_time.time_relative      0.967     0.935     0.951      31.0
                I-class_type      1.000     1.000     1.000      17.0
               B-flight_stop      1.000     1.000     1.000      21.0
          I-arrive_time.time      0.971     0.971     0.971      35.0
              B-airline_code      1.000     0.912     0.954      34.0
    I-depart_date.day_number      1.000     1.000     1.000      15.0
      I-fromloc.airport_name      0.395     1.000     0.566      15.0
      B-fromloc.airport_name      0.360     0.750     0.486      12.0
      B-arrive_date.day_name      0.786     1.000     0.880      11.0
          B-toloc.state_code      1.000     1.000     1.000      18.0
B-depart_date.today_relative      1.000     1.000     1.000       9.0
             B-flight_number      0.524     1.000     0.688      11.0
 B-depart_date.date_relative      0.944     1.000     0.971      17.0
          B-toloc.state_name      0.862     0.893     0.877      28.0
           B-fare_basis_code      0.917     0.647     0.759      17.0
               B-flight_time      1.000     1.000     1.000       1.0
                        B-or      0.750     1.000     0.857       3.0
 B-arrive_time.period_of_day      0.667     1.000     0.800       6.0
          B-meal_description      1.000     0.900     0.947      10.0
             I-cost_relative      1.000     0.667     0.800       3.0
              I-airport_name      0.833     0.345     0.488      29.0
               B-fare_amount      1.000     1.000     1.000       2.0
               I-fare_amount      1.000     1.000     1.000       2.0
                 I-city_name      1.000     0.533     0.696      30.0
        I-toloc.airport_name      1.000     1.000     1.000       3.0
            B-transport_type      0.833     1.000     0.909      10.0
    B-arrive_date.month_name      0.714     0.833     0.769       6.0
    B-arrive_date.day_number      0.833     0.833     0.833       6.0
         I-stoploc.city_name      0.909     1.000     0.952      10.0
                      B-meal      1.000     1.000     1.000      16.0
        B-fromloc.state_code      1.000     1.000     1.000      23.0
    B-depart_time.period_mod      1.000     1.000     1.000       5.0
                   B-connect      1.000     1.000     1.000       6.0
               B-flight_days      1.000     1.000     1.000      10.0
        B-toloc.airport_name      1.000     1.000     1.000       3.0
        B-fromloc.state_name      1.000     0.941     0.970      17.0
              B-airport_name      0.875     0.333     0.483      21.0
                   B-economy      1.000     1.000     1.000       6.0
               I-flight_time      1.000     1.000     1.000       1.0
             B-aircraft_code      0.958     0.697     0.807      33.0
                       B-mod      1.000     0.500     0.667       2.0
              B-airport_code      0.667     0.444     0.533       9.0
    B-depart_time.start_time      0.400     0.667     0.500       3.0
      B-depart_time.end_time      1.000     0.667     0.800       3.0
          B-depart_date.year      0.600     1.000     0.750       3.0
            I-transport_type      1.000     1.000     1.000       1.0
          B-restriction_code      1.000     1.000     1.000       4.0
    B-arrive_time.start_time      0.889     1.000     0.941       8.0
        B-toloc.airport_code      1.000     0.750     0.857       4.0
      B-arrive_time.end_time      0.875     0.875     0.875       8.0
      I-arrive_time.end_time      0.875     0.875     0.875       8.0
      I-depart_time.end_time      1.000     0.667     0.800       3.0
               I-flight_stop      0.000     0.000     0.000       0.0
      B-fromloc.airport_code      0.833     1.000     0.909       5.0
          I-restriction_code      1.000     1.000     1.000       3.0
    I-depart_time.start_time      1.000     1.000     1.000       1.0
          I-toloc.state_name      1.000     1.000     1.000       1.0
I-depart_date.today_relative      0.000     0.000     0.000       0.0
 B-arrive_date.date_relative      1.000     1.000     1.000       2.0
                I-flight_mod      1.000     0.167     0.286       6.0
                   I-economy      0.000     0.000     0.000       0.0
 B-return_date.date_relative      0.000     0.000     0.000       3.0
        I-fromloc.state_name      1.000     1.000     1.000       1.0
                B-state_code      1.000     1.000     1.000       1.0
    I-arrive_time.start_time      1.000     1.000     1.000       1.0
    I-arrive_date.day_number      0.000     0.000     0.000       0.0
                 B-meal_code      0.000     0.000     0.000       1.0
 I-depart_time.period_of_day      1.000     1.000     1.000       1.0
                  B-day_name      1.000     0.500     0.667       2.0
             B-period_of_day      1.000     0.750     0.857       4.0
        B-stoploc.state_code      0.000     0.000     0.000       0.0
    B-return_date.month_name      0.000     0.000     0.000       0.0
    B-return_date.day_number      0.000     0.000     0.000       0.0
    B-arrive_time.period_mod      0.000     0.000     0.000       0.0
                 I-meal_code      0.000     0.000     0.000       0.0
        B-toloc.country_name      1.000     1.000     1.000       1.0
                 B-days_code      1.000     1.000     1.000       1.0
 I-arrive_time.period_of_day      0.000     0.000     0.000       0.0
            I-today_relative      0.000     0.000     0.000       0.0
 B-return_time.period_of_day      0.000     0.000     0.000       0.0
                      B-time      0.000     0.000     0.000       0.0
           I-fare_basis_code      0.000     0.000     0.000       0.0
 I-arrive_time.time_relative      1.000     1.000     1.000       4.0
 I-depart_time.time_relative      0.000     0.000     0.000       1.0
            B-today_relative      0.000     0.000     0.000       0.0
                B-state_name      0.750     0.333     0.462       9.0
B-arrive_date.today_relative      0.000     0.000     0.000       0.0
    B-return_time.period_mod      0.000     0.000     0.000       0.0
                B-month_name      0.000     0.000     0.000       0.0
                B-day_number      0.000     0.000     0.000       0.0
 I-return_date.date_relative      0.667     0.667     0.667       3.0
I-return_date.today_relative      0.000     0.000     0.000       0.0
      B-stoploc.airport_name      0.000     0.000     0.000       0.0
             B-time_relative      0.000     0.000     0.000       0.0
                      I-time      0.000     0.000     0.000       0.0
    I-return_date.day_number      0.000     0.000     0.000       0.0
          I-meal_description      0.000     0.000     0.000       0.0
B-return_date.today_relative      0.000     0.000     0.000       0.0
      B-return_date.day_name      0.000     0.000     0.000       2.0

                   micro avg      0.950     0.952     0.951    3657.0
                   macro avg      0.685     0.663     0.661    3657.0
                weighted avg      0.959     0.952     0.949    3657.0

I0715 07:19:53.042534 140369481070464 interactiveshell.py:2882] Best Slot F1: 0.956
Reading ../data/atis.train.w-intent.iob
I0715 07:19:54.592176 140369481070464 interactiveshell.py:2882] Step 14050 | Loss: 0.0005 | Loss_intent: 0.0002 | Loss_slots: 0.0003 | Spent: 5.9 secs | LR: 0.000068
I0715 07:19:58.373170 140369481070464 interactiveshell.py:2882] Step 14100 | Loss: 0.0075 | Loss_intent: 0.0000 | Loss_slots: 0.0075 | Spent: 3.8 secs | LR: 0.000068
I0715 07:20:02.113836 140369481070464 interactiveshell.py:2882] Step 14150 | Loss: 0.0022 | Loss_intent: 0.0000 | Loss_slots: 0.0022 | Spent: 3.7 secs | LR: 0.000068
I0715 07:20:05.872124 140369481070464 interactiveshell.py:2882] Step 14200 | Loss: 0.0021 | Loss_intent: 0.0000 | Loss_slots: 0.0021 | Spent: 3.8 secs | LR: 0.000067
I0715 07:20:09.596041 140369481070464 interactiveshell.py:2882] Step 14250 | Loss: 0.0003 | Loss_intent: 0.0000 | Loss_slots: 0.0003 | Spent: 3.7 secs | LR: 0.000067
I0715 07:20:13.808855 140369481070464 interactiveshell.py:2882] Step 14300 | Loss: 0.0014 | Loss_intent: 0.0000 | Loss_slots: 0.0014 | Spent: 4.2 secs | LR: 0.000066
I0715 07:20:19.905692 140369481070464 interactiveshell.py:2882] Step 14350 | Loss: 0.0002 | Loss_intent: 0.0000 | Loss_slots: 0.0002 | Spent: 6.1 secs | LR: 0.000066
Reading ../data/atis.test.w-intent.iob
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.
  'precision', 'predicted', average, warn_for)
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1439: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples.
  'recall', 'true', average, warn_for)
I0715 07:20:21.794082 140369481070464 interactiveshell.py:2882] 
                                          precision    recall  f1-score   support

                             atis_flight      0.975     0.989     0.982       632
                            atis_airfare      0.960     1.000     0.980        48
                     atis_ground_service      1.000     1.000     1.000        36
                            atis_airline      1.000     1.000     1.000        38
                       atis_abbreviation      1.000     1.000     1.000        33
                           atis_aircraft      1.000     0.889     0.941         9
                        atis_flight_time      1.000     1.000     1.000         1
                           atis_quantity      0.429     1.000     0.600         3
                atis_flight#atis_airfare      0.800     0.333     0.471        12
                            atis_airport      1.000     0.944     0.971        18
                           atis_distance      1.000     1.000     1.000        10
                               atis_city      0.667     0.667     0.667         6
                        atis_ground_fare      1.000     0.714     0.833         7
                           atis_capacity      0.955     1.000     0.977        21
                          atis_flight_no      1.000     1.000     1.000         8
                               atis_meal      1.000     0.833     0.909         6
                        atis_restriction      0.000     0.000     0.000         0
             atis_airline#atis_flight_no      0.000     0.000     0.000         0
    atis_ground_service#atis_ground_fare      0.000     0.000     0.000         0
           atis_airfare#atis_flight_time      0.000     0.000     0.000         0
                           atis_cheapest      0.000     0.000     0.000         0
atis_aircraft#atis_flight#atis_flight_no      0.000     0.000     0.000         0

                               micro avg      0.970     0.975     0.972       888
                               macro avg      0.672     0.653     0.651       888
                            weighted avg      0.972     0.975     0.972       888

I0715 07:20:21.832974 140369481070464 interactiveshell.py:2882] 
                              precision    recall  f1-score   support

                           O      0.000     0.000     0.000       0.0
           B-toloc.city_name      0.971     0.994     0.983     716.0
         B-fromloc.city_name      0.985     0.999     0.992     704.0
           I-toloc.city_name      0.977     0.981     0.979     265.0
      B-depart_date.day_name      0.959     0.986     0.972     212.0
              B-airline_name      0.981     1.000     0.990     101.0
         I-fromloc.city_name      0.989     0.983     0.986     177.0
 B-depart_time.period_of_day      1.000     0.877     0.934     130.0
              I-airline_name      1.000     1.000     1.000      65.0
    B-depart_date.day_number      0.981     0.964     0.972      55.0
    B-depart_date.month_name      0.982     0.964     0.973      56.0
          B-depart_time.time      0.838     1.000     0.912      57.0
                B-round_trip      1.000     0.986     0.993      73.0
             B-cost_relative      1.000     0.973     0.986      37.0
                I-round_trip      1.000     0.986     0.993      71.0
                B-flight_mod      0.828     1.000     0.906      24.0
 B-depart_time.time_relative      0.984     0.954     0.969      65.0
          I-depart_time.time      0.981     1.000     0.990      52.0
         B-stoploc.city_name      1.000     1.000     1.000      20.0
                 B-city_name      0.933     0.491     0.644      57.0
                B-class_type      0.960     1.000     0.980      24.0
          B-arrive_time.time      0.943     0.971     0.957      34.0
 B-arrive_time.time_relative      0.968     0.968     0.968      31.0
                I-class_type      1.000     1.000     1.000      17.0
               B-flight_stop      1.000     1.000     1.000      21.0
          I-arrive_time.time      0.971     0.971     0.971      35.0
              B-airline_code      0.939     0.912     0.925      34.0
    I-depart_date.day_number      1.000     0.933     0.966      15.0
      I-fromloc.airport_name      0.417     1.000     0.588      15.0
      B-fromloc.airport_name      0.387     1.000     0.558      12.0
      B-arrive_date.day_name      0.786     1.000     0.880      11.0
          B-toloc.state_code      1.000     1.000     1.000      18.0
B-depart_date.today_relative      1.000     0.889     0.941       9.0
             B-flight_number      0.440     1.000     0.611      11.0
 B-depart_date.date_relative      0.944     1.000     0.971      17.0
          B-toloc.state_name      0.893     0.893     0.893      28.0
           B-fare_basis_code      0.941     0.941     0.941      17.0
               B-flight_time      1.000     1.000     1.000       1.0
                        B-or      1.000     1.000     1.000       3.0
 B-arrive_time.period_of_day      0.667     1.000     0.800       6.0
          B-meal_description      1.000     0.900     0.947      10.0
             I-cost_relative      1.000     0.667     0.800       3.0
              I-airport_name      0.917     0.379     0.537      29.0
               B-fare_amount      1.000     1.000     1.000       2.0
               I-fare_amount      1.000     1.000     1.000       2.0
                 I-city_name      0.750     0.600     0.667      30.0
        I-toloc.airport_name      1.000     1.000     1.000       3.0
            B-transport_type      1.000     1.000     1.000      10.0
    B-arrive_date.month_name      0.714     0.833     0.769       6.0
    B-arrive_date.day_number      0.714     0.833     0.769       6.0
         I-stoploc.city_name      0.900     0.900     0.900      10.0
                      B-meal      1.000     1.000     1.000      16.0
        B-fromloc.state_code      1.000     1.000     1.000      23.0
    B-depart_time.period_mod      1.000     1.000     1.000       5.0
                   B-connect      1.000     1.000     1.000       6.0
               B-flight_days      1.000     1.000     1.000      10.0
        B-toloc.airport_name      1.000     1.000     1.000       3.0
        B-fromloc.state_name      1.000     0.941     0.970      17.0
              B-airport_name      0.778     0.333     0.467      21.0
                   B-economy      1.000     1.000     1.000       6.0
               I-flight_time      1.000     1.000     1.000       1.0
             B-aircraft_code      0.920     0.697     0.793      33.0
                       B-mod      1.000     0.500     0.667       2.0
              B-airport_code      0.000     0.000     0.000       9.0
    B-depart_time.start_time      0.500     0.667     0.571       3.0
      B-depart_time.end_time      1.000     0.667     0.800       3.0
          B-depart_date.year      1.000     1.000     1.000       3.0
            I-transport_type      1.000     1.000     1.000       1.0
          B-restriction_code      1.000     1.000     1.000       4.0
    B-arrive_time.start_time      0.875     0.875     0.875       8.0
        B-toloc.airport_code      1.000     0.750     0.857       4.0
      B-arrive_time.end_time      0.875     0.875     0.875       8.0
      I-arrive_time.end_time      0.875     0.875     0.875       8.0
      I-depart_time.end_time      1.000     0.667     0.800       3.0
               I-flight_stop      0.000     0.000     0.000       0.0
      B-fromloc.airport_code      0.833     1.000     0.909       5.0
          I-restriction_code      1.000     1.000     1.000       3.0
    I-depart_time.start_time      1.000     1.000     1.000       1.0
          I-toloc.state_name      1.000     1.000     1.000       1.0
I-depart_date.today_relative      0.000     0.000     0.000       0.0
 B-arrive_date.date_relative      1.000     1.000     1.000       2.0
                I-flight_mod      1.000     0.167     0.286       6.0
                   I-economy      0.000     0.000     0.000       0.0
 B-return_date.date_relative      1.000     0.333     0.500       3.0
        I-fromloc.state_name      1.000     1.000     1.000       1.0
                B-state_code      1.000     1.000     1.000       1.0
    I-arrive_time.start_time      1.000     1.000     1.000       1.0
    I-arrive_date.day_number      0.000     0.000     0.000       0.0
                 B-meal_code      0.000     0.000     0.000       1.0
 I-depart_time.period_of_day      1.000     1.000     1.000       1.0
                  B-day_name      1.000     0.500     0.667       2.0
             B-period_of_day      1.000     0.750     0.857       4.0
        B-stoploc.state_code      0.000     0.000     0.000       0.0
    B-return_date.month_name      0.000     0.000     0.000       0.0
    B-return_date.day_number      0.000     0.000     0.000       0.0
    B-arrive_time.period_mod      0.000     0.000     0.000       0.0
                 I-meal_code      0.000     0.000     0.000       0.0
        B-toloc.country_name      1.000     1.000     1.000       1.0
                 B-days_code      1.000     1.000     1.000       1.0
 I-arrive_time.period_of_day      0.000     0.000     0.000       0.0
            I-today_relative      0.000     0.000     0.000       0.0
 B-return_time.period_of_day      0.000     0.000     0.000       0.0
                      B-time      0.000     0.000     0.000       0.0
           I-fare_basis_code      0.000     0.000     0.000       0.0
 I-arrive_time.time_relative      1.000     1.000     1.000       4.0
 I-depart_time.time_relative      0.000     0.000     0.000       1.0
            B-today_relative      0.000     0.000     0.000       0.0
                B-state_name      0.800     0.444     0.571       9.0
B-arrive_date.today_relative      0.000     0.000     0.000       0.0
    B-return_time.period_mod      0.000     0.000     0.000       0.0
                B-month_name      0.000     0.000     0.000       0.0
                B-day_number      0.000     0.000     0.000       0.0
 I-return_date.date_relative      0.667     0.667     0.667       3.0
I-return_date.today_relative      0.000     0.000     0.000       0.0
      B-stoploc.airport_name      0.000     0.000     0.000       0.0
             B-time_relative      0.000     0.000     0.000       0.0
                      I-time      0.000     0.000     0.000       0.0
    I-return_date.day_number      0.000     0.000     0.000       0.0
          I-meal_description      0.000     0.000     0.000       0.0
B-return_date.today_relative      0.000     0.000     0.000       0.0
      B-return_date.day_name      0.000     0.000     0.000       2.0

                   micro avg      0.949     0.951     0.950    3657.0
                   macro avg      0.692     0.665     0.665    3657.0
                weighted avg      0.958     0.951     0.949    3657.0

I0715 07:20:21.833842 140369481070464 interactiveshell.py:2882] Best Slot F1: 0.956
Reading ../data/atis.train.w-intent.iob
I0715 07:20:26.229943 140369481070464 interactiveshell.py:2882] Step 14400 | Loss: 0.0002 | Loss_intent: 0.0000 | Loss_slots: 0.0002 | Spent: 6.3 secs | LR: 0.000066
I0715 07:20:29.978739 140369481070464 interactiveshell.py:2882] Step 14450 | Loss: 0.0029 | Loss_intent: 0.0000 | Loss_slots: 0.0029 | Spent: 3.7 secs | LR: 0.000065
I0715 07:20:33.734402 140369481070464 interactiveshell.py:2882] Step 14500 | Loss: 0.0006 | Loss_intent: 0.0000 | Loss_slots: 0.0006 | Spent: 3.8 secs | LR: 0.000065
I0715 07:20:37.468424 140369481070464 interactiveshell.py:2882] Step 14550 | Loss: 0.0025 | Loss_intent: 0.0000 | Loss_slots: 0.0025 | Spent: 3.7 secs | LR: 0.000065
I0715 07:20:41.203136 140369481070464 interactiveshell.py:2882] Step 14600 | Loss: 0.0007 | Loss_intent: 0.0000 | Loss_slots: 0.0007 | Spent: 3.7 secs | LR: 0.000064
I0715 07:20:44.968080 140369481070464 interactiveshell.py:2882] Step 14650 | Loss: 0.0009 | Loss_intent: 0.0000 | Loss_slots: 0.0009 | Spent: 3.8 secs | LR: 0.000064
Reading ../data/atis.test.w-intent.iob
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.
  'precision', 'predicted', average, warn_for)
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1439: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples.
  'recall', 'true', average, warn_for)
I0715 07:20:47.397363 140369481070464 interactiveshell.py:2882] 
                                          precision    recall  f1-score   support

                             atis_flight      0.978     0.989     0.983       632
                            atis_airfare      0.941     1.000     0.970        48
                     atis_ground_service      1.000     1.000     1.000        36
                            atis_airline      1.000     1.000     1.000        38
                       atis_abbreviation      1.000     1.000     1.000        33
                           atis_aircraft      1.000     0.889     0.941         9
                        atis_flight_time      1.000     1.000     1.000         1
                           atis_quantity      0.429     1.000     0.600         3
                atis_flight#atis_airfare      0.833     0.417     0.556        12
                            atis_airport      1.000     0.944     0.971        18
                           atis_distance      1.000     1.000     1.000        10
                               atis_city      0.667     0.667     0.667         6
                        atis_ground_fare      1.000     0.714     0.833         7
                           atis_capacity      0.955     1.000     0.977        21
                          atis_flight_no      0.889     1.000     0.941         8
                               atis_meal      1.000     0.833     0.909         6
                        atis_restriction      0.000     0.000     0.000         0
             atis_airline#atis_flight_no      0.000     0.000     0.000         0
    atis_ground_service#atis_ground_fare      0.000     0.000     0.000         0
           atis_airfare#atis_flight_time      0.000     0.000     0.000         0
                           atis_cheapest      0.000     0.000     0.000         0
atis_aircraft#atis_flight#atis_flight_no      0.000     0.000     0.000         0

                               micro avg      0.971     0.976     0.974       888
                               macro avg      0.668     0.657     0.652       888
                            weighted avg      0.973     0.976     0.973       888

I0715 07:20:47.438967 140369481070464 interactiveshell.py:2882] 
                              precision    recall  f1-score   support

                           O      0.000     0.000     0.000       0.0
           B-toloc.city_name      0.971     0.994     0.983     716.0
         B-fromloc.city_name      0.987     0.999     0.993     704.0
           I-toloc.city_name      0.977     0.977     0.977     265.0
      B-depart_date.day_name      0.963     0.986     0.974     212.0
              B-airline_name      0.980     0.990     0.985     101.0
         I-fromloc.city_name      0.978     0.983     0.980     177.0
 B-depart_time.period_of_day      1.000     0.869     0.930     130.0
              I-airline_name      1.000     1.000     1.000      65.0
    B-depart_date.day_number      0.981     0.964     0.972      55.0
    B-depart_date.month_name      0.982     0.964     0.973      56.0
          B-depart_time.time      0.812     0.982     0.889      57.0
                B-round_trip      1.000     0.986     0.993      73.0
             B-cost_relative      1.000     0.973     0.986      37.0
                I-round_trip      1.000     1.000     1.000      71.0
                B-flight_mod      0.828     1.000     0.906      24.0
 B-depart_time.time_relative      0.984     0.969     0.977      65.0
          I-depart_time.time      0.981     1.000     0.990      52.0
         B-stoploc.city_name      1.000     1.000     1.000      20.0
                 B-city_name      0.931     0.474     0.628      57.0
                B-class_type      0.960     1.000     0.980      24.0
          B-arrive_time.time      0.892     0.971     0.930      34.0
 B-arrive_time.time_relative      0.968     0.968     0.968      31.0
                I-class_type      1.000     1.000     1.000      17.0
               B-flight_stop      1.000     1.000     1.000      21.0
          I-arrive_time.time      0.971     0.971     0.971      35.0
              B-airline_code      0.969     0.912     0.939      34.0
    I-depart_date.day_number      1.000     0.933     0.966      15.0
      I-fromloc.airport_name      0.405     1.000     0.577      15.0
      B-fromloc.airport_name      0.400     1.000     0.571      12.0
      B-arrive_date.day_name      0.733     1.000     0.846      11.0
          B-toloc.state_code      1.000     1.000     1.000      18.0
B-depart_date.today_relative      1.000     0.889     0.941       9.0
             B-flight_number      0.550     1.000     0.710      11.0
 B-depart_date.date_relative      0.944     1.000     0.971      17.0
          B-toloc.state_name      0.926     0.893     0.909      28.0
           B-fare_basis_code      0.944     1.000     0.971      17.0
               B-flight_time      1.000     1.000     1.000       1.0
                        B-or      1.000     1.000     1.000       3.0
 B-arrive_time.period_of_day      0.600     1.000     0.750       6.0
          B-meal_description      1.000     0.900     0.947      10.0
             I-cost_relative      1.000     0.667     0.800       3.0
              I-airport_name      0.909     0.345     0.500      29.0
               B-fare_amount      1.000     1.000     1.000       2.0
               I-fare_amount      1.000     1.000     1.000       2.0
                 I-city_name      0.720     0.600     0.655      30.0
        I-toloc.airport_name      1.000     1.000     1.000       3.0
            B-transport_type      1.000     1.000     1.000      10.0
    B-arrive_date.month_name      0.714     0.833     0.769       6.0
    B-arrive_date.day_number      0.714     0.833     0.769       6.0
         I-stoploc.city_name      1.000     0.900     0.947      10.0
                      B-meal      1.000     1.000     1.000      16.0
        B-fromloc.state_code      1.000     1.000     1.000      23.0
    B-depart_time.period_mod      1.000     1.000     1.000       5.0
                   B-connect      1.000     1.000     1.000       6.0
               B-flight_days      1.000     1.000     1.000      10.0
        B-toloc.airport_name      1.000     1.000     1.000       3.0
        B-fromloc.state_name      1.000     1.000     1.000      17.0
              B-airport_name      0.700     0.333     0.452      21.0
                   B-economy      1.000     1.000     1.000       6.0
               I-flight_time      1.000     1.000     1.000       1.0
             B-aircraft_code      1.000     0.788     0.881      33.0
                       B-mod      1.000     0.500     0.667       2.0
              B-airport_code      0.000     0.000     0.000       9.0
    B-depart_time.start_time      0.500     0.667     0.571       3.0
      B-depart_time.end_time      1.000     1.000     1.000       3.0
          B-depart_date.year      0.429     1.000     0.600       3.0
            I-transport_type      0.000     0.000     0.000       1.0
          B-restriction_code      1.000     1.000     1.000       4.0
    B-arrive_time.start_time      0.889     1.000     0.941       8.0
        B-toloc.airport_code      1.000     0.750     0.857       4.0
      B-arrive_time.end_time      1.000     0.875     0.933       8.0
      I-arrive_time.end_time      0.875     0.875     0.875       8.0
      I-depart_time.end_time      1.000     0.667     0.800       3.0
               I-flight_stop      0.000     0.000     0.000       0.0
      B-fromloc.airport_code      0.833     1.000     0.909       5.0
          I-restriction_code      1.000     1.000     1.000       3.0
    I-depart_time.start_time      1.000     1.000     1.000       1.0
          I-toloc.state_name      1.000     1.000     1.000       1.0
I-depart_date.today_relative      0.000     0.000     0.000       0.0
 B-arrive_date.date_relative      1.000     1.000     1.000       2.0
                I-flight_mod      1.000     0.167     0.286       6.0
                   I-economy      0.000     0.000     0.000       0.0
 B-return_date.date_relative      1.000     0.333     0.500       3.0
        I-fromloc.state_name      1.000     1.000     1.000       1.0
                B-state_code      1.000     1.000     1.000       1.0
    I-arrive_time.start_time      1.000     1.000     1.000       1.0
    I-arrive_date.day_number      0.000     0.000     0.000       0.0
                 B-meal_code      0.000     0.000     0.000       1.0
 I-depart_time.period_of_day      1.000     1.000     1.000       1.0
                  B-day_name      1.000     0.500     0.667       2.0
             B-period_of_day      1.000     0.500     0.667       4.0
        B-stoploc.state_code      0.000     0.000     0.000       0.0
    B-return_date.month_name      0.000     0.000     0.000       0.0
    B-return_date.day_number      0.000     0.000     0.000       0.0
    B-arrive_time.period_mod      0.000     0.000     0.000       0.0
                 I-meal_code      0.000     0.000     0.000       0.0
        B-toloc.country_name      1.000     1.000     1.000       1.0
                 B-days_code      1.000     1.000     1.000       1.0
 I-arrive_time.period_of_day      0.000     0.000     0.000       0.0
            I-today_relative      0.000     0.000     0.000       0.0
 B-return_time.period_of_day      0.000     0.000     0.000       0.0
                      B-time      0.000     0.000     0.000       0.0
           I-fare_basis_code      0.000     0.000     0.000       0.0
 I-arrive_time.time_relative      1.000     1.000     1.000       4.0
 I-depart_time.time_relative      0.000     0.000     0.000       1.0
            B-today_relative      0.000     0.000     0.000       0.0
                B-state_name      0.800     0.444     0.571       9.0
B-arrive_date.today_relative      0.000     0.000     0.000       0.0
    B-return_time.period_mod      0.000     0.000     0.000       0.0
                B-month_name      0.000     0.000     0.000       0.0
                B-day_number      0.000     0.000     0.000       0.0
 I-return_date.date_relative      0.667     0.667     0.667       3.0
I-return_date.today_relative      0.000     0.000     0.000       0.0
      B-stoploc.airport_name      0.000     0.000     0.000       0.0
             B-time_relative      0.000     0.000     0.000       0.0
                      I-time      0.000     0.000     0.000       0.0
    I-return_date.day_number      0.000     0.000     0.000       0.0
          I-meal_description      0.000     0.000     0.000       0.0
B-return_date.today_relative      0.000     0.000     0.000       0.0
      B-return_date.day_name      0.000     0.000     0.000       2.0

                   micro avg      0.950     0.951     0.951    3657.0
                   macro avg      0.681     0.659     0.656    3657.0
                weighted avg      0.958     0.951     0.949    3657.0

I0715 07:20:47.439842 140369481070464 interactiveshell.py:2882] Best Slot F1: 0.956
Reading ../data/atis.train.w-intent.iob
I0715 07:20:50.906966 140369481070464 interactiveshell.py:2882] Step 14700 | Loss: 0.0013 | Loss_intent: 0.0000 | Loss_slots: 0.0013 | Spent: 5.9 secs | LR: 0.000064
I0715 07:20:54.687942 140369481070464 interactiveshell.py:2882] Step 14750 | Loss: 0.0006 | Loss_intent: 0.0000 | Loss_slots: 0.0006 | Spent: 3.8 secs | LR: 0.000063
I0715 07:20:58.524708 140369481070464 interactiveshell.py:2882] Step 14800 | Loss: 0.0013 | Loss_intent: 0.0000 | Loss_slots: 0.0013 | Spent: 3.8 secs | LR: 0.000063
I0715 07:21:02.326875 140369481070464 interactiveshell.py:2882] Step 14850 | Loss: 0.0018 | Loss_intent: 0.0000 | Loss_slots: 0.0018 | Spent: 3.8 secs | LR: 0.000063
I0715 07:21:06.075740 140369481070464 interactiveshell.py:2882] Step 14900 | Loss: 0.0018 | Loss_intent: 0.0001 | Loss_slots: 0.0017 | Spent: 3.7 secs | LR: 0.000062
I0715 07:21:09.823340 140369481070464 interactiveshell.py:2882] Step 14950 | Loss: 0.0004 | Loss_intent: 0.0000 | Loss_slots: 0.0004 | Spent: 3.7 secs | LR: 0.000062
Reading ../data/atis.test.w-intent.iob
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.
  'precision', 'predicted', average, warn_for)
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1439: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples.
  'recall', 'true', average, warn_for)
I0715 07:21:13.132009 140369481070464 interactiveshell.py:2882] 
                                          precision    recall  f1-score   support

                             atis_flight      0.978     0.989     0.983       632
                            atis_airfare      0.941     1.000     0.970        48
                     atis_ground_service      1.000     1.000     1.000        36
                            atis_airline      1.000     1.000     1.000        38
                       atis_abbreviation      1.000     1.000     1.000        33
                           atis_aircraft      1.000     0.889     0.941         9
                        atis_flight_time      1.000     1.000     1.000         1
                           atis_quantity      0.429     1.000     0.600         3
                atis_flight#atis_airfare      0.833     0.417     0.556        12
                            atis_airport      1.000     0.944     0.971        18
                           atis_distance      1.000     1.000     1.000        10
                               atis_city      0.667     0.667     0.667         6
                        atis_ground_fare      1.000     0.714     0.833         7
                           atis_capacity      0.955     1.000     0.977        21
                          atis_flight_no      0.889     1.000     0.941         8
                               atis_meal      1.000     0.833     0.909         6
                        atis_restriction      0.000     0.000     0.000         0
             atis_airline#atis_flight_no      0.000     0.000     0.000         0
    atis_ground_service#atis_ground_fare      0.000     0.000     0.000         0
           atis_airfare#atis_flight_time      0.000     0.000     0.000         0
                           atis_cheapest      0.000     0.000     0.000         0
atis_aircraft#atis_flight#atis_flight_no      0.000     0.000     0.000         0

                               micro avg      0.971     0.976     0.974       888
                               macro avg      0.668     0.657     0.652       888
                            weighted avg      0.973     0.976     0.973       888

I0715 07:21:13.171366 140369481070464 interactiveshell.py:2882] 
                              precision    recall  f1-score   support

                           O      0.000     0.000     0.000       0.0
           B-toloc.city_name      0.970     0.996     0.983     716.0
         B-fromloc.city_name      0.985     0.999     0.992     704.0
           I-toloc.city_name      0.978     0.996     0.987     265.0
      B-depart_date.day_name      0.959     0.986     0.972     212.0
              B-airline_name      1.000     1.000     1.000     101.0
         I-fromloc.city_name      0.983     0.983     0.983     177.0
 B-depart_time.period_of_day      1.000     0.869     0.930     130.0
              I-airline_name      1.000     1.000     1.000      65.0
    B-depart_date.day_number      0.981     0.964     0.972      55.0
    B-depart_date.month_name      0.982     0.964     0.973      56.0
          B-depart_time.time      0.848     0.982     0.911      57.0
                B-round_trip      1.000     0.986     0.993      73.0
             B-cost_relative      1.000     0.973     0.986      37.0
                I-round_trip      1.000     1.000     1.000      71.0
                B-flight_mod      0.828     1.000     0.906      24.0
 B-depart_time.time_relative      0.984     0.954     0.969      65.0
          I-depart_time.time      0.981     1.000     0.990      52.0
         B-stoploc.city_name      1.000     1.000     1.000      20.0
                 B-city_name      0.929     0.456     0.612      57.0
                B-class_type      0.960     1.000     0.980      24.0
          B-arrive_time.time      0.917     0.971     0.943      34.0
 B-arrive_time.time_relative      0.882     0.968     0.923      31.0
                I-class_type      1.000     1.000     1.000      17.0
               B-flight_stop      1.000     1.000     1.000      21.0
          I-arrive_time.time      0.971     0.971     0.971      35.0
              B-airline_code      1.000     0.912     0.954      34.0
    I-depart_date.day_number      1.000     0.933     0.966      15.0
      I-fromloc.airport_name      0.405     1.000     0.577      15.0
      B-fromloc.airport_name      0.357     0.833     0.500      12.0
      B-arrive_date.day_name      0.786     1.000     0.880      11.0
          B-toloc.state_code      1.000     1.000     1.000      18.0
B-depart_date.today_relative      1.000     1.000     1.000       9.0
             B-flight_number      0.647     1.000     0.786      11.0
 B-depart_date.date_relative      0.944     1.000     0.971      17.0
          B-toloc.state_name      0.781     0.893     0.833      28.0
           B-fare_basis_code      0.944     1.000     0.971      17.0
               B-flight_time      1.000     1.000     1.000       1.0
                        B-or      1.000     1.000     1.000       3.0
 B-arrive_time.period_of_day      0.545     1.000     0.706       6.0
          B-meal_description      1.000     0.900     0.947      10.0
             I-cost_relative      1.000     0.667     0.800       3.0
              I-airport_name      0.833     0.345     0.488      29.0
               B-fare_amount      0.500     1.000     0.667       2.0
               I-fare_amount      1.000     1.000     1.000       2.0
                 I-city_name      0.857     0.600     0.706      30.0
        I-toloc.airport_name      1.000     1.000     1.000       3.0
            B-transport_type      0.833     1.000     0.909      10.0
    B-arrive_date.month_name      0.714     0.833     0.769       6.0
    B-arrive_date.day_number      0.714     0.833     0.769       6.0
         I-stoploc.city_name      1.000     0.900     0.947      10.0
                      B-meal      1.000     0.938     0.968      16.0
        B-fromloc.state_code      1.000     1.000     1.000      23.0
    B-depart_time.period_mod      1.000     1.000     1.000       5.0
                   B-connect      1.000     1.000     1.000       6.0
               B-flight_days      1.000     1.000     1.000      10.0
        B-toloc.airport_name      1.000     1.000     1.000       3.0
        B-fromloc.state_name      1.000     0.941     0.970      17.0
              B-airport_name      0.875     0.333     0.483      21.0
                   B-economy      1.000     1.000     1.000       6.0
               I-flight_time      1.000     1.000     1.000       1.0
             B-aircraft_code      0.862     0.758     0.806      33.0
                       B-mod      1.000     0.500     0.667       2.0
              B-airport_code      0.571     0.444     0.500       9.0
    B-depart_time.start_time      0.500     0.667     0.571       3.0
      B-depart_time.end_time      1.000     0.667     0.800       3.0
          B-depart_date.year      1.000     1.000     1.000       3.0
            I-transport_type      1.000     1.000     1.000       1.0
          B-restriction_code      1.000     1.000     1.000       4.0
    B-arrive_time.start_time      0.889     1.000     0.941       8.0
        B-toloc.airport_code      1.000     0.750     0.857       4.0
      B-arrive_time.end_time      0.875     0.875     0.875       8.0
      I-arrive_time.end_time      0.875     0.875     0.875       8.0
      I-depart_time.end_time      1.000     0.667     0.800       3.0
               I-flight_stop      0.000     0.000     0.000       0.0
      B-fromloc.airport_code      0.833     1.000     0.909       5.0
          I-restriction_code      1.000     1.000     1.000       3.0
    I-depart_time.start_time      1.000     1.000     1.000       1.0
          I-toloc.state_name      1.000     1.000     1.000       1.0
I-depart_date.today_relative      0.000     0.000     0.000       0.0
 B-arrive_date.date_relative      1.000     1.000     1.000       2.0
                I-flight_mod      1.000     0.167     0.286       6.0
                   I-economy      0.000     0.000     0.000       0.0
 B-return_date.date_relative      0.000     0.000     0.000       3.0
        I-fromloc.state_name      1.000     1.000     1.000       1.0
                B-state_code      1.000     1.000     1.000       1.0
    I-arrive_time.start_time      1.000     1.000     1.000       1.0
    I-arrive_date.day_number      0.000     0.000     0.000       0.0
                 B-meal_code      0.000     0.000     0.000       1.0
 I-depart_time.period_of_day      1.000     1.000     1.000       1.0
                  B-day_name      1.000     0.500     0.667       2.0
             B-period_of_day      1.000     0.500     0.667       4.0
        B-stoploc.state_code      0.000     0.000     0.000       0.0
    B-return_date.month_name      0.000     0.000     0.000       0.0
    B-return_date.day_number      0.000     0.000     0.000       0.0
    B-arrive_time.period_mod      0.000     0.000     0.000       0.0
                 I-meal_code      0.000     0.000     0.000       0.0
        B-toloc.country_name      1.000     1.000     1.000       1.0
                 B-days_code      0.000     0.000     0.000       1.0
 I-arrive_time.period_of_day      0.000     0.000     0.000       0.0
            I-today_relative      0.000     0.000     0.000       0.0
 B-return_time.period_of_day      0.000     0.000     0.000       0.0
                      B-time      0.000     0.000     0.000       0.0
           I-fare_basis_code      0.000     0.000     0.000       0.0
 I-arrive_time.time_relative      1.000     0.500     0.667       4.0
 I-depart_time.time_relative      0.000     0.000     0.000       1.0
            B-today_relative      0.000     0.000     0.000       0.0
                B-state_name      0.667     0.222     0.333       9.0
B-arrive_date.today_relative      0.000     0.000     0.000       0.0
    B-return_time.period_mod      0.000     0.000     0.000       0.0
                B-month_name      0.000     0.000     0.000       0.0
                B-day_number      0.000     0.000     0.000       0.0
 I-return_date.date_relative      0.667     0.667     0.667       3.0
I-return_date.today_relative      0.000     0.000     0.000       0.0
      B-stoploc.airport_name      0.000     0.000     0.000       0.0
             B-time_relative      0.000     0.000     0.000       0.0
                      I-time      0.000     0.000     0.000       0.0
    I-return_date.day_number      0.000     0.000     0.000       0.0
          I-meal_description      0.000     0.000     0.000       0.0
B-return_date.today_relative      0.000     0.000     0.000       0.0
      B-return_date.day_name      0.000     0.000     0.000       2.0

                   micro avg      0.949     0.951     0.950    3657.0
                   macro avg      0.675     0.650     0.648    3657.0
                weighted avg      0.957     0.951     0.948    3657.0

I0715 07:21:13.172256 140369481070464 interactiveshell.py:2882] Best Slot F1: 0.956
Reading ../data/atis.train.w-intent.iob
I0715 07:21:15.755464 140369481070464 interactiveshell.py:2882] Step 15000 | Loss: 0.0017 | Loss_intent: 0.0000 | Loss_slots: 0.0017 | Spent: 5.9 secs | LR: 0.000062
I0715 07:21:19.688506 140369481070464 interactiveshell.py:2882] Step 15050 | Loss: 0.0006 | Loss_intent: 0.0000 | Loss_slots: 0.0006 | Spent: 3.9 secs | LR: 0.000061
I0715 07:21:23.585297 140369481070464 interactiveshell.py:2882] Step 15100 | Loss: 0.0008 | Loss_intent: 0.0000 | Loss_slots: 0.0008 | Spent: 3.9 secs | LR: 0.000061
I0715 07:21:27.327171 140369481070464 interactiveshell.py:2882] Step 15150 | Loss: 0.0004 | Loss_intent: 0.0000 | Loss_slots: 0.0004 | Spent: 3.7 secs | LR: 0.000061
I0715 07:21:31.096049 140369481070464 interactiveshell.py:2882] Step 15200 | Loss: 0.0010 | Loss_intent: 0.0000 | Loss_slots: 0.0010 | Spent: 3.8 secs | LR: 0.000060
I0715 07:21:34.854764 140369481070464 interactiveshell.py:2882] Step 15250 | Loss: 0.0063 | Loss_intent: 0.0000 | Loss_slots: 0.0063 | Spent: 3.8 secs | LR: 0.000060
Reading ../data/atis.test.w-intent.iob
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.
  'precision', 'predicted', average, warn_for)
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1439: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples.
  'recall', 'true', average, warn_for)
I0715 07:21:39.078171 140369481070464 interactiveshell.py:2882] 
                                          precision    recall  f1-score   support

                             atis_flight      0.978     0.987     0.983       632
                            atis_airfare      0.941     1.000     0.970        48
                     atis_ground_service      1.000     1.000     1.000        36
                            atis_airline      1.000     1.000     1.000        38
                       atis_abbreviation      1.000     1.000     1.000        33
                           atis_aircraft      1.000     0.889     0.941         9
                        atis_flight_time      1.000     1.000     1.000         1
                           atis_quantity      0.429     1.000     0.600         3
                atis_flight#atis_airfare      0.833     0.417     0.556        12
                            atis_airport      1.000     0.944     0.971        18
                           atis_distance      1.000     1.000     1.000        10
                               atis_city      0.571     0.667     0.615         6
                        atis_ground_fare      1.000     0.714     0.833         7
                           atis_capacity      0.955     1.000     0.977        21
                          atis_flight_no      0.889     1.000     0.941         8
                               atis_meal      1.000     0.833     0.909         6
                        atis_restriction      0.000     0.000     0.000         0
             atis_airline#atis_flight_no      0.000     0.000     0.000         0
    atis_ground_service#atis_ground_fare      0.000     0.000     0.000         0
           atis_airfare#atis_flight_time      0.000     0.000     0.000         0
                           atis_cheapest      0.000     0.000     0.000         0
atis_aircraft#atis_flight#atis_flight_no      0.000     0.000     0.000         0

                               micro avg      0.970     0.975     0.972       888
                               macro avg      0.663     0.657     0.650       888
                            weighted avg      0.972     0.975     0.972       888

I0715 07:21:39.118217 140369481070464 interactiveshell.py:2882] 
                              precision    recall  f1-score   support

                           O      0.000     0.000     0.000       0.0
           B-toloc.city_name      0.975     0.994     0.985     716.0
         B-fromloc.city_name      0.983     1.000     0.992     704.0
           I-toloc.city_name      0.978     0.996     0.987     265.0
      B-depart_date.day_name      0.959     0.986     0.972     212.0
              B-airline_name      1.000     1.000     1.000     101.0
         I-fromloc.city_name      0.983     0.983     0.983     177.0
 B-depart_time.period_of_day      1.000     0.877     0.934     130.0
              I-airline_name      1.000     1.000     1.000      65.0
    B-depart_date.day_number      0.981     0.964     0.972      55.0
    B-depart_date.month_name      0.982     0.964     0.973      56.0
          B-depart_time.time      0.862     0.982     0.918      57.0
                B-round_trip      1.000     0.986     0.993      73.0
             B-cost_relative      1.000     0.973     0.986      37.0
                I-round_trip      1.000     1.000     1.000      71.0
                B-flight_mod      0.821     0.958     0.885      24.0
 B-depart_time.time_relative      0.984     0.954     0.969      65.0
          I-depart_time.time      0.981     0.981     0.981      52.0
         B-stoploc.city_name      1.000     1.000     1.000      20.0
                 B-city_name      0.939     0.544     0.689      57.0
                B-class_type      0.960     1.000     0.980      24.0
          B-arrive_time.time      0.892     0.971     0.930      34.0
 B-arrive_time.time_relative      0.909     0.968     0.937      31.0
                I-class_type      1.000     1.000     1.000      17.0
               B-flight_stop      1.000     1.000     1.000      21.0
          I-arrive_time.time      0.944     0.971     0.958      35.0
              B-airline_code      1.000     0.912     0.954      34.0
    I-depart_date.day_number      1.000     0.933     0.966      15.0
      I-fromloc.airport_name      0.395     1.000     0.566      15.0
      B-fromloc.airport_name      0.375     0.750     0.500      12.0
      B-arrive_date.day_name      0.786     1.000     0.880      11.0
          B-toloc.state_code      1.000     1.000     1.000      18.0
B-depart_date.today_relative      1.000     1.000     1.000       9.0
             B-flight_number      0.440     1.000     0.611      11.0
 B-depart_date.date_relative      0.895     1.000     0.944      17.0
          B-toloc.state_name      0.800     1.000     0.889      28.0
           B-fare_basis_code      0.944     1.000     0.971      17.0
               B-flight_time      1.000     1.000     1.000       1.0
                        B-or      1.000     1.000     1.000       3.0
 B-arrive_time.period_of_day      0.667     1.000     0.800       6.0
          B-meal_description      1.000     0.900     0.947      10.0
             I-cost_relative      1.000     0.667     0.800       3.0
              I-airport_name      0.909     0.345     0.500      29.0
               B-fare_amount      0.500     1.000     0.667       2.0
               I-fare_amount      1.000     1.000     1.000       2.0
                 I-city_name      0.864     0.633     0.731      30.0
        I-toloc.airport_name      1.000     1.000     1.000       3.0
            B-transport_type      0.833     1.000     0.909      10.0
    B-arrive_date.month_name      0.714     0.833     0.769       6.0
    B-arrive_date.day_number      0.714     0.833     0.769       6.0
         I-stoploc.city_name      1.000     0.900     0.947      10.0
                      B-meal      1.000     1.000     1.000      16.0
        B-fromloc.state_code      1.000     1.000     1.000      23.0
    B-depart_time.period_mod      1.000     1.000     1.000       5.0
                   B-connect      1.000     1.000     1.000       6.0
               B-flight_days      1.000     1.000     1.000      10.0
        B-toloc.airport_name      1.000     1.000     1.000       3.0
        B-fromloc.state_name      1.000     0.941     0.970      17.0
              B-airport_name      0.778     0.333     0.467      21.0
                   B-economy      1.000     1.000     1.000       6.0
               I-flight_time      1.000     1.000     1.000       1.0
             B-aircraft_code      0.958     0.697     0.807      33.0
                       B-mod      1.000     0.500     0.667       2.0
              B-airport_code      0.000     0.000     0.000       9.0
    B-depart_time.start_time      0.400     0.667     0.500       3.0
      B-depart_time.end_time      1.000     0.667     0.800       3.0
          B-depart_date.year      0.750     1.000     0.857       3.0
            I-transport_type      0.000     0.000     0.000       1.0
          B-restriction_code      1.000     1.000     1.000       4.0
    B-arrive_time.start_time      0.889     1.000     0.941       8.0
        B-toloc.airport_code      1.000     0.750     0.857       4.0
      B-arrive_time.end_time      0.875     0.875     0.875       8.0
      I-arrive_time.end_time      0.875     0.875     0.875       8.0
      I-depart_time.end_time      1.000     0.667     0.800       3.0
               I-flight_stop      0.000     0.000     0.000       0.0
      B-fromloc.airport_code      0.833     1.000     0.909       5.0
          I-restriction_code      1.000     1.000     1.000       3.0
    I-depart_time.start_time      1.000     1.000     1.000       1.0
          I-toloc.state_name      1.000     1.000     1.000       1.0
I-depart_date.today_relative      0.000     0.000     0.000       0.0
 B-arrive_date.date_relative      1.000     1.000     1.000       2.0
                I-flight_mod      1.000     0.167     0.286       6.0
                   I-economy      0.000     0.000     0.000       0.0
 B-return_date.date_relative      0.000     0.000     0.000       3.0
        I-fromloc.state_name      1.000     1.000     1.000       1.0
                B-state_code      1.000     1.000     1.000       1.0
    I-arrive_time.start_time      1.000     1.000     1.000       1.0
    I-arrive_date.day_number      0.000     0.000     0.000       0.0
                 B-meal_code      0.000     0.000     0.000       1.0
 I-depart_time.period_of_day      1.000     1.000     1.000       1.0
                  B-day_name      1.000     0.500     0.667       2.0
             B-period_of_day      1.000     0.500     0.667       4.0
        B-stoploc.state_code      0.000     0.000     0.000       0.0
    B-return_date.month_name      0.000     0.000     0.000       0.0
    B-return_date.day_number      0.000     0.000     0.000       0.0
    B-arrive_time.period_mod      0.000     0.000     0.000       0.0
                 I-meal_code      0.000     0.000     0.000       0.0
        B-toloc.country_name      1.000     1.000     1.000       1.0
                 B-days_code      1.000     1.000     1.000       1.0
 I-arrive_time.period_of_day      0.000     0.000     0.000       0.0
            I-today_relative      0.000     0.000     0.000       0.0
 B-return_time.period_of_day      0.000     0.000     0.000       0.0
                      B-time      0.000     0.000     0.000       0.0
           I-fare_basis_code      0.000     0.000     0.000       0.0
 I-arrive_time.time_relative      1.000     0.750     0.857       4.0
 I-depart_time.time_relative      0.000     0.000     0.000       1.0
            B-today_relative      0.000     0.000     0.000       0.0
                B-state_name      0.667     0.222     0.333       9.0
B-arrive_date.today_relative      0.000     0.000     0.000       0.0
    B-return_time.period_mod      0.000     0.000     0.000       0.0
                B-month_name      0.000     0.000     0.000       0.0
                B-day_number      0.000     0.000     0.000       0.0
 I-return_date.date_relative      0.667     0.667     0.667       3.0
I-return_date.today_relative      0.000     0.000     0.000       0.0
      B-stoploc.airport_name      0.000     0.000     0.000       0.0
             B-time_relative      0.000     0.000     0.000       0.0
                      I-time      0.000     0.000     0.000       0.0
    I-return_date.day_number      0.000     0.000     0.000       0.0
          I-meal_description      0.000     0.000     0.000       0.0
B-return_date.today_relative      0.000     0.000     0.000       0.0
      B-return_date.day_name      0.000     0.000     0.000       2.0

                   micro avg      0.950     0.952     0.951    3657.0
                   macro avg      0.667     0.649     0.644    3657.0
                weighted avg      0.957     0.952     0.949    3657.0

I0715 07:21:39.119026 140369481070464 interactiveshell.py:2882] Best Slot F1: 0.956
Reading ../data/atis.train.w-intent.iob
I0715 07:21:40.782833 140369481070464 interactiveshell.py:2882] Step 15300 | Loss: 0.0009 | Loss_intent: 0.0000 | Loss_slots: 0.0009 | Spent: 5.9 secs | LR: 0.000060
I0715 07:21:44.530207 140369481070464 interactiveshell.py:2882] Step 15350 | Loss: 0.0025 | Loss_intent: 0.0000 | Loss_slots: 0.0025 | Spent: 3.7 secs | LR: 0.000060
I0715 07:21:48.287205 140369481070464 interactiveshell.py:2882] Step 15400 | Loss: 0.0001 | Loss_intent: 0.0000 | Loss_slots: 0.0001 | Spent: 3.8 secs | LR: 0.000059
I0715 07:21:52.021978 140369481070464 interactiveshell.py:2882] Step 15450 | Loss: 0.0027 | Loss_intent: 0.0000 | Loss_slots: 0.0027 | Spent: 3.7 secs | LR: 0.000059
I0715 07:21:55.780242 140369481070464 interactiveshell.py:2882] Step 15500 | Loss: 0.0006 | Loss_intent: 0.0000 | Loss_slots: 0.0006 | Spent: 3.8 secs | LR: 0.000059
I0715 07:21:59.521985 140369481070464 interactiveshell.py:2882] Step 15550 | Loss: 0.0008 | Loss_intent: 0.0000 | Loss_slots: 0.0008 | Spent: 3.7 secs | LR: 0.000058
Reading ../data/atis.test.w-intent.iob
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.
  'precision', 'predicted', average, warn_for)
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1439: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples.
  'recall', 'true', average, warn_for)
I0715 07:22:04.647487 140369481070464 interactiveshell.py:2882] 
                                          precision    recall  f1-score   support

                             atis_flight      0.977     0.987     0.982       632
                            atis_airfare      0.960     1.000     0.980        48
                     atis_ground_service      1.000     1.000     1.000        36
                            atis_airline      1.000     1.000     1.000        38
                       atis_abbreviation      1.000     1.000     1.000        33
                           atis_aircraft      1.000     0.889     0.941         9
                        atis_flight_time      1.000     1.000     1.000         1
                           atis_quantity      0.429     1.000     0.600         3
                atis_flight#atis_airfare      0.800     0.333     0.471        12
                            atis_airport      1.000     0.944     0.971        18
                           atis_distance      1.000     1.000     1.000        10
                               atis_city      0.571     0.667     0.615         6
                        atis_ground_fare      1.000     0.857     0.923         7
                           atis_capacity      0.955     1.000     0.977        21
                          atis_flight_no      0.889     1.000     0.941         8
                               atis_meal      1.000     0.833     0.909         6
                        atis_restriction      0.000     0.000     0.000         0
             atis_airline#atis_flight_no      0.000     0.000     0.000         0
    atis_ground_service#atis_ground_fare      0.000     0.000     0.000         0
           atis_airfare#atis_flight_time      0.000     0.000     0.000         0
                           atis_cheapest      0.000     0.000     0.000         0
atis_aircraft#atis_flight#atis_flight_no      0.000     0.000     0.000         0

                               micro avg      0.970     0.975     0.972       888
                               macro avg      0.663     0.660     0.650       888
                            weighted avg      0.972     0.975     0.971       888

I0715 07:22:04.694835 140369481070464 interactiveshell.py:2882] 
                              precision    recall  f1-score   support

                           O      0.000     0.000     0.000       0.0
           B-toloc.city_name      0.970     0.994     0.982     716.0
         B-fromloc.city_name      0.983     1.000     0.992     704.0
           I-toloc.city_name      0.978     0.992     0.985     265.0
      B-depart_date.day_name      0.959     0.986     0.972     212.0
              B-airline_name      1.000     1.000     1.000     101.0
         I-fromloc.city_name      0.989     0.983     0.986     177.0
 B-depart_time.period_of_day      1.000     0.869     0.930     130.0
              I-airline_name      1.000     1.000     1.000      65.0
    B-depart_date.day_number      0.981     0.964     0.972      55.0
    B-depart_date.month_name      0.982     0.964     0.973      56.0
          B-depart_time.time      0.851     1.000     0.919      57.0
                B-round_trip      1.000     0.986     0.993      73.0
             B-cost_relative      1.000     0.973     0.986      37.0
                I-round_trip      1.000     1.000     1.000      71.0
                B-flight_mod      0.821     0.958     0.885      24.0
 B-depart_time.time_relative      0.984     0.969     0.977      65.0
          I-depart_time.time      0.981     1.000     0.990      52.0
         B-stoploc.city_name      1.000     1.000     1.000      20.0
                 B-city_name      0.833     0.526     0.645      57.0
                B-class_type      0.960     1.000     0.980      24.0
          B-arrive_time.time      0.943     0.971     0.957      34.0
 B-arrive_time.time_relative      0.966     0.903     0.933      31.0
                I-class_type      1.000     1.000     1.000      17.0
               B-flight_stop      1.000     1.000     1.000      21.0
          I-arrive_time.time      0.971     0.971     0.971      35.0
              B-airline_code      0.969     0.912     0.939      34.0
    I-depart_date.day_number      1.000     0.933     0.966      15.0
      I-fromloc.airport_name      0.417     1.000     0.588      15.0
      B-fromloc.airport_name      0.360     0.750     0.486      12.0
      B-arrive_date.day_name      0.786     1.000     0.880      11.0
          B-toloc.state_code      1.000     1.000     1.000      18.0
B-depart_date.today_relative      1.000     1.000     1.000       9.0
             B-flight_number      0.440     1.000     0.611      11.0
 B-depart_date.date_relative      0.895     1.000     0.944      17.0
          B-toloc.state_name      0.962     0.893     0.926      28.0
           B-fare_basis_code      0.944     1.000     0.971      17.0
               B-flight_time      1.000     1.000     1.000       1.0
                        B-or      1.000     1.000     1.000       3.0
 B-arrive_time.period_of_day      0.600     1.000     0.750       6.0
          B-meal_description      1.000     0.900     0.947      10.0
             I-cost_relative      1.000     0.667     0.800       3.0
              I-airport_name      0.917     0.379     0.537      29.0
               B-fare_amount      0.500     1.000     0.667       2.0
               I-fare_amount      1.000     1.000     1.000       2.0
                 I-city_name      0.900     0.600     0.720      30.0
        I-toloc.airport_name      1.000     1.000     1.000       3.0
            B-transport_type      1.000     1.000     1.000      10.0
    B-arrive_date.month_name      0.714     0.833     0.769       6.0
    B-arrive_date.day_number      0.714     0.833     0.769       6.0
         I-stoploc.city_name      0.909     1.000     0.952      10.0
                      B-meal      1.000     1.000     1.000      16.0
        B-fromloc.state_code      1.000     1.000     1.000      23.0
    B-depart_time.period_mod      1.000     1.000     1.000       5.0
                   B-connect      1.000     1.000     1.000       6.0
               B-flight_days      1.000     1.000     1.000      10.0
        B-toloc.airport_name      1.000     1.000     1.000       3.0
        B-fromloc.state_name      1.000     1.000     1.000      17.0
              B-airport_name      0.778     0.333     0.467      21.0
                   B-economy      1.000     1.000     1.000       6.0
               I-flight_time      1.000     1.000     1.000       1.0
             B-aircraft_code      0.923     0.727     0.814      33.0
                       B-mod      1.000     0.500     0.667       2.0
              B-airport_code      0.000     0.000     0.000       9.0
    B-depart_time.start_time      0.400     0.667     0.500       3.0
      B-depart_time.end_time      1.000     1.000     1.000       3.0
          B-depart_date.year      1.000     1.000     1.000       3.0
            I-transport_type      0.000     0.000     0.000       1.0
          B-restriction_code      1.000     1.000     1.000       4.0
    B-arrive_time.start_time      0.889     1.000     0.941       8.0
        B-toloc.airport_code      1.000     0.750     0.857       4.0
      B-arrive_time.end_time      1.000     0.875     0.933       8.0
      I-arrive_time.end_time      0.875     0.875     0.875       8.0
      I-depart_time.end_time      1.000     0.667     0.800       3.0
               I-flight_stop      0.000     0.000     0.000       0.0
      B-fromloc.airport_code      0.833     1.000     0.909       5.0
          I-restriction_code      1.000     1.000     1.000       3.0
    I-depart_time.start_time      1.000     1.000     1.000       1.0
          I-toloc.state_name      1.000     1.000     1.000       1.0
I-depart_date.today_relative      0.000     0.000     0.000       0.0
 B-arrive_date.date_relative      1.000     1.000     1.000       2.0
                I-flight_mod      1.000     0.167     0.286       6.0
                   I-economy      0.000     0.000     0.000       0.0
 B-return_date.date_relative      1.000     0.333     0.500       3.0
        I-fromloc.state_name      1.000     1.000     1.000       1.0
                B-state_code      1.000     1.000     1.000       1.0
    I-arrive_time.start_time      1.000     1.000     1.000       1.0
    I-arrive_date.day_number      0.000     0.000     0.000       0.0
                 B-meal_code      0.000     0.000     0.000       1.0
 I-depart_time.period_of_day      1.000     1.000     1.000       1.0
                  B-day_name      1.000     0.500     0.667       2.0
             B-period_of_day      1.000     0.500     0.667       4.0
        B-stoploc.state_code      0.000     0.000     0.000       0.0
    B-return_date.month_name      0.000     0.000     0.000       0.0
    B-return_date.day_number      0.000     0.000     0.000       0.0
    B-arrive_time.period_mod      0.000     0.000     0.000       0.0
                 I-meal_code      0.000     0.000     0.000       0.0
        B-toloc.country_name      1.000     1.000     1.000       1.0
                 B-days_code      1.000     1.000     1.000       1.0
 I-arrive_time.period_of_day      0.000     0.000     0.000       0.0
            I-today_relative      0.000     0.000     0.000       0.0
 B-return_time.period_of_day      0.000     0.000     0.000       0.0
                      B-time      0.000     0.000     0.000       0.0
           I-fare_basis_code      0.000     0.000     0.000       0.0
 I-arrive_time.time_relative      0.667     1.000     0.800       4.0
 I-depart_time.time_relative      0.000     0.000     0.000       1.0
            B-today_relative      0.000     0.000     0.000       0.0
                B-state_name      0.750     0.333     0.462       9.0
B-arrive_date.today_relative      0.000     0.000     0.000       0.0
    B-return_time.period_mod      0.000     0.000     0.000       0.0
                B-month_name      0.000     0.000     0.000       0.0
                B-day_number      0.000     0.000     0.000       0.0
 I-return_date.date_relative      0.667     0.667     0.667       3.0
I-return_date.today_relative      0.000     0.000     0.000       0.0
      B-stoploc.airport_name      0.000     0.000     0.000       0.0
             B-time_relative      0.000     0.000     0.000       0.0
                      I-time      0.000     0.000     0.000       0.0
    I-return_date.day_number      0.000     0.000     0.000       0.0
          I-meal_description      0.000     0.000     0.000       0.0
B-return_date.today_relative      0.000     0.000     0.000       0.0
      B-return_date.day_name      0.000     0.000     0.000       2.0

                   micro avg      0.951     0.952     0.952    3657.0
                   macro avg      0.677     0.658     0.653    3657.0
                weighted avg      0.958     0.952     0.950    3657.0

I0715 07:22:04.695844 140369481070464 interactiveshell.py:2882] Best Slot F1: 0.956
Reading ../data/atis.train.w-intent.iob
I0715 07:22:05.456815 140369481070464 interactiveshell.py:2882] Step 15600 | Loss: 0.0000 | Loss_intent: 0.0000 | Loss_slots: 0.0000 | Spent: 5.9 secs | LR: 0.000058
I0715 07:22:09.199487 140369481070464 interactiveshell.py:2882] Step 15650 | Loss: 0.0008 | Loss_intent: 0.0000 | Loss_slots: 0.0008 | Spent: 3.7 secs | LR: 0.000058
I0715 07:22:12.946363 140369481070464 interactiveshell.py:2882] Step 15700 | Loss: 0.0033 | Loss_intent: 0.0000 | Loss_slots: 0.0033 | Spent: 3.7 secs | LR: 0.000057
I0715 07:22:16.756994 140369481070464 interactiveshell.py:2882] Step 15750 | Loss: 0.0004 | Loss_intent: 0.0000 | Loss_slots: 0.0004 | Spent: 3.8 secs | LR: 0.000057
I0715 07:22:20.573420 140369481070464 interactiveshell.py:2882] Step 15800 | Loss: 0.0018 | Loss_intent: 0.0000 | Loss_slots: 0.0018 | Spent: 3.8 secs | LR: 0.000057
I0715 07:22:24.374532 140369481070464 interactiveshell.py:2882] Step 15850 | Loss: 0.0012 | Loss_intent: 0.0000 | Loss_slots: 0.0012 | Spent: 3.8 secs | LR: 0.000056
I0715 07:22:28.115760 140369481070464 interactiveshell.py:2882] Step 15900 | Loss: 0.0007 | Loss_intent: 0.0000 | Loss_slots: 0.0007 | Spent: 3.7 secs | LR: 0.000056
Reading ../data/atis.test.w-intent.iob
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.
  'precision', 'predicted', average, warn_for)
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1439: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples.
  'recall', 'true', average, warn_for)
I0715 07:22:30.410586 140369481070464 interactiveshell.py:2882] 
                                          precision    recall  f1-score   support

                             atis_flight      0.978     0.987     0.983       632
                            atis_airfare      0.941     1.000     0.970        48
                     atis_ground_service      1.000     1.000     1.000        36
                            atis_airline      1.000     1.000     1.000        38
                       atis_abbreviation      1.000     1.000     1.000        33
                           atis_aircraft      1.000     0.889     0.941         9
                        atis_flight_time      1.000     1.000     1.000         1
                           atis_quantity      0.429     1.000     0.600         3
                atis_flight#atis_airfare      0.833     0.417     0.556        12
                            atis_airport      1.000     0.944     0.971        18
                           atis_distance      1.000     1.000     1.000        10
                               atis_city      0.571     0.667     0.615         6
                        atis_ground_fare      1.000     0.714     0.833         7
                           atis_capacity      0.955     1.000     0.977        21
                          atis_flight_no      0.889     1.000     0.941         8
                               atis_meal      1.000     0.833     0.909         6
                        atis_restriction      0.000     0.000     0.000         0
             atis_airline#atis_flight_no      0.000     0.000     0.000         0
    atis_ground_service#atis_ground_fare      0.000     0.000     0.000         0
           atis_airfare#atis_flight_time      0.000     0.000     0.000         0
                           atis_cheapest      0.000     0.000     0.000         0
atis_aircraft#atis_flight#atis_flight_no      0.000     0.000     0.000         0

                               micro avg      0.970     0.975     0.972       888
                               macro avg      0.663     0.657     0.650       888
                            weighted avg      0.972     0.975     0.972       888

I0715 07:22:30.451331 140369481070464 interactiveshell.py:2882] 
                              precision    recall  f1-score   support

                           O      0.000     0.000     0.000       0.0
           B-toloc.city_name      0.971     0.994     0.983     716.0
         B-fromloc.city_name      0.987     0.999     0.993     704.0
           I-toloc.city_name      0.978     0.992     0.985     265.0
      B-depart_date.day_name      0.959     0.986     0.972     212.0
              B-airline_name      1.000     1.000     1.000     101.0
         I-fromloc.city_name      0.983     0.994     0.989     177.0
 B-depart_time.period_of_day      1.000     0.869     0.930     130.0
              I-airline_name      1.000     1.000     1.000      65.0
    B-depart_date.day_number      0.981     0.964     0.972      55.0
    B-depart_date.month_name      0.982     0.964     0.973      56.0
          B-depart_time.time      0.851     1.000     0.919      57.0
                B-round_trip      1.000     0.986     0.993      73.0
             B-cost_relative      1.000     0.973     0.986      37.0
                I-round_trip      1.000     1.000     1.000      71.0
                B-flight_mod      0.821     0.958     0.885      24.0
 B-depart_time.time_relative      0.984     0.969     0.977      65.0
          I-depart_time.time      0.981     1.000     0.990      52.0
         B-stoploc.city_name      1.000     1.000     1.000      20.0
                 B-city_name      0.886     0.544     0.674      57.0
                B-class_type      0.960     1.000     0.980      24.0
          B-arrive_time.time      0.943     0.971     0.957      34.0
 B-arrive_time.time_relative      0.968     0.968     0.968      31.0
                I-class_type      1.000     1.000     1.000      17.0
               B-flight_stop      1.000     1.000     1.000      21.0
          I-arrive_time.time      0.971     0.971     0.971      35.0
              B-airline_code      1.000     0.912     0.954      34.0
    I-depart_date.day_number      1.000     0.933     0.966      15.0
      I-fromloc.airport_name      0.385     1.000     0.556      15.0
      B-fromloc.airport_name      0.400     0.833     0.541      12.0
      B-arrive_date.day_name      0.786     1.000     0.880      11.0
          B-toloc.state_code      1.000     1.000     1.000      18.0
B-depart_date.today_relative      1.000     0.889     0.941       9.0
             B-flight_number      0.647     1.000     0.786      11.0
 B-depart_date.date_relative      0.895     1.000     0.944      17.0
          B-toloc.state_name      0.926     0.893     0.909      28.0
           B-fare_basis_code      0.917     0.647     0.759      17.0
               B-flight_time      1.000     1.000     1.000       1.0
                        B-or      1.000     1.000     1.000       3.0
 B-arrive_time.period_of_day      0.545     1.000     0.706       6.0
          B-meal_description      1.000     0.900     0.947      10.0
             I-cost_relative      1.000     0.667     0.800       3.0
              I-airport_name      0.909     0.345     0.500      29.0
               B-fare_amount      0.500     1.000     0.667       2.0
               I-fare_amount      1.000     1.000     1.000       2.0
                 I-city_name      1.000     0.600     0.750      30.0
        I-toloc.airport_name      1.000     1.000     1.000       3.0
            B-transport_type      1.000     0.900     0.947      10.0
    B-arrive_date.month_name      0.714     0.833     0.769       6.0
    B-arrive_date.day_number      0.714     0.833     0.769       6.0
         I-stoploc.city_name      1.000     1.000     1.000      10.0
                      B-meal      1.000     1.000     1.000      16.0
        B-fromloc.state_code      1.000     1.000     1.000      23.0
    B-depart_time.period_mod      1.000     1.000     1.000       5.0
                   B-connect      1.000     1.000     1.000       6.0
               B-flight_days      1.000     1.000     1.000      10.0
        B-toloc.airport_name      1.000     1.000     1.000       3.0
        B-fromloc.state_name      1.000     1.000     1.000      17.0
              B-airport_name      0.778     0.333     0.467      21.0
                   B-economy      1.000     1.000     1.000       6.0
               I-flight_time      1.000     1.000     1.000       1.0
             B-aircraft_code      0.871     0.818     0.844      33.0
                       B-mod      1.000     0.500     0.667       2.0
              B-airport_code      0.667     0.444     0.533       9.0
    B-depart_time.start_time      0.400     0.667     0.500       3.0
      B-depart_time.end_time      1.000     0.667     0.800       3.0
          B-depart_date.year      1.000     1.000     1.000       3.0
            I-transport_type      0.000     0.000     0.000       1.0
          B-restriction_code      1.000     1.000     1.000       4.0
    B-arrive_time.start_time      0.889     1.000     0.941       8.0
        B-toloc.airport_code      1.000     0.750     0.857       4.0
      B-arrive_time.end_time      0.875     0.875     0.875       8.0
      I-arrive_time.end_time      0.875     0.875     0.875       8.0
      I-depart_time.end_time      1.000     0.667     0.800       3.0
               I-flight_stop      0.000     0.000     0.000       0.0
      B-fromloc.airport_code      0.625     1.000     0.769       5.0
          I-restriction_code      1.000     1.000     1.000       3.0
    I-depart_time.start_time      1.000     1.000     1.000       1.0
          I-toloc.state_name      1.000     1.000     1.000       1.0
I-depart_date.today_relative      0.000     0.000     0.000       0.0
 B-arrive_date.date_relative      1.000     1.000     1.000       2.0
                I-flight_mod      1.000     0.167     0.286       6.0
                   I-economy      0.000     0.000     0.000       0.0
 B-return_date.date_relative      0.000     0.000     0.000       3.0
        I-fromloc.state_name      1.000     1.000     1.000       1.0
                B-state_code      1.000     1.000     1.000       1.0
    I-arrive_time.start_time      1.000     1.000     1.000       1.0
    I-arrive_date.day_number      0.000     0.000     0.000       0.0
                 B-meal_code      0.000     0.000     0.000       1.0
 I-depart_time.period_of_day      1.000     1.000     1.000       1.0
                  B-day_name      1.000     0.500     0.667       2.0
             B-period_of_day      1.000     0.500     0.667       4.0
        B-stoploc.state_code      0.000     0.000     0.000       0.0
    B-return_date.month_name      0.000     0.000     0.000       0.0
    B-return_date.day_number      0.000     0.000     0.000       0.0
    B-arrive_time.period_mod      0.000     0.000     0.000       0.0
                 I-meal_code      0.000     0.000     0.000       0.0
        B-toloc.country_name      1.000     1.000     1.000       1.0
                 B-days_code      1.000     1.000     1.000       1.0
 I-arrive_time.period_of_day      0.000     0.000     0.000       0.0
            I-today_relative      0.000     0.000     0.000       0.0
 B-return_time.period_of_day      0.000     0.000     0.000       0.0
                      B-time      0.000     0.000     0.000       0.0
           I-fare_basis_code      0.000     0.000     0.000       0.0
 I-arrive_time.time_relative      1.000     1.000     1.000       4.0
 I-depart_time.time_relative      0.000     0.000     0.000       1.0
            B-today_relative      0.000     0.000     0.000       0.0
                B-state_name      0.800     0.444     0.571       9.0
B-arrive_date.today_relative      0.000     0.000     0.000       0.0
    B-return_time.period_mod      0.000     0.000     0.000       0.0
                B-month_name      0.000     0.000     0.000       0.0
                B-day_number      0.000     0.000     0.000       0.0
 I-return_date.date_relative      0.667     0.667     0.667       3.0
I-return_date.today_relative      0.000     0.000     0.000       0.0
      B-stoploc.airport_name      0.000     0.000     0.000       0.0
             B-time_relative      0.000     0.000     0.000       0.0
                      I-time      0.000     0.000     0.000       0.0
    I-return_date.day_number      0.000     0.000     0.000       0.0
          I-meal_description      0.000     0.000     0.000       0.0
B-return_date.today_relative      0.000     0.000     0.000       0.0
      B-return_date.day_name      0.000     0.000     0.000       2.0

                   micro avg      0.951     0.953     0.952    3657.0
                   macro avg      0.678     0.654     0.653    3657.0
                weighted avg      0.961     0.953     0.952    3657.0

I0715 07:22:30.452180 140369481070464 interactiveshell.py:2882] Best Slot F1: 0.956
Reading ../data/atis.train.w-intent.iob
I0715 07:22:34.080724 140369481070464 interactiveshell.py:2882] Step 15950 | Loss: 0.0001 | Loss_intent: 0.0000 | Loss_slots: 0.0001 | Spent: 6.0 secs | LR: 0.000056
I0715 07:22:37.841538 140369481070464 interactiveshell.py:2882] Step 16000 | Loss: 0.0005 | Loss_intent: 0.0000 | Loss_slots: 0.0005 | Spent: 3.8 secs | LR: 0.000056
I0715 07:22:41.594272 140369481070464 interactiveshell.py:2882] Step 16050 | Loss: 0.0000 | Loss_intent: 0.0000 | Loss_slots: 0.0000 | Spent: 3.8 secs | LR: 0.000055
I0715 07:22:45.344685 140369481070464 interactiveshell.py:2882] Step 16100 | Loss: 0.0002 | Loss_intent: 0.0000 | Loss_slots: 0.0002 | Spent: 3.7 secs | LR: 0.000055
I0715 07:22:49.119834 140369481070464 interactiveshell.py:2882] Step 16150 | Loss: 0.0002 | Loss_intent: 0.0000 | Loss_slots: 0.0002 | Spent: 3.8 secs | LR: 0.000055
I0715 07:22:52.855886 140369481070464 interactiveshell.py:2882] Step 16200 | Loss: 0.0003 | Loss_intent: 0.0000 | Loss_slots: 0.0003 | Spent: 3.7 secs | LR: 0.000054
Reading ../data/atis.test.w-intent.iob
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.
  'precision', 'predicted', average, warn_for)
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1439: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples.
  'recall', 'true', average, warn_for)
I0715 07:22:56.018197 140369481070464 interactiveshell.py:2882] 
                                          precision    recall  f1-score   support

                             atis_flight      0.978     0.987     0.983       632
                            atis_airfare      0.941     1.000     0.970        48
                     atis_ground_service      1.000     1.000     1.000        36
                            atis_airline      1.000     1.000     1.000        38
                       atis_abbreviation      1.000     1.000     1.000        33
                           atis_aircraft      1.000     0.889     0.941         9
                        atis_flight_time      1.000     1.000     1.000         1
                           atis_quantity      0.429     1.000     0.600         3
                atis_flight#atis_airfare      0.833     0.417     0.556        12
                            atis_airport      1.000     0.944     0.971        18
                           atis_distance      1.000     1.000     1.000        10
                               atis_city      0.571     0.667     0.615         6
                        atis_ground_fare      1.000     0.714     0.833         7
                           atis_capacity      0.955     1.000     0.977        21
                          atis_flight_no      0.889     1.000     0.941         8
                               atis_meal      1.000     0.833     0.909         6
                        atis_restriction      0.000     0.000     0.000         0
             atis_airline#atis_flight_no      0.000     0.000     0.000         0
    atis_ground_service#atis_ground_fare      0.000     0.000     0.000         0
           atis_airfare#atis_flight_time      0.000     0.000     0.000         0
                           atis_cheapest      0.000     0.000     0.000         0
atis_aircraft#atis_flight#atis_flight_no      0.000     0.000     0.000         0

                               micro avg      0.970     0.975     0.972       888
                               macro avg      0.663     0.657     0.650       888
                            weighted avg      0.972     0.975     0.972       888

I0715 07:22:56.058583 140369481070464 interactiveshell.py:2882] 
                              precision    recall  f1-score   support

                           O      0.000     0.000     0.000       0.0
           B-toloc.city_name      0.971     0.994     0.983     716.0
         B-fromloc.city_name      0.986     0.997     0.992     704.0
           I-toloc.city_name      0.985     0.977     0.981     265.0
      B-depart_date.day_name      0.963     0.986     0.974     212.0
              B-airline_name      1.000     1.000     1.000     101.0
         I-fromloc.city_name      0.978     0.994     0.986     177.0
 B-depart_time.period_of_day      0.991     0.877     0.931     130.0
              I-airline_name      1.000     1.000     1.000      65.0
    B-depart_date.day_number      0.982     0.982     0.982      55.0
    B-depart_date.month_name      0.982     0.964     0.973      56.0
          B-depart_time.time      0.851     1.000     0.919      57.0
                B-round_trip      1.000     0.986     0.993      73.0
             B-cost_relative      1.000     0.973     0.986      37.0
                I-round_trip      1.000     1.000     1.000      71.0
                B-flight_mod      0.828     1.000     0.906      24.0
 B-depart_time.time_relative      0.984     0.969     0.977      65.0
          I-depart_time.time      0.963     1.000     0.981      52.0
         B-stoploc.city_name      0.952     1.000     0.976      20.0
                 B-city_name      0.842     0.561     0.674      57.0
                B-class_type      0.960     1.000     0.980      24.0
          B-arrive_time.time      0.943     0.971     0.957      34.0
 B-arrive_time.time_relative      0.909     0.968     0.937      31.0
                I-class_type      1.000     1.000     1.000      17.0
               B-flight_stop      1.000     1.000     1.000      21.0
          I-arrive_time.time      0.971     0.971     0.971      35.0
              B-airline_code      0.969     0.912     0.939      34.0
    I-depart_date.day_number      1.000     1.000     1.000      15.0
      I-fromloc.airport_name      0.405     1.000     0.577      15.0
      B-fromloc.airport_name      0.379     0.917     0.537      12.0
      B-arrive_date.day_name      0.786     1.000     0.880      11.0
          B-toloc.state_code      1.000     1.000     1.000      18.0
B-depart_date.today_relative      1.000     1.000     1.000       9.0
             B-flight_number      0.524     1.000     0.688      11.0
 B-depart_date.date_relative      0.944     1.000     0.971      17.0
          B-toloc.state_name      0.962     0.893     0.926      28.0
           B-fare_basis_code      0.917     0.647     0.759      17.0
               B-flight_time      1.000     1.000     1.000       1.0
                        B-or      1.000     1.000     1.000       3.0
 B-arrive_time.period_of_day      0.667     1.000     0.800       6.0
          B-meal_description      1.000     0.900     0.947      10.0
             I-cost_relative      1.000     0.667     0.800       3.0
              I-airport_name      0.909     0.345     0.500      29.0
               B-fare_amount      1.000     1.000     1.000       2.0
               I-fare_amount      1.000     1.000     1.000       2.0
                 I-city_name      0.833     0.667     0.741      30.0
        I-toloc.airport_name      1.000     1.000     1.000       3.0
            B-transport_type      0.818     0.900     0.857      10.0
    B-arrive_date.month_name      0.714     0.833     0.769       6.0
    B-arrive_date.day_number      0.833     0.833     0.833       6.0
         I-stoploc.city_name      1.000     0.900     0.947      10.0
                      B-meal      1.000     1.000     1.000      16.0
        B-fromloc.state_code      1.000     1.000     1.000      23.0
    B-depart_time.period_mod      1.000     1.000     1.000       5.0
                   B-connect      1.000     1.000     1.000       6.0
               B-flight_days      1.000     1.000     1.000      10.0
        B-toloc.airport_name      1.000     1.000     1.000       3.0
        B-fromloc.state_name      1.000     1.000     1.000      17.0
              B-airport_name      0.778     0.333     0.467      21.0
                   B-economy      1.000     1.000     1.000       6.0
               I-flight_time      1.000     1.000     1.000       1.0
             B-aircraft_code      1.000     0.667     0.800      33.0
                       B-mod      1.000     0.500     0.667       2.0
              B-airport_code      0.667     0.444     0.533       9.0
    B-depart_time.start_time      0.500     0.667     0.571       3.0
      B-depart_time.end_time      1.000     0.667     0.800       3.0
          B-depart_date.year      0.429     1.000     0.600       3.0
            I-transport_type      0.000     0.000     0.000       1.0
          B-restriction_code      1.000     1.000     1.000       4.0
    B-arrive_time.start_time      0.889     1.000     0.941       8.0
        B-toloc.airport_code      1.000     0.750     0.857       4.0
      B-arrive_time.end_time      0.875     0.875     0.875       8.0
      I-arrive_time.end_time      0.875     0.875     0.875       8.0
      I-depart_time.end_time      1.000     0.667     0.800       3.0
               I-flight_stop      0.000     0.000     0.000       0.0
      B-fromloc.airport_code      0.833     1.000     0.909       5.0
          I-restriction_code      1.000     1.000     1.000       3.0
    I-depart_time.start_time      1.000     1.000     1.000       1.0
          I-toloc.state_name      1.000     1.000     1.000       1.0
I-depart_date.today_relative      0.000     0.000     0.000       0.0
 B-arrive_date.date_relative      1.000     1.000     1.000       2.0
                I-flight_mod      1.000     0.167     0.286       6.0
                   I-economy      0.000     0.000     0.000       0.0
 B-return_date.date_relative      0.000     0.000     0.000       3.0
        I-fromloc.state_name      1.000     1.000     1.000       1.0
                B-state_code      1.000     1.000     1.000       1.0
    I-arrive_time.start_time      1.000     1.000     1.000       1.0
    I-arrive_date.day_number      0.000     0.000     0.000       0.0
                 B-meal_code      0.000     0.000     0.000       1.0
 I-depart_time.period_of_day      1.000     1.000     1.000       1.0
                  B-day_name      1.000     0.500     0.667       2.0
             B-period_of_day      1.000     0.500     0.667       4.0
        B-stoploc.state_code      0.000     0.000     0.000       0.0
    B-return_date.month_name      0.000     0.000     0.000       0.0
    B-return_date.day_number      0.000     0.000     0.000       0.0
    B-arrive_time.period_mod      0.000     0.000     0.000       0.0
                 I-meal_code      0.000     0.000     0.000       0.0
        B-toloc.country_name      1.000     1.000     1.000       1.0
                 B-days_code      1.000     1.000     1.000       1.0
 I-arrive_time.period_of_day      0.000     0.000     0.000       0.0
            I-today_relative      0.000     0.000     0.000       0.0
 B-return_time.period_of_day      0.000     0.000     0.000       0.0
                      B-time      0.000     0.000     0.000       0.0
           I-fare_basis_code      0.000     0.000     0.000       0.0
 I-arrive_time.time_relative      1.000     0.500     0.667       4.0
 I-depart_time.time_relative      0.000     0.000     0.000       1.0
            B-today_relative      0.000     0.000     0.000       0.0
                B-state_name      0.750     0.333     0.462       9.0
B-arrive_date.today_relative      0.000     0.000     0.000       0.0
    B-return_time.period_mod      0.000     0.000     0.000       0.0
                B-month_name      0.000     0.000     0.000       0.0
                B-day_number      0.000     0.000     0.000       0.0
 I-return_date.date_relative      0.667     0.667     0.667       3.0
I-return_date.today_relative      0.000     0.000     0.000       0.0
      B-stoploc.airport_name      0.000     0.000     0.000       0.0
             B-time_relative      0.000     0.000     0.000       0.0
                      I-time      0.000     0.000     0.000       0.0
    I-return_date.day_number      0.000     0.000     0.000       0.0
          I-meal_description      0.000     0.000     0.000       0.0
B-return_date.today_relative      0.000     0.000     0.000       0.0
      B-return_date.day_name      0.000     0.000     0.000       2.0

                   micro avg      0.950     0.952     0.951    3657.0
                   macro avg      0.677     0.650     0.650    3657.0
                weighted avg      0.959     0.952     0.950    3657.0

I0715 07:22:56.059628 140369481070464 interactiveshell.py:2882] Best Slot F1: 0.956
Reading ../data/atis.train.w-intent.iob
I0715 07:22:58.771536 140369481070464 interactiveshell.py:2882] Step 16250 | Loss: 0.0010 | Loss_intent: 0.0000 | Loss_slots: 0.0010 | Spent: 5.9 secs | LR: 0.000054
I0715 07:23:02.521914 140369481070464 interactiveshell.py:2882] Step 16300 | Loss: 0.0015 | Loss_intent: 0.0000 | Loss_slots: 0.0015 | Spent: 3.7 secs | LR: 0.000054
I0715 07:23:06.286699 140369481070464 interactiveshell.py:2882] Step 16350 | Loss: 0.0002 | Loss_intent: 0.0000 | Loss_slots: 0.0002 | Spent: 3.8 secs | LR: 0.000054
I0715 07:23:10.025820 140369481070464 interactiveshell.py:2882] Step 16400 | Loss: 0.0005 | Loss_intent: 0.0000 | Loss_slots: 0.0005 | Spent: 3.7 secs | LR: 0.000053
I0715 07:23:13.780222 140369481070464 interactiveshell.py:2882] Step 16450 | Loss: 0.0025 | Loss_intent: 0.0000 | Loss_slots: 0.0025 | Spent: 3.8 secs | LR: 0.000053
I0715 07:23:17.516925 140369481070464 interactiveshell.py:2882] Step 16500 | Loss: 0.0002 | Loss_intent: 0.0000 | Loss_slots: 0.0002 | Spent: 3.7 secs | LR: 0.000053
Reading ../data/atis.test.w-intent.iob
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.
  'precision', 'predicted', average, warn_for)
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1439: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples.
  'recall', 'true', average, warn_for)
I0715 07:23:21.598379 140369481070464 interactiveshell.py:2882] 
                                          precision    recall  f1-score   support

                             atis_flight      0.981     0.987     0.984       632
                            atis_airfare      0.960     1.000     0.980        48
                     atis_ground_service      1.000     1.000     1.000        36
                            atis_airline      1.000     1.000     1.000        38
                       atis_abbreviation      0.943     1.000     0.971        33
                           atis_aircraft      1.000     0.889     0.941         9
                        atis_flight_time      1.000     1.000     1.000         1
                           atis_quantity      0.429     1.000     0.600         3
                atis_flight#atis_airfare      0.833     0.417     0.556        12
                            atis_airport      1.000     0.944     0.971        18
                           atis_distance      1.000     1.000     1.000        10
                               atis_city      0.571     0.667     0.615         6
                        atis_ground_fare      1.000     0.857     0.923         7
                           atis_capacity      0.955     1.000     0.977        21
                          atis_flight_no      0.889     1.000     0.941         8
                               atis_meal      1.000     0.833     0.909         6
                        atis_restriction      0.000     0.000     0.000         0
             atis_airline#atis_flight_no      0.000     0.000     0.000         0
    atis_ground_service#atis_ground_fare      0.000     0.000     0.000         0
           atis_airfare#atis_flight_time      0.000     0.000     0.000         0
                           atis_cheapest      0.000     0.000     0.000         0
atis_aircraft#atis_flight#atis_flight_no      0.000     0.000     0.000         0

                               micro avg      0.971     0.976     0.974       888
                               macro avg      0.662     0.663     0.653       888
                            weighted avg      0.973     0.976     0.973       888

I0715 07:23:21.639486 140369481070464 interactiveshell.py:2882] 
                              precision    recall  f1-score   support

                           O      0.000     0.000     0.000       0.0
           B-toloc.city_name      0.971     0.994     0.983     716.0
         B-fromloc.city_name      0.985     0.999     0.992     704.0
           I-toloc.city_name      0.985     1.000     0.993     265.0
      B-depart_date.day_name      0.981     0.986     0.984     212.0
              B-airline_name      1.000     1.000     1.000     101.0
         I-fromloc.city_name      0.989     0.983     0.986     177.0
 B-depart_time.period_of_day      0.991     0.892     0.939     130.0
              I-airline_name      1.000     1.000     1.000      65.0
    B-depart_date.day_number      0.981     0.964     0.972      55.0
    B-depart_date.month_name      0.982     0.964     0.973      56.0
          B-depart_time.time      0.851     1.000     0.919      57.0
                B-round_trip      1.000     0.986     0.993      73.0
             B-cost_relative      1.000     0.973     0.986      37.0
                I-round_trip      1.000     1.000     1.000      71.0
                B-flight_mod      0.828     1.000     0.906      24.0
 B-depart_time.time_relative      0.984     0.969     0.977      65.0
          I-depart_time.time      0.929     1.000     0.963      52.0
         B-stoploc.city_name      1.000     1.000     1.000      20.0
                 B-city_name      0.941     0.561     0.703      57.0
                B-class_type      0.960     1.000     0.980      24.0
          B-arrive_time.time      0.943     0.971     0.957      34.0
 B-arrive_time.time_relative      0.968     0.968     0.968      31.0
                I-class_type      1.000     1.000     1.000      17.0
               B-flight_stop      1.000     1.000     1.000      21.0
          I-arrive_time.time      0.971     0.971     0.971      35.0
              B-airline_code      1.000     0.912     0.954      34.0
    I-depart_date.day_number      1.000     0.933     0.966      15.0
      I-fromloc.airport_name      0.395     1.000     0.566      15.0
      B-fromloc.airport_name      0.360     0.750     0.486      12.0
      B-arrive_date.day_name      0.786     1.000     0.880      11.0
          B-toloc.state_code      1.000     1.000     1.000      18.0
B-depart_date.today_relative      1.000     0.889     0.941       9.0
             B-flight_number      0.524     1.000     0.688      11.0
 B-depart_date.date_relative      0.944     1.000     0.971      17.0
          B-toloc.state_name      0.806     0.893     0.847      28.0
           B-fare_basis_code      0.944     1.000     0.971      17.0
               B-flight_time      1.000     1.000     1.000       1.0
                        B-or      1.000     1.000     1.000       3.0
 B-arrive_time.period_of_day      0.667     1.000     0.800       6.0
          B-meal_description      1.000     0.900     0.947      10.0
             I-cost_relative      1.000     0.667     0.800       3.0
              I-airport_name      0.909     0.345     0.500      29.0
               B-fare_amount      1.000     1.000     1.000       2.0
               I-fare_amount      1.000     1.000     1.000       2.0
                 I-city_name      0.913     0.700     0.792      30.0
        I-toloc.airport_name      1.000     1.000     1.000       3.0
            B-transport_type      0.818     0.900     0.857      10.0
    B-arrive_date.month_name      0.714     0.833     0.769       6.0
    B-arrive_date.day_number      0.714     0.833     0.769       6.0
         I-stoploc.city_name      1.000     0.900     0.947      10.0
                      B-meal      1.000     1.000     1.000      16.0
        B-fromloc.state_code      1.000     1.000     1.000      23.0
    B-depart_time.period_mod      1.000     1.000     1.000       5.0
                   B-connect      1.000     1.000     1.000       6.0
               B-flight_days      1.000     1.000     1.000      10.0
        B-toloc.airport_name      1.000     1.000     1.000       3.0
        B-fromloc.state_name      1.000     0.941     0.970      17.0
              B-airport_name      0.778     0.333     0.467      21.0
                   B-economy      1.000     1.000     1.000       6.0
               I-flight_time      1.000     1.000     1.000       1.0
             B-aircraft_code      1.000     0.697     0.821      33.0
                       B-mod      1.000     0.500     0.667       2.0
              B-airport_code      0.667     0.444     0.533       9.0
    B-depart_time.start_time      0.400     0.667     0.500       3.0
      B-depart_time.end_time      1.000     0.667     0.800       3.0
          B-depart_date.year      0.429     1.000     0.600       3.0
            I-transport_type      0.000     0.000     0.000       1.0
          B-restriction_code      1.000     1.000     1.000       4.0
    B-arrive_time.start_time      0.889     1.000     0.941       8.0
        B-toloc.airport_code      1.000     0.750     0.857       4.0
      B-arrive_time.end_time      0.875     0.875     0.875       8.0
      I-arrive_time.end_time      0.875     0.875     0.875       8.0
      I-depart_time.end_time      1.000     0.667     0.800       3.0
               I-flight_stop      0.000     0.000     0.000       0.0
      B-fromloc.airport_code      0.714     1.000     0.833       5.0
          I-restriction_code      1.000     1.000     1.000       3.0
    I-depart_time.start_time      1.000     1.000     1.000       1.0
          I-toloc.state_name      1.000     1.000     1.000       1.0
I-depart_date.today_relative      0.000     0.000     0.000       0.0
 B-arrive_date.date_relative      1.000     1.000     1.000       2.0
                I-flight_mod      1.000     0.167     0.286       6.0
                   I-economy      0.000     0.000     0.000       0.0
 B-return_date.date_relative      0.000     0.000     0.000       3.0
        I-fromloc.state_name      1.000     1.000     1.000       1.0
                B-state_code      1.000     1.000     1.000       1.0
    I-arrive_time.start_time      1.000     1.000     1.000       1.0
    I-arrive_date.day_number      0.000     0.000     0.000       0.0
                 B-meal_code      0.000     0.000     0.000       1.0
 I-depart_time.period_of_day      1.000     1.000     1.000       1.0
                  B-day_name      1.000     0.500     0.667       2.0
             B-period_of_day      1.000     0.500     0.667       4.0
        B-stoploc.state_code      0.000     0.000     0.000       0.0
    B-return_date.month_name      0.000     0.000     0.000       0.0
    B-return_date.day_number      0.000     0.000     0.000       0.0
    B-arrive_time.period_mod      0.000     0.000     0.000       0.0
                 I-meal_code      0.000     0.000     0.000       0.0
        B-toloc.country_name      1.000     1.000     1.000       1.0
                 B-days_code      1.000     1.000     1.000       1.0
 I-arrive_time.period_of_day      0.000     0.000     0.000       0.0
            I-today_relative      0.000     0.000     0.000       0.0
 B-return_time.period_of_day      0.000     0.000     0.000       0.0
                      B-time      0.000     0.000     0.000       0.0
           I-fare_basis_code      0.000     0.000     0.000       0.0
 I-arrive_time.time_relative      1.000     1.000     1.000       4.0
 I-depart_time.time_relative      0.000     0.000     0.000       1.0
            B-today_relative      0.000     0.000     0.000       0.0
                B-state_name      0.750     0.333     0.462       9.0
B-arrive_date.today_relative      0.000     0.000     0.000       0.0
    B-return_time.period_mod      0.000     0.000     0.000       0.0
                B-month_name      0.000     0.000     0.000       0.0
                B-day_number      0.000     0.000     0.000       0.0
 I-return_date.date_relative      0.667     0.667     0.667       3.0
I-return_date.today_relative      0.000     0.000     0.000       0.0
      B-stoploc.airport_name      0.000     0.000     0.000       0.0
             B-time_relative      0.000     0.000     0.000       0.0
                      I-time      0.000     0.000     0.000       0.0
    I-return_date.day_number      0.000     0.000     0.000       0.0
          I-meal_description      0.000     0.000     0.000       0.0
B-return_date.today_relative      0.000     0.000     0.000       0.0
      B-return_date.day_name      0.000     0.000     0.000       2.0

                   micro avg      0.953     0.955     0.954    3657.0
                   macro avg      0.676     0.655     0.652    3657.0
                weighted avg      0.961     0.955     0.953    3657.0

I0715 07:23:21.643017 140369481070464 interactiveshell.py:2882] Best Slot F1: 0.956
Reading ../data/atis.train.w-intent.iob
I0715 07:23:23.463915 140369481070464 interactiveshell.py:2882] Step 16550 | Loss: 0.0006 | Loss_intent: 0.0000 | Loss_slots: 0.0006 | Spent: 5.9 secs | LR: 0.000052
I0715 07:23:27.241161 140369481070464 interactiveshell.py:2882] Step 16600 | Loss: 0.0002 | Loss_intent: 0.0000 | Loss_slots: 0.0002 | Spent: 3.8 secs | LR: 0.000052
I0715 07:23:31.007467 140369481070464 interactiveshell.py:2882] Step 16650 | Loss: 0.0024 | Loss_intent: 0.0000 | Loss_slots: 0.0024 | Spent: 3.8 secs | LR: 0.000052
I0715 07:23:34.802139 140369481070464 interactiveshell.py:2882] Step 16700 | Loss: 0.0002 | Loss_intent: 0.0000 | Loss_slots: 0.0002 | Spent: 3.8 secs | LR: 0.000052
I0715 07:23:38.616781 140369481070464 interactiveshell.py:2882] Step 16750 | Loss: 0.0004 | Loss_intent: 0.0000 | Loss_slots: 0.0004 | Spent: 3.8 secs | LR: 0.000051
I0715 07:23:42.439882 140369481070464 interactiveshell.py:2882] Step 16800 | Loss: 0.0028 | Loss_intent: 0.0000 | Loss_slots: 0.0028 | Spent: 3.8 secs | LR: 0.000051
Reading ../data/atis.test.w-intent.iob
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.
  'precision', 'predicted', average, warn_for)
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1439: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples.
  'recall', 'true', average, warn_for)
I0715 07:23:47.415583 140369481070464 interactiveshell.py:2882] 
                                          precision    recall  f1-score   support

                             atis_flight      0.981     0.989     0.985       632
                            atis_airfare      0.960     1.000     0.980        48
                     atis_ground_service      1.000     1.000     1.000        36
                            atis_airline      1.000     1.000     1.000        38
                       atis_abbreviation      0.943     1.000     0.971        33
                           atis_aircraft      1.000     0.889     0.941         9
                        atis_flight_time      1.000     1.000     1.000         1
                           atis_quantity      0.429     1.000     0.600         3
                atis_flight#atis_airfare      0.833     0.417     0.556        12
                            atis_airport      1.000     0.944     0.971        18
                           atis_distance      1.000     1.000     1.000        10
                               atis_city      0.667     0.667     0.667         6
                        atis_ground_fare      1.000     0.857     0.923         7
                           atis_capacity      0.955     1.000     0.977        21
                          atis_flight_no      0.889     1.000     0.941         8
                               atis_meal      1.000     0.833     0.909         6
                        atis_restriction      0.000     0.000     0.000         0
             atis_airline#atis_flight_no      0.000     0.000     0.000         0
    atis_ground_service#atis_ground_fare      0.000     0.000     0.000         0
           atis_airfare#atis_flight_time      0.000     0.000     0.000         0
                           atis_cheapest      0.000     0.000     0.000         0
atis_aircraft#atis_flight#atis_flight_no      0.000     0.000     0.000         0

                               micro avg      0.972     0.977     0.975       888
                               macro avg      0.666     0.663     0.655       888
                            weighted avg      0.974     0.977     0.974       888

I0715 07:23:47.456410 140369481070464 interactiveshell.py:2882] 
                              precision    recall  f1-score   support

                           O      0.000     0.000     0.000       0.0
           B-toloc.city_name      0.974     0.996     0.985     716.0
         B-fromloc.city_name      0.989     0.997     0.993     704.0
           I-toloc.city_name      0.978     1.000     0.989     265.0
      B-depart_date.day_name      0.981     0.986     0.984     212.0
              B-airline_name      1.000     0.990     0.995     101.0
         I-fromloc.city_name      0.994     0.949     0.971     177.0
 B-depart_time.period_of_day      0.991     0.892     0.939     130.0
              I-airline_name      1.000     1.000     1.000      65.0
    B-depart_date.day_number      0.981     0.964     0.972      55.0
    B-depart_date.month_name      0.982     0.964     0.973      56.0
          B-depart_time.time      0.851     1.000     0.919      57.0
                B-round_trip      1.000     0.986     0.993      73.0
             B-cost_relative      1.000     0.973     0.986      37.0
                I-round_trip      1.000     1.000     1.000      71.0
                B-flight_mod      0.828     1.000     0.906      24.0
 B-depart_time.time_relative      0.984     0.969     0.977      65.0
          I-depart_time.time      0.929     1.000     0.963      52.0
         B-stoploc.city_name      0.952     1.000     0.976      20.0
                 B-city_name      0.909     0.526     0.667      57.0
                B-class_type      0.960     1.000     0.980      24.0
          B-arrive_time.time      0.943     0.971     0.957      34.0
 B-arrive_time.time_relative      0.968     0.968     0.968      31.0
                I-class_type      1.000     1.000     1.000      17.0
               B-flight_stop      1.000     1.000     1.000      21.0
          I-arrive_time.time      1.000     0.971     0.986      35.0
              B-airline_code      1.000     0.912     0.954      34.0
    I-depart_date.day_number      1.000     0.933     0.966      15.0
      I-fromloc.airport_name      0.405     1.000     0.577      15.0
      B-fromloc.airport_name      0.367     0.917     0.524      12.0
      B-arrive_date.day_name      0.786     1.000     0.880      11.0
          B-toloc.state_code      1.000     1.000     1.000      18.0
B-depart_date.today_relative      1.000     1.000     1.000       9.0
             B-flight_number      0.440     1.000     0.611      11.0
 B-depart_date.date_relative      0.944     1.000     0.971      17.0
          B-toloc.state_name      0.778     1.000     0.875      28.0
           B-fare_basis_code      0.944     1.000     0.971      17.0
               B-flight_time      1.000     1.000     1.000       1.0
                        B-or      1.000     1.000     1.000       3.0
 B-arrive_time.period_of_day      0.667     1.000     0.800       6.0
          B-meal_description      1.000     0.900     0.947      10.0
             I-cost_relative      1.000     0.667     0.800       3.0
              I-airport_name      0.909     0.345     0.500      29.0
               B-fare_amount      0.500     1.000     0.667       2.0
               I-fare_amount      1.000     1.000     1.000       2.0
                 I-city_name      0.778     0.700     0.737      30.0
        I-toloc.airport_name      1.000     1.000     1.000       3.0
            B-transport_type      1.000     0.900     0.947      10.0
    B-arrive_date.month_name      0.714     0.833     0.769       6.0
    B-arrive_date.day_number      0.714     0.833     0.769       6.0
         I-stoploc.city_name      1.000     1.000     1.000      10.0
                      B-meal      1.000     1.000     1.000      16.0
        B-fromloc.state_code      1.000     1.000     1.000      23.0
    B-depart_time.period_mod      1.000     1.000     1.000       5.0
                   B-connect      1.000     1.000     1.000       6.0
               B-flight_days      1.000     1.000     1.000      10.0
        B-toloc.airport_name      1.000     1.000     1.000       3.0
        B-fromloc.state_name      1.000     0.941     0.970      17.0
              B-airport_name      0.778     0.333     0.467      21.0
                   B-economy      1.000     1.000     1.000       6.0
               I-flight_time      1.000     1.000     1.000       1.0
             B-aircraft_code      1.000     0.788     0.881      33.0
                       B-mod      1.000     0.500     0.667       2.0
              B-airport_code      0.000     0.000     0.000       9.0
    B-depart_time.start_time      0.500     1.000     0.667       3.0
      B-depart_time.end_time      1.000     1.000     1.000       3.0
          B-depart_date.year      0.600     1.000     0.750       3.0
            I-transport_type      0.000     0.000     0.000       1.0
          B-restriction_code      1.000     1.000     1.000       4.0
    B-arrive_time.start_time      1.000     1.000     1.000       8.0
        B-toloc.airport_code      1.000     0.750     0.857       4.0
      B-arrive_time.end_time      1.000     0.875     0.933       8.0
      I-arrive_time.end_time      0.889     1.000     0.941       8.0
      I-depart_time.end_time      1.000     0.667     0.800       3.0
               I-flight_stop      0.000     0.000     0.000       0.0
      B-fromloc.airport_code      1.000     1.000     1.000       5.0
          I-restriction_code      1.000     1.000     1.000       3.0
    I-depart_time.start_time      1.000     1.000     1.000       1.0
          I-toloc.state_name      1.000     1.000     1.000       1.0
I-depart_date.today_relative      0.000     0.000     0.000       0.0
 B-arrive_date.date_relative      1.000     1.000     1.000       2.0
                I-flight_mod      1.000     0.167     0.286       6.0
                   I-economy      0.000     0.000     0.000       0.0
 B-return_date.date_relative      0.000     0.000     0.000       3.0
        I-fromloc.state_name      1.000     1.000     1.000       1.0
                B-state_code      1.000     1.000     1.000       1.0
    I-arrive_time.start_time      1.000     1.000     1.000       1.0
    I-arrive_date.day_number      0.000     0.000     0.000       0.0
                 B-meal_code      0.000     0.000     0.000       1.0
 I-depart_time.period_of_day      1.000     1.000     1.000       1.0
                  B-day_name      1.000     0.500     0.667       2.0
             B-period_of_day      1.000     0.500     0.667       4.0
        B-stoploc.state_code      0.000     0.000     0.000       0.0
    B-return_date.month_name      0.000     0.000     0.000       0.0
    B-return_date.day_number      0.000     0.000     0.000       0.0
    B-arrive_time.period_mod      0.000     0.000     0.000       0.0
                 I-meal_code      0.000     0.000     0.000       0.0
        B-toloc.country_name      1.000     1.000     1.000       1.0
                 B-days_code      1.000     1.000     1.000       1.0
 I-arrive_time.period_of_day      0.000     0.000     0.000       0.0
            I-today_relative      0.000     0.000     0.000       0.0
 B-return_time.period_of_day      0.000     0.000     0.000       0.0
                      B-time      0.000     0.000     0.000       0.0
           I-fare_basis_code      0.000     0.000     0.000       0.0
 I-arrive_time.time_relative      1.000     1.000     1.000       4.0
 I-depart_time.time_relative      0.000     0.000     0.000       1.0
            B-today_relative      0.000     0.000     0.000       0.0
                B-state_name      0.500     0.111     0.182       9.0
B-arrive_date.today_relative      0.000     0.000     0.000       0.0
    B-return_time.period_mod      0.000     0.000     0.000       0.0
                B-month_name      0.000     0.000     0.000       0.0
                B-day_number      0.000     0.000     0.000       0.0
 I-return_date.date_relative      0.667     0.667     0.667       3.0
I-return_date.today_relative      0.000     0.000     0.000       0.0
      B-stoploc.airport_name      0.000     0.000     0.000       0.0
             B-time_relative      0.000     0.000     0.000       0.0
                      I-time      0.000     0.000     0.000       0.0
    I-return_date.day_number      0.000     0.000     0.000       0.0
          I-meal_description      0.000     0.000     0.000       0.0
B-return_date.today_relative      0.000     0.000     0.000       0.0
      B-return_date.day_name      0.000     0.000     0.000       2.0

                   micro avg      0.952     0.954     0.953    3657.0
                   macro avg      0.670     0.660     0.651    3657.0
                weighted avg      0.960     0.954     0.951    3657.0

I0715 07:23:47.457370 140369481070464 interactiveshell.py:2882] Best Slot F1: 0.956
Reading ../data/atis.train.w-intent.iob
I0715 07:23:48.368273 140369481070464 interactiveshell.py:2882] Step 16850 | Loss: 0.0009 | Loss_intent: 0.0000 | Loss_slots: 0.0009 | Spent: 5.9 secs | LR: 0.000051
I0715 07:23:52.114522 140369481070464 interactiveshell.py:2882] Step 16900 | Loss: 0.0006 | Loss_intent: 0.0000 | Loss_slots: 0.0006 | Spent: 3.7 secs | LR: 0.000051
I0715 07:23:55.885294 140369481070464 interactiveshell.py:2882] Step 16950 | Loss: 0.0000 | Loss_intent: 0.0000 | Loss_slots: 0.0000 | Spent: 3.8 secs | LR: 0.000050
I0715 07:23:59.649181 140369481070464 interactiveshell.py:2882] Step 17000 | Loss: 0.0003 | Loss_intent: 0.0000 | Loss_slots: 0.0003 | Spent: 3.8 secs | LR: 0.000050
I0715 07:24:03.381897 140369481070464 interactiveshell.py:2882] Step 17050 | Loss: 0.0000 | Loss_intent: 0.0000 | Loss_slots: 0.0000 | Spent: 3.7 secs | LR: 0.000050
I0715 07:24:07.162316 140369481070464 interactiveshell.py:2882] Step 17100 | Loss: 0.0003 | Loss_intent: 0.0000 | Loss_slots: 0.0003 | Spent: 3.8 secs | LR: 0.000050
I0715 07:24:10.941869 140369481070464 interactiveshell.py:2882] Step 17150 | Loss: 0.0001 | Loss_intent: 0.0000 | Loss_slots: 0.0001 | Spent: 3.8 secs | LR: 0.000049
Reading ../data/atis.test.w-intent.iob
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.
  'precision', 'predicted', average, warn_for)
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1439: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples.
  'recall', 'true', average, warn_for)
I0715 07:24:13.060645 140369481070464 interactiveshell.py:2882] 
                                          precision    recall  f1-score   support

                             atis_flight      0.981     0.987     0.984       632
                            atis_airfare      0.960     1.000     0.980        48
                     atis_ground_service      1.000     1.000     1.000        36
                            atis_airline      1.000     1.000     1.000        38
                       atis_abbreviation      0.943     1.000     0.971        33
                           atis_aircraft      1.000     0.889     0.941         9
                        atis_flight_time      1.000     1.000     1.000         1
                           atis_quantity      0.429     1.000     0.600         3
                atis_flight#atis_airfare      0.833     0.417     0.556        12
                            atis_airport      1.000     0.944     0.971        18
                           atis_distance      1.000     1.000     1.000        10
                               atis_city      0.571     0.667     0.615         6
                        atis_ground_fare      1.000     0.857     0.923         7
                           atis_capacity      0.955     1.000     0.977        21
                          atis_flight_no      0.889     1.000     0.941         8
                               atis_meal      1.000     0.833     0.909         6
                        atis_restriction      0.000     0.000     0.000         0
             atis_airline#atis_flight_no      0.000     0.000     0.000         0
    atis_ground_service#atis_ground_fare      0.000     0.000     0.000         0
           atis_airfare#atis_flight_time      0.000     0.000     0.000         0
                           atis_cheapest      0.000     0.000     0.000         0
atis_aircraft#atis_flight#atis_flight_no      0.000     0.000     0.000         0

                               micro avg      0.971     0.976     0.974       888
                               macro avg      0.662     0.663     0.653       888
                            weighted avg      0.973     0.976     0.973       888

I0715 07:24:13.099493 140369481070464 interactiveshell.py:2882] 
                              precision    recall  f1-score   support

                           O      0.000     0.000     0.000       0.0
           B-toloc.city_name      0.973     0.993     0.983     716.0
         B-fromloc.city_name      0.983     1.000     0.992     704.0
           I-toloc.city_name      0.978     1.000     0.989     265.0
      B-depart_date.day_name      0.977     0.986     0.981     212.0
              B-airline_name      1.000     0.990     0.995     101.0
         I-fromloc.city_name      0.994     0.983     0.989     177.0
 B-depart_time.period_of_day      1.000     0.885     0.939     130.0
              I-airline_name      1.000     1.000     1.000      65.0
    B-depart_date.day_number      0.981     0.964     0.972      55.0
    B-depart_date.month_name      0.982     0.964     0.973      56.0
          B-depart_time.time      0.851     1.000     0.919      57.0
                B-round_trip      1.000     0.986     0.993      73.0
             B-cost_relative      1.000     0.973     0.986      37.0
                I-round_trip      1.000     1.000     1.000      71.0
                B-flight_mod      0.828     1.000     0.906      24.0
 B-depart_time.time_relative      0.984     0.969     0.977      65.0
          I-depart_time.time      0.945     1.000     0.972      52.0
         B-stoploc.city_name      1.000     1.000     1.000      20.0
                 B-city_name      0.882     0.526     0.659      57.0
                B-class_type      0.960     1.000     0.980      24.0
          B-arrive_time.time      0.943     0.971     0.957      34.0
 B-arrive_time.time_relative      0.968     0.968     0.968      31.0
                I-class_type      1.000     1.000     1.000      17.0
               B-flight_stop      1.000     1.000     1.000      21.0
          I-arrive_time.time      0.971     0.971     0.971      35.0
              B-airline_code      1.000     0.912     0.954      34.0
    I-depart_date.day_number      1.000     0.933     0.966      15.0
      I-fromloc.airport_name      0.405     1.000     0.577      15.0
      B-fromloc.airport_name      0.360     0.750     0.486      12.0
      B-arrive_date.day_name      0.786     1.000     0.880      11.0
          B-toloc.state_code      1.000     1.000     1.000      18.0
B-depart_date.today_relative      1.000     0.889     0.941       9.0
             B-flight_number      0.524     1.000     0.688      11.0
 B-depart_date.date_relative      0.944     1.000     0.971      17.0
          B-toloc.state_name      0.794     0.964     0.871      28.0
           B-fare_basis_code      0.944     1.000     0.971      17.0
               B-flight_time      1.000     1.000     1.000       1.0
                        B-or      1.000     1.000     1.000       3.0
 B-arrive_time.period_of_day      0.545     1.000     0.706       6.0
          B-meal_description      1.000     0.900     0.947      10.0
             I-cost_relative      1.000     0.667     0.800       3.0
              I-airport_name      0.909     0.345     0.500      29.0
               B-fare_amount      0.500     1.000     0.667       2.0
               I-fare_amount      1.000     1.000     1.000       2.0
                 I-city_name      0.900     0.600     0.720      30.0
        I-toloc.airport_name      1.000     1.000     1.000       3.0
            B-transport_type      1.000     0.900     0.947      10.0
    B-arrive_date.month_name      0.714     0.833     0.769       6.0
    B-arrive_date.day_number      0.714     0.833     0.769       6.0
         I-stoploc.city_name      1.000     1.000     1.000      10.0
                      B-meal      1.000     1.000     1.000      16.0
        B-fromloc.state_code      1.000     1.000     1.000      23.0
    B-depart_time.period_mod      1.000     1.000     1.000       5.0
                   B-connect      1.000     1.000     1.000       6.0
               B-flight_days      1.000     1.000     1.000      10.0
        B-toloc.airport_name      1.000     1.000     1.000       3.0
        B-fromloc.state_name      1.000     1.000     1.000      17.0
              B-airport_name      0.778     0.333     0.467      21.0
                   B-economy      1.000     1.000     1.000       6.0
               I-flight_time      1.000     1.000     1.000       1.0
             B-aircraft_code      1.000     0.788     0.881      33.0
                       B-mod      1.000     0.500     0.667       2.0
              B-airport_code      0.667     0.444     0.533       9.0
    B-depart_time.start_time      0.500     1.000     0.667       3.0
      B-depart_time.end_time      1.000     1.000     1.000       3.0
          B-depart_date.year      0.600     1.000     0.750       3.0
            I-transport_type      0.000     0.000     0.000       1.0
          B-restriction_code      1.000     1.000     1.000       4.0
    B-arrive_time.start_time      1.000     1.000     1.000       8.0
        B-toloc.airport_code      1.000     0.750     0.857       4.0
      B-arrive_time.end_time      1.000     0.875     0.933       8.0
      I-arrive_time.end_time      0.875     0.875     0.875       8.0
      I-depart_time.end_time      1.000     0.667     0.800       3.0
               I-flight_stop      0.000     0.000     0.000       0.0
      B-fromloc.airport_code      0.833     1.000     0.909       5.0
          I-restriction_code      1.000     1.000     1.000       3.0
    I-depart_time.start_time      1.000     1.000     1.000       1.0
          I-toloc.state_name      1.000     1.000     1.000       1.0
I-depart_date.today_relative      0.000     0.000     0.000       0.0
 B-arrive_date.date_relative      1.000     1.000     1.000       2.0
                I-flight_mod      1.000     0.167     0.286       6.0
                   I-economy      0.000     0.000     0.000       0.0
 B-return_date.date_relative      0.000     0.000     0.000       3.0
        I-fromloc.state_name      1.000     1.000     1.000       1.0
                B-state_code      1.000     1.000     1.000       1.0
    I-arrive_time.start_time      1.000     1.000     1.000       1.0
    I-arrive_date.day_number      0.000     0.000     0.000       0.0
                 B-meal_code      0.000     0.000     0.000       1.0
 I-depart_time.period_of_day      1.000     1.000     1.000       1.0
                  B-day_name      1.000     0.500     0.667       2.0
             B-period_of_day      1.000     0.500     0.667       4.0
        B-stoploc.state_code      0.000     0.000     0.000       0.0
    B-return_date.month_name      0.000     0.000     0.000       0.0
    B-return_date.day_number      0.000     0.000     0.000       0.0
    B-arrive_time.period_mod      0.000     0.000     0.000       0.0
                 I-meal_code      0.000     0.000     0.000       0.0
        B-toloc.country_name      1.000     1.000     1.000       1.0
                 B-days_code      1.000     1.000     1.000       1.0
 I-arrive_time.period_of_day      0.000     0.000     0.000       0.0
            I-today_relative      0.000     0.000     0.000       0.0
 B-return_time.period_of_day      0.000     0.000     0.000       0.0
                      B-time      0.000     0.000     0.000       0.0
           I-fare_basis_code      0.000     0.000     0.000       0.0
 I-arrive_time.time_relative      1.000     1.000     1.000       4.0
 I-depart_time.time_relative      0.000     0.000     0.000       1.0
            B-today_relative      0.000     0.000     0.000       0.0
                B-state_name      0.000     0.000     0.000       9.0
B-arrive_date.today_relative      0.000     0.000     0.000       0.0
    B-return_time.period_mod      0.000     0.000     0.000       0.0
                B-month_name      0.000     0.000     0.000       0.0
                B-day_number      0.000     0.000     0.000       0.0
 I-return_date.date_relative      0.667     0.667     0.667       3.0
I-return_date.today_relative      0.000     0.000     0.000       0.0
      B-stoploc.airport_name      0.000     0.000     0.000       0.0
             B-time_relative      0.000     0.000     0.000       0.0
                      I-time      0.000     0.000     0.000       0.0
    I-return_date.day_number      0.000     0.000     0.000       0.0
          I-meal_description      0.000     0.000     0.000       0.0
B-return_date.today_relative      0.000     0.000     0.000       0.0
      B-return_date.day_name      0.000     0.000     0.000       2.0

                   micro avg      0.953     0.954     0.954    3657.0
                   macro avg      0.671     0.659     0.652    3657.0
                weighted avg      0.959     0.954     0.952    3657.0

I0715 07:24:13.100525 140369481070464 interactiveshell.py:2882] Best Slot F1: 0.956
Reading ../data/atis.train.w-intent.iob
I0715 07:24:16.853723 140369481070464 interactiveshell.py:2882] Step 17200 | Loss: 0.0001 | Loss_intent: 0.0000 | Loss_slots: 0.0001 | Spent: 5.9 secs | LR: 0.000049
I0715 07:24:20.634905 140369481070464 interactiveshell.py:2882] Step 17250 | Loss: 0.0006 | Loss_intent: 0.0000 | Loss_slots: 0.0006 | Spent: 3.8 secs | LR: 0.000049
I0715 07:24:24.406111 140369481070464 interactiveshell.py:2882] Step 17300 | Loss: 0.0001 | Loss_intent: 0.0000 | Loss_slots: 0.0001 | Spent: 3.8 secs | LR: 0.000048
I0715 07:24:28.395929 140369481070464 interactiveshell.py:2882] Step 17350 | Loss: 0.0001 | Loss_intent: 0.0000 | Loss_slots: 0.0001 | Spent: 4.0 secs | LR: 0.000048
I0715 07:24:32.537505 140369481070464 interactiveshell.py:2882] Step 17400 | Loss: 0.0002 | Loss_intent: 0.0000 | Loss_slots: 0.0002 | Spent: 4.1 secs | LR: 0.000048
I0715 07:24:36.681712 140369481070464 interactiveshell.py:2882] Step 17450 | Loss: 0.0003 | Loss_intent: 0.0000 | Loss_slots: 0.0003 | Spent: 4.1 secs | LR: 0.000048
Reading ../data/atis.test.w-intent.iob
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.
  'precision', 'predicted', average, warn_for)
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1439: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples.
  'recall', 'true', average, warn_for)
I0715 07:24:39.709429 140369481070464 interactiveshell.py:2882] 
                                          precision    recall  f1-score   support

                             atis_flight      0.978     0.989     0.983       632
                            atis_airfare      0.960     1.000     0.980        48
                     atis_ground_service      1.000     1.000     1.000        36
                            atis_airline      1.000     1.000     1.000        38
                       atis_abbreviation      1.000     1.000     1.000        33
                           atis_aircraft      1.000     0.889     0.941         9
                        atis_flight_time      1.000     1.000     1.000         1
                           atis_quantity      0.429     1.000     0.600         3
                atis_flight#atis_airfare      0.833     0.417     0.556        12
                            atis_airport      1.000     0.944     0.971        18
                           atis_distance      1.000     1.000     1.000        10
                               atis_city      0.667     0.667     0.667         6
                        atis_ground_fare      1.000     0.857     0.923         7
                           atis_capacity      0.955     1.000     0.977        21
                          atis_flight_no      0.889     1.000     0.941         8
                               atis_meal      1.000     0.833     0.909         6
                        atis_restriction      0.000     0.000     0.000         0
             atis_airline#atis_flight_no      0.000     0.000     0.000         0
    atis_ground_service#atis_ground_fare      0.000     0.000     0.000         0
           atis_airfare#atis_flight_time      0.000     0.000     0.000         0
                           atis_cheapest      0.000     0.000     0.000         0
atis_aircraft#atis_flight#atis_flight_no      0.000     0.000     0.000         0

                               micro avg      0.972     0.977     0.975       888
                               macro avg      0.669     0.663     0.657       888
                            weighted avg      0.974     0.977     0.974       888

I0715 07:24:39.749296 140369481070464 interactiveshell.py:2882] 
                              precision    recall  f1-score   support

                           O      0.000     0.000     0.000       0.0
           B-toloc.city_name      0.973     0.994     0.983     716.0
         B-fromloc.city_name      0.985     1.000     0.992     704.0
           I-toloc.city_name      0.981     0.996     0.989     265.0
      B-depart_date.day_name      0.977     0.986     0.981     212.0
              B-airline_name      1.000     0.990     0.995     101.0
         I-fromloc.city_name      0.989     0.994     0.992     177.0
 B-depart_time.period_of_day      1.000     0.885     0.939     130.0
              I-airline_name      1.000     1.000     1.000      65.0
    B-depart_date.day_number      0.981     0.964     0.972      55.0
    B-depart_date.month_name      0.982     0.964     0.973      56.0
          B-depart_time.time      0.864     1.000     0.927      57.0
                B-round_trip      1.000     0.986     0.993      73.0
             B-cost_relative      1.000     0.973     0.986      37.0
                I-round_trip      1.000     1.000     1.000      71.0
                B-flight_mod      0.828     1.000     0.906      24.0
 B-depart_time.time_relative      0.969     0.969     0.969      65.0
          I-depart_time.time      0.945     1.000     0.972      52.0
         B-stoploc.city_name      1.000     1.000     1.000      20.0
                 B-city_name      0.909     0.526     0.667      57.0
                B-class_type      0.960     1.000     0.980      24.0
          B-arrive_time.time      0.917     0.971     0.943      34.0
 B-arrive_time.time_relative      0.906     0.935     0.921      31.0
                I-class_type      1.000     1.000     1.000      17.0
               B-flight_stop      1.000     1.000     1.000      21.0
          I-arrive_time.time      0.971     0.971     0.971      35.0
              B-airline_code      1.000     0.912     0.954      34.0
    I-depart_date.day_number      1.000     0.933     0.966      15.0
      I-fromloc.airport_name      0.405     1.000     0.577      15.0
      B-fromloc.airport_name      0.370     0.833     0.513      12.0
      B-arrive_date.day_name      0.786     1.000     0.880      11.0
          B-toloc.state_code      1.000     1.000     1.000      18.0
B-depart_date.today_relative      1.000     0.889     0.941       9.0
             B-flight_number      0.611     1.000     0.759      11.0
 B-depart_date.date_relative      0.944     1.000     0.971      17.0
          B-toloc.state_name      0.844     0.964     0.900      28.0
           B-fare_basis_code      0.944     1.000     0.971      17.0
               B-flight_time      1.000     1.000     1.000       1.0
                        B-or      1.000     1.000     1.000       3.0
 B-arrive_time.period_of_day      0.545     1.000     0.706       6.0
          B-meal_description      1.000     0.900     0.947      10.0
             I-cost_relative      1.000     0.667     0.800       3.0
              I-airport_name      0.909     0.345     0.500      29.0
               B-fare_amount      0.500     1.000     0.667       2.0
               I-fare_amount      1.000     1.000     1.000       2.0
                 I-city_name      1.000     0.667     0.800      30.0
        I-toloc.airport_name      1.000     1.000     1.000       3.0
            B-transport_type      1.000     1.000     1.000      10.0
    B-arrive_date.month_name      0.714     0.833     0.769       6.0
    B-arrive_date.day_number      0.714     0.833     0.769       6.0
         I-stoploc.city_name      1.000     1.000     1.000      10.0
                      B-meal      1.000     1.000     1.000      16.0
        B-fromloc.state_code      1.000     1.000     1.000      23.0
    B-depart_time.period_mod      1.000     1.000     1.000       5.0
                   B-connect      1.000     1.000     1.000       6.0
               B-flight_days      1.000     1.000     1.000      10.0
        B-toloc.airport_name      1.000     1.000     1.000       3.0
        B-fromloc.state_name      1.000     1.000     1.000      17.0
              B-airport_name      0.700     0.333     0.452      21.0
                   B-economy      1.000     1.000     1.000       6.0
               I-flight_time      1.000     1.000     1.000       1.0
             B-aircraft_code      0.929     0.788     0.852      33.0
                       B-mod      1.000     0.500     0.667       2.0
              B-airport_code      0.667     0.444     0.533       9.0
    B-depart_time.start_time      0.600     1.000     0.750       3.0
      B-depart_time.end_time      1.000     1.000     1.000       3.0
          B-depart_date.year      0.600     1.000     0.750       3.0
            I-transport_type      0.000     0.000     0.000       1.0
          B-restriction_code      1.000     1.000     1.000       4.0
    B-arrive_time.start_time      1.000     1.000     1.000       8.0
        B-toloc.airport_code      1.000     0.750     0.857       4.0
      B-arrive_time.end_time      1.000     0.875     0.933       8.0
      I-arrive_time.end_time      0.875     0.875     0.875       8.0
      I-depart_time.end_time      1.000     0.667     0.800       3.0
               I-flight_stop      0.000     0.000     0.000       0.0
      B-fromloc.airport_code      0.833     1.000     0.909       5.0
          I-restriction_code      1.000     1.000     1.000       3.0
    I-depart_time.start_time      1.000     1.000     1.000       1.0
          I-toloc.state_name      1.000     1.000     1.000       1.0
I-depart_date.today_relative      0.000     0.000     0.000       0.0
 B-arrive_date.date_relative      1.000     1.000     1.000       2.0
                I-flight_mod      1.000     0.167     0.286       6.0
                   I-economy      0.000     0.000     0.000       0.0
 B-return_date.date_relative      0.000     0.000     0.000       3.0
        I-fromloc.state_name      1.000     1.000     1.000       1.0
                B-state_code      1.000     1.000     1.000       1.0
    I-arrive_time.start_time      1.000     1.000     1.000       1.0
    I-arrive_date.day_number      0.000     0.000     0.000       0.0
                 B-meal_code      0.000     0.000     0.000       1.0
 I-depart_time.period_of_day      1.000     1.000     1.000       1.0
                  B-day_name      1.000     0.500     0.667       2.0
             B-period_of_day      1.000     0.500     0.667       4.0
        B-stoploc.state_code      0.000     0.000     0.000       0.0
    B-return_date.month_name      0.000     0.000     0.000       0.0
    B-return_date.day_number      0.000     0.000     0.000       0.0
    B-arrive_time.period_mod      0.000     0.000     0.000       0.0
                 I-meal_code      0.000     0.000     0.000       0.0
        B-toloc.country_name      1.000     1.000     1.000       1.0
                 B-days_code      1.000     1.000     1.000       1.0
 I-arrive_time.period_of_day      0.000     0.000     0.000       0.0
            I-today_relative      0.000     0.000     0.000       0.0
 B-return_time.period_of_day      0.000     0.000     0.000       0.0
                      B-time      0.000     0.000     0.000       0.0
           I-fare_basis_code      0.000     0.000     0.000       0.0
 I-arrive_time.time_relative      1.000     0.500     0.667       4.0
 I-depart_time.time_relative      0.000     0.000     0.000       1.0
            B-today_relative      0.000     0.000     0.000       0.0
                B-state_name      0.667     0.222     0.333       9.0
B-arrive_date.today_relative      0.000     0.000     0.000       0.0
    B-return_time.period_mod      0.000     0.000     0.000       0.0
                B-month_name      0.000     0.000     0.000       0.0
                B-day_number      0.000     0.000     0.000       0.0
 I-return_date.date_relative      0.667     0.667     0.667       3.0
I-return_date.today_relative      0.000     0.000     0.000       0.0
      B-stoploc.airport_name      0.000     0.000     0.000       0.0
             B-time_relative      0.000     0.000     0.000       0.0
                      I-time      0.000     0.000     0.000       0.0
    I-return_date.day_number      0.000     0.000     0.000       0.0
          I-meal_description      0.000     0.000     0.000       0.0
B-return_date.today_relative      0.000     0.000     0.000       0.0
      B-return_date.day_name      0.000     0.000     0.000       2.0

                   micro avg      0.954     0.956     0.955    3657.0
                   macro avg      0.677     0.658     0.654    3657.0
                weighted avg      0.961     0.956     0.953    3657.0

I0715 07:24:39.750338 140369481070464 interactiveshell.py:2882] Best Slot F1: 0.956
Reading ../data/atis.train.w-intent.iob
I0715 07:24:42.629687 140369481070464 interactiveshell.py:2882] Step 17500 | Loss: 0.0005 | Loss_intent: 0.0004 | Loss_slots: 0.0002 | Spent: 5.9 secs | LR: 0.000047
I0715 07:24:46.393825 140369481070464 interactiveshell.py:2882] Step 17550 | Loss: 0.0003 | Loss_intent: 0.0000 | Loss_slots: 0.0003 | Spent: 3.8 secs | LR: 0.000047
I0715 07:24:50.181716 140369481070464 interactiveshell.py:2882] Step 17600 | Loss: 0.0091 | Loss_intent: 0.0000 | Loss_slots: 0.0091 | Spent: 3.8 secs | LR: 0.000047
I0715 07:24:53.954009 140369481070464 interactiveshell.py:2882] Step 17650 | Loss: 0.0002 | Loss_intent: 0.0000 | Loss_slots: 0.0002 | Spent: 3.8 secs | LR: 0.000047
I0715 07:24:57.789540 140369481070464 interactiveshell.py:2882] Step 17700 | Loss: 0.0010 | Loss_intent: 0.0000 | Loss_slots: 0.0010 | Spent: 3.8 secs | LR: 0.000046
I0715 07:25:01.608942 140369481070464 interactiveshell.py:2882] Step 17750 | Loss: 0.0001 | Loss_intent: 0.0000 | Loss_slots: 0.0001 | Spent: 3.8 secs | LR: 0.000046
Reading ../data/atis.test.w-intent.iob
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.
  'precision', 'predicted', average, warn_for)
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1439: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples.
  'recall', 'true', average, warn_for)
I0715 07:25:05.580747 140369481070464 interactiveshell.py:2882] 
                                          precision    recall  f1-score   support

                             atis_flight      0.978     0.987     0.983       632
                            atis_airfare      0.960     1.000     0.980        48
                     atis_ground_service      1.000     1.000     1.000        36
                            atis_airline      1.000     1.000     1.000        38
                       atis_abbreviation      1.000     1.000     1.000        33
                           atis_aircraft      1.000     0.889     0.941         9
                        atis_flight_time      1.000     1.000     1.000         1
                           atis_quantity      0.429     1.000     0.600         3
                atis_flight#atis_airfare      0.833     0.417     0.556        12
                            atis_airport      1.000     0.944     0.971        18
                           atis_distance      1.000     1.000     1.000        10
                               atis_city      0.571     0.667     0.615         6
                        atis_ground_fare      1.000     0.857     0.923         7
                           atis_capacity      0.955     1.000     0.977        21
                          atis_flight_no      0.889     1.000     0.941         8
                               atis_meal      1.000     0.833     0.909         6
                        atis_restriction      0.000     0.000     0.000         0
             atis_airline#atis_flight_no      0.000     0.000     0.000         0
    atis_ground_service#atis_ground_fare      0.000     0.000     0.000         0
           atis_airfare#atis_flight_time      0.000     0.000     0.000         0
                           atis_cheapest      0.000     0.000     0.000         0
atis_aircraft#atis_flight#atis_flight_no      0.000     0.000     0.000         0

                               micro avg      0.971     0.976     0.974       888
                               macro avg      0.664     0.663     0.654       888
                            weighted avg      0.973     0.976     0.973       888

I0715 07:25:05.620797 140369481070464 interactiveshell.py:2882] 
                              precision    recall  f1-score   support

                           O      0.000     0.000     0.000       0.0
           B-toloc.city_name      0.975     0.992     0.983     716.0
         B-fromloc.city_name      0.987     1.000     0.994     704.0
           I-toloc.city_name      0.974     0.996     0.985     265.0
      B-depart_date.day_name      0.981     0.986     0.984     212.0
              B-airline_name      1.000     1.000     1.000     101.0
         I-fromloc.city_name      0.989     0.994     0.992     177.0
 B-depart_time.period_of_day      1.000     0.885     0.939     130.0
              I-airline_name      1.000     1.000     1.000      65.0
    B-depart_date.day_number      0.981     0.964     0.972      55.0
    B-depart_date.month_name      0.982     0.964     0.973      56.0
          B-depart_time.time      0.864     1.000     0.927      57.0
                B-round_trip      1.000     0.986     0.993      73.0
             B-cost_relative      1.000     0.973     0.986      37.0
                I-round_trip      1.000     1.000     1.000      71.0
                B-flight_mod      0.828     1.000     0.906      24.0
 B-depart_time.time_relative      0.984     0.954     0.969      65.0
          I-depart_time.time      0.945     1.000     0.972      52.0
         B-stoploc.city_name      0.952     1.000     0.976      20.0
                 B-city_name      0.914     0.561     0.696      57.0
                B-class_type      0.960     1.000     0.980      24.0
          B-arrive_time.time      0.917     0.971     0.943      34.0
 B-arrive_time.time_relative      0.882     0.968     0.923      31.0
                I-class_type      1.000     1.000     1.000      17.0
               B-flight_stop      1.000     1.000     1.000      21.0
          I-arrive_time.time      0.971     0.971     0.971      35.0
              B-airline_code      1.000     0.912     0.954      34.0
    I-depart_date.day_number      1.000     0.933     0.966      15.0
      I-fromloc.airport_name      0.405     1.000     0.577      15.0
      B-fromloc.airport_name      0.385     0.833     0.526      12.0
      B-arrive_date.day_name      0.733     1.000     0.846      11.0
          B-toloc.state_code      1.000     1.000     1.000      18.0
B-depart_date.today_relative      1.000     0.889     0.941       9.0
             B-flight_number      0.550     1.000     0.710      11.0
 B-depart_date.date_relative      0.944     1.000     0.971      17.0
          B-toloc.state_name      0.867     0.929     0.897      28.0
           B-fare_basis_code      0.944     1.000     0.971      17.0
               B-flight_time      1.000     1.000     1.000       1.0
                        B-or      1.000     1.000     1.000       3.0
 B-arrive_time.period_of_day      0.545     1.000     0.706       6.0
          B-meal_description      1.000     0.900     0.947      10.0
             I-cost_relative      1.000     0.667     0.800       3.0
              I-airport_name      0.909     0.345     0.500      29.0
               B-fare_amount      0.500     1.000     0.667       2.0
               I-fare_amount      1.000     1.000     1.000       2.0
                 I-city_name      1.000     0.600     0.750      30.0
        I-toloc.airport_name      1.000     1.000     1.000       3.0
            B-transport_type      1.000     0.900     0.947      10.0
    B-arrive_date.month_name      0.714     0.833     0.769       6.0
    B-arrive_date.day_number      0.714     0.833     0.769       6.0
         I-stoploc.city_name      1.000     1.000     1.000      10.0
                      B-meal      1.000     1.000     1.000      16.0
        B-fromloc.state_code      1.000     1.000     1.000      23.0
    B-depart_time.period_mod      1.000     1.000     1.000       5.0
                   B-connect      1.000     1.000     1.000       6.0
               B-flight_days      1.000     1.000     1.000      10.0
        B-toloc.airport_name      0.750     1.000     0.857       3.0
        B-fromloc.state_name      1.000     1.000     1.000      17.0
              B-airport_name      0.778     0.333     0.467      21.0
                   B-economy      1.000     1.000     1.000       6.0
               I-flight_time      1.000     1.000     1.000       1.0
             B-aircraft_code      0.962     0.758     0.847      33.0
                       B-mod      1.000     0.500     0.667       2.0
              B-airport_code      0.667     0.444     0.533       9.0
    B-depart_time.start_time      0.500     0.667     0.571       3.0
      B-depart_time.end_time      1.000     1.000     1.000       3.0
          B-depart_date.year      0.600     1.000     0.750       3.0
            I-transport_type      0.000     0.000     0.000       1.0
          B-restriction_code      1.000     1.000     1.000       4.0
    B-arrive_time.start_time      0.889     1.000     0.941       8.0
        B-toloc.airport_code      1.000     0.750     0.857       4.0
      B-arrive_time.end_time      1.000     0.875     0.933       8.0
      I-arrive_time.end_time      0.875     0.875     0.875       8.0
      I-depart_time.end_time      1.000     0.667     0.800       3.0
               I-flight_stop      0.000     0.000     0.000       0.0
      B-fromloc.airport_code      0.556     1.000     0.714       5.0
          I-restriction_code      1.000     1.000     1.000       3.0
    I-depart_time.start_time      1.000     1.000     1.000       1.0
          I-toloc.state_name      1.000     1.000     1.000       1.0
I-depart_date.today_relative      0.000     0.000     0.000       0.0
 B-arrive_date.date_relative      1.000     1.000     1.000       2.0
                I-flight_mod      1.000     0.167     0.286       6.0
                   I-economy      0.000     0.000     0.000       0.0
 B-return_date.date_relative      0.000     0.000     0.000       3.0
        I-fromloc.state_name      1.000     1.000     1.000       1.0
                B-state_code      1.000     1.000     1.000       1.0
    I-arrive_time.start_time      1.000     1.000     1.000       1.0
    I-arrive_date.day_number      0.000     0.000     0.000       0.0
                 B-meal_code      0.000     0.000     0.000       1.0
 I-depart_time.period_of_day      1.000     1.000     1.000       1.0
                  B-day_name      1.000     0.500     0.667       2.0
             B-period_of_day      1.000     0.500     0.667       4.0
        B-stoploc.state_code      0.000     0.000     0.000       0.0
    B-return_date.month_name      0.000     0.000     0.000       0.0
    B-return_date.day_number      0.000     0.000     0.000       0.0
    B-arrive_time.period_mod      0.000     0.000     0.000       0.0
                 I-meal_code      0.000     0.000     0.000       0.0
        B-toloc.country_name      1.000     1.000     1.000       1.0
                 B-days_code      1.000     1.000     1.000       1.0
 I-arrive_time.period_of_day      0.000     0.000     0.000       0.0
            I-today_relative      0.000     0.000     0.000       0.0
 B-return_time.period_of_day      0.000     0.000     0.000       0.0
                      B-time      0.000     0.000     0.000       0.0
           I-fare_basis_code      0.000     0.000     0.000       0.0
 I-arrive_time.time_relative      1.000     0.500     0.667       4.0
 I-depart_time.time_relative      0.000     0.000     0.000       1.0
            B-today_relative      0.000     0.000     0.000       0.0
                B-state_name      0.800     0.444     0.571       9.0
B-arrive_date.today_relative      0.000     0.000     0.000       0.0
    B-return_time.period_mod      0.000     0.000     0.000       0.0
                B-month_name      0.000     0.000     0.000       0.0
                B-day_number      0.000     0.000     0.000       0.0
 I-return_date.date_relative      0.667     0.667     0.667       3.0
I-return_date.today_relative      0.000     0.000     0.000       0.0
      B-stoploc.airport_name      0.000     0.000     0.000       0.0
             B-time_relative      0.000     0.000     0.000       0.0
                      I-time      0.000     0.000     0.000       0.0
    I-return_date.day_number      0.000     0.000     0.000       0.0
          I-meal_description      0.000     0.000     0.000       0.0
B-return_date.today_relative      0.000     0.000     0.000       0.0
      B-return_date.day_name      0.000     0.000     0.000       2.0

                   micro avg      0.953     0.955     0.954    3657.0
                   macro avg      0.672     0.656     0.650    3657.0
                weighted avg      0.962     0.955     0.953    3657.0

I0715 07:25:05.621670 140369481070464 interactiveshell.py:2882] Best Slot F1: 0.956
Reading ../data/atis.train.w-intent.iob
I0715 07:25:07.593933 140369481070464 interactiveshell.py:2882] Step 17800 | Loss: 0.0012 | Loss_intent: 0.0000 | Loss_slots: 0.0012 | Spent: 6.0 secs | LR: 0.000046
I0715 07:25:11.365247 140369481070464 interactiveshell.py:2882] Step 17850 | Loss: 0.0002 | Loss_intent: 0.0000 | Loss_slots: 0.0002 | Spent: 3.8 secs | LR: 0.000046
I0715 07:25:15.113974 140369481070464 interactiveshell.py:2882] Step 17900 | Loss: 0.0060 | Loss_intent: 0.0001 | Loss_slots: 0.0060 | Spent: 3.7 secs | LR: 0.000046
I0715 07:25:18.880216 140369481070464 interactiveshell.py:2882] Step 17950 | Loss: 0.0000 | Loss_intent: 0.0000 | Loss_slots: 0.0000 | Spent: 3.8 secs | LR: 0.000045
I0715 07:25:22.602116 140369481070464 interactiveshell.py:2882] Step 18000 | Loss: 0.0002 | Loss_intent: 0.0000 | Loss_slots: 0.0002 | Spent: 3.7 secs | LR: 0.000045
I0715 07:25:26.351345 140369481070464 interactiveshell.py:2882] Step 18050 | Loss: 0.0018 | Loss_intent: 0.0000 | Loss_slots: 0.0018 | Spent: 3.7 secs | LR: 0.000045
Reading ../data/atis.test.w-intent.iob
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.
  'precision', 'predicted', average, warn_for)
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1439: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples.
  'recall', 'true', average, warn_for)
I0715 07:25:31.193309 140369481070464 interactiveshell.py:2882] 
                                          precision    recall  f1-score   support

                             atis_flight      0.980     0.992     0.986       632
                            atis_airfare      0.960     1.000     0.980        48
                     atis_ground_service      1.000     1.000     1.000        36
                            atis_airline      1.000     1.000     1.000        38
                       atis_abbreviation      0.943     1.000     0.971        33
                           atis_aircraft      1.000     0.889     0.941         9
                        atis_flight_time      1.000     1.000     1.000         1
                           atis_quantity      0.429     1.000     0.600         3
                atis_flight#atis_airfare      0.833     0.417     0.556        12
                            atis_airport      1.000     0.889     0.941        18
                           atis_distance      1.000     1.000     1.000        10
                               atis_city      1.000     0.667     0.800         6
                        atis_ground_fare      1.000     0.857     0.923         7
                           atis_capacity      0.955     1.000     0.977        21
                          atis_flight_no      0.889     1.000     0.941         8
                               atis_meal      1.000     0.833     0.909         6
                        atis_restriction      0.000     0.000     0.000         0
             atis_airline#atis_flight_no      0.000     0.000     0.000         0
    atis_ground_service#atis_ground_fare      0.000     0.000     0.000         0
           atis_airfare#atis_flight_time      0.000     0.000     0.000         0
                           atis_cheapest      0.000     0.000     0.000         0
atis_aircraft#atis_flight#atis_flight_no      0.000     0.000     0.000         0

                               micro avg      0.973     0.979     0.976       888
                               macro avg      0.681     0.661     0.660       888
                            weighted avg      0.975     0.979     0.975       888

I0715 07:25:31.236798 140369481070464 interactiveshell.py:2882] 
                              precision    recall  f1-score   support

                           O      0.000     0.000     0.000       0.0
           B-toloc.city_name      0.973     0.993     0.983     716.0
         B-fromloc.city_name      0.983     1.000     0.992     704.0
           I-toloc.city_name      0.978     0.985     0.981     265.0
      B-depart_date.day_name      0.981     0.986     0.984     212.0
              B-airline_name      1.000     1.000     1.000     101.0
         I-fromloc.city_name      0.978     0.994     0.986     177.0
 B-depart_time.period_of_day      0.991     0.885     0.935     130.0
              I-airline_name      0.984     0.969     0.977      65.0
    B-depart_date.day_number      0.981     0.964     0.972      55.0
    B-depart_date.month_name      0.982     0.964     0.973      56.0
          B-depart_time.time      0.864     1.000     0.927      57.0
                B-round_trip      1.000     0.986     0.993      73.0
             B-cost_relative      1.000     0.973     0.986      37.0
                I-round_trip      0.986     1.000     0.993      71.0
                B-flight_mod      0.828     1.000     0.906      24.0
 B-depart_time.time_relative      0.984     0.969     0.977      65.0
          I-depart_time.time      0.945     1.000     0.972      52.0
         B-stoploc.city_name      1.000     1.000     1.000      20.0
                 B-city_name      0.938     0.526     0.674      57.0
                B-class_type      0.960     1.000     0.980      24.0
          B-arrive_time.time      0.917     0.971     0.943      34.0
 B-arrive_time.time_relative      0.909     0.968     0.937      31.0
                I-class_type      1.000     1.000     1.000      17.0
               B-flight_stop      1.000     1.000     1.000      21.0
          I-arrive_time.time      1.000     0.971     0.986      35.0
              B-airline_code      1.000     0.912     0.954      34.0
    I-depart_date.day_number      1.000     0.933     0.966      15.0
      I-fromloc.airport_name      0.417     1.000     0.588      15.0
      B-fromloc.airport_name      0.346     0.750     0.474      12.0
      B-arrive_date.day_name      0.733     1.000     0.846      11.0
          B-toloc.state_code      1.000     1.000     1.000      18.0
B-depart_date.today_relative      1.000     0.889     0.941       9.0
             B-flight_number      0.579     1.000     0.733      11.0
 B-depart_date.date_relative      0.944     1.000     0.971      17.0
          B-toloc.state_name      0.931     0.964     0.947      28.0
           B-fare_basis_code      0.944     1.000     0.971      17.0
               B-flight_time      1.000     1.000     1.000       1.0
                        B-or      1.000     1.000     1.000       3.0
 B-arrive_time.period_of_day      0.545     1.000     0.706       6.0
          B-meal_description      1.000     0.900     0.947      10.0
             I-cost_relative      1.000     0.667     0.800       3.0
              I-airport_name      0.909     0.345     0.500      29.0
               B-fare_amount      0.500     1.000     0.667       2.0
               I-fare_amount      1.000     1.000     1.000       2.0
                 I-city_name      1.000     0.633     0.776      30.0
        I-toloc.airport_name      1.000     1.000     1.000       3.0
            B-transport_type      1.000     1.000     1.000      10.0
    B-arrive_date.month_name      0.714     0.833     0.769       6.0
    B-arrive_date.day_number      0.714     0.833     0.769       6.0
         I-stoploc.city_name      0.833     1.000     0.909      10.0
                      B-meal      1.000     1.000     1.000      16.0
        B-fromloc.state_code      1.000     1.000     1.000      23.0
    B-depart_time.period_mod      1.000     0.800     0.889       5.0
                   B-connect      1.000     1.000     1.000       6.0
               B-flight_days      1.000     1.000     1.000      10.0
        B-toloc.airport_name      1.000     1.000     1.000       3.0
        B-fromloc.state_name      1.000     1.000     1.000      17.0
              B-airport_name      0.778     0.333     0.467      21.0
                   B-economy      1.000     1.000     1.000       6.0
               I-flight_time      1.000     1.000     1.000       1.0
             B-aircraft_code      0.926     0.758     0.833      33.0
                       B-mod      1.000     0.500     0.667       2.0
              B-airport_code      0.667     0.444     0.533       9.0
    B-depart_time.start_time      0.750     1.000     0.857       3.0
      B-depart_time.end_time      1.000     0.667     0.800       3.0
          B-depart_date.year      0.600     1.000     0.750       3.0
            I-transport_type      0.000     0.000     0.000       1.0
          B-restriction_code      1.000     1.000     1.000       4.0
    B-arrive_time.start_time      1.000     1.000     1.000       8.0
        B-toloc.airport_code      1.000     0.750     0.857       4.0
      B-arrive_time.end_time      0.875     0.875     0.875       8.0
      I-arrive_time.end_time      0.889     1.000     0.941       8.0
      I-depart_time.end_time      1.000     0.667     0.800       3.0
               I-flight_stop      0.000     0.000     0.000       0.0
      B-fromloc.airport_code      0.556     1.000     0.714       5.0
          I-restriction_code      1.000     1.000     1.000       3.0
    I-depart_time.start_time      1.000     1.000     1.000       1.0
          I-toloc.state_name      1.000     1.000     1.000       1.0
I-depart_date.today_relative      0.000     0.000     0.000       0.0
 B-arrive_date.date_relative      1.000     1.000     1.000       2.0
                I-flight_mod      1.000     0.167     0.286       6.0
                   I-economy      0.000     0.000     0.000       0.0
 B-return_date.date_relative      0.000     0.000     0.000       3.0
        I-fromloc.state_name      1.000     1.000     1.000       1.0
                B-state_code      1.000     1.000     1.000       1.0
    I-arrive_time.start_time      1.000     1.000     1.000       1.0
    I-arrive_date.day_number      0.000     0.000     0.000       0.0
                 B-meal_code      0.000     0.000     0.000       1.0
 I-depart_time.period_of_day      1.000     1.000     1.000       1.0
                  B-day_name      1.000     0.500     0.667       2.0
             B-period_of_day      1.000     0.500     0.667       4.0
        B-stoploc.state_code      0.000     0.000     0.000       0.0
    B-return_date.month_name      0.000     0.000     0.000       0.0
    B-return_date.day_number      0.000     0.000     0.000       0.0
    B-arrive_time.period_mod      0.000     0.000     0.000       0.0
                 I-meal_code      0.000     0.000     0.000       0.0
        B-toloc.country_name      1.000     1.000     1.000       1.0
                 B-days_code      1.000     1.000     1.000       1.0
 I-arrive_time.period_of_day      0.000     0.000     0.000       0.0
            I-today_relative      0.000     0.000     0.000       0.0
 B-return_time.period_of_day      0.000     0.000     0.000       0.0
                      B-time      0.000     0.000     0.000       0.0
           I-fare_basis_code      0.000     0.000     0.000       0.0
 I-arrive_time.time_relative      1.000     0.500     0.667       4.0
 I-depart_time.time_relative      0.000     0.000     0.000       1.0
            B-today_relative      0.000     0.000     0.000       0.0
                B-state_name      0.857     0.667     0.750       9.0
B-arrive_date.today_relative      0.000     0.000     0.000       0.0
    B-return_time.period_mod      0.000     0.000     0.000       0.0
                B-month_name      0.000     0.000     0.000       0.0
                B-day_number      0.000     0.000     0.000       0.0
 I-return_date.date_relative      0.667     0.667     0.667       3.0
I-return_date.today_relative      0.000     0.000     0.000       0.0
      B-stoploc.airport_name      0.000     0.000     0.000       0.0
             B-time_relative      0.000     0.000     0.000       0.0
                      I-time      0.000     0.000     0.000       0.0
    I-return_date.day_number      0.000     0.000     0.000       0.0
          I-meal_description      0.000     0.000     0.000       0.0
B-return_date.today_relative      0.000     0.000     0.000       0.0
      B-return_date.day_name      0.000     0.000     0.000       2.0

                   micro avg      0.953     0.955     0.954    3657.0
                   macro avg      0.676     0.658     0.653    3657.0
                weighted avg      0.961     0.955     0.952    3657.0

I0715 07:25:31.237628 140369481070464 interactiveshell.py:2882] Best Slot F1: 0.956
Reading ../data/atis.train.w-intent.iob
I0715 07:25:32.311006 140369481070464 interactiveshell.py:2882] Step 18100 | Loss: 0.0004 | Loss_intent: 0.0000 | Loss_slots: 0.0004 | Spent: 6.0 secs | LR: 0.000045
I0715 07:25:36.075880 140369481070464 interactiveshell.py:2882] Step 18150 | Loss: 0.0010 | Loss_intent: 0.0000 | Loss_slots: 0.0010 | Spent: 3.8 secs | LR: 0.000044
I0715 07:25:39.860780 140369481070464 interactiveshell.py:2882] Step 18200 | Loss: 0.0002 | Loss_intent: 0.0000 | Loss_slots: 0.0002 | Spent: 3.8 secs | LR: 0.000044
I0715 07:25:43.624301 140369481070464 interactiveshell.py:2882] Step 18250 | Loss: 0.0008 | Loss_intent: 0.0000 | Loss_slots: 0.0008 | Spent: 3.8 secs | LR: 0.000044
I0715 07:25:47.369841 140369481070464 interactiveshell.py:2882] Step 18300 | Loss: 0.0003 | Loss_intent: 0.0000 | Loss_slots: 0.0003 | Spent: 3.7 secs | LR: 0.000044
I0715 07:25:51.140519 140369481070464 interactiveshell.py:2882] Step 18350 | Loss: 0.0003 | Loss_intent: 0.0000 | Loss_slots: 0.0003 | Spent: 3.8 secs | LR: 0.000043
I0715 07:25:54.923210 140369481070464 interactiveshell.py:2882] Step 18400 | Loss: 0.0007 | Loss_intent: 0.0000 | Loss_slots: 0.0007 | Spent: 3.8 secs | LR: 0.000043
Reading ../data/atis.test.w-intent.iob
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.
  'precision', 'predicted', average, warn_for)
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1439: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples.
  'recall', 'true', average, warn_for)
I0715 07:25:56.889292 140369481070464 interactiveshell.py:2882] 
                                          precision    recall  f1-score   support

                             atis_flight      0.980     0.992     0.986       632
                            atis_airfare      0.960     1.000     0.980        48
                     atis_ground_service      1.000     1.000     1.000        36
                            atis_airline      1.000     1.000     1.000        38
                       atis_abbreviation      0.943     1.000     0.971        33
                           atis_aircraft      1.000     0.889     0.941         9
                        atis_flight_time      1.000     1.000     1.000         1
                           atis_quantity      0.429     1.000     0.600         3
                atis_flight#atis_airfare      0.833     0.417     0.556        12
                            atis_airport      1.000     0.889     0.941        18
                           atis_distance      1.000     1.000     1.000        10
                               atis_city      1.000     0.667     0.800         6
                        atis_ground_fare      1.000     0.857     0.923         7
                           atis_capacity      0.955     1.000     0.977        21
                          atis_flight_no      0.889     1.000     0.941         8
                               atis_meal      1.000     0.833     0.909         6
                        atis_restriction      0.000     0.000     0.000         0
             atis_airline#atis_flight_no      0.000     0.000     0.000         0
    atis_ground_service#atis_ground_fare      0.000     0.000     0.000         0
           atis_airfare#atis_flight_time      0.000     0.000     0.000         0
                           atis_cheapest      0.000     0.000     0.000         0
atis_aircraft#atis_flight#atis_flight_no      0.000     0.000     0.000         0

                               micro avg      0.973     0.979     0.976       888
                               macro avg      0.681     0.661     0.660       888
                            weighted avg      0.975     0.979     0.975       888

I0715 07:25:56.929153 140369481070464 interactiveshell.py:2882] 
                              precision    recall  f1-score   support

                           O      0.000     0.000     0.000       0.0
           B-toloc.city_name      0.978     0.993     0.985     716.0
         B-fromloc.city_name      0.987     0.997     0.992     704.0
           I-toloc.city_name      0.978     0.996     0.987     265.0
      B-depart_date.day_name      0.986     0.986     0.986     212.0
              B-airline_name      1.000     1.000     1.000     101.0
         I-fromloc.city_name      0.983     0.994     0.989     177.0
 B-depart_time.period_of_day      0.991     0.892     0.939     130.0
              I-airline_name      0.985     1.000     0.992      65.0
    B-depart_date.day_number      0.981     0.964     0.972      55.0
    B-depart_date.month_name      0.982     0.964     0.973      56.0
          B-depart_time.time      0.864     1.000     0.927      57.0
                B-round_trip      1.000     0.986     0.993      73.0
             B-cost_relative      1.000     0.973     0.986      37.0
                I-round_trip      1.000     1.000     1.000      71.0
                B-flight_mod      0.828     1.000     0.906      24.0
 B-depart_time.time_relative      0.984     0.969     0.977      65.0
          I-depart_time.time      0.945     1.000     0.972      52.0
         B-stoploc.city_name      0.952     1.000     0.976      20.0
                 B-city_name      0.889     0.561     0.688      57.0
                B-class_type      0.960     1.000     0.980      24.0
          B-arrive_time.time      0.917     0.971     0.943      34.0
 B-arrive_time.time_relative      0.968     0.968     0.968      31.0
                I-class_type      1.000     1.000     1.000      17.0
               B-flight_stop      1.000     1.000     1.000      21.0
          I-arrive_time.time      1.000     0.971     0.986      35.0
              B-airline_code      1.000     0.912     0.954      34.0
    I-depart_date.day_number      1.000     0.933     0.966      15.0
      I-fromloc.airport_name      0.417     1.000     0.588      15.0
      B-fromloc.airport_name      0.400     1.000     0.571      12.0
      B-arrive_date.day_name      0.733     1.000     0.846      11.0
          B-toloc.state_code      1.000     1.000     1.000      18.0
B-depart_date.today_relative      1.000     1.000     1.000       9.0
             B-flight_number      0.550     1.000     0.710      11.0
 B-depart_date.date_relative      0.944     1.000     0.971      17.0
          B-toloc.state_name      0.800     1.000     0.889      28.0
           B-fare_basis_code      0.944     1.000     0.971      17.0
               B-flight_time      1.000     1.000     1.000       1.0
                        B-or      1.000     1.000     1.000       3.0
 B-arrive_time.period_of_day      0.600     1.000     0.750       6.0
          B-meal_description      1.000     0.900     0.947      10.0
             I-cost_relative      1.000     0.667     0.800       3.0
              I-airport_name      0.909     0.345     0.500      29.0
               B-fare_amount      0.500     1.000     0.667       2.0
               I-fare_amount      1.000     1.000     1.000       2.0
                 I-city_name      1.000     0.633     0.776      30.0
        I-toloc.airport_name      1.000     1.000     1.000       3.0
            B-transport_type      1.000     1.000     1.000      10.0
    B-arrive_date.month_name      0.714     0.833     0.769       6.0
    B-arrive_date.day_number      0.714     0.833     0.769       6.0
         I-stoploc.city_name      1.000     0.900     0.947      10.0
                      B-meal      1.000     1.000     1.000      16.0
        B-fromloc.state_code      1.000     1.000     1.000      23.0
    B-depart_time.period_mod      1.000     1.000     1.000       5.0
                   B-connect      1.000     1.000     1.000       6.0
               B-flight_days      1.000     1.000     1.000      10.0
        B-toloc.airport_name      0.750     1.000     0.857       3.0
        B-fromloc.state_name      1.000     0.941     0.970      17.0
              B-airport_name      0.778     0.333     0.467      21.0
                   B-economy      1.000     1.000     1.000       6.0
               I-flight_time      1.000     1.000     1.000       1.0
             B-aircraft_code      0.958     0.697     0.807      33.0
                       B-mod      1.000     0.500     0.667       2.0
              B-airport_code      0.667     0.444     0.533       9.0
    B-depart_time.start_time      0.600     1.000     0.750       3.0
      B-depart_time.end_time      1.000     0.667     0.800       3.0
          B-depart_date.year      0.600     1.000     0.750       3.0
            I-transport_type      0.000     0.000     0.000       1.0
          B-restriction_code      1.000     1.000     1.000       4.0
    B-arrive_time.start_time      1.000     1.000     1.000       8.0
        B-toloc.airport_code      1.000     0.750     0.857       4.0
      B-arrive_time.end_time      0.875     0.875     0.875       8.0
      I-arrive_time.end_time      0.889     1.000     0.941       8.0
      I-depart_time.end_time      1.000     0.667     0.800       3.0
               I-flight_stop      0.000     0.000     0.000       0.0
      B-fromloc.airport_code      0.714     1.000     0.833       5.0
          I-restriction_code      1.000     1.000     1.000       3.0
    I-depart_time.start_time      1.000     1.000     1.000       1.0
          I-toloc.state_name      1.000     1.000     1.000       1.0
I-depart_date.today_relative      0.000     0.000     0.000       0.0
 B-arrive_date.date_relative      1.000     1.000     1.000       2.0
                I-flight_mod      1.000     0.167     0.286       6.0
                   I-economy      0.000     0.000     0.000       0.0
 B-return_date.date_relative      0.000     0.000     0.000       3.0
        I-fromloc.state_name      1.000     1.000     1.000       1.0
                B-state_code      1.000     1.000     1.000       1.0
    I-arrive_time.start_time      1.000     1.000     1.000       1.0
    I-arrive_date.day_number      0.000     0.000     0.000       0.0
                 B-meal_code      0.000     0.000     0.000       1.0
 I-depart_time.period_of_day      1.000     1.000     1.000       1.0
                  B-day_name      1.000     0.500     0.667       2.0
             B-period_of_day      1.000     0.500     0.667       4.0
        B-stoploc.state_code      0.000     0.000     0.000       0.0
    B-return_date.month_name      0.000     0.000     0.000       0.0
    B-return_date.day_number      0.000     0.000     0.000       0.0
    B-arrive_time.period_mod      0.000     0.000     0.000       0.0
                 I-meal_code      0.000     0.000     0.000       0.0
        B-toloc.country_name      1.000     1.000     1.000       1.0
                 B-days_code      1.000     1.000     1.000       1.0
 I-arrive_time.period_of_day      0.000     0.000     0.000       0.0
            I-today_relative      0.000     0.000     0.000       0.0
 B-return_time.period_of_day      0.000     0.000     0.000       0.0
                      B-time      0.000     0.000     0.000       0.0
           I-fare_basis_code      0.000     0.000     0.000       0.0
 I-arrive_time.time_relative      1.000     1.000     1.000       4.0
 I-depart_time.time_relative      0.000     0.000     0.000       1.0
            B-today_relative      0.000     0.000     0.000       0.0
                B-state_name      0.500     0.111     0.182       9.0
B-arrive_date.today_relative      0.000     0.000     0.000       0.0
    B-return_time.period_mod      0.000     0.000     0.000       0.0
                B-month_name      0.000     0.000     0.000       0.0
                B-day_number      0.000     0.000     0.000       0.0
 I-return_date.date_relative      0.667     0.667     0.667       3.0
I-return_date.today_relative      0.000     0.000     0.000       0.0
      B-stoploc.airport_name      0.000     0.000     0.000       0.0
             B-time_relative      0.000     0.000     0.000       0.0
                      I-time      0.000     0.000     0.000       0.0
    I-return_date.day_number      0.000     0.000     0.000       0.0
          I-meal_description      0.000     0.000     0.000       0.0
B-return_date.today_relative      0.000     0.000     0.000       0.0
      B-return_date.day_name      0.000     0.000     0.000       2.0

                   micro avg      0.954     0.956     0.955    3657.0
                   macro avg      0.673     0.661     0.652    3657.0
                weighted avg      0.962     0.956     0.953    3657.0

I0715 07:25:56.930126 140369481070464 interactiveshell.py:2882] Best Slot F1: 0.956
Reading ../data/atis.train.w-intent.iob
I0715 07:26:01.700378 140369481070464 interactiveshell.py:2882] Step 18450 | Loss: 0.0000 | Loss_intent: 0.0000 | Loss_slots: 0.0000 | Spent: 6.8 secs | LR: 0.000043
I0715 07:26:06.545595 140369481070464 interactiveshell.py:2882] Step 18500 | Loss: 0.0001 | Loss_intent: 0.0000 | Loss_slots: 0.0001 | Spent: 4.8 secs | LR: 0.000043
I0715 07:26:10.319256 140369481070464 interactiveshell.py:2882] Step 18550 | Loss: 0.0005 | Loss_intent: 0.0000 | Loss_slots: 0.0005 | Spent: 3.8 secs | LR: 0.000042
I0715 07:26:14.102504 140369481070464 interactiveshell.py:2882] Step 18600 | Loss: 0.0005 | Loss_intent: 0.0000 | Loss_slots: 0.0005 | Spent: 3.8 secs | LR: 0.000042
I0715 07:26:17.932804 140369481070464 interactiveshell.py:2882] Step 18650 | Loss: 0.0013 | Loss_intent: 0.0000 | Loss_slots: 0.0013 | Spent: 3.8 secs | LR: 0.000042
I0715 07:26:21.749472 140369481070464 interactiveshell.py:2882] Step 18700 | Loss: 0.0014 | Loss_intent: 0.0000 | Loss_slots: 0.0014 | Spent: 3.8 secs | LR: 0.000042
Reading ../data/atis.test.w-intent.iob
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.
  'precision', 'predicted', average, warn_for)
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1439: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples.
  'recall', 'true', average, warn_for)
I0715 07:26:24.816116 140369481070464 interactiveshell.py:2882] 
                                          precision    recall  f1-score   support

                             atis_flight      0.981     0.989     0.985       632
                            atis_airfare      0.960     1.000     0.980        48
                     atis_ground_service      1.000     1.000     1.000        36
                            atis_airline      1.000     1.000     1.000        38
                       atis_abbreviation      0.943     1.000     0.971        33
                           atis_aircraft      1.000     0.889     0.941         9
                        atis_flight_time      1.000     1.000     1.000         1
                           atis_quantity      0.429     1.000     0.600         3
                atis_flight#atis_airfare      0.833     0.417     0.556        12
                            atis_airport      1.000     0.944     0.971        18
                           atis_distance      1.000     1.000     1.000        10
                               atis_city      0.667     0.667     0.667         6
                        atis_ground_fare      1.000     0.857     0.923         7
                           atis_capacity      0.955     1.000     0.977        21
                          atis_flight_no      0.889     1.000     0.941         8
                               atis_meal      1.000     0.833     0.909         6
                        atis_restriction      0.000     0.000     0.000         0
             atis_airline#atis_flight_no      0.000     0.000     0.000         0
    atis_ground_service#atis_ground_fare      0.000     0.000     0.000         0
           atis_airfare#atis_flight_time      0.000     0.000     0.000         0
                           atis_cheapest      0.000     0.000     0.000         0
atis_aircraft#atis_flight#atis_flight_no      0.000     0.000     0.000         0

                               micro avg      0.972     0.977     0.975       888
                               macro avg      0.666     0.663     0.655       888
                            weighted avg      0.974     0.977     0.974       888

I0715 07:26:24.856269 140369481070464 interactiveshell.py:2882] 
                              precision    recall  f1-score   support

                           O      0.000     0.000     0.000       0.0
           B-toloc.city_name      0.974     0.994     0.984     716.0
         B-fromloc.city_name      0.987     0.997     0.992     704.0
           I-toloc.city_name      0.981     0.996     0.989     265.0
      B-depart_date.day_name      0.981     0.986     0.984     212.0
              B-airline_name      1.000     1.000     1.000     101.0
         I-fromloc.city_name      0.983     0.994     0.989     177.0
 B-depart_time.period_of_day      0.991     0.885     0.935     130.0
              I-airline_name      1.000     0.985     0.992      65.0
    B-depart_date.day_number      0.981     0.964     0.972      55.0
    B-depart_date.month_name      0.982     0.964     0.973      56.0
          B-depart_time.time      0.851     1.000     0.919      57.0
                B-round_trip      1.000     0.986     0.993      73.0
             B-cost_relative      1.000     0.973     0.986      37.0
                I-round_trip      1.000     1.000     1.000      71.0
                B-flight_mod      0.828     1.000     0.906      24.0
 B-depart_time.time_relative      0.984     0.969     0.977      65.0
          I-depart_time.time      0.945     1.000     0.972      52.0
         B-stoploc.city_name      0.952     1.000     0.976      20.0
                 B-city_name      0.938     0.526     0.674      57.0
                B-class_type      0.960     1.000     0.980      24.0
          B-arrive_time.time      0.943     0.971     0.957      34.0
 B-arrive_time.time_relative      0.968     0.968     0.968      31.0
                I-class_type      1.000     1.000     1.000      17.0
               B-flight_stop      1.000     1.000     1.000      21.0
          I-arrive_time.time      1.000     0.971     0.986      35.0
              B-airline_code      1.000     0.912     0.954      34.0
    I-depart_date.day_number      1.000     0.933     0.966      15.0
      I-fromloc.airport_name      0.405     1.000     0.577      15.0
      B-fromloc.airport_name      0.400     1.000     0.571      12.0
      B-arrive_date.day_name      0.733     1.000     0.846      11.0
          B-toloc.state_code      1.000     1.000     1.000      18.0
B-depart_date.today_relative      1.000     1.000     1.000       9.0
             B-flight_number      0.524     1.000     0.688      11.0
 B-depart_date.date_relative      0.944     1.000     0.971      17.0
          B-toloc.state_name      0.800     1.000     0.889      28.0
           B-fare_basis_code      0.944     1.000     0.971      17.0
               B-flight_time      1.000     1.000     1.000       1.0
                        B-or      1.000     1.000     1.000       3.0
 B-arrive_time.period_of_day      0.600     1.000     0.750       6.0
          B-meal_description      1.000     0.900     0.947      10.0
             I-cost_relative      1.000     0.667     0.800       3.0
              I-airport_name      0.909     0.345     0.500      29.0
               B-fare_amount      0.500     1.000     0.667       2.0
               I-fare_amount      1.000     1.000     1.000       2.0
                 I-city_name      1.000     0.700     0.824      30.0
        I-toloc.airport_name      1.000     1.000     1.000       3.0
            B-transport_type      0.833     1.000     0.909      10.0
    B-arrive_date.month_name      0.714     0.833     0.769       6.0
    B-arrive_date.day_number      0.714     0.833     0.769       6.0
         I-stoploc.city_name      0.909     1.000     0.952      10.0
                      B-meal      1.000     1.000     1.000      16.0
        B-fromloc.state_code      1.000     1.000     1.000      23.0
    B-depart_time.period_mod      1.000     1.000     1.000       5.0
                   B-connect      1.000     1.000     1.000       6.0
               B-flight_days      1.000     1.000     1.000      10.0
        B-toloc.airport_name      1.000     1.000     1.000       3.0
        B-fromloc.state_name      1.000     0.941     0.970      17.0
              B-airport_name      0.778     0.333     0.467      21.0
                   B-economy      1.000     1.000     1.000       6.0
               I-flight_time      1.000     1.000     1.000       1.0
             B-aircraft_code      1.000     0.697     0.821      33.0
                       B-mod      1.000     0.500     0.667       2.0
              B-airport_code      0.667     0.444     0.533       9.0
    B-depart_time.start_time      0.600     1.000     0.750       3.0
      B-depart_time.end_time      1.000     0.667     0.800       3.0
          B-depart_date.year      0.750     1.000     0.857       3.0
            I-transport_type      0.000     0.000     0.000       1.0
          B-restriction_code      1.000     1.000     1.000       4.0
    B-arrive_time.start_time      1.000     1.000     1.000       8.0
        B-toloc.airport_code      1.000     0.750     0.857       4.0
      B-arrive_time.end_time      0.875     0.875     0.875       8.0
      I-arrive_time.end_time      0.889     1.000     0.941       8.0
      I-depart_time.end_time      1.000     0.667     0.800       3.0
               I-flight_stop      0.000     0.000     0.000       0.0
      B-fromloc.airport_code      0.714     1.000     0.833       5.0
          I-restriction_code      1.000     1.000     1.000       3.0
    I-depart_time.start_time      1.000     1.000     1.000       1.0
          I-toloc.state_name      1.000     1.000     1.000       1.0
I-depart_date.today_relative      0.000     0.000     0.000       0.0
 B-arrive_date.date_relative      1.000     1.000     1.000       2.0
                I-flight_mod      1.000     0.167     0.286       6.0
                   I-economy      0.000     0.000     0.000       0.0
 B-return_date.date_relative      0.000     0.000     0.000       3.0
        I-fromloc.state_name      1.000     1.000     1.000       1.0
                B-state_code      1.000     1.000     1.000       1.0
    I-arrive_time.start_time      1.000     1.000     1.000       1.0
    I-arrive_date.day_number      0.000     0.000     0.000       0.0
                 B-meal_code      0.000     0.000     0.000       1.0
 I-depart_time.period_of_day      1.000     1.000     1.000       1.0
                  B-day_name      1.000     0.500     0.667       2.0
             B-period_of_day      1.000     0.500     0.667       4.0
        B-stoploc.state_code      0.000     0.000     0.000       0.0
    B-return_date.month_name      0.000     0.000     0.000       0.0
    B-return_date.day_number      0.000     0.000     0.000       0.0
    B-arrive_time.period_mod      0.000     0.000     0.000       0.0
                 I-meal_code      0.000     0.000     0.000       0.0
        B-toloc.country_name      1.000     1.000     1.000       1.0
                 B-days_code      1.000     1.000     1.000       1.0
 I-arrive_time.period_of_day      0.000     0.000     0.000       0.0
            I-today_relative      0.000     0.000     0.000       0.0
 B-return_time.period_of_day      0.000     0.000     0.000       0.0
                      B-time      0.000     0.000     0.000       0.0
           I-fare_basis_code      0.000     0.000     0.000       0.0
 I-arrive_time.time_relative      1.000     1.000     1.000       4.0
 I-depart_time.time_relative      0.000     0.000     0.000       1.0
            B-today_relative      0.000     0.000     0.000       0.0
                B-state_name      0.500     0.111     0.182       9.0
B-arrive_date.today_relative      0.000     0.000     0.000       0.0
    B-return_time.period_mod      0.000     0.000     0.000       0.0
                B-month_name      0.000     0.000     0.000       0.0
                B-day_number      0.000     0.000     0.000       0.0
 I-return_date.date_relative      0.667     0.667     0.667       3.0
I-return_date.today_relative      0.000     0.000     0.000       0.0
      B-stoploc.airport_name      0.000     0.000     0.000       0.0
             B-time_relative      0.000     0.000     0.000       0.0
                      I-time      0.000     0.000     0.000       0.0
    I-return_date.day_number      0.000     0.000     0.000       0.0
          I-meal_description      0.000     0.000     0.000       0.0
B-return_date.today_relative      0.000     0.000     0.000       0.0
      B-return_date.day_name      0.000     0.000     0.000       2.0

                   micro avg      0.954     0.956     0.955    3657.0
                   macro avg      0.674     0.662     0.653    3657.0
                weighted avg      0.962     0.956     0.953    3657.0

I0715 07:26:24.857285 140369481070464 interactiveshell.py:2882] Best Slot F1: 0.956
Reading ../data/atis.train.w-intent.iob
I0715 07:26:27.999701 140369481070464 interactiveshell.py:2882] Step 18750 | Loss: 0.0001 | Loss_intent: 0.0000 | Loss_slots: 0.0001 | Spent: 6.2 secs | LR: 0.000042
I0715 07:26:31.772235 140369481070464 interactiveshell.py:2882] Step 18800 | Loss: 0.0003 | Loss_intent: 0.0000 | Loss_slots: 0.0003 | Spent: 3.8 secs | LR: 0.000041
I0715 07:26:35.547659 140369481070464 interactiveshell.py:2882] Step 18850 | Loss: 0.0002 | Loss_intent: 0.0000 | Loss_slots: 0.0002 | Spent: 3.8 secs | LR: 0.000041
I0715 07:26:39.323085 140369481070464 interactiveshell.py:2882] Step 18900 | Loss: 0.0008 | Loss_intent: 0.0000 | Loss_slots: 0.0008 | Spent: 3.8 secs | LR: 0.000041
I0715 07:26:43.095206 140369481070464 interactiveshell.py:2882] Step 18950 | Loss: 0.0000 | Loss_intent: 0.0000 | Loss_slots: 0.0000 | Spent: 3.8 secs | LR: 0.000041
I0715 07:26:46.882210 140369481070464 interactiveshell.py:2882] Step 19000 | Loss: 0.0001 | Loss_intent: 0.0000 | Loss_slots: 0.0001 | Spent: 3.8 secs | LR: 0.000041
Reading ../data/atis.test.w-intent.iob
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.
  'precision', 'predicted', average, warn_for)
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1439: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples.
  'recall', 'true', average, warn_for)
I0715 07:26:50.843009 140369481070464 interactiveshell.py:2882] 
                                          precision    recall  f1-score   support

                             atis_flight      0.981     0.989     0.985       632
                            atis_airfare      0.960     1.000     0.980        48
                     atis_ground_service      1.000     1.000     1.000        36
                            atis_airline      1.000     1.000     1.000        38
                       atis_abbreviation      0.943     1.000     0.971        33
                           atis_aircraft      1.000     0.889     0.941         9
                        atis_flight_time      1.000     1.000     1.000         1
                           atis_quantity      0.429     1.000     0.600         3
                atis_flight#atis_airfare      0.833     0.417     0.556        12
                            atis_airport      1.000     0.944     0.971        18
                           atis_distance      1.000     1.000     1.000        10
                               atis_city      0.667     0.667     0.667         6
                        atis_ground_fare      1.000     0.857     0.923         7
                           atis_capacity      0.955     1.000     0.977        21
                          atis_flight_no      0.889     1.000     0.941         8
                               atis_meal      1.000     0.833     0.909         6
                        atis_restriction      0.000     0.000     0.000         0
             atis_airline#atis_flight_no      0.000     0.000     0.000         0
    atis_ground_service#atis_ground_fare      0.000     0.000     0.000         0
           atis_airfare#atis_flight_time      0.000     0.000     0.000         0
                           atis_cheapest      0.000     0.000     0.000         0
atis_aircraft#atis_flight#atis_flight_no      0.000     0.000     0.000         0

                               micro avg      0.972     0.977     0.975       888
                               macro avg      0.666     0.663     0.655       888
                            weighted avg      0.974     0.977     0.974       888

I0715 07:26:50.882802 140369481070464 interactiveshell.py:2882] 
                              precision    recall  f1-score   support

                           O      0.000     0.000     0.000       0.0
           B-toloc.city_name      0.970     0.993     0.981     716.0
         B-fromloc.city_name      0.987     1.000     0.994     704.0
           I-toloc.city_name      0.981     0.996     0.989     265.0
      B-depart_date.day_name      0.981     0.986     0.984     212.0
              B-airline_name      1.000     1.000     1.000     101.0
         I-fromloc.city_name      0.978     0.994     0.986     177.0
 B-depart_time.period_of_day      0.991     0.885     0.935     130.0
              I-airline_name      1.000     0.985     0.992      65.0
    B-depart_date.day_number      0.964     0.964     0.964      55.0
    B-depart_date.month_name      0.982     0.964     0.973      56.0
          B-depart_time.time      0.864     1.000     0.927      57.0
                B-round_trip      1.000     0.986     0.993      73.0
             B-cost_relative      1.000     0.973     0.986      37.0
                I-round_trip      1.000     1.000     1.000      71.0
                B-flight_mod      0.828     1.000     0.906      24.0
 B-depart_time.time_relative      0.984     0.969     0.977      65.0
          I-depart_time.time      0.945     1.000     0.972      52.0
         B-stoploc.city_name      1.000     1.000     1.000      20.0
                 B-city_name      0.909     0.526     0.667      57.0
                B-class_type      0.960     1.000     0.980      24.0
          B-arrive_time.time      0.917     0.971     0.943      34.0
 B-arrive_time.time_relative      0.968     0.968     0.968      31.0
                I-class_type      1.000     1.000     1.000      17.0
               B-flight_stop      1.000     1.000     1.000      21.0
          I-arrive_time.time      0.971     0.971     0.971      35.0
              B-airline_code      1.000     0.912     0.954      34.0
    I-depart_date.day_number      1.000     0.933     0.966      15.0
      I-fromloc.airport_name      0.400     0.933     0.560      15.0
      B-fromloc.airport_name      0.400     1.000     0.571      12.0
      B-arrive_date.day_name      0.733     1.000     0.846      11.0
          B-toloc.state_code      1.000     1.000     1.000      18.0
B-depart_date.today_relative      1.000     0.889     0.941       9.0
             B-flight_number      0.524     1.000     0.688      11.0
 B-depart_date.date_relative      0.944     1.000     0.971      17.0
          B-toloc.state_name      0.833     0.893     0.862      28.0
           B-fare_basis_code      0.944     1.000     0.971      17.0
               B-flight_time      1.000     1.000     1.000       1.0
                        B-or      1.000     1.000     1.000       3.0
 B-arrive_time.period_of_day      0.600     1.000     0.750       6.0
          B-meal_description      1.000     0.900     0.947      10.0
             I-cost_relative      1.000     0.667     0.800       3.0
              I-airport_name      0.909     0.345     0.500      29.0
               B-fare_amount      0.500     1.000     0.667       2.0
               I-fare_amount      1.000     1.000     1.000       2.0
                 I-city_name      1.000     0.667     0.800      30.0
        I-toloc.airport_name      1.000     1.000     1.000       3.0
            B-transport_type      1.000     0.900     0.947      10.0
    B-arrive_date.month_name      0.714     0.833     0.769       6.0
    B-arrive_date.day_number      0.714     0.833     0.769       6.0
         I-stoploc.city_name      0.900     0.900     0.900      10.0
                      B-meal      1.000     1.000     1.000      16.0
        B-fromloc.state_code      1.000     1.000     1.000      23.0
    B-depart_time.period_mod      1.000     1.000     1.000       5.0
                   B-connect      1.000     1.000     1.000       6.0
               B-flight_days      1.000     1.000     1.000      10.0
        B-toloc.airport_name      1.000     1.000     1.000       3.0
        B-fromloc.state_name      1.000     0.941     0.970      17.0
              B-airport_name      0.778     0.333     0.467      21.0
                   B-economy      1.000     1.000     1.000       6.0
               I-flight_time      1.000     1.000     1.000       1.0
             B-aircraft_code      1.000     0.697     0.821      33.0
                       B-mod      1.000     0.500     0.667       2.0
              B-airport_code      0.667     0.444     0.533       9.0
    B-depart_time.start_time      0.600     1.000     0.750       3.0
      B-depart_time.end_time      1.000     0.667     0.800       3.0
          B-depart_date.year      0.750     1.000     0.857       3.0
            I-transport_type      0.000     0.000     0.000       1.0
          B-restriction_code      1.000     1.000     1.000       4.0
    B-arrive_time.start_time      1.000     1.000     1.000       8.0
        B-toloc.airport_code      1.000     0.750     0.857       4.0
      B-arrive_time.end_time      0.875     0.875     0.875       8.0
      I-arrive_time.end_time      0.875     0.875     0.875       8.0
      I-depart_time.end_time      1.000     0.667     0.800       3.0
               I-flight_stop      0.000     0.000     0.000       0.0
      B-fromloc.airport_code      0.833     1.000     0.909       5.0
          I-restriction_code      1.000     1.000     1.000       3.0
    I-depart_time.start_time      1.000     1.000     1.000       1.0
          I-toloc.state_name      1.000     1.000     1.000       1.0
I-depart_date.today_relative      0.000     0.000     0.000       0.0
 B-arrive_date.date_relative      1.000     1.000     1.000       2.0
                I-flight_mod      1.000     0.167     0.286       6.0
                   I-economy      0.000     0.000     0.000       0.0
 B-return_date.date_relative      0.000     0.000     0.000       3.0
        I-fromloc.state_name      1.000     1.000     1.000       1.0
                B-state_code      1.000     1.000     1.000       1.0
    I-arrive_time.start_time      1.000     1.000     1.000       1.0
    I-arrive_date.day_number      0.000     0.000     0.000       0.0
                 B-meal_code      0.000     0.000     0.000       1.0
 I-depart_time.period_of_day      1.000     1.000     1.000       1.0
                  B-day_name      1.000     0.500     0.667       2.0
             B-period_of_day      1.000     0.500     0.667       4.0
        B-stoploc.state_code      0.000     0.000     0.000       0.0
    B-return_date.month_name      0.000     0.000     0.000       0.0
    B-return_date.day_number      0.000     0.000     0.000       0.0
    B-arrive_time.period_mod      0.000     0.000     0.000       0.0
                 I-meal_code      0.000     0.000     0.000       0.0
        B-toloc.country_name      1.000     1.000     1.000       1.0
                 B-days_code      1.000     1.000     1.000       1.0
 I-arrive_time.period_of_day      0.000     0.000     0.000       0.0
            I-today_relative      0.000     0.000     0.000       0.0
 B-return_time.period_of_day      0.000     0.000     0.000       0.0
                      B-time      0.000     0.000     0.000       0.0
           I-fare_basis_code      0.000     0.000     0.000       0.0
 I-arrive_time.time_relative      1.000     1.000     1.000       4.0
 I-depart_time.time_relative      0.000     0.000     0.000       1.0
            B-today_relative      0.000     0.000     0.000       0.0
                B-state_name      0.750     0.333     0.462       9.0
B-arrive_date.today_relative      0.000     0.000     0.000       0.0
    B-return_time.period_mod      0.000     0.000     0.000       0.0
                B-month_name      0.000     0.000     0.000       0.0
                B-day_number      0.000     0.000     0.000       0.0
 I-return_date.date_relative      0.667     0.667     0.667       3.0
I-return_date.today_relative      0.000     0.000     0.000       0.0
      B-stoploc.airport_name      0.000     0.000     0.000       0.0
             B-time_relative      0.000     0.000     0.000       0.0
                      I-time      0.000     0.000     0.000       0.0
    I-return_date.day_number      0.000     0.000     0.000       0.0
          I-meal_description      0.000     0.000     0.000       0.0
B-return_date.today_relative      0.000     0.000     0.000       0.0
      B-return_date.day_name      0.000     0.000     0.000       2.0

                   micro avg      0.953     0.954     0.954    3657.0
                   macro avg      0.678     0.658     0.655    3657.0
                weighted avg      0.962     0.954     0.953    3657.0

I0715 07:26:50.883623 140369481070464 interactiveshell.py:2882] Best Slot F1: 0.956
Reading ../data/atis.train.w-intent.iob
I0715 07:26:53.006810 140369481070464 interactiveshell.py:2882] Step 19050 | Loss: 0.0014 | Loss_intent: 0.0000 | Loss_slots: 0.0014 | Spent: 6.1 secs | LR: 0.000040
I0715 07:26:56.793436 140369481070464 interactiveshell.py:2882] Step 19100 | Loss: 0.0011 | Loss_intent: 0.0000 | Loss_slots: 0.0011 | Spent: 3.8 secs | LR: 0.000040
I0715 07:27:00.572014 140369481070464 interactiveshell.py:2882] Step 19150 | Loss: 0.0006 | Loss_intent: 0.0000 | Loss_slots: 0.0006 | Spent: 3.8 secs | LR: 0.000040
I0715 07:27:04.343267 140369481070464 interactiveshell.py:2882] Step 19200 | Loss: 0.0011 | Loss_intent: 0.0000 | Loss_slots: 0.0011 | Spent: 3.8 secs | LR: 0.000040
I0715 07:27:08.132665 140369481070464 interactiveshell.py:2882] Step 19250 | Loss: 0.0007 | Loss_intent: 0.0000 | Loss_slots: 0.0007 | Spent: 3.8 secs | LR: 0.000039
I0715 07:27:11.891001 140369481070464 interactiveshell.py:2882] Step 19300 | Loss: 0.0002 | Loss_intent: 0.0000 | Loss_slots: 0.0002 | Spent: 3.8 secs | LR: 0.000039
Reading ../data/atis.test.w-intent.iob
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.
  'precision', 'predicted', average, warn_for)
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1439: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples.
  'recall', 'true', average, warn_for)
I0715 07:27:16.562302 140369481070464 interactiveshell.py:2882] 
                                          precision    recall  f1-score   support

                             atis_flight      0.977     0.992     0.984       632
                            atis_airfare      0.960     1.000     0.980        48
                     atis_ground_service      1.000     1.000     1.000        36
                            atis_airline      1.000     1.000     1.000        38
                       atis_abbreviation      1.000     1.000     1.000        33
                           atis_aircraft      1.000     0.889     0.941         9
                        atis_flight_time      1.000     1.000     1.000         1
                           atis_quantity      0.429     1.000     0.600         3
                atis_flight#atis_airfare      0.833     0.417     0.556        12
                            atis_airport      1.000     0.889     0.941        18
                           atis_distance      1.000     1.000     1.000        10
                               atis_city      1.000     0.667     0.800         6
                        atis_ground_fare      1.000     0.857     0.923         7
                           atis_capacity      0.955     1.000     0.977        21
                          atis_flight_no      0.889     1.000     0.941         8
                               atis_meal      1.000     0.833     0.909         6
                        atis_restriction      0.000     0.000     0.000         0
             atis_airline#atis_flight_no      0.000     0.000     0.000         0
    atis_ground_service#atis_ground_fare      0.000     0.000     0.000         0
           atis_airfare#atis_flight_time      0.000     0.000     0.000         0
                           atis_cheapest      0.000     0.000     0.000         0
atis_aircraft#atis_flight#atis_flight_no      0.000     0.000     0.000         0

                               micro avg      0.973     0.979     0.976       888
                               macro avg      0.684     0.661     0.661       888
                            weighted avg      0.975     0.979     0.975       888

I0715 07:27:16.602632 140369481070464 interactiveshell.py:2882] 
                              precision    recall  f1-score   support

                           O      0.000     0.000     0.000       0.0
           B-toloc.city_name      0.973     0.993     0.983     716.0
         B-fromloc.city_name      0.987     1.000     0.994     704.0
           I-toloc.city_name      0.981     0.989     0.985     265.0
      B-depart_date.day_name      0.977     0.986     0.981     212.0
              B-airline_name      1.000     1.000     1.000     101.0
         I-fromloc.city_name      0.967     0.994     0.981     177.0
 B-depart_time.period_of_day      1.000     0.885     0.939     130.0
              I-airline_name      1.000     0.985     0.992      65.0
    B-depart_date.day_number      0.982     0.982     0.982      55.0
    B-depart_date.month_name      0.982     0.964     0.973      56.0
          B-depart_time.time      0.864     1.000     0.927      57.0
                B-round_trip      1.000     0.986     0.993      73.0
             B-cost_relative      1.000     0.973     0.986      37.0
                I-round_trip      1.000     1.000     1.000      71.0
                B-flight_mod      0.828     1.000     0.906      24.0
 B-depart_time.time_relative      0.984     0.954     0.969      65.0
          I-depart_time.time      0.963     1.000     0.981      52.0
         B-stoploc.city_name      1.000     1.000     1.000      20.0
                 B-city_name      0.909     0.526     0.667      57.0
                B-class_type      0.960     1.000     0.980      24.0
          B-arrive_time.time      0.917     0.971     0.943      34.0
 B-arrive_time.time_relative      0.968     0.968     0.968      31.0
                I-class_type      1.000     1.000     1.000      17.0
               B-flight_stop      1.000     1.000     1.000      21.0
          I-arrive_time.time      0.971     0.971     0.971      35.0
              B-airline_code      1.000     0.912     0.954      34.0
    I-depart_date.day_number      1.000     0.933     0.966      15.0
      I-fromloc.airport_name      0.405     1.000     0.577      15.0
      B-fromloc.airport_name      0.414     1.000     0.585      12.0
      B-arrive_date.day_name      0.733     1.000     0.846      11.0
          B-toloc.state_code      1.000     1.000     1.000      18.0
B-depart_date.today_relative      1.000     0.889     0.941       9.0
             B-flight_number      0.500     1.000     0.667      11.0
 B-depart_date.date_relative      0.944     1.000     0.971      17.0
          B-toloc.state_name      0.818     0.964     0.885      28.0
           B-fare_basis_code      0.944     1.000     0.971      17.0
               B-flight_time      1.000     1.000     1.000       1.0
                        B-or      0.750     1.000     0.857       3.0
 B-arrive_time.period_of_day      0.545     1.000     0.706       6.0
          B-meal_description      1.000     0.900     0.947      10.0
             I-cost_relative      1.000     0.667     0.800       3.0
              I-airport_name      0.909     0.345     0.500      29.0
               B-fare_amount      1.000     1.000     1.000       2.0
               I-fare_amount      1.000     1.000     1.000       2.0
                 I-city_name      1.000     0.700     0.824      30.0
        I-toloc.airport_name      1.000     1.000     1.000       3.0
            B-transport_type      1.000     1.000     1.000      10.0
    B-arrive_date.month_name      0.714     0.833     0.769       6.0
    B-arrive_date.day_number      0.833     0.833     0.833       6.0
         I-stoploc.city_name      0.900     0.900     0.900      10.0
                      B-meal      1.000     1.000     1.000      16.0
        B-fromloc.state_code      1.000     1.000     1.000      23.0
    B-depart_time.period_mod      1.000     1.000     1.000       5.0
                   B-connect      1.000     1.000     1.000       6.0
               B-flight_days      1.000     1.000     1.000      10.0
        B-toloc.airport_name      1.000     1.000     1.000       3.0
        B-fromloc.state_name      1.000     1.000     1.000      17.0
              B-airport_name      0.778     0.333     0.467      21.0
                   B-economy      1.000     1.000     1.000       6.0
               I-flight_time      1.000     1.000     1.000       1.0
             B-aircraft_code      1.000     0.818     0.900      33.0
                       B-mod      1.000     0.500     0.667       2.0
              B-airport_code      0.667     0.444     0.533       9.0
    B-depart_time.start_time      0.750     1.000     0.857       3.0
      B-depart_time.end_time      1.000     0.667     0.800       3.0
          B-depart_date.year      0.750     1.000     0.857       3.0
            I-transport_type      0.000     0.000     0.000       1.0
          B-restriction_code      1.000     1.000     1.000       4.0
    B-arrive_time.start_time      1.000     1.000     1.000       8.0
        B-toloc.airport_code      1.000     0.750     0.857       4.0
      B-arrive_time.end_time      0.875     0.875     0.875       8.0
      I-arrive_time.end_time      0.875     0.875     0.875       8.0
      I-depart_time.end_time      1.000     0.667     0.800       3.0
               I-flight_stop      0.000     0.000     0.000       0.0
      B-fromloc.airport_code      0.833     1.000     0.909       5.0
          I-restriction_code      1.000     1.000     1.000       3.0
    I-depart_time.start_time      0.500     1.000     0.667       1.0
          I-toloc.state_name      1.000     1.000     1.000       1.0
I-depart_date.today_relative      0.000     0.000     0.000       0.0
 B-arrive_date.date_relative      1.000     1.000     1.000       2.0
                I-flight_mod      1.000     0.167     0.286       6.0
                   I-economy      0.000     0.000     0.000       0.0
 B-return_date.date_relative      0.000     0.000     0.000       3.0
        I-fromloc.state_name      1.000     1.000     1.000       1.0
                B-state_code      1.000     1.000     1.000       1.0
    I-arrive_time.start_time      1.000     1.000     1.000       1.0
    I-arrive_date.day_number      0.000     0.000     0.000       0.0
                 B-meal_code      0.000     0.000     0.000       1.0
 I-depart_time.period_of_day      1.000     1.000     1.000       1.0
                  B-day_name      1.000     0.500     0.667       2.0
             B-period_of_day      1.000     0.500     0.667       4.0
        B-stoploc.state_code      0.000     0.000     0.000       0.0
    B-return_date.month_name      0.000     0.000     0.000       0.0
    B-return_date.day_number      0.000     0.000     0.000       0.0
    B-arrive_time.period_mod      0.000     0.000     0.000       0.0
                 I-meal_code      0.000     0.000     0.000       0.0
        B-toloc.country_name      1.000     1.000     1.000       1.0
                 B-days_code      1.000     1.000     1.000       1.0
 I-arrive_time.period_of_day      0.000     0.000     0.000       0.0
            I-today_relative      0.000     0.000     0.000       0.0
 B-return_time.period_of_day      0.000     0.000     0.000       0.0
                      B-time      0.000     0.000     0.000       0.0
           I-fare_basis_code      0.000     0.000     0.000       0.0
 I-arrive_time.time_relative      1.000     1.000     1.000       4.0
 I-depart_time.time_relative      0.000     0.000     0.000       1.0
            B-today_relative      0.000     0.000     0.000       0.0
                B-state_name      0.500     0.111     0.182       9.0
B-arrive_date.today_relative      0.000     0.000     0.000       0.0
    B-return_time.period_mod      0.000     0.000     0.000       0.0
                B-month_name      0.000     0.000     0.000       0.0
                B-day_number      0.000     0.000     0.000       0.0
 I-return_date.date_relative      0.667     0.667     0.667       3.0
I-return_date.today_relative      0.000     0.000     0.000       0.0
      B-stoploc.airport_name      0.000     0.000     0.000       0.0
             B-time_relative      0.000     0.000     0.000       0.0
                      I-time      0.000     0.000     0.000       0.0
    I-return_date.day_number      0.000     0.000     0.000       0.0
          I-meal_description      0.000     0.000     0.000       0.0
B-return_date.today_relative      0.000     0.000     0.000       0.0
      B-return_date.day_name      0.000     0.000     0.000       2.0

                   micro avg      0.954     0.956     0.955    3657.0
                   macro avg      0.676     0.660     0.654    3657.0
                weighted avg      0.962     0.956     0.953    3657.0

I0715 07:27:16.603616 140369481070464 interactiveshell.py:2882] Best Slot F1: 0.956
Reading ../data/atis.train.w-intent.iob
I0715 07:27:17.838776 140369481070464 interactiveshell.py:2882] Step 19350 | Loss: 0.0010 | Loss_intent: 0.0000 | Loss_slots: 0.0010 | Spent: 5.9 secs | LR: 0.000039
I0715 07:27:21.602943 140369481070464 interactiveshell.py:2882] Step 19400 | Loss: 0.0014 | Loss_intent: 0.0000 | Loss_slots: 0.0014 | Spent: 3.8 secs | LR: 0.000039
I0715 07:27:25.368445 140369481070464 interactiveshell.py:2882] Step 19450 | Loss: 0.0002 | Loss_intent: 0.0000 | Loss_slots: 0.0002 | Spent: 3.8 secs | LR: 0.000039
I0715 07:27:29.156307 140369481070464 interactiveshell.py:2882] Step 19500 | Loss: 0.0007 | Loss_intent: 0.0000 | Loss_slots: 0.0007 | Spent: 3.8 secs | LR: 0.000038
I0715 07:27:32.932975 140369481070464 interactiveshell.py:2882] Step 19550 | Loss: 0.0007 | Loss_intent: 0.0004 | Loss_slots: 0.0003 | Spent: 3.8 secs | LR: 0.000038
I0715 07:27:36.748561 140369481070464 interactiveshell.py:2882] Step 19600 | Loss: 0.0004 | Loss_intent: 0.0000 | Loss_slots: 0.0004 | Spent: 3.8 secs | LR: 0.000038
I0715 07:27:40.591642 140369481070464 interactiveshell.py:2882] Step 19650 | Loss: 0.0005 | Loss_intent: 0.0000 | Loss_slots: 0.0005 | Spent: 3.8 secs | LR: 0.000038
Reading ../data/atis.test.w-intent.iob
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.
  'precision', 'predicted', average, warn_for)
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1439: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples.
  'recall', 'true', average, warn_for)
I0715 07:27:42.448599 140369481070464 interactiveshell.py:2882] 
                                          precision    recall  f1-score   support

                             atis_flight      0.978     0.992     0.985       632
                            atis_airfare      0.960     1.000     0.980        48
                     atis_ground_service      1.000     1.000     1.000        36
                            atis_airline      1.000     1.000     1.000        38
                       atis_abbreviation      1.000     1.000     1.000        33
                           atis_aircraft      1.000     0.889     0.941         9
                        atis_flight_time      1.000     1.000     1.000         1
                           atis_quantity      0.429     1.000     0.600         3
                atis_flight#atis_airfare      0.833     0.417     0.556        12
                            atis_airport      1.000     0.944     0.971        18
                           atis_distance      1.000     1.000     1.000        10
                               atis_city      1.000     0.667     0.800         6
                        atis_ground_fare      1.000     0.857     0.923         7
                           atis_capacity      0.955     1.000     0.977        21
                          atis_flight_no      0.889     1.000     0.941         8
                               atis_meal      1.000     0.833     0.909         6
                        atis_restriction      0.000     0.000     0.000         0
             atis_airline#atis_flight_no      0.000     0.000     0.000         0
    atis_ground_service#atis_ground_fare      0.000     0.000     0.000         0
           atis_airfare#atis_flight_time      0.000     0.000     0.000         0
                           atis_cheapest      0.000     0.000     0.000         0
atis_aircraft#atis_flight#atis_flight_no      0.000     0.000     0.000         0

                               micro avg      0.974     0.980     0.977       888
                               macro avg      0.684     0.664     0.663       888
                            weighted avg      0.976     0.980     0.976       888

I0715 07:27:42.487317 140369481070464 interactiveshell.py:2882] 
                              precision    recall  f1-score   support

                           O      0.000     0.000     0.000       0.0
           B-toloc.city_name      0.971     0.993     0.982     716.0
         B-fromloc.city_name      0.987     0.999     0.993     704.0
           I-toloc.city_name      0.981     0.992     0.987     265.0
      B-depart_date.day_name      0.977     0.986     0.981     212.0
              B-airline_name      1.000     1.000     1.000     101.0
         I-fromloc.city_name      0.972     0.994     0.983     177.0
 B-depart_time.period_of_day      1.000     0.877     0.934     130.0
              I-airline_name      1.000     0.985     0.992      65.0
    B-depart_date.day_number      0.982     0.982     0.982      55.0
    B-depart_date.month_name      0.982     0.964     0.973      56.0
          B-depart_time.time      0.851     1.000     0.919      57.0
                B-round_trip      1.000     0.986     0.993      73.0
             B-cost_relative      1.000     0.973     0.986      37.0
                I-round_trip      1.000     1.000     1.000      71.0
                B-flight_mod      0.828     1.000     0.906      24.0
 B-depart_time.time_relative      0.984     0.969     0.977      65.0
          I-depart_time.time      0.945     1.000     0.972      52.0
         B-stoploc.city_name      1.000     1.000     1.000      20.0
                 B-city_name      0.938     0.526     0.674      57.0
                B-class_type      0.960     1.000     0.980      24.0
          B-arrive_time.time      0.943     0.971     0.957      34.0
 B-arrive_time.time_relative      0.967     0.935     0.951      31.0
                I-class_type      1.000     1.000     1.000      17.0
               B-flight_stop      1.000     1.000     1.000      21.0
          I-arrive_time.time      1.000     0.971     0.986      35.0
              B-airline_code      1.000     0.912     0.954      34.0
    I-depart_date.day_number      1.000     0.933     0.966      15.0
      I-fromloc.airport_name      0.405     1.000     0.577      15.0
      B-fromloc.airport_name      0.400     1.000     0.571      12.0
      B-arrive_date.day_name      0.786     1.000     0.880      11.0
          B-toloc.state_code      1.000     1.000     1.000      18.0
B-depart_date.today_relative      1.000     0.889     0.941       9.0
             B-flight_number      0.500     1.000     0.667      11.0
 B-depart_date.date_relative      0.895     1.000     0.944      17.0
          B-toloc.state_name      0.929     0.929     0.929      28.0
           B-fare_basis_code      0.944     1.000     0.971      17.0
               B-flight_time      1.000     1.000     1.000       1.0
                        B-or      1.000     1.000     1.000       3.0
 B-arrive_time.period_of_day      0.600     1.000     0.750       6.0
          B-meal_description      1.000     0.900     0.947      10.0
             I-cost_relative      1.000     0.667     0.800       3.0
              I-airport_name      0.909     0.345     0.500      29.0
               B-fare_amount      0.500     1.000     0.667       2.0
               I-fare_amount      1.000     1.000     1.000       2.0
                 I-city_name      1.000     0.700     0.824      30.0
        I-toloc.airport_name      1.000     1.000     1.000       3.0
            B-transport_type      1.000     1.000     1.000      10.0
    B-arrive_date.month_name      0.714     0.833     0.769       6.0
    B-arrive_date.day_number      0.833     0.833     0.833       6.0
         I-stoploc.city_name      0.900     0.900     0.900      10.0
                      B-meal      1.000     1.000     1.000      16.0
        B-fromloc.state_code      1.000     1.000     1.000      23.0
    B-depart_time.period_mod      0.833     1.000     0.909       5.0
                   B-connect      1.000     1.000     1.000       6.0
               B-flight_days      1.000     1.000     1.000      10.0
        B-toloc.airport_name      1.000     1.000     1.000       3.0
        B-fromloc.state_name      1.000     1.000     1.000      17.0
              B-airport_name      0.778     0.333     0.467      21.0
                   B-economy      1.000     1.000     1.000       6.0
               I-flight_time      1.000     1.000     1.000       1.0
             B-aircraft_code      1.000     0.758     0.862      33.0
                       B-mod      1.000     0.500     0.667       2.0
              B-airport_code      0.800     0.444     0.571       9.0
    B-depart_time.start_time      0.600     1.000     0.750       3.0
      B-depart_time.end_time      1.000     0.667     0.800       3.0
          B-depart_date.year      0.750     1.000     0.857       3.0
            I-transport_type      0.000     0.000     0.000       1.0
          B-restriction_code      1.000     1.000     1.000       4.0
    B-arrive_time.start_time      1.000     1.000     1.000       8.0
        B-toloc.airport_code      1.000     0.750     0.857       4.0
      B-arrive_time.end_time      0.875     0.875     0.875       8.0
      I-arrive_time.end_time      0.889     1.000     0.941       8.0
      I-depart_time.end_time      1.000     0.667     0.800       3.0
               I-flight_stop      0.000     0.000     0.000       0.0
      B-fromloc.airport_code      1.000     1.000     1.000       5.0
          I-restriction_code      1.000     1.000     1.000       3.0
    I-depart_time.start_time      1.000     1.000     1.000       1.0
          I-toloc.state_name      1.000     1.000     1.000       1.0
I-depart_date.today_relative      0.000     0.000     0.000       0.0
 B-arrive_date.date_relative      1.000     1.000     1.000       2.0
                I-flight_mod      1.000     0.167     0.286       6.0
                   I-economy      0.000     0.000     0.000       0.0
 B-return_date.date_relative      0.000     0.000     0.000       3.0
        I-fromloc.state_name      1.000     1.000     1.000       1.0
                B-state_code      1.000     1.000     1.000       1.0
    I-arrive_time.start_time      1.000     1.000     1.000       1.0
    I-arrive_date.day_number      0.000     0.000     0.000       0.0
                 B-meal_code      0.000     0.000     0.000       1.0
 I-depart_time.period_of_day      1.000     1.000     1.000       1.0
                  B-day_name      1.000     0.500     0.667       2.0
             B-period_of_day      1.000     0.500     0.667       4.0
        B-stoploc.state_code      0.000     0.000     0.000       0.0
    B-return_date.month_name      0.000     0.000     0.000       0.0
    B-return_date.day_number      0.000     0.000     0.000       0.0
    B-arrive_time.period_mod      0.000     0.000     0.000       0.0
                 I-meal_code      0.000     0.000     0.000       0.0
        B-toloc.country_name      1.000     1.000     1.000       1.0
                 B-days_code      0.500     1.000     0.667       1.0
 I-arrive_time.period_of_day      0.000     0.000     0.000       0.0
            I-today_relative      0.000     0.000     0.000       0.0
 B-return_time.period_of_day      0.000     0.000     0.000       0.0
                      B-time      0.000     0.000     0.000       0.0
           I-fare_basis_code      0.000     0.000     0.000       0.0
 I-arrive_time.time_relative      1.000     1.000     1.000       4.0
 I-depart_time.time_relative      0.000     0.000     0.000       1.0
            B-today_relative      0.000     0.000     0.000       0.0
                B-state_name      0.875     0.778     0.824       9.0
B-arrive_date.today_relative      0.000     0.000     0.000       0.0
    B-return_time.period_mod      0.000     0.000     0.000       0.0
                B-month_name      0.000     0.000     0.000       0.0
                B-day_number      0.000     0.000     0.000       0.0
 I-return_date.date_relative      0.667     0.667     0.667       3.0
I-return_date.today_relative      0.000     0.000     0.000       0.0
      B-stoploc.airport_name      0.000     0.000     0.000       0.0
             B-time_relative      0.000     0.000     0.000       0.0
                      I-time      0.000     0.000     0.000       0.0
    I-return_date.day_number      0.000     0.000     0.000       0.0
          I-meal_description      0.000     0.000     0.000       0.0
B-return_date.today_relative      0.000     0.000     0.000       0.0
      B-return_date.day_name      0.000     0.000     0.000       2.0

                   micro avg      0.955     0.957     0.956    3657.0
                   macro avg      0.679     0.666     0.658    3657.0
                weighted avg      0.964     0.957     0.955    3657.0

I0715 07:27:42.488277 140369481070464 interactiveshell.py:2882] Best Slot F1: 0.956
Reading ../data/atis.train.w-intent.iob
I0715 07:27:46.594006 140369481070464 interactiveshell.py:2882] Step 19700 | Loss: 0.0001 | Loss_intent: 0.0000 | Loss_slots: 0.0001 | Spent: 6.0 secs | LR: 0.000038
I0715 07:27:50.361381 140369481070464 interactiveshell.py:2882] Step 19750 | Loss: 0.0003 | Loss_intent: 0.0000 | Loss_slots: 0.0003 | Spent: 3.8 secs | LR: 0.000037
I0715 07:27:54.118489 140369481070464 interactiveshell.py:2882] Step 19800 | Loss: 0.0001 | Loss_intent: 0.0000 | Loss_slots: 0.0001 | Spent: 3.8 secs | LR: 0.000037
I0715 07:27:57.899159 140369481070464 interactiveshell.py:2882] Step 19850 | Loss: 0.0001 | Loss_intent: 0.0000 | Loss_slots: 0.0001 | Spent: 3.8 secs | LR: 0.000037
I0715 07:28:01.676317 140369481070464 interactiveshell.py:2882] Step 19900 | Loss: 0.0001 | Loss_intent: 0.0000 | Loss_slots: 0.0001 | Spent: 3.8 secs | LR: 0.000037
I0715 07:28:05.441524 140369481070464 interactiveshell.py:2882] Step 19950 | Loss: 0.0004 | Loss_intent: 0.0000 | Loss_slots: 0.0004 | Spent: 3.8 secs | LR: 0.000037
Reading ../data/atis.test.w-intent.iob
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.
  'precision', 'predicted', average, warn_for)
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1439: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples.
  'recall', 'true', average, warn_for)
I0715 07:28:08.172999 140369481070464 interactiveshell.py:2882] 
                                          precision    recall  f1-score   support

                             atis_flight      0.981     0.992     0.987       632
                            atis_airfare      0.960     1.000     0.980        48
                     atis_ground_service      1.000     1.000     1.000        36
                            atis_airline      1.000     1.000     1.000        38
                       atis_abbreviation      0.943     1.000     0.971        33
                           atis_aircraft      1.000     0.889     0.941         9
                        atis_flight_time      1.000     1.000     1.000         1
                           atis_quantity      0.429     1.000     0.600         3
                atis_flight#atis_airfare      0.833     0.417     0.556        12
                            atis_airport      1.000     0.944     0.971        18
                           atis_distance      1.000     1.000     1.000        10
                               atis_city      1.000     0.667     0.800         6
                        atis_ground_fare      1.000     0.857     0.923         7
                           atis_capacity      0.955     1.000     0.977        21
                          atis_flight_no      0.889     1.000     0.941         8
                               atis_meal      1.000     0.833     0.909         6
                        atis_restriction      0.000     0.000     0.000         0
             atis_airline#atis_flight_no      0.000     0.000     0.000         0
    atis_ground_service#atis_ground_fare      0.000     0.000     0.000         0
           atis_airfare#atis_flight_time      0.000     0.000     0.000         0
                           atis_cheapest      0.000     0.000     0.000         0
atis_aircraft#atis_flight#atis_flight_no      0.000     0.000     0.000         0

                               micro avg      0.974     0.980     0.977       888
                               macro avg      0.681     0.664     0.662       888
                            weighted avg      0.976     0.980     0.976       888

I0715 07:28:08.217309 140369481070464 interactiveshell.py:2882] 
                              precision    recall  f1-score   support

                           O      0.000     0.000     0.000       0.0
           B-toloc.city_name      0.975     0.993     0.984     716.0
         B-fromloc.city_name      0.987     0.999     0.993     704.0
           I-toloc.city_name      0.981     0.992     0.987     265.0
      B-depart_date.day_name      0.977     0.986     0.981     212.0
              B-airline_name      1.000     1.000     1.000     101.0
         I-fromloc.city_name      0.972     0.994     0.983     177.0
 B-depart_time.period_of_day      1.000     0.885     0.939     130.0
              I-airline_name      1.000     0.985     0.992      65.0
    B-depart_date.day_number      0.982     0.982     0.982      55.0
    B-depart_date.month_name      0.982     0.964     0.973      56.0
          B-depart_time.time      0.864     1.000     0.927      57.0
                B-round_trip      1.000     0.986     0.993      73.0
             B-cost_relative      1.000     0.973     0.986      37.0
                I-round_trip      1.000     1.000     1.000      71.0
                B-flight_mod      0.828     1.000     0.906      24.0
 B-depart_time.time_relative      0.984     0.969     0.977      65.0
          I-depart_time.time      0.945     1.000     0.972      52.0
         B-stoploc.city_name      1.000     1.000     1.000      20.0
                 B-city_name      0.886     0.544     0.674      57.0
                B-class_type      0.960     1.000     0.980      24.0
          B-arrive_time.time      0.917     0.971     0.943      34.0
 B-arrive_time.time_relative      0.968     0.968     0.968      31.0
                I-class_type      1.000     1.000     1.000      17.0
               B-flight_stop      1.000     1.000     1.000      21.0
          I-arrive_time.time      1.000     0.971     0.986      35.0
              B-airline_code      1.000     0.912     0.954      34.0
    I-depart_date.day_number      1.000     0.933     0.966      15.0
      I-fromloc.airport_name      0.405     1.000     0.577      15.0
      B-fromloc.airport_name      0.414     1.000     0.585      12.0
      B-arrive_date.day_name      0.786     1.000     0.880      11.0
          B-toloc.state_code      1.000     1.000     1.000      18.0
B-depart_date.today_relative      1.000     0.889     0.941       9.0
             B-flight_number      0.500     1.000     0.667      11.0
 B-depart_date.date_relative      0.944     1.000     0.971      17.0
          B-toloc.state_name      0.903     1.000     0.949      28.0
           B-fare_basis_code      0.944     1.000     0.971      17.0
               B-flight_time      1.000     1.000     1.000       1.0
                        B-or      1.000     1.000     1.000       3.0
 B-arrive_time.period_of_day      0.600     1.000     0.750       6.0
          B-meal_description      1.000     0.900     0.947      10.0
             I-cost_relative      1.000     0.667     0.800       3.0
              I-airport_name      0.909     0.345     0.500      29.0
               B-fare_amount      0.500     1.000     0.667       2.0
               I-fare_amount      1.000     1.000     1.000       2.0
                 I-city_name      1.000     0.667     0.800      30.0
        I-toloc.airport_name      1.000     1.000     1.000       3.0
            B-transport_type      1.000     0.900     0.947      10.0
    B-arrive_date.month_name      0.714     0.833     0.769       6.0
    B-arrive_date.day_number      0.833     0.833     0.833       6.0
         I-stoploc.city_name      0.900     0.900     0.900      10.0
                      B-meal      1.000     1.000     1.000      16.0
        B-fromloc.state_code      1.000     1.000     1.000      23.0
    B-depart_time.period_mod      1.000     1.000     1.000       5.0
                   B-connect      1.000     1.000     1.000       6.0
               B-flight_days      1.000     1.000     1.000      10.0
        B-toloc.airport_name      1.000     1.000     1.000       3.0
        B-fromloc.state_name      1.000     1.000     1.000      17.0
              B-airport_name      0.750     0.286     0.414      21.0
                   B-economy      1.000     1.000     1.000       6.0
               I-flight_time      1.000     1.000     1.000       1.0
             B-aircraft_code      1.000     0.758     0.862      33.0
                       B-mod      1.000     0.500     0.667       2.0
              B-airport_code      0.800     0.444     0.571       9.0
    B-depart_time.start_time      0.600     1.000     0.750       3.0
      B-depart_time.end_time      1.000     0.667     0.800       3.0
          B-depart_date.year      0.750     1.000     0.857       3.0
            I-transport_type      0.000     0.000     0.000       1.0
          B-restriction_code      1.000     1.000     1.000       4.0
    B-arrive_time.start_time      1.000     1.000     1.000       8.0
        B-toloc.airport_code      1.000     0.750     0.857       4.0
      B-arrive_time.end_time      0.875     0.875     0.875       8.0
      I-arrive_time.end_time      0.889     1.000     0.941       8.0
      I-depart_time.end_time      1.000     0.667     0.800       3.0
               I-flight_stop      0.000     0.000     0.000       0.0
      B-fromloc.airport_code      0.833     1.000     0.909       5.0
          I-restriction_code      1.000     1.000     1.000       3.0
    I-depart_time.start_time      1.000     1.000     1.000       1.0
          I-toloc.state_name      1.000     1.000     1.000       1.0
I-depart_date.today_relative      0.000     0.000     0.000       0.0
 B-arrive_date.date_relative      1.000     1.000     1.000       2.0
                I-flight_mod      1.000     0.167     0.286       6.0
                   I-economy      0.000     0.000     0.000       0.0
 B-return_date.date_relative      0.000     0.000     0.000       3.0
        I-fromloc.state_name      1.000     1.000     1.000       1.0
                B-state_code      1.000     1.000     1.000       1.0
    I-arrive_time.start_time      1.000     1.000     1.000       1.0
    I-arrive_date.day_number      0.000     0.000     0.000       0.0
                 B-meal_code      0.000     0.000     0.000       1.0
 I-depart_time.period_of_day      1.000     1.000     1.000       1.0
                  B-day_name      1.000     0.500     0.667       2.0
             B-period_of_day      1.000     0.500     0.667       4.0
        B-stoploc.state_code      0.000     0.000     0.000       0.0
    B-return_date.month_name      0.000     0.000     0.000       0.0
    B-return_date.day_number      0.000     0.000     0.000       0.0
    B-arrive_time.period_mod      0.000     0.000     0.000       0.0
                 I-meal_code      0.000     0.000     0.000       0.0
        B-toloc.country_name      1.000     1.000     1.000       1.0
                 B-days_code      0.500     1.000     0.667       1.0
 I-arrive_time.period_of_day      0.000     0.000     0.000       0.0
            I-today_relative      0.000     0.000     0.000       0.0
 B-return_time.period_of_day      0.000     0.000     0.000       0.0
                      B-time      0.000     0.000     0.000       0.0
           I-fare_basis_code      0.000     0.000     0.000       0.0
 I-arrive_time.time_relative      1.000     1.000     1.000       4.0
 I-depart_time.time_relative      0.000     0.000     0.000       1.0
            B-today_relative      0.000     0.000     0.000       0.0
                B-state_name      0.833     0.556     0.667       9.0
B-arrive_date.today_relative      0.000     0.000     0.000       0.0
    B-return_time.period_mod      0.000     0.000     0.000       0.0
                B-month_name      0.000     0.000     0.000       0.0
                B-day_number      0.000     0.000     0.000       0.0
 I-return_date.date_relative      0.667     0.667     0.667       3.0
I-return_date.today_relative      0.000     0.000     0.000       0.0
      B-stoploc.airport_name      0.000     0.000     0.000       0.0
             B-time_relative      0.000     0.000     0.000       0.0
                      I-time      0.000     0.000     0.000       0.0
    I-return_date.day_number      0.000     0.000     0.000       0.0
          I-meal_description      0.000     0.000     0.000       0.0
B-return_date.today_relative      0.000     0.000     0.000       0.0
      B-return_date.day_name      0.000     0.000     0.000       2.0

                   micro avg      0.955     0.957     0.956    3657.0
                   macro avg      0.678     0.663     0.656    3657.0
                weighted avg      0.964     0.957     0.955    3657.0

I0715 07:28:08.218070 140369481070464 interactiveshell.py:2882] Best Slot F1: 0.956
Reading ../data/atis.train.w-intent.iob
I0715 07:28:11.394210 140369481070464 interactiveshell.py:2882] Step 20000 | Loss: 0.0002 | Loss_intent: 0.0000 | Loss_slots: 0.0002 | Spent: 6.0 secs | LR: 0.000036
I0715 07:28:15.192836 140369481070464 interactiveshell.py:2882] Step 20050 | Loss: 0.0002 | Loss_intent: 0.0000 | Loss_slots: 0.0002 | Spent: 3.8 secs | LR: 0.000036
I0715 07:28:18.969057 140369481070464 interactiveshell.py:2882] Step 20100 | Loss: 0.0007 | Loss_intent: 0.0000 | Loss_slots: 0.0007 | Spent: 3.8 secs | LR: 0.000036
I0715 07:28:22.767053 140369481070464 interactiveshell.py:2882] Step 20150 | Loss: 0.0001 | Loss_intent: 0.0000 | Loss_slots: 0.0001 | Spent: 3.8 secs | LR: 0.000036
I0715 07:28:26.556641 140369481070464 interactiveshell.py:2882] Step 20200 | Loss: 0.0073 | Loss_intent: 0.0000 | Loss_slots: 0.0073 | Spent: 3.8 secs | LR: 0.000036
I0715 07:28:30.334181 140369481070464 interactiveshell.py:2882] Step 20250 | Loss: 0.0027 | Loss_intent: 0.0000 | Loss_slots: 0.0027 | Spent: 3.8 secs | LR: 0.000036
Reading ../data/atis.test.w-intent.iob
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.
  'precision', 'predicted', average, warn_for)
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1439: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples.
  'recall', 'true', average, warn_for)
I0715 07:28:33.994818 140369481070464 interactiveshell.py:2882] 
                                          precision    recall  f1-score   support

                             atis_flight      0.981     0.992     0.987       632
                            atis_airfare      0.960     1.000     0.980        48
                     atis_ground_service      1.000     1.000     1.000        36
                            atis_airline      1.000     1.000     1.000        38
                       atis_abbreviation      0.943     1.000     0.971        33
                           atis_aircraft      1.000     0.889     0.941         9
                        atis_flight_time      1.000     1.000     1.000         1
                           atis_quantity      0.429     1.000     0.600         3
                atis_flight#atis_airfare      0.833     0.417     0.556        12
                            atis_airport      1.000     0.944     0.971        18
                           atis_distance      1.000     1.000     1.000        10
                               atis_city      1.000     0.667     0.800         6
                        atis_ground_fare      1.000     0.857     0.923         7
                           atis_capacity      0.955     1.000     0.977        21
                          atis_flight_no      0.889     1.000     0.941         8
                               atis_meal      1.000     0.833     0.909         6
                        atis_restriction      0.000     0.000     0.000         0
             atis_airline#atis_flight_no      0.000     0.000     0.000         0
    atis_ground_service#atis_ground_fare      0.000     0.000     0.000         0
           atis_airfare#atis_flight_time      0.000     0.000     0.000         0
                           atis_cheapest      0.000     0.000     0.000         0
atis_aircraft#atis_flight#atis_flight_no      0.000     0.000     0.000         0

                               micro avg      0.974     0.980     0.977       888
                               macro avg      0.681     0.664     0.662       888
                            weighted avg      0.976     0.980     0.976       888

I0715 07:28:34.036046 140369481070464 interactiveshell.py:2882] 
                              precision    recall  f1-score   support

                           O      0.000     0.000     0.000       0.0
           B-toloc.city_name      0.974     0.993     0.983     716.0
         B-fromloc.city_name      0.987     1.000     0.994     704.0
           I-toloc.city_name      0.981     0.992     0.987     265.0
      B-depart_date.day_name      0.977     0.986     0.981     212.0
              B-airline_name      1.000     1.000     1.000     101.0
         I-fromloc.city_name      0.978     0.994     0.986     177.0
 B-depart_time.period_of_day      0.991     0.892     0.939     130.0
              I-airline_name      1.000     0.969     0.984      65.0
    B-depart_date.day_number      0.982     0.982     0.982      55.0
    B-depart_date.month_name      0.982     0.964     0.973      56.0
          B-depart_time.time      0.864     1.000     0.927      57.0
                B-round_trip      1.000     0.986     0.993      73.0
             B-cost_relative      1.000     0.973     0.986      37.0
                I-round_trip      1.000     1.000     1.000      71.0
                B-flight_mod      0.828     1.000     0.906      24.0
 B-depart_time.time_relative      0.984     0.969     0.977      65.0
          I-depart_time.time      0.945     1.000     0.972      52.0
         B-stoploc.city_name      1.000     1.000     1.000      20.0
                 B-city_name      0.909     0.526     0.667      57.0
                B-class_type      0.960     1.000     0.980      24.0
          B-arrive_time.time      0.917     0.971     0.943      34.0
 B-arrive_time.time_relative      0.967     0.935     0.951      31.0
                I-class_type      1.000     1.000     1.000      17.0
               B-flight_stop      1.000     1.000     1.000      21.0
          I-arrive_time.time      0.971     0.971     0.971      35.0
              B-airline_code      1.000     0.912     0.954      34.0
    I-depart_date.day_number      1.000     1.000     1.000      15.0
      I-fromloc.airport_name      0.405     1.000     0.577      15.0
      B-fromloc.airport_name      0.414     1.000     0.585      12.0
      B-arrive_date.day_name      0.786     1.000     0.880      11.0
          B-toloc.state_code      1.000     1.000     1.000      18.0
B-depart_date.today_relative      1.000     0.889     0.941       9.0
             B-flight_number      0.688     1.000     0.815      11.0
 B-depart_date.date_relative      0.944     1.000     0.971      17.0
          B-toloc.state_name      0.933     1.000     0.966      28.0
           B-fare_basis_code      0.944     1.000     0.971      17.0
               B-flight_time      1.000     1.000     1.000       1.0
                        B-or      1.000     1.000     1.000       3.0
 B-arrive_time.period_of_day      0.667     1.000     0.800       6.0
          B-meal_description      1.000     0.900     0.947      10.0
             I-cost_relative      1.000     0.667     0.800       3.0
              I-airport_name      0.909     0.345     0.500      29.0
               B-fare_amount      1.000     1.000     1.000       2.0
               I-fare_amount      1.000     1.000     1.000       2.0
                 I-city_name      1.000     0.667     0.800      30.0
        I-toloc.airport_name      1.000     1.000     1.000       3.0
            B-transport_type      1.000     1.000     1.000      10.0
    B-arrive_date.month_name      0.714     0.833     0.769       6.0
    B-arrive_date.day_number      0.833     0.833     0.833       6.0
         I-stoploc.city_name      1.000     1.000     1.000      10.0
                      B-meal      1.000     1.000     1.000      16.0
        B-fromloc.state_code      1.000     1.000     1.000      23.0
    B-depart_time.period_mod      1.000     1.000     1.000       5.0
                   B-connect      1.000     1.000     1.000       6.0
               B-flight_days      1.000     1.000     1.000      10.0
        B-toloc.airport_name      1.000     1.000     1.000       3.0
        B-fromloc.state_name      1.000     1.000     1.000      17.0
              B-airport_name      0.778     0.333     0.467      21.0
                   B-economy      1.000     1.000     1.000       6.0
               I-flight_time      1.000     1.000     1.000       1.0
             B-aircraft_code      0.909     0.909     0.909      33.0
                       B-mod      1.000     0.500     0.667       2.0
              B-airport_code      0.800     0.444     0.571       9.0
    B-depart_time.start_time      0.600     1.000     0.750       3.0
      B-depart_time.end_time      1.000     0.667     0.800       3.0
          B-depart_date.year      0.750     1.000     0.857       3.0
            I-transport_type      0.000     0.000     0.000       1.0
          B-restriction_code      1.000     1.000     1.000       4.0
    B-arrive_time.start_time      1.000     1.000     1.000       8.0
        B-toloc.airport_code      1.000     0.750     0.857       4.0
      B-arrive_time.end_time      0.875     0.875     0.875       8.0
      I-arrive_time.end_time      0.875     0.875     0.875       8.0
      I-depart_time.end_time      1.000     0.667     0.800       3.0
               I-flight_stop      0.000     0.000     0.000       0.0
      B-fromloc.airport_code      0.714     1.000     0.833       5.0
          I-restriction_code      1.000     1.000     1.000       3.0
    I-depart_time.start_time      1.000     1.000     1.000       1.0
          I-toloc.state_name      1.000     1.000     1.000       1.0
I-depart_date.today_relative      0.000     0.000     0.000       0.0
 B-arrive_date.date_relative      1.000     1.000     1.000       2.0
                I-flight_mod      1.000     0.167     0.286       6.0
                   I-economy      0.000     0.000     0.000       0.0
 B-return_date.date_relative      0.000     0.000     0.000       3.0
        I-fromloc.state_name      1.000     1.000     1.000       1.0
                B-state_code      1.000     1.000     1.000       1.0
    I-arrive_time.start_time      1.000     1.000     1.000       1.0
    I-arrive_date.day_number      0.000     0.000     0.000       0.0
                 B-meal_code      0.000     0.000     0.000       1.0
 I-depart_time.period_of_day      1.000     1.000     1.000       1.0
                  B-day_name      1.000     0.500     0.667       2.0
             B-period_of_day      1.000     0.500     0.667       4.0
        B-stoploc.state_code      0.000     0.000     0.000       0.0
    B-return_date.month_name      0.000     0.000     0.000       0.0
    B-return_date.day_number      0.000     0.000     0.000       0.0
    B-arrive_time.period_mod      0.000     0.000     0.000       0.0
                 I-meal_code      0.000     0.000     0.000       0.0
        B-toloc.country_name      1.000     1.000     1.000       1.0
                 B-days_code      0.500     1.000     0.667       1.0
 I-arrive_time.period_of_day      0.000     0.000     0.000       0.0
            I-today_relative      0.000     0.000     0.000       0.0
 B-return_time.period_of_day      0.000     0.000     0.000       0.0
                      B-time      0.000     0.000     0.000       0.0
           I-fare_basis_code      0.000     0.000     0.000       0.0
 I-arrive_time.time_relative      1.000     1.000     1.000       4.0
 I-depart_time.time_relative      0.000     0.000     0.000       1.0
            B-today_relative      0.000     0.000     0.000       0.0
                B-state_name      0.833     0.556     0.667       9.0
B-arrive_date.today_relative      0.000     0.000     0.000       0.0
    B-return_time.period_mod      0.000     0.000     0.000       0.0
                B-month_name      0.000     0.000     0.000       0.0
                B-day_number      0.000     0.000     0.000       0.0
 I-return_date.date_relative      0.667     0.667     0.667       3.0
I-return_date.today_relative      0.000     0.000     0.000       0.0
      B-stoploc.airport_name      0.000     0.000     0.000       0.0
             B-time_relative      0.000     0.000     0.000       0.0
                      I-time      0.000     0.000     0.000       0.0
    I-return_date.day_number      0.000     0.000     0.000       0.0
          I-meal_description      0.000     0.000     0.000       0.0
B-return_date.today_relative      0.000     0.000     0.000       0.0
      B-return_date.day_name      0.000     0.000     0.000       2.0

                   micro avg      0.957     0.959     0.958    3657.0
                   macro avg      0.684     0.666     0.662    3657.0
                weighted avg      0.964     0.959     0.956    3657.0

I0715 07:28:34.036870 140369481070464 interactiveshell.py:2882] Best Slot F1: 0.958
Reading ../data/atis.train.w-intent.iob
I0715 07:28:36.308607 140369481070464 interactiveshell.py:2882] Step 20300 | Loss: 0.0003 | Loss_intent: 0.0000 | Loss_slots: 0.0003 | Spent: 6.0 secs | LR: 0.000035
I0715 07:28:40.109869 140369481070464 interactiveshell.py:2882] Step 20350 | Loss: 0.0000 | Loss_intent: 0.0000 | Loss_slots: 0.0000 | Spent: 3.8 secs | LR: 0.000035
I0715 07:28:43.884165 140369481070464 interactiveshell.py:2882] Step 20400 | Loss: 0.0001 | Loss_intent: 0.0000 | Loss_slots: 0.0001 | Spent: 3.8 secs | LR: 0.000035
I0715 07:28:47.671161 140369481070464 interactiveshell.py:2882] Step 20450 | Loss: 0.0000 | Loss_intent: 0.0000 | Loss_slots: 0.0000 | Spent: 3.8 secs | LR: 0.000035
I0715 07:28:51.445531 140369481070464 interactiveshell.py:2882] Step 20500 | Loss: 0.0003 | Loss_intent: 0.0000 | Loss_slots: 0.0003 | Spent: 3.8 secs | LR: 0.000035
I0715 07:28:55.198265 140369481070464 interactiveshell.py:2882] Step 20550 | Loss: 0.0000 | Loss_intent: 0.0000 | Loss_slots: 0.0000 | Spent: 3.8 secs | LR: 0.000034
Reading ../data/atis.test.w-intent.iob
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.
  'precision', 'predicted', average, warn_for)
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1439: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples.
  'recall', 'true', average, warn_for)
I0715 07:28:59.830219 140369481070464 interactiveshell.py:2882] 
                                          precision    recall  f1-score   support

                             atis_flight      0.981     0.992     0.987       632
                            atis_airfare      0.960     1.000     0.980        48
                     atis_ground_service      1.000     1.000     1.000        36
                            atis_airline      1.000     1.000     1.000        38
                       atis_abbreviation      0.943     1.000     0.971        33
                           atis_aircraft      1.000     0.889     0.941         9
                        atis_flight_time      1.000     1.000     1.000         1
                           atis_quantity      0.429     1.000     0.600         3
                atis_flight#atis_airfare      0.833     0.417     0.556        12
                            atis_airport      1.000     0.944     0.971        18
                           atis_distance      1.000     1.000     1.000        10
                               atis_city      1.000     0.667     0.800         6
                        atis_ground_fare      1.000     0.857     0.923         7
                           atis_capacity      0.955     1.000     0.977        21
                          atis_flight_no      0.889     1.000     0.941         8
                               atis_meal      1.000     0.833     0.909         6
                        atis_restriction      0.000     0.000     0.000         0
             atis_airline#atis_flight_no      0.000     0.000     0.000         0
    atis_ground_service#atis_ground_fare      0.000     0.000     0.000         0
           atis_airfare#atis_flight_time      0.000     0.000     0.000         0
                           atis_cheapest      0.000     0.000     0.000         0
atis_aircraft#atis_flight#atis_flight_no      0.000     0.000     0.000         0

                               micro avg      0.974     0.980     0.977       888
                               macro avg      0.681     0.664     0.662       888
                            weighted avg      0.976     0.980     0.976       888

I0715 07:28:59.870824 140369481070464 interactiveshell.py:2882] 
                              precision    recall  f1-score   support

                           O      0.000     0.000     0.000       0.0
           B-toloc.city_name      0.971     0.993     0.982     716.0
         B-fromloc.city_name      0.987     0.999     0.993     704.0
           I-toloc.city_name      0.981     0.992     0.987     265.0
      B-depart_date.day_name      0.981     0.986     0.984     212.0
              B-airline_name      1.000     1.000     1.000     101.0
         I-fromloc.city_name      0.978     0.994     0.986     177.0
 B-depart_time.period_of_day      1.000     0.885     0.939     130.0
              I-airline_name      1.000     0.985     0.992      65.0
    B-depart_date.day_number      0.981     0.964     0.972      55.0
    B-depart_date.month_name      0.982     0.964     0.973      56.0
          B-depart_time.time      0.864     1.000     0.927      57.0
                B-round_trip      1.000     0.986     0.993      73.0
             B-cost_relative      1.000     0.973     0.986      37.0
                I-round_trip      1.000     1.000     1.000      71.0
                B-flight_mod      0.828     1.000     0.906      24.0
 B-depart_time.time_relative      0.984     0.969     0.977      65.0
          I-depart_time.time      0.945     1.000     0.972      52.0
         B-stoploc.city_name      0.952     1.000     0.976      20.0
                 B-city_name      0.909     0.526     0.667      57.0
                B-class_type      0.960     1.000     0.980      24.0
          B-arrive_time.time      0.917     0.971     0.943      34.0
 B-arrive_time.time_relative      0.968     0.968     0.968      31.0
                I-class_type      1.000     1.000     1.000      17.0
               B-flight_stop      1.000     1.000     1.000      21.0
          I-arrive_time.time      0.971     0.971     0.971      35.0
              B-airline_code      1.000     0.912     0.954      34.0
    I-depart_date.day_number      1.000     0.933     0.966      15.0
      I-fromloc.airport_name      0.405     1.000     0.577      15.0
      B-fromloc.airport_name      0.414     1.000     0.585      12.0
      B-arrive_date.day_name      0.733     1.000     0.846      11.0
          B-toloc.state_code      1.000     1.000     1.000      18.0
B-depart_date.today_relative      1.000     0.889     0.941       9.0
             B-flight_number      0.688     1.000     0.815      11.0
 B-depart_date.date_relative      0.944     1.000     0.971      17.0
          B-toloc.state_name      0.839     0.929     0.881      28.0
           B-fare_basis_code      0.944     1.000     0.971      17.0
               B-flight_time      1.000     1.000     1.000       1.0
                        B-or      1.000     1.000     1.000       3.0
 B-arrive_time.period_of_day      0.545     1.000     0.706       6.0
          B-meal_description      1.000     0.900     0.947      10.0
             I-cost_relative      1.000     0.667     0.800       3.0
              I-airport_name      0.909     0.345     0.500      29.0
               B-fare_amount      1.000     1.000     1.000       2.0
               I-fare_amount      1.000     1.000     1.000       2.0
                 I-city_name      1.000     0.667     0.800      30.0
        I-toloc.airport_name      1.000     1.000     1.000       3.0
            B-transport_type      1.000     1.000     1.000      10.0
    B-arrive_date.month_name      0.714     0.833     0.769       6.0
    B-arrive_date.day_number      0.714     0.833     0.769       6.0
         I-stoploc.city_name      1.000     1.000     1.000      10.0
                      B-meal      1.000     1.000     1.000      16.0
        B-fromloc.state_code      1.000     1.000     1.000      23.0
    B-depart_time.period_mod      1.000     1.000     1.000       5.0
                   B-connect      1.000     1.000     1.000       6.0
               B-flight_days      1.000     1.000     1.000      10.0
        B-toloc.airport_name      1.000     1.000     1.000       3.0
        B-fromloc.state_name      1.000     1.000     1.000      17.0
              B-airport_name      0.778     0.333     0.467      21.0
                   B-economy      1.000     1.000     1.000       6.0
               I-flight_time      1.000     1.000     1.000       1.0
             B-aircraft_code      0.882     0.909     0.896      33.0
                       B-mod      1.000     0.500     0.667       2.0
              B-airport_code      0.800     0.444     0.571       9.0
    B-depart_time.start_time      0.600     1.000     0.750       3.0
      B-depart_time.end_time      1.000     0.667     0.800       3.0
          B-depart_date.year      1.000     1.000     1.000       3.0
            I-transport_type      0.000     0.000     0.000       1.0
          B-restriction_code      1.000     1.000     1.000       4.0
    B-arrive_time.start_time      1.000     1.000     1.000       8.0
        B-toloc.airport_code      1.000     0.750     0.857       4.0
      B-arrive_time.end_time      0.875     0.875     0.875       8.0
      I-arrive_time.end_time      0.875     0.875     0.875       8.0
      I-depart_time.end_time      1.000     0.667     0.800       3.0
               I-flight_stop      0.000     0.000     0.000       0.0
      B-fromloc.airport_code      0.714     1.000     0.833       5.0
          I-restriction_code      1.000     1.000     1.000       3.0
    I-depart_time.start_time      1.000     1.000     1.000       1.0
          I-toloc.state_name      1.000     1.000     1.000       1.0
I-depart_date.today_relative      0.000     0.000     0.000       0.0
 B-arrive_date.date_relative      1.000     1.000     1.000       2.0
                I-flight_mod      1.000     0.167     0.286       6.0
                   I-economy      0.000     0.000     0.000       0.0
 B-return_date.date_relative      0.000     0.000     0.000       3.0
        I-fromloc.state_name      1.000     1.000     1.000       1.0
                B-state_code      1.000     1.000     1.000       1.0
    I-arrive_time.start_time      1.000     1.000     1.000       1.0
    I-arrive_date.day_number      0.000     0.000     0.000       0.0
                 B-meal_code      0.000     0.000     0.000       1.0
 I-depart_time.period_of_day      1.000     1.000     1.000       1.0
                  B-day_name      1.000     0.500     0.667       2.0
             B-period_of_day      1.000     0.500     0.667       4.0
        B-stoploc.state_code      0.000     0.000     0.000       0.0
    B-return_date.month_name      0.000     0.000     0.000       0.0
    B-return_date.day_number      0.000     0.000     0.000       0.0
    B-arrive_time.period_mod      0.000     0.000     0.000       0.0
                 I-meal_code      0.000     0.000     0.000       0.0
        B-toloc.country_name      1.000     1.000     1.000       1.0
                 B-days_code      0.500     1.000     0.667       1.0
 I-arrive_time.period_of_day      0.000     0.000     0.000       0.0
            I-today_relative      0.000     0.000     0.000       0.0
 B-return_time.period_of_day      0.000     0.000     0.000       0.0
                      B-time      0.000     0.000     0.000       0.0
           I-fare_basis_code      0.000     0.000     0.000       0.0
 I-arrive_time.time_relative      1.000     1.000     1.000       4.0
 I-depart_time.time_relative      0.000     0.000     0.000       1.0
            B-today_relative      0.000     0.000     0.000       0.0
                B-state_name      0.667     0.222     0.333       9.0
B-arrive_date.today_relative      0.000     0.000     0.000       0.0
    B-return_time.period_mod      0.000     0.000     0.000       0.0
                B-month_name      0.000     0.000     0.000       0.0
                B-day_number      0.000     0.000     0.000       0.0
 I-return_date.date_relative      0.667     0.667     0.667       3.0
I-return_date.today_relative      0.000     0.000     0.000       0.0
      B-stoploc.airport_name      0.000     0.000     0.000       0.0
             B-time_relative      0.000     0.000     0.000       0.0
                      I-time      0.000     0.000     0.000       0.0
    I-return_date.day_number      0.000     0.000     0.000       0.0
          I-meal_description      0.000     0.000     0.000       0.0
B-return_date.today_relative      0.000     0.000     0.000       0.0
      B-return_date.day_name      0.000     0.000     0.000       2.0

                   micro avg      0.955     0.957     0.956    3657.0
                   macro avg      0.681     0.662     0.657    3657.0
                weighted avg      0.963     0.957     0.954    3657.0

I0715 07:28:59.871813 140369481070464 interactiveshell.py:2882] Best Slot F1: 0.958
Reading ../data/atis.train.w-intent.iob
I0715 07:29:01.254766 140369481070464 interactiveshell.py:2882] Step 20600 | Loss: 0.0000 | Loss_intent: 0.0000 | Loss_slots: 0.0000 | Spent: 6.1 secs | LR: 0.000034
I0715 07:29:05.093160 140369481070464 interactiveshell.py:2882] Step 20650 | Loss: 0.0001 | Loss_intent: 0.0000 | Loss_slots: 0.0001 | Spent: 3.8 secs | LR: 0.000034
I0715 07:29:08.907574 140369481070464 interactiveshell.py:2882] Step 20700 | Loss: 0.0006 | Loss_intent: 0.0000 | Loss_slots: 0.0006 | Spent: 3.8 secs | LR: 0.000034
I0715 07:29:12.690350 140369481070464 interactiveshell.py:2882] Step 20750 | Loss: 0.0023 | Loss_intent: 0.0000 | Loss_slots: 0.0023 | Spent: 3.8 secs | LR: 0.000034
I0715 07:29:16.475701 140369481070464 interactiveshell.py:2882] Step 20800 | Loss: 0.0002 | Loss_intent: 0.0000 | Loss_slots: 0.0002 | Spent: 3.8 secs | LR: 0.000034
I0715 07:29:20.295903 140369481070464 interactiveshell.py:2882] Step 20850 | Loss: 0.0001 | Loss_intent: 0.0000 | Loss_slots: 0.0001 | Spent: 3.8 secs | LR: 0.000033
I0715 07:29:24.090823 140369481070464 interactiveshell.py:2882] Step 20900 | Loss: 0.0008 | Loss_intent: 0.0000 | Loss_slots: 0.0008 | Spent: 3.8 secs | LR: 0.000033
Reading ../data/atis.test.w-intent.iob
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.
  'precision', 'predicted', average, warn_for)
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1439: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples.
  'recall', 'true', average, warn_for)
I0715 07:29:25.767622 140369481070464 interactiveshell.py:2882] 
                                          precision    recall  f1-score   support

                             atis_flight      0.981     0.992     0.987       632
                            atis_airfare      0.960     1.000     0.980        48
                     atis_ground_service      1.000     1.000     1.000        36
                            atis_airline      1.000     1.000     1.000        38
                       atis_abbreviation      0.943     1.000     0.971        33
                           atis_aircraft      1.000     0.889     0.941         9
                        atis_flight_time      1.000     1.000     1.000         1
                           atis_quantity      0.429     1.000     0.600         3
                atis_flight#atis_airfare      0.833     0.417     0.556        12
                            atis_airport      1.000     0.944     0.971        18
                           atis_distance      1.000     1.000     1.000        10
                               atis_city      1.000     0.667     0.800         6
                        atis_ground_fare      1.000     0.857     0.923         7
                           atis_capacity      0.955     1.000     0.977        21
                          atis_flight_no      0.889     1.000     0.941         8
                               atis_meal      1.000     0.833     0.909         6
                        atis_restriction      0.000     0.000     0.000         0
             atis_airline#atis_flight_no      0.000     0.000     0.000         0
    atis_ground_service#atis_ground_fare      0.000     0.000     0.000         0
           atis_airfare#atis_flight_time      0.000     0.000     0.000         0
                           atis_cheapest      0.000     0.000     0.000         0
atis_aircraft#atis_flight#atis_flight_no      0.000     0.000     0.000         0

                               micro avg      0.974     0.980     0.977       888
                               macro avg      0.681     0.664     0.662       888
                            weighted avg      0.976     0.980     0.976       888

I0715 07:29:25.808356 140369481070464 interactiveshell.py:2882] 
                              precision    recall  f1-score   support

                           O      0.000     0.000     0.000       0.0
           B-toloc.city_name      0.974     0.993     0.983     716.0
         B-fromloc.city_name      0.987     0.999     0.993     704.0
           I-toloc.city_name      0.981     0.989     0.985     265.0
      B-depart_date.day_name      0.981     0.986     0.984     212.0
              B-airline_name      1.000     1.000     1.000     101.0
         I-fromloc.city_name      0.972     0.994     0.983     177.0
 B-depart_time.period_of_day      0.992     0.900     0.944     130.0
              I-airline_name      1.000     1.000     1.000      65.0
    B-depart_date.day_number      0.981     0.964     0.972      55.0
    B-depart_date.month_name      0.982     0.964     0.973      56.0
          B-depart_time.time      0.864     1.000     0.927      57.0
                B-round_trip      1.000     0.986     0.993      73.0
             B-cost_relative      1.000     0.973     0.986      37.0
                I-round_trip      1.000     1.000     1.000      71.0
                B-flight_mod      0.828     1.000     0.906      24.0
 B-depart_time.time_relative      0.984     0.969     0.977      65.0
          I-depart_time.time      0.945     1.000     0.972      52.0
         B-stoploc.city_name      1.000     1.000     1.000      20.0
                 B-city_name      0.909     0.526     0.667      57.0
                B-class_type      0.960     1.000     0.980      24.0
          B-arrive_time.time      0.943     0.971     0.957      34.0
 B-arrive_time.time_relative      0.968     0.968     0.968      31.0
                I-class_type      1.000     1.000     1.000      17.0
               B-flight_stop      1.000     1.000     1.000      21.0
          I-arrive_time.time      0.971     0.971     0.971      35.0
              B-airline_code      1.000     0.912     0.954      34.0
    I-depart_date.day_number      1.000     0.933     0.966      15.0
      I-fromloc.airport_name      0.405     1.000     0.577      15.0
      B-fromloc.airport_name      0.414     1.000     0.585      12.0
      B-arrive_date.day_name      0.733     1.000     0.846      11.0
          B-toloc.state_code      1.000     1.000     1.000      18.0
B-depart_date.today_relative      1.000     0.889     0.941       9.0
             B-flight_number      0.524     1.000     0.688      11.0
 B-depart_date.date_relative      0.944     1.000     0.971      17.0
          B-toloc.state_name      0.794     0.964     0.871      28.0
           B-fare_basis_code      0.944     1.000     0.971      17.0
               B-flight_time      1.000     1.000     1.000       1.0
                        B-or      1.000     1.000     1.000       3.0
 B-arrive_time.period_of_day      0.750     1.000     0.857       6.0
          B-meal_description      1.000     0.900     0.947      10.0
             I-cost_relative      1.000     0.667     0.800       3.0
              I-airport_name      0.909     0.345     0.500      29.0
               B-fare_amount      1.000     1.000     1.000       2.0
               I-fare_amount      1.000     1.000     1.000       2.0
                 I-city_name      1.000     0.667     0.800      30.0
        I-toloc.airport_name      1.000     1.000     1.000       3.0
            B-transport_type      1.000     0.900     0.947      10.0
    B-arrive_date.month_name      0.714     0.833     0.769       6.0
    B-arrive_date.day_number      0.714     0.833     0.769       6.0
         I-stoploc.city_name      1.000     0.900     0.947      10.0
                      B-meal      1.000     1.000     1.000      16.0
        B-fromloc.state_code      1.000     1.000     1.000      23.0
    B-depart_time.period_mod      1.000     1.000     1.000       5.0
                   B-connect      1.000     1.000     1.000       6.0
               B-flight_days      1.000     1.000     1.000      10.0
        B-toloc.airport_name      1.000     1.000     1.000       3.0
        B-fromloc.state_name      1.000     1.000     1.000      17.0
              B-airport_name      0.778     0.333     0.467      21.0
                   B-economy      1.000     1.000     1.000       6.0
               I-flight_time      1.000     1.000     1.000       1.0
             B-aircraft_code      0.966     0.848     0.903      33.0
                       B-mod      1.000     0.500     0.667       2.0
              B-airport_code      0.800     0.444     0.571       9.0
    B-depart_time.start_time      0.600     1.000     0.750       3.0
      B-depart_time.end_time      1.000     0.667     0.800       3.0
          B-depart_date.year      0.750     1.000     0.857       3.0
            I-transport_type      0.000     0.000     0.000       1.0
          B-restriction_code      1.000     1.000     1.000       4.0
    B-arrive_time.start_time      1.000     1.000     1.000       8.0
        B-toloc.airport_code      1.000     1.000     1.000       4.0
      B-arrive_time.end_time      0.889     1.000     0.941       8.0
      I-arrive_time.end_time      0.875     0.875     0.875       8.0
      I-depart_time.end_time      1.000     0.667     0.800       3.0
               I-flight_stop      0.000     0.000     0.000       0.0
      B-fromloc.airport_code      0.833     1.000     0.909       5.0
          I-restriction_code      1.000     1.000     1.000       3.0
    I-depart_time.start_time      1.000     1.000     1.000       1.0
          I-toloc.state_name      1.000     1.000     1.000       1.0
I-depart_date.today_relative      0.000     0.000     0.000       0.0
 B-arrive_date.date_relative      1.000     1.000     1.000       2.0
                I-flight_mod      1.000     0.167     0.286       6.0
                   I-economy      0.000     0.000     0.000       0.0
 B-return_date.date_relative      1.000     0.333     0.500       3.0
        I-fromloc.state_name      1.000     1.000     1.000       1.0
                B-state_code      1.000     1.000     1.000       1.0
    I-arrive_time.start_time      1.000     1.000     1.000       1.0
    I-arrive_date.day_number      0.000     0.000     0.000       0.0
                 B-meal_code      0.000     0.000     0.000       1.0
 I-depart_time.period_of_day      0.000     0.000     0.000       1.0
                  B-day_name      1.000     0.500     0.667       2.0
             B-period_of_day      1.000     0.500     0.667       4.0
        B-stoploc.state_code      0.000     0.000     0.000       0.0
    B-return_date.month_name      0.000     0.000     0.000       0.0
    B-return_date.day_number      0.000     0.000     0.000       0.0
    B-arrive_time.period_mod      0.000     0.000     0.000       0.0
                 I-meal_code      0.000     0.000     0.000       0.0
        B-toloc.country_name      1.000     1.000     1.000       1.0
                 B-days_code      0.500     1.000     0.667       1.0
 I-arrive_time.period_of_day      0.000     0.000     0.000       0.0
            I-today_relative      0.000     0.000     0.000       0.0
 B-return_time.period_of_day      0.000     0.000     0.000       0.0
                      B-time      0.000     0.000     0.000       0.0
           I-fare_basis_code      0.000     0.000     0.000       0.0
 I-arrive_time.time_relative      1.000     1.000     1.000       4.0
 I-depart_time.time_relative      0.000     0.000     0.000       1.0
            B-today_relative      0.000     0.000     0.000       0.0
                B-state_name      0.500     0.111     0.182       9.0
B-arrive_date.today_relative      0.000     0.000     0.000       0.0
    B-return_time.period_mod      0.000     0.000     0.000       0.0
                B-month_name      0.000     0.000     0.000       0.0
                B-day_number      0.000     0.000     0.000       0.0
 I-return_date.date_relative      0.667     0.667     0.667       3.0
I-return_date.today_relative      0.000     0.000     0.000       0.0
      B-stoploc.airport_name      0.000     0.000     0.000       0.0
             B-time_relative      0.000     0.000     0.000       0.0
                      I-time      0.000     0.000     0.000       0.0
    I-return_date.day_number      0.000     0.000     0.000       0.0
          I-meal_description      0.000     0.000     0.000       0.0
B-return_date.today_relative      0.000     0.000     0.000       0.0
      B-return_date.day_name      0.000     0.000     0.000       2.0

                   micro avg      0.955     0.957     0.956    3657.0
                   macro avg      0.680     0.657     0.653    3657.0
                weighted avg      0.963     0.957     0.954    3657.0

I0715 07:29:25.809161 140369481070464 interactiveshell.py:2882] Best Slot F1: 0.958
Reading ../data/atis.train.w-intent.iob
I0715 07:29:30.074550 140369481070464 interactiveshell.py:2882] Step 20950 | Loss: 0.0001 | Loss_intent: 0.0000 | Loss_slots: 0.0001 | Spent: 6.0 secs | LR: 0.000033
I0715 07:29:33.853501 140369481070464 interactiveshell.py:2882] Step 21000 | Loss: 0.0001 | Loss_intent: 0.0000 | Loss_slots: 0.0001 | Spent: 3.8 secs | LR: 0.000033
I0715 07:29:37.739344 140369481070464 interactiveshell.py:2882] Step 21050 | Loss: 0.0002 | Loss_intent: 0.0000 | Loss_slots: 0.0002 | Spent: 3.9 secs | LR: 0.000033
I0715 07:29:41.942936 140369481070464 interactiveshell.py:2882] Step 21100 | Loss: 0.0000 | Loss_intent: 0.0000 | Loss_slots: 0.0000 | Spent: 4.2 secs | LR: 0.000032
I0715 07:29:46.117192 140369481070464 interactiveshell.py:2882] Step 21150 | Loss: 0.0009 | Loss_intent: 0.0000 | Loss_slots: 0.0009 | Spent: 4.2 secs | LR: 0.000032
I0715 07:29:50.631770 140369481070464 interactiveshell.py:2882] Step 21200 | Loss: 0.0005 | Loss_intent: 0.0000 | Loss_slots: 0.0005 | Spent: 4.5 secs | LR: 0.000032
Reading ../data/atis.test.w-intent.iob
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.
  'precision', 'predicted', average, warn_for)
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1439: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples.
  'recall', 'true', average, warn_for)
I0715 07:29:53.333986 140369481070464 interactiveshell.py:2882] 
                                          precision    recall  f1-score   support

                             atis_flight      0.981     0.992     0.987       632
                            atis_airfare      0.960     1.000     0.980        48
                     atis_ground_service      1.000     1.000     1.000        36
                            atis_airline      1.000     1.000     1.000        38
                       atis_abbreviation      0.943     1.000     0.971        33
                           atis_aircraft      1.000     0.889     0.941         9
                        atis_flight_time      1.000     1.000     1.000         1
                           atis_quantity      0.429     1.000     0.600         3
                atis_flight#atis_airfare      0.833     0.417     0.556        12
                            atis_airport      1.000     0.944     0.971        18
                           atis_distance      1.000     1.000     1.000        10
                               atis_city      1.000     0.667     0.800         6
                        atis_ground_fare      1.000     0.857     0.923         7
                           atis_capacity      0.955     1.000     0.977        21
                          atis_flight_no      0.889     1.000     0.941         8
                               atis_meal      1.000     0.833     0.909         6
                        atis_restriction      0.000     0.000     0.000         0
             atis_airline#atis_flight_no      0.000     0.000     0.000         0
    atis_ground_service#atis_ground_fare      0.000     0.000     0.000         0
           atis_airfare#atis_flight_time      0.000     0.000     0.000         0
                           atis_cheapest      0.000     0.000     0.000         0
atis_aircraft#atis_flight#atis_flight_no      0.000     0.000     0.000         0

                               micro avg      0.974     0.980     0.977       888
                               macro avg      0.681     0.664     0.662       888
                            weighted avg      0.976     0.980     0.976       888

I0715 07:29:53.374760 140369481070464 interactiveshell.py:2882] 
                              precision    recall  f1-score   support

                           O      0.000     0.000     0.000       0.0
           B-toloc.city_name      0.973     0.993     0.983     716.0
         B-fromloc.city_name      0.987     1.000     0.994     704.0
           I-toloc.city_name      0.978     0.992     0.985     265.0
      B-depart_date.day_name      0.986     0.986     0.986     212.0
              B-airline_name      1.000     1.000     1.000     101.0
         I-fromloc.city_name      0.978     0.994     0.986     177.0
 B-depart_time.period_of_day      1.000     0.892     0.943     130.0
              I-airline_name      1.000     1.000     1.000      65.0
    B-depart_date.day_number      0.981     0.964     0.972      55.0
    B-depart_date.month_name      0.982     0.964     0.973      56.0
          B-depart_time.time      0.864     1.000     0.927      57.0
                B-round_trip      1.000     0.986     0.993      73.0
             B-cost_relative      1.000     0.973     0.986      37.0
                I-round_trip      1.000     1.000     1.000      71.0
                B-flight_mod      0.828     1.000     0.906      24.0
 B-depart_time.time_relative      0.984     0.969     0.977      65.0
          I-depart_time.time      0.945     1.000     0.972      52.0
         B-stoploc.city_name      1.000     1.000     1.000      20.0
                 B-city_name      0.882     0.526     0.659      57.0
                B-class_type      0.960     1.000     0.980      24.0
          B-arrive_time.time      0.917     0.971     0.943      34.0
 B-arrive_time.time_relative      0.968     0.968     0.968      31.0
                I-class_type      1.000     1.000     1.000      17.0
               B-flight_stop      1.000     1.000     1.000      21.0
          I-arrive_time.time      0.971     0.971     0.971      35.0
              B-airline_code      1.000     0.912     0.954      34.0
    I-depart_date.day_number      1.000     0.933     0.966      15.0
      I-fromloc.airport_name      0.405     1.000     0.577      15.0
      B-fromloc.airport_name      0.393     0.917     0.550      12.0
      B-arrive_date.day_name      0.733     1.000     0.846      11.0
          B-toloc.state_code      1.000     1.000     1.000      18.0
B-depart_date.today_relative      1.000     0.889     0.941       9.0
             B-flight_number      0.550     1.000     0.710      11.0
 B-depart_date.date_relative      0.944     1.000     0.971      17.0
          B-toloc.state_name      0.871     0.964     0.915      28.0
           B-fare_basis_code      0.944     1.000     0.971      17.0
               B-flight_time      1.000     1.000     1.000       1.0
                        B-or      1.000     1.000     1.000       3.0
 B-arrive_time.period_of_day      0.600     1.000     0.750       6.0
          B-meal_description      1.000     0.900     0.947      10.0
             I-cost_relative      1.000     0.667     0.800       3.0
              I-airport_name      0.909     0.345     0.500      29.0
               B-fare_amount      1.000     1.000     1.000       2.0
               I-fare_amount      1.000     1.000     1.000       2.0
                 I-city_name      1.000     0.600     0.750      30.0
        I-toloc.airport_name      1.000     1.000     1.000       3.0
            B-transport_type      1.000     0.900     0.947      10.0
    B-arrive_date.month_name      0.714     0.833     0.769       6.0
    B-arrive_date.day_number      0.714     0.833     0.769       6.0
         I-stoploc.city_name      1.000     0.900     0.947      10.0
                      B-meal      1.000     1.000     1.000      16.0
        B-fromloc.state_code      1.000     1.000     1.000      23.0
    B-depart_time.period_mod      1.000     1.000     1.000       5.0
                   B-connect      1.000     1.000     1.000       6.0
               B-flight_days      1.000     1.000     1.000      10.0
        B-toloc.airport_name      1.000     1.000     1.000       3.0
        B-fromloc.state_name      1.000     1.000     1.000      17.0
              B-airport_name      0.778     0.333     0.467      21.0
                   B-economy      1.000     1.000     1.000       6.0
               I-flight_time      1.000     1.000     1.000       1.0
             B-aircraft_code      0.966     0.848     0.903      33.0
                       B-mod      1.000     0.500     0.667       2.0
              B-airport_code      0.800     0.444     0.571       9.0
    B-depart_time.start_time      0.500     1.000     0.667       3.0
      B-depart_time.end_time      1.000     0.667     0.800       3.0
          B-depart_date.year      0.600     1.000     0.750       3.0
            I-transport_type      0.000     0.000     0.000       1.0
          B-restriction_code      1.000     1.000     1.000       4.0
    B-arrive_time.start_time      1.000     1.000     1.000       8.0
        B-toloc.airport_code      1.000     0.750     0.857       4.0
      B-arrive_time.end_time      0.875     0.875     0.875       8.0
      I-arrive_time.end_time      0.875     0.875     0.875       8.0
      I-depart_time.end_time      1.000     0.667     0.800       3.0
               I-flight_stop      0.000     0.000     0.000       0.0
      B-fromloc.airport_code      0.833     1.000     0.909       5.0
          I-restriction_code      1.000     1.000     1.000       3.0
    I-depart_time.start_time      1.000     1.000     1.000       1.0
          I-toloc.state_name      1.000     1.000     1.000       1.0
I-depart_date.today_relative      0.000     0.000     0.000       0.0
 B-arrive_date.date_relative      1.000     1.000     1.000       2.0
                I-flight_mod      1.000     0.167     0.286       6.0
                   I-economy      0.000     0.000     0.000       0.0
 B-return_date.date_relative      1.000     0.333     0.500       3.0
        I-fromloc.state_name      1.000     1.000     1.000       1.0
                B-state_code      1.000     1.000     1.000       1.0
    I-arrive_time.start_time      1.000     1.000     1.000       1.0
    I-arrive_date.day_number      0.000     0.000     0.000       0.0
                 B-meal_code      0.000     0.000     0.000       1.0
 I-depart_time.period_of_day      1.000     1.000     1.000       1.0
                  B-day_name      1.000     0.500     0.667       2.0
             B-period_of_day      1.000     0.500     0.667       4.0
        B-stoploc.state_code      0.000     0.000     0.000       0.0
    B-return_date.month_name      0.000     0.000     0.000       0.0
    B-return_date.day_number      0.000     0.000     0.000       0.0
    B-arrive_time.period_mod      0.000     0.000     0.000       0.0
                 I-meal_code      0.000     0.000     0.000       0.0
        B-toloc.country_name      1.000     1.000     1.000       1.0
                 B-days_code      0.500     1.000     0.667       1.0
 I-arrive_time.period_of_day      0.000     0.000     0.000       0.0
            I-today_relative      0.000     0.000     0.000       0.0
 B-return_time.period_of_day      0.000     0.000     0.000       0.0
                      B-time      0.000     0.000     0.000       0.0
           I-fare_basis_code      0.000     0.000     0.000       0.0
 I-arrive_time.time_relative      1.000     1.000     1.000       4.0
 I-depart_time.time_relative      0.000     0.000     0.000       1.0
            B-today_relative      0.000     0.000     0.000       0.0
                B-state_name      0.750     0.333     0.462       9.0
B-arrive_date.today_relative      0.000     0.000     0.000       0.0
    B-return_time.period_mod      0.000     0.000     0.000       0.0
                B-month_name      0.000     0.000     0.000       0.0
                B-day_number      0.000     0.000     0.000       0.0
 I-return_date.date_relative      0.667     0.667     0.667       3.0
I-return_date.today_relative      0.000     0.000     0.000       0.0
      B-stoploc.airport_name      0.000     0.000     0.000       0.0
             B-time_relative      0.000     0.000     0.000       0.0
                      I-time      0.000     0.000     0.000       0.0
    I-return_date.day_number      0.000     0.000     0.000       0.0
          I-meal_description      0.000     0.000     0.000       0.0
B-return_date.today_relative      0.000     0.000     0.000       0.0
      B-return_date.day_name      0.000     0.000     0.000       2.0

                   micro avg      0.955     0.957     0.956    3657.0
                   macro avg      0.687     0.663     0.659    3657.0
                weighted avg      0.964     0.957     0.955    3657.0

I0715 07:29:53.375755 140369481070464 interactiveshell.py:2882] Best Slot F1: 0.958
Reading ../data/atis.train.w-intent.iob
I0715 07:29:56.753245 140369481070464 interactiveshell.py:2882] Step 21250 | Loss: 0.0000 | Loss_intent: 0.0000 | Loss_slots: 0.0000 | Spent: 6.1 secs | LR: 0.000032
I0715 07:30:00.622763 140369481070464 interactiveshell.py:2882] Step 21300 | Loss: 0.0005 | Loss_intent: 0.0000 | Loss_slots: 0.0005 | Spent: 3.9 secs | LR: 0.000032
I0715 07:30:05.804350 140369481070464 interactiveshell.py:2882] Step 21350 | Loss: 0.0008 | Loss_intent: 0.0000 | Loss_slots: 0.0008 | Spent: 5.2 secs | LR: 0.000032
I0715 07:30:09.713340 140369481070464 interactiveshell.py:2882] Step 21400 | Loss: 0.0000 | Loss_intent: 0.0000 | Loss_slots: 0.0000 | Spent: 3.9 secs | LR: 0.000031
I0715 07:30:13.572124 140369481070464 interactiveshell.py:2882] Step 21450 | Loss: 0.0004 | Loss_intent: 0.0000 | Loss_slots: 0.0004 | Spent: 3.9 secs | LR: 0.000031
I0715 07:30:17.418920 140369481070464 interactiveshell.py:2882] Step 21500 | Loss: 0.0001 | Loss_intent: 0.0000 | Loss_slots: 0.0001 | Spent: 3.8 secs | LR: 0.000031
Reading ../data/atis.test.w-intent.iob
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.
  'precision', 'predicted', average, warn_for)
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1439: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples.
  'recall', 'true', average, warn_for)
I0715 07:30:21.006066 140369481070464 interactiveshell.py:2882] 
                                          precision    recall  f1-score   support

                             atis_flight      0.981     0.992     0.987       632
                            atis_airfare      0.960     1.000     0.980        48
                     atis_ground_service      1.000     1.000     1.000        36
                            atis_airline      1.000     1.000     1.000        38
                       atis_abbreviation      0.943     1.000     0.971        33
                           atis_aircraft      1.000     0.889     0.941         9
                        atis_flight_time      1.000     1.000     1.000         1
                           atis_quantity      0.429     1.000     0.600         3
                atis_flight#atis_airfare      0.833     0.417     0.556        12
                            atis_airport      1.000     0.944     0.971        18
                           atis_distance      1.000     1.000     1.000        10
                               atis_city      1.000     0.667     0.800         6
                        atis_ground_fare      1.000     0.857     0.923         7
                           atis_capacity      0.955     1.000     0.977        21
                          atis_flight_no      0.889     1.000     0.941         8
                               atis_meal      1.000     0.833     0.909         6
                        atis_restriction      0.000     0.000     0.000         0
             atis_airline#atis_flight_no      0.000     0.000     0.000         0
    atis_ground_service#atis_ground_fare      0.000     0.000     0.000         0
           atis_airfare#atis_flight_time      0.000     0.000     0.000         0
                           atis_cheapest      0.000     0.000     0.000         0
atis_aircraft#atis_flight#atis_flight_no      0.000     0.000     0.000         0

                               micro avg      0.974     0.980     0.977       888
                               macro avg      0.681     0.664     0.662       888
                            weighted avg      0.976     0.980     0.976       888

I0715 07:30:21.046893 140369481070464 interactiveshell.py:2882] 
                              precision    recall  f1-score   support

                           O      0.000     0.000     0.000       0.0
           B-toloc.city_name      0.974     0.993     0.983     716.0
         B-fromloc.city_name      0.986     1.000     0.993     704.0
           I-toloc.city_name      0.978     0.992     0.985     265.0
      B-depart_date.day_name      0.986     0.986     0.986     212.0
              B-airline_name      1.000     1.000     1.000     101.0
         I-fromloc.city_name      0.978     0.994     0.986     177.0
 B-depart_time.period_of_day      1.000     0.892     0.943     130.0
              I-airline_name      1.000     1.000     1.000      65.0
    B-depart_date.day_number      0.981     0.964     0.972      55.0
    B-depart_date.month_name      0.982     0.964     0.973      56.0
          B-depart_time.time      0.851     1.000     0.919      57.0
                B-round_trip      1.000     0.986     0.993      73.0
             B-cost_relative      1.000     0.973     0.986      37.0
                I-round_trip      1.000     1.000     1.000      71.0
                B-flight_mod      0.828     1.000     0.906      24.0
 B-depart_time.time_relative      0.984     0.969     0.977      65.0
          I-depart_time.time      0.945     1.000     0.972      52.0
         B-stoploc.city_name      1.000     1.000     1.000      20.0
                 B-city_name      0.886     0.544     0.674      57.0
                B-class_type      0.960     1.000     0.980      24.0
          B-arrive_time.time      0.943     0.971     0.957      34.0
 B-arrive_time.time_relative      0.968     0.968     0.968      31.0
                I-class_type      1.000     1.000     1.000      17.0
               B-flight_stop      1.000     1.000     1.000      21.0
          I-arrive_time.time      0.971     0.971     0.971      35.0
              B-airline_code      1.000     0.912     0.954      34.0
    I-depart_date.day_number      1.000     0.933     0.966      15.0
      I-fromloc.airport_name      0.405     1.000     0.577      15.0
      B-fromloc.airport_name      0.414     1.000     0.585      12.0
      B-arrive_date.day_name      0.733     1.000     0.846      11.0
          B-toloc.state_code      1.000     1.000     1.000      18.0
B-depart_date.today_relative      1.000     0.889     0.941       9.0
             B-flight_number      0.524     1.000     0.688      11.0
 B-depart_date.date_relative      0.944     1.000     0.971      17.0
          B-toloc.state_name      0.964     0.964     0.964      28.0
           B-fare_basis_code      0.944     1.000     0.971      17.0
               B-flight_time      1.000     1.000     1.000       1.0
                        B-or      1.000     1.000     1.000       3.0
 B-arrive_time.period_of_day      0.600     1.000     0.750       6.0
          B-meal_description      1.000     0.900     0.947      10.0
             I-cost_relative      1.000     0.667     0.800       3.0
              I-airport_name      0.909     0.345     0.500      29.0
               B-fare_amount      1.000     1.000     1.000       2.0
               I-fare_amount      1.000     1.000     1.000       2.0
                 I-city_name      1.000     0.600     0.750      30.0
        I-toloc.airport_name      1.000     1.000     1.000       3.0
            B-transport_type      1.000     0.900     0.947      10.0
    B-arrive_date.month_name      0.714     0.833     0.769       6.0
    B-arrive_date.day_number      0.714     0.833     0.769       6.0
         I-stoploc.city_name      1.000     0.900     0.947      10.0
                      B-meal      1.000     1.000     1.000      16.0
        B-fromloc.state_code      1.000     1.000     1.000      23.0
    B-depart_time.period_mod      1.000     1.000     1.000       5.0
                   B-connect      1.000     1.000     1.000       6.0
               B-flight_days      1.000     1.000     1.000      10.0
        B-toloc.airport_name      1.000     1.000     1.000       3.0
        B-fromloc.state_name      1.000     1.000     1.000      17.0
              B-airport_name      0.778     0.333     0.467      21.0
                   B-economy      1.000     1.000     1.000       6.0
               I-flight_time      1.000     1.000     1.000       1.0
             B-aircraft_code      0.964     0.818     0.885      33.0
                       B-mod      1.000     0.500     0.667       2.0
              B-airport_code      0.667     0.444     0.533       9.0
    B-depart_time.start_time      0.600     1.000     0.750       3.0
      B-depart_time.end_time      1.000     1.000     1.000       3.0
          B-depart_date.year      0.600     1.000     0.750       3.0
            I-transport_type      0.000     0.000     0.000       1.0
          B-restriction_code      1.000     1.000     1.000       4.0
    B-arrive_time.start_time      1.000     0.875     0.933       8.0
        B-toloc.airport_code      1.000     0.750     0.857       4.0
      B-arrive_time.end_time      1.000     0.875     0.933       8.0
      I-arrive_time.end_time      0.875     0.875     0.875       8.0
      I-depart_time.end_time      1.000     0.667     0.800       3.0
               I-flight_stop      0.000     0.000     0.000       0.0
      B-fromloc.airport_code      0.833     1.000     0.909       5.0
          I-restriction_code      1.000     1.000     1.000       3.0
    I-depart_time.start_time      1.000     1.000     1.000       1.0
          I-toloc.state_name      1.000     1.000     1.000       1.0
I-depart_date.today_relative      0.000     0.000     0.000       0.0
 B-arrive_date.date_relative      1.000     1.000     1.000       2.0
                I-flight_mod      1.000     0.167     0.286       6.0
                   I-economy      0.000     0.000     0.000       0.0
 B-return_date.date_relative      1.000     0.333     0.500       3.0
        I-fromloc.state_name      1.000     1.000     1.000       1.0
                B-state_code      1.000     1.000     1.000       1.0
    I-arrive_time.start_time      1.000     1.000     1.000       1.0
    I-arrive_date.day_number      0.000     0.000     0.000       0.0
                 B-meal_code      0.000     0.000     0.000       1.0
 I-depart_time.period_of_day      1.000     1.000     1.000       1.0
                  B-day_name      1.000     0.500     0.667       2.0
             B-period_of_day      1.000     0.500     0.667       4.0
        B-stoploc.state_code      0.000     0.000     0.000       0.0
    B-return_date.month_name      0.000     0.000     0.000       0.0
    B-return_date.day_number      0.000     0.000     0.000       0.0
    B-arrive_time.period_mod      0.000     0.000     0.000       0.0
                 I-meal_code      0.000     0.000     0.000       0.0
        B-toloc.country_name      1.000     1.000     1.000       1.0
                 B-days_code      1.000     1.000     1.000       1.0
 I-arrive_time.period_of_day      0.000     0.000     0.000       0.0
            I-today_relative      0.000     0.000     0.000       0.0
 B-return_time.period_of_day      0.000     0.000     0.000       0.0
                      B-time      0.000     0.000     0.000       0.0
           I-fare_basis_code      0.000     0.000     0.000       0.0
 I-arrive_time.time_relative      1.000     1.000     1.000       4.0
 I-depart_time.time_relative      0.000     0.000     0.000       1.0
            B-today_relative      0.000     0.000     0.000       0.0
                B-state_name      0.857     0.667     0.750       9.0
B-arrive_date.today_relative      0.000     0.000     0.000       0.0
    B-return_time.period_mod      0.000     0.000     0.000       0.0
                B-month_name      0.000     0.000     0.000       0.0
                B-day_number      0.000     0.000     0.000       0.0
 I-return_date.date_relative      0.667     0.667     0.667       3.0
I-return_date.today_relative      0.000     0.000     0.000       0.0
      B-stoploc.airport_name      0.000     0.000     0.000       0.0
             B-time_relative      0.000     0.000     0.000       0.0
                      I-time      0.000     0.000     0.000       0.0
    I-return_date.day_number      0.000     0.000     0.000       0.0
          I-meal_description      0.000     0.000     0.000       0.0
B-return_date.today_relative      0.000     0.000     0.000       0.0
      B-return_date.day_name      0.000     0.000     0.000       2.0

                   micro avg      0.956     0.958     0.957    3657.0
                   macro avg      0.693     0.668     0.667    3657.0
                weighted avg      0.965     0.958     0.956    3657.0

I0715 07:30:21.047963 140369481070464 interactiveshell.py:2882] Best Slot F1: 0.958
Reading ../data/atis.train.w-intent.iob
I0715 07:30:23.615166 140369481070464 interactiveshell.py:2882] Step 21550 | Loss: 0.0001 | Loss_intent: 0.0000 | Loss_slots: 0.0001 | Spent: 6.2 secs | LR: 0.000031
I0715 07:30:28.039842 140369481070464 interactiveshell.py:2882] Step 21600 | Loss: 0.0003 | Loss_intent: 0.0000 | Loss_slots: 0.0003 | Spent: 4.4 secs | LR: 0.000031
I0715 07:30:32.224263 140369481070464 interactiveshell.py:2882] Step 21650 | Loss: 0.0001 | Loss_intent: 0.0000 | Loss_slots: 0.0001 | Spent: 4.2 secs | LR: 0.000031
I0715 07:30:36.027277 140369481070464 interactiveshell.py:2882] Step 21700 | Loss: 0.0001 | Loss_intent: 0.0000 | Loss_slots: 0.0001 | Spent: 3.8 secs | LR: 0.000030
I0715 07:30:39.881754 140369481070464 interactiveshell.py:2882] Step 21750 | Loss: 0.0000 | Loss_intent: 0.0000 | Loss_slots: 0.0000 | Spent: 3.9 secs | LR: 0.000030
I0715 07:30:43.735874 140369481070464 interactiveshell.py:2882] Step 21800 | Loss: 0.0001 | Loss_intent: 0.0000 | Loss_slots: 0.0001 | Spent: 3.9 secs | LR: 0.000030
Reading ../data/atis.test.w-intent.iob
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.
  'precision', 'predicted', average, warn_for)
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1439: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples.
  'recall', 'true', average, warn_for)
I0715 07:30:48.590317 140369481070464 interactiveshell.py:2882] 
                                          precision    recall  f1-score   support

                             atis_flight      0.981     0.992     0.987       632
                            atis_airfare      0.960     1.000     0.980        48
                     atis_ground_service      1.000     1.000     1.000        36
                            atis_airline      1.000     1.000     1.000        38
                       atis_abbreviation      0.943     1.000     0.971        33
                           atis_aircraft      1.000     0.889     0.941         9
                        atis_flight_time      1.000     1.000     1.000         1
                           atis_quantity      0.429     1.000     0.600         3
                atis_flight#atis_airfare      0.833     0.417     0.556        12
                            atis_airport      1.000     0.944     0.971        18
                           atis_distance      1.000     1.000     1.000        10
                               atis_city      1.000     0.667     0.800         6
                        atis_ground_fare      1.000     0.857     0.923         7
                           atis_capacity      0.955     1.000     0.977        21
                          atis_flight_no      0.889     1.000     0.941         8
                               atis_meal      1.000     0.833     0.909         6
                        atis_restriction      0.000     0.000     0.000         0
             atis_airline#atis_flight_no      0.000     0.000     0.000         0
    atis_ground_service#atis_ground_fare      0.000     0.000     0.000         0
           atis_airfare#atis_flight_time      0.000     0.000     0.000         0
                           atis_cheapest      0.000     0.000     0.000         0
atis_aircraft#atis_flight#atis_flight_no      0.000     0.000     0.000         0

                               micro avg      0.974     0.980     0.977       888
                               macro avg      0.681     0.664     0.662       888
                            weighted avg      0.976     0.980     0.976       888

I0715 07:30:48.631514 140369481070464 interactiveshell.py:2882] 
                              precision    recall  f1-score   support

                           O      0.000     0.000     0.000       0.0
           B-toloc.city_name      0.971     0.993     0.982     716.0
         B-fromloc.city_name      0.986     0.999     0.992     704.0
           I-toloc.city_name      0.978     0.992     0.985     265.0
      B-depart_date.day_name      0.981     0.986     0.984     212.0
              B-airline_name      1.000     1.000     1.000     101.0
         I-fromloc.city_name      0.972     0.994     0.983     177.0
 B-depart_time.period_of_day      1.000     0.892     0.943     130.0
              I-airline_name      1.000     1.000     1.000      65.0
    B-depart_date.day_number      0.981     0.964     0.972      55.0
    B-depart_date.month_name      0.982     0.964     0.973      56.0
          B-depart_time.time      0.851     1.000     0.919      57.0
                B-round_trip      1.000     0.986     0.993      73.0
             B-cost_relative      1.000     0.973     0.986      37.0
                I-round_trip      1.000     1.000     1.000      71.0
                B-flight_mod      0.828     1.000     0.906      24.0
 B-depart_time.time_relative      0.984     0.969     0.977      65.0
          I-depart_time.time      0.945     1.000     0.972      52.0
         B-stoploc.city_name      0.952     1.000     0.976      20.0
                 B-city_name      0.838     0.544     0.660      57.0
                B-class_type      0.960     1.000     0.980      24.0
          B-arrive_time.time      0.943     0.971     0.957      34.0
 B-arrive_time.time_relative      0.968     0.968     0.968      31.0
                I-class_type      1.000     1.000     1.000      17.0
               B-flight_stop      1.000     1.000     1.000      21.0
          I-arrive_time.time      0.971     0.971     0.971      35.0
              B-airline_code      1.000     0.912     0.954      34.0
    I-depart_date.day_number      1.000     0.933     0.966      15.0
      I-fromloc.airport_name      0.417     1.000     0.588      15.0
      B-fromloc.airport_name      0.414     1.000     0.585      12.0
      B-arrive_date.day_name      0.786     1.000     0.880      11.0
          B-toloc.state_code      1.000     1.000     1.000      18.0
B-depart_date.today_relative      1.000     0.889     0.941       9.0
             B-flight_number      0.500     1.000     0.667      11.0
 B-depart_date.date_relative      0.944     1.000     0.971      17.0
          B-toloc.state_name      0.926     0.893     0.909      28.0
           B-fare_basis_code      0.944     1.000     0.971      17.0
               B-flight_time      1.000     1.000     1.000       1.0
                        B-or      1.000     1.000     1.000       3.0
 B-arrive_time.period_of_day      0.600     1.000     0.750       6.0
          B-meal_description      1.000     0.900     0.947      10.0
             I-cost_relative      1.000     0.667     0.800       3.0
              I-airport_name      0.909     0.345     0.500      29.0
               B-fare_amount      1.000     1.000     1.000       2.0
               I-fare_amount      1.000     1.000     1.000       2.0
                 I-city_name      1.000     0.600     0.750      30.0
        I-toloc.airport_name      1.000     1.000     1.000       3.0
            B-transport_type      1.000     0.900     0.947      10.0
    B-arrive_date.month_name      0.714     0.833     0.769       6.0
    B-arrive_date.day_number      0.714     0.833     0.769       6.0
         I-stoploc.city_name      1.000     0.900     0.947      10.0
                      B-meal      1.000     1.000     1.000      16.0
        B-fromloc.state_code      1.000     1.000     1.000      23.0
    B-depart_time.period_mod      1.000     1.000     1.000       5.0
                   B-connect      1.000     1.000     1.000       6.0
               B-flight_days      1.000     1.000     1.000      10.0
        B-toloc.airport_name      1.000     1.000     1.000       3.0
        B-fromloc.state_name      1.000     1.000     1.000      17.0
              B-airport_name      0.778     0.333     0.467      21.0
                   B-economy      1.000     1.000     1.000       6.0
               I-flight_time      1.000     1.000     1.000       1.0
             B-aircraft_code      1.000     0.818     0.900      33.0
                       B-mod      1.000     0.500     0.667       2.0
              B-airport_code      0.667     0.444     0.533       9.0
    B-depart_time.start_time      0.600     1.000     0.750       3.0
      B-depart_time.end_time      1.000     0.667     0.800       3.0
          B-depart_date.year      0.600     1.000     0.750       3.0
            I-transport_type      0.000     0.000     0.000       1.0
          B-restriction_code      1.000     1.000     1.000       4.0
    B-arrive_time.start_time      1.000     0.875     0.933       8.0
        B-toloc.airport_code      1.000     0.750     0.857       4.0
      B-arrive_time.end_time      0.875     0.875     0.875       8.0
      I-arrive_time.end_time      0.875     0.875     0.875       8.0
      I-depart_time.end_time      1.000     0.667     0.800       3.0
               I-flight_stop      0.000     0.000     0.000       0.0
      B-fromloc.airport_code      0.833     1.000     0.909       5.0
          I-restriction_code      1.000     1.000     1.000       3.0
    I-depart_time.start_time      1.000     1.000     1.000       1.0
          I-toloc.state_name      1.000     1.000     1.000       1.0
I-depart_date.today_relative      0.000     0.000     0.000       0.0
 B-arrive_date.date_relative      1.000     1.000     1.000       2.0
                I-flight_mod      1.000     0.167     0.286       6.0
                   I-economy      0.000     0.000     0.000       0.0
 B-return_date.date_relative      1.000     0.333     0.500       3.0
        I-fromloc.state_name      1.000     1.000     1.000       1.0
                B-state_code      1.000     1.000     1.000       1.0
    I-arrive_time.start_time      1.000     1.000     1.000       1.0
    I-arrive_date.day_number      0.000     0.000     0.000       0.0
                 B-meal_code      0.000     0.000     0.000       1.0
 I-depart_time.period_of_day      1.000     1.000     1.000       1.0
                  B-day_name      1.000     0.500     0.667       2.0
             B-period_of_day      1.000     0.500     0.667       4.0
        B-stoploc.state_code      0.000     0.000     0.000       0.0
    B-return_date.month_name      0.000     0.000     0.000       0.0
    B-return_date.day_number      0.000     0.000     0.000       0.0
    B-arrive_time.period_mod      0.000     0.000     0.000       0.0
                 I-meal_code      0.000     0.000     0.000       0.0
        B-toloc.country_name      1.000     1.000     1.000       1.0
                 B-days_code      1.000     1.000     1.000       1.0
 I-arrive_time.period_of_day      0.000     0.000     0.000       0.0
            I-today_relative      0.000     0.000     0.000       0.0
 B-return_time.period_of_day      0.000     0.000     0.000       0.0
                      B-time      0.000     0.000     0.000       0.0
           I-fare_basis_code      0.000     0.000     0.000       0.0
 I-arrive_time.time_relative      1.000     1.000     1.000       4.0
 I-depart_time.time_relative      0.000     0.000     0.000       1.0
            B-today_relative      0.000     0.000     0.000       0.0
                B-state_name      0.750     0.333     0.462       9.0
B-arrive_date.today_relative      0.000     0.000     0.000       0.0
    B-return_time.period_mod      0.000     0.000     0.000       0.0
                B-month_name      0.000     0.000     0.000       0.0
                B-day_number      0.000     0.000     0.000       0.0
 I-return_date.date_relative      0.667     0.667     0.667       3.0
I-return_date.today_relative      0.000     0.000     0.000       0.0
      B-stoploc.airport_name      0.000     0.000     0.000       0.0
             B-time_relative      0.000     0.000     0.000       0.0
                      I-time      0.000     0.000     0.000       0.0
    I-return_date.day_number      0.000     0.000     0.000       0.0
          I-meal_description      0.000     0.000     0.000       0.0
B-return_date.today_relative      0.000     0.000     0.000       0.0
      B-return_date.day_name      0.000     0.000     0.000       2.0

                   micro avg      0.954     0.956     0.955    3657.0
                   macro avg      0.691     0.662     0.662    3657.0
                weighted avg      0.963     0.956     0.954    3657.0

I0715 07:30:48.632349 140369481070464 interactiveshell.py:2882] Best Slot F1: 0.958
Reading ../data/atis.train.w-intent.iob
I0715 07:30:50.159929 140369481070464 interactiveshell.py:2882] Step 21850 | Loss: 0.0002 | Loss_intent: 0.0000 | Loss_slots: 0.0002 | Spent: 6.4 secs | LR: 0.000030
I0715 07:30:53.994699 140369481070464 interactiveshell.py:2882] Step 21900 | Loss: 0.0002 | Loss_intent: 0.0000 | Loss_slots: 0.0002 | Spent: 3.8 secs | LR: 0.000030
I0715 07:30:57.824352 140369481070464 interactiveshell.py:2882] Step 21950 | Loss: 0.0003 | Loss_intent: 0.0000 | Loss_slots: 0.0003 | Spent: 3.8 secs | LR: 0.000030
I0715 07:31:01.647092 140369481070464 interactiveshell.py:2882] Step 22000 | Loss: 0.0000 | Loss_intent: 0.0000 | Loss_slots: 0.0000 | Spent: 3.8 secs | LR: 0.000030
I0715 07:31:05.477034 140369481070464 interactiveshell.py:2882] Step 22050 | Loss: 0.0001 | Loss_intent: 0.0000 | Loss_slots: 0.0001 | Spent: 3.8 secs | LR: 0.000029
I0715 07:31:09.288417 140369481070464 interactiveshell.py:2882] Step 22100 | Loss: 0.0005 | Loss_intent: 0.0000 | Loss_slots: 0.0005 | Spent: 3.8 secs | LR: 0.000029
I0715 07:31:13.098365 140369481070464 interactiveshell.py:2882] Step 22150 | Loss: 0.0000 | Loss_intent: 0.0000 | Loss_slots: 0.0000 | Spent: 3.8 secs | LR: 0.000029
Reading ../data/atis.test.w-intent.iob
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.
  'precision', 'predicted', average, warn_for)
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1439: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples.
  'recall', 'true', average, warn_for)
I0715 07:31:14.624684 140369481070464 interactiveshell.py:2882] 
                                          precision    recall  f1-score   support

                             atis_flight      0.980     0.992     0.986       632
                            atis_airfare      0.960     1.000     0.980        48
                     atis_ground_service      1.000     1.000     1.000        36
                            atis_airline      1.000     1.000     1.000        38
                       atis_abbreviation      0.943     1.000     0.971        33
                           atis_aircraft      1.000     0.889     0.941         9
                        atis_flight_time      1.000     1.000     1.000         1
                           atis_quantity      0.429     1.000     0.600         3
                atis_flight#atis_airfare      0.833     0.417     0.556        12
                            atis_airport      1.000     0.889     0.941        18
                           atis_distance      1.000     1.000     1.000        10
                               atis_city      1.000     0.667     0.800         6
                        atis_ground_fare      1.000     0.857     0.923         7
                           atis_capacity      0.955     1.000     0.977        21
                          atis_flight_no      0.889     1.000     0.941         8
                               atis_meal      1.000     0.833     0.909         6
                        atis_restriction      0.000     0.000     0.000         0
             atis_airline#atis_flight_no      0.000     0.000     0.000         0
    atis_ground_service#atis_ground_fare      0.000     0.000     0.000         0
           atis_airfare#atis_flight_time      0.000     0.000     0.000         0
                           atis_cheapest      0.000     0.000     0.000         0
atis_aircraft#atis_flight#atis_flight_no      0.000     0.000     0.000         0

                               micro avg      0.973     0.979     0.976       888
                               macro avg      0.681     0.661     0.660       888
                            weighted avg      0.975     0.979     0.975       888

I0715 07:31:14.663879 140369481070464 interactiveshell.py:2882] 
                              precision    recall  f1-score   support

                           O      0.000     0.000     0.000       0.0
           B-toloc.city_name      0.970     0.993     0.981     716.0
         B-fromloc.city_name      0.987     0.999     0.993     704.0
           I-toloc.city_name      0.978     0.992     0.985     265.0
      B-depart_date.day_name      0.986     0.986     0.986     212.0
              B-airline_name      1.000     1.000     1.000     101.0
         I-fromloc.city_name      0.978     0.994     0.986     177.0
 B-depart_time.period_of_day      1.000     0.900     0.947     130.0
              I-airline_name      1.000     0.985     0.992      65.0
    B-depart_date.day_number      0.981     0.964     0.972      55.0
    B-depart_date.month_name      0.982     0.964     0.973      56.0
          B-depart_time.time      0.851     1.000     0.919      57.0
                B-round_trip      1.000     0.986     0.993      73.0
             B-cost_relative      1.000     0.973     0.986      37.0
                I-round_trip      1.000     1.000     1.000      71.0
                B-flight_mod      0.828     1.000     0.906      24.0
 B-depart_time.time_relative      0.984     0.969     0.977      65.0
          I-depart_time.time      0.945     1.000     0.972      52.0
         B-stoploc.city_name      1.000     1.000     1.000      20.0
                 B-city_name      0.912     0.544     0.681      57.0
                B-class_type      0.960     1.000     0.980      24.0
          B-arrive_time.time      0.943     0.971     0.957      34.0
 B-arrive_time.time_relative      0.968     0.968     0.968      31.0
                I-class_type      1.000     1.000     1.000      17.0
               B-flight_stop      1.000     1.000     1.000      21.0
          I-arrive_time.time      0.971     0.971     0.971      35.0
              B-airline_code      1.000     0.912     0.954      34.0
    I-depart_date.day_number      1.000     0.933     0.966      15.0
      I-fromloc.airport_name      0.405     1.000     0.577      15.0
      B-fromloc.airport_name      0.400     1.000     0.571      12.0
      B-arrive_date.day_name      0.733     1.000     0.846      11.0
          B-toloc.state_code      1.000     1.000     1.000      18.0
B-depart_date.today_relative      1.000     0.889     0.941       9.0
             B-flight_number      0.500     1.000     0.667      11.0
 B-depart_date.date_relative      0.944     1.000     0.971      17.0
          B-toloc.state_name      0.962     0.893     0.926      28.0
           B-fare_basis_code      0.944     1.000     0.971      17.0
               B-flight_time      1.000     1.000     1.000       1.0
                        B-or      1.000     1.000     1.000       3.0
 B-arrive_time.period_of_day      0.667     1.000     0.800       6.0
          B-meal_description      1.000     0.900     0.947      10.0
             I-cost_relative      1.000     0.667     0.800       3.0
              I-airport_name      0.909     0.345     0.500      29.0
               B-fare_amount      1.000     1.000     1.000       2.0
               I-fare_amount      1.000     1.000     1.000       2.0
                 I-city_name      1.000     0.667     0.800      30.0
        I-toloc.airport_name      1.000     1.000     1.000       3.0
            B-transport_type      0.833     1.000     0.909      10.0
    B-arrive_date.month_name      0.714     0.833     0.769       6.0
    B-arrive_date.day_number      0.714     0.833     0.769       6.0
         I-stoploc.city_name      1.000     1.000     1.000      10.0
                      B-meal      1.000     1.000     1.000      16.0
        B-fromloc.state_code      1.000     1.000     1.000      23.0
    B-depart_time.period_mod      1.000     1.000     1.000       5.0
                   B-connect      1.000     1.000     1.000       6.0
               B-flight_days      1.000     1.000     1.000      10.0
        B-toloc.airport_name      1.000     1.000     1.000       3.0
        B-fromloc.state_name      1.000     1.000     1.000      17.0
              B-airport_name      0.778     0.333     0.467      21.0
                   B-economy      1.000     1.000     1.000       6.0
               I-flight_time      1.000     1.000     1.000       1.0
             B-aircraft_code      1.000     0.758     0.862      33.0
                       B-mod      1.000     0.500     0.667       2.0
              B-airport_code      0.667     0.444     0.533       9.0
    B-depart_time.start_time      0.600     1.000     0.750       3.0
      B-depart_time.end_time      1.000     0.667     0.800       3.0
          B-depart_date.year      0.600     1.000     0.750       3.0
            I-transport_type      0.000     0.000     0.000       1.0
          B-restriction_code      1.000     1.000     1.000       4.0
    B-arrive_time.start_time      1.000     1.000     1.000       8.0
        B-toloc.airport_code      1.000     0.750     0.857       4.0
      B-arrive_time.end_time      0.875     0.875     0.875       8.0
      I-arrive_time.end_time      0.875     0.875     0.875       8.0
      I-depart_time.end_time      1.000     0.667     0.800       3.0
               I-flight_stop      0.000     0.000     0.000       0.0
      B-fromloc.airport_code      0.833     1.000     0.909       5.0
          I-restriction_code      1.000     1.000     1.000       3.0
    I-depart_time.start_time      1.000     1.000     1.000       1.0
          I-toloc.state_name      1.000     1.000     1.000       1.0
I-depart_date.today_relative      0.000     0.000     0.000       0.0
 B-arrive_date.date_relative      1.000     1.000     1.000       2.0
                I-flight_mod      1.000     0.167     0.286       6.0
                   I-economy      0.000     0.000     0.000       0.0
 B-return_date.date_relative      1.000     0.333     0.500       3.0
        I-fromloc.state_name      1.000     1.000     1.000       1.0
                B-state_code      1.000     1.000     1.000       1.0
    I-arrive_time.start_time      1.000     1.000     1.000       1.0
    I-arrive_date.day_number      0.000     0.000     0.000       0.0
                 B-meal_code      0.000     0.000     0.000       1.0
 I-depart_time.period_of_day      1.000     1.000     1.000       1.0
                  B-day_name      1.000     0.500     0.667       2.0
             B-period_of_day      1.000     0.500     0.667       4.0
        B-stoploc.state_code      0.000     0.000     0.000       0.0
    B-return_date.month_name      0.000     0.000     0.000       0.0
    B-return_date.day_number      0.000     0.000     0.000       0.0
    B-arrive_time.period_mod      0.000     0.000     0.000       0.0
                 I-meal_code      0.000     0.000     0.000       0.0
        B-toloc.country_name      1.000     1.000     1.000       1.0
                 B-days_code      1.000     1.000     1.000       1.0
 I-arrive_time.period_of_day      0.000     0.000     0.000       0.0
            I-today_relative      0.000     0.000     0.000       0.0
 B-return_time.period_of_day      0.000     0.000     0.000       0.0
                      B-time      0.000     0.000     0.000       0.0
           I-fare_basis_code      0.000     0.000     0.000       0.0
 I-arrive_time.time_relative      1.000     1.000     1.000       4.0
 I-depart_time.time_relative      0.000     0.000     0.000       1.0
            B-today_relative      0.000     0.000     0.000       0.0
                B-state_name      0.857     0.667     0.750       9.0
B-arrive_date.today_relative      0.000     0.000     0.000       0.0
    B-return_time.period_mod      0.000     0.000     0.000       0.0
                B-month_name      0.000     0.000     0.000       0.0
                B-day_number      0.000     0.000     0.000       0.0
 I-return_date.date_relative      0.667     0.667     0.667       3.0
I-return_date.today_relative      0.000     0.000     0.000       0.0
      B-stoploc.airport_name      0.000     0.000     0.000       0.0
             B-time_relative      0.000     0.000     0.000       0.0
                      I-time      0.000     0.000     0.000       0.0
    I-return_date.day_number      0.000     0.000     0.000       0.0
          I-meal_description      0.000     0.000     0.000       0.0
B-return_date.today_relative      0.000     0.000     0.000       0.0
      B-return_date.day_name      0.000     0.000     0.000       2.0

                   micro avg      0.956     0.957     0.957    3657.0
                   macro avg      0.692     0.667     0.665    3657.0
                weighted avg      0.965     0.957     0.956    3657.0

I0715 07:31:14.664699 140369481070464 interactiveshell.py:2882] Best Slot F1: 0.958
Reading ../data/atis.train.w-intent.iob
I0715 07:31:19.101779 140369481070464 interactiveshell.py:2882] Step 22200 | Loss: 0.0002 | Loss_intent: 0.0000 | Loss_slots: 0.0002 | Spent: 6.0 secs | LR: 0.000029
I0715 07:31:22.886196 140369481070464 interactiveshell.py:2882] Step 22250 | Loss: 0.0007 | Loss_intent: 0.0000 | Loss_slots: 0.0007 | Spent: 3.8 secs | LR: 0.000029
I0715 07:31:26.730889 140369481070464 interactiveshell.py:2882] Step 22300 | Loss: 0.0003 | Loss_intent: 0.0000 | Loss_slots: 0.0003 | Spent: 3.8 secs | LR: 0.000029
I0715 07:31:30.734112 140369481070464 interactiveshell.py:2882] Step 22350 | Loss: 0.0000 | Loss_intent: 0.0000 | Loss_slots: 0.0000 | Spent: 4.0 secs | LR: 0.000028
I0715 07:31:34.638639 140369481070464 interactiveshell.py:2882] Step 22400 | Loss: 0.0006 | Loss_intent: 0.0000 | Loss_slots: 0.0006 | Spent: 3.9 secs | LR: 0.000028
I0715 07:31:38.521017 140369481070464 interactiveshell.py:2882] Step 22450 | Loss: 0.0000 | Loss_intent: 0.0000 | Loss_slots: 0.0000 | Spent: 3.9 secs | LR: 0.000028
Reading ../data/atis.test.w-intent.iob
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.
  'precision', 'predicted', average, warn_for)
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1439: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples.
  'recall', 'true', average, warn_for)
I0715 07:31:41.020962 140369481070464 interactiveshell.py:2882] 
                                          precision    recall  f1-score   support

                             atis_flight      0.981     0.987     0.984       632
                            atis_airfare      0.960     1.000     0.980        48
                     atis_ground_service      1.000     1.000     1.000        36
                            atis_airline      1.000     1.000     1.000        38
                       atis_abbreviation      0.943     1.000     0.971        33
                           atis_aircraft      1.000     0.889     0.941         9
                        atis_flight_time      1.000     1.000     1.000         1
                           atis_quantity      0.429     1.000     0.600         3
                atis_flight#atis_airfare      0.833     0.417     0.556        12
                            atis_airport      1.000     0.944     0.971        18
                           atis_distance      1.000     1.000     1.000        10
                               atis_city      0.571     0.667     0.615         6
                        atis_ground_fare      1.000     0.857     0.923         7
                           atis_capacity      0.955     1.000     0.977        21
                          atis_flight_no      0.889     1.000     0.941         8
                               atis_meal      1.000     0.833     0.909         6
                        atis_restriction      0.000     0.000     0.000         0
             atis_airline#atis_flight_no      0.000     0.000     0.000         0
    atis_ground_service#atis_ground_fare      0.000     0.000     0.000         0
           atis_airfare#atis_flight_time      0.000     0.000     0.000         0
                           atis_cheapest      0.000     0.000     0.000         0
atis_aircraft#atis_flight#atis_flight_no      0.000     0.000     0.000         0

                               micro avg      0.971     0.976     0.974       888
                               macro avg      0.662     0.663     0.653       888
                            weighted avg      0.973     0.976     0.973       888

I0715 07:31:41.072254 140369481070464 interactiveshell.py:2882] 
                              precision    recall  f1-score   support

                           O      0.000     0.000     0.000       0.0
           B-toloc.city_name      0.973     0.993     0.983     716.0
         B-fromloc.city_name      0.987     0.997     0.992     704.0
           I-toloc.city_name      0.978     0.992     0.985     265.0
      B-depart_date.day_name      0.981     0.986     0.984     212.0
              B-airline_name      1.000     1.000     1.000     101.0
         I-fromloc.city_name      0.978     0.994     0.986     177.0
 B-depart_time.period_of_day      1.000     0.892     0.943     130.0
              I-airline_name      1.000     0.985     0.992      65.0
    B-depart_date.day_number      0.981     0.964     0.972      55.0
    B-depart_date.month_name      0.982     0.964     0.973      56.0
          B-depart_time.time      0.851     1.000     0.919      57.0
                B-round_trip      1.000     0.986     0.993      73.0
             B-cost_relative      1.000     0.973     0.986      37.0
                I-round_trip      1.000     1.000     1.000      71.0
                B-flight_mod      0.828     1.000     0.906      24.0
 B-depart_time.time_relative      0.969     0.969     0.969      65.0
          I-depart_time.time      0.945     1.000     0.972      52.0
         B-stoploc.city_name      0.952     1.000     0.976      20.0
                 B-city_name      0.857     0.526     0.652      57.0
                B-class_type      0.960     1.000     0.980      24.0
          B-arrive_time.time      0.943     0.971     0.957      34.0
 B-arrive_time.time_relative      0.967     0.935     0.951      31.0
                I-class_type      1.000     1.000     1.000      17.0
               B-flight_stop      1.000     1.000     1.000      21.0
          I-arrive_time.time      0.971     0.971     0.971      35.0
              B-airline_code      1.000     0.912     0.954      34.0
    I-depart_date.day_number      1.000     0.933     0.966      15.0
      I-fromloc.airport_name      0.405     1.000     0.577      15.0
      B-fromloc.airport_name      0.400     1.000     0.571      12.0
      B-arrive_date.day_name      0.733     1.000     0.846      11.0
          B-toloc.state_code      1.000     1.000     1.000      18.0
B-depart_date.today_relative      1.000     0.889     0.941       9.0
             B-flight_number      0.550     1.000     0.710      11.0
 B-depart_date.date_relative      0.944     1.000     0.971      17.0
          B-toloc.state_name      0.929     0.929     0.929      28.0
           B-fare_basis_code      0.944     1.000     0.971      17.0
               B-flight_time      1.000     1.000     1.000       1.0
                        B-or      1.000     1.000     1.000       3.0
 B-arrive_time.period_of_day      0.667     1.000     0.800       6.0
          B-meal_description      1.000     0.900     0.947      10.0
             I-cost_relative      1.000     0.667     0.800       3.0
              I-airport_name      0.909     0.345     0.500      29.0
               B-fare_amount      1.000     1.000     1.000       2.0
               I-fare_amount      1.000     1.000     1.000       2.0
                 I-city_name      1.000     0.633     0.776      30.0
        I-toloc.airport_name      1.000     1.000     1.000       3.0
            B-transport_type      1.000     0.900     0.947      10.0
    B-arrive_date.month_name      0.714     0.833     0.769       6.0
    B-arrive_date.day_number      0.714     0.833     0.769       6.0
         I-stoploc.city_name      1.000     1.000     1.000      10.0
                      B-meal      1.000     1.000     1.000      16.0
        B-fromloc.state_code      1.000     1.000     1.000      23.0
    B-depart_time.period_mod      1.000     1.000     1.000       5.0
                   B-connect      1.000     1.000     1.000       6.0
               B-flight_days      1.000     1.000     1.000      10.0
        B-toloc.airport_name      1.000     1.000     1.000       3.0
        B-fromloc.state_name      1.000     1.000     1.000      17.0
              B-airport_name      0.778     0.333     0.467      21.0
                   B-economy      1.000     1.000     1.000       6.0
               I-flight_time      1.000     1.000     1.000       1.0
             B-aircraft_code      0.931     0.818     0.871      33.0
                       B-mod      1.000     0.500     0.667       2.0
              B-airport_code      0.667     0.444     0.533       9.0
    B-depart_time.start_time      0.600     1.000     0.750       3.0
      B-depart_time.end_time      1.000     1.000     1.000       3.0
          B-depart_date.year      0.600     1.000     0.750       3.0
            I-transport_type      0.000     0.000     0.000       1.0
          B-restriction_code      1.000     1.000     1.000       4.0
    B-arrive_time.start_time      1.000     1.000     1.000       8.0
        B-toloc.airport_code      1.000     1.000     1.000       4.0
      B-arrive_time.end_time      1.000     0.875     0.933       8.0
      I-arrive_time.end_time      0.875     0.875     0.875       8.0
      I-depart_time.end_time      1.000     0.667     0.800       3.0
               I-flight_stop      0.000     0.000     0.000       0.0
      B-fromloc.airport_code      0.833     1.000     0.909       5.0
          I-restriction_code      1.000     1.000     1.000       3.0
    I-depart_time.start_time      1.000     1.000     1.000       1.0
          I-toloc.state_name      1.000     1.000     1.000       1.0
I-depart_date.today_relative      0.000     0.000     0.000       0.0
 B-arrive_date.date_relative      1.000     1.000     1.000       2.0
                I-flight_mod      1.000     0.167     0.286       6.0
                   I-economy      0.000     0.000     0.000       0.0
 B-return_date.date_relative      1.000     0.333     0.500       3.0
        I-fromloc.state_name      1.000     1.000     1.000       1.0
                B-state_code      1.000     1.000     1.000       1.0
    I-arrive_time.start_time      1.000     1.000     1.000       1.0
    I-arrive_date.day_number      0.000     0.000     0.000       0.0
                 B-meal_code      0.000     0.000     0.000       1.0
 I-depart_time.period_of_day      1.000     1.000     1.000       1.0
                  B-day_name      1.000     0.500     0.667       2.0
             B-period_of_day      1.000     0.500     0.667       4.0
        B-stoploc.state_code      0.000     0.000     0.000       0.0
    B-return_date.month_name      0.000     0.000     0.000       0.0
    B-return_date.day_number      0.000     0.000     0.000       0.0
    B-arrive_time.period_mod      0.000     0.000     0.000       0.0
                 I-meal_code      0.000     0.000     0.000       0.0
        B-toloc.country_name      1.000     1.000     1.000       1.0
                 B-days_code      1.000     1.000     1.000       1.0
 I-arrive_time.period_of_day      0.000     0.000     0.000       0.0
            I-today_relative      0.000     0.000     0.000       0.0
 B-return_time.period_of_day      0.000     0.000     0.000       0.0
                      B-time      0.000     0.000     0.000       0.0
           I-fare_basis_code      0.000     0.000     0.000       0.0
 I-arrive_time.time_relative      1.000     1.000     1.000       4.0
 I-depart_time.time_relative      0.000     0.000     0.000       1.0
            B-today_relative      0.000     0.000     0.000       0.0
                B-state_name      0.750     0.333     0.462       9.0
B-arrive_date.today_relative      0.000     0.000     0.000       0.0
    B-return_time.period_mod      0.000     0.000     0.000       0.0
                B-month_name      0.000     0.000     0.000       0.0
                B-day_number      0.000     0.000     0.000       0.0
 I-return_date.date_relative      0.667     0.667     0.667       3.0
I-return_date.today_relative      0.000     0.000     0.000       0.0
      B-stoploc.airport_name      0.000     0.000     0.000       0.0
             B-time_relative      0.000     0.000     0.000       0.0
                      I-time      0.000     0.000     0.000       0.0
    I-return_date.day_number      0.000     0.000     0.000       0.0
          I-meal_description      0.000     0.000     0.000       0.0
B-return_date.today_relative      0.000     0.000     0.000       0.0
      B-return_date.day_name      0.000     0.000     0.000       2.0

                   micro avg