In [1]:
"""
We use following lines because we are running on Google Colab
If you are running notebook on a local computer, you don't need this cell
"""
from google.colab import drive
drive.mount('/content/gdrive')
import os
os.chdir('/content/gdrive/My Drive/finch/tensorflow2/spoken_language_understanding/atis/main')
Drive already mounted at /content/gdrive; to attempt to forcibly remount, call drive.mount("/content/gdrive", force_remount=True).
In [2]:
!pip install tensorflow-gpu==2.0.0-beta1
Requirement already satisfied: tensorflow-gpu==2.0.0-beta1 in /usr/local/lib/python3.6/dist-packages (2.0.0b1)
Requirement already satisfied: absl-py>=0.7.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow-gpu==2.0.0-beta1) (0.7.1)
Requirement already satisfied: six>=1.10.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow-gpu==2.0.0-beta1) (1.12.0)
Requirement already satisfied: wheel>=0.26 in /usr/local/lib/python3.6/dist-packages (from tensorflow-gpu==2.0.0-beta1) (0.33.4)
Requirement already satisfied: google-pasta>=0.1.6 in /usr/local/lib/python3.6/dist-packages (from tensorflow-gpu==2.0.0-beta1) (0.1.7)
Requirement already satisfied: tf-estimator-nightly<1.14.0.dev2019060502,>=1.14.0.dev2019060501 in /usr/local/lib/python3.6/dist-packages (from tensorflow-gpu==2.0.0-beta1) (1.14.0.dev2019060501)
Requirement already satisfied: grpcio>=1.8.6 in /usr/local/lib/python3.6/dist-packages (from tensorflow-gpu==2.0.0-beta1) (1.15.0)
Requirement already satisfied: protobuf>=3.6.1 in /usr/local/lib/python3.6/dist-packages (from tensorflow-gpu==2.0.0-beta1) (3.7.1)
Requirement already satisfied: gast>=0.2.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow-gpu==2.0.0-beta1) (0.2.2)
Requirement already satisfied: tb-nightly<1.14.0a20190604,>=1.14.0a20190603 in /usr/local/lib/python3.6/dist-packages (from tensorflow-gpu==2.0.0-beta1) (1.14.0a20190603)
Requirement already satisfied: keras-preprocessing>=1.0.5 in /usr/local/lib/python3.6/dist-packages (from tensorflow-gpu==2.0.0-beta1) (1.1.0)
Requirement already satisfied: termcolor>=1.1.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow-gpu==2.0.0-beta1) (1.1.0)
Requirement already satisfied: numpy<2.0,>=1.14.5 in /usr/local/lib/python3.6/dist-packages (from tensorflow-gpu==2.0.0-beta1) (1.16.4)
Requirement already satisfied: keras-applications>=1.0.6 in /usr/local/lib/python3.6/dist-packages (from tensorflow-gpu==2.0.0-beta1) (1.0.8)
Requirement already satisfied: wrapt>=1.11.1 in /usr/local/lib/python3.6/dist-packages (from tensorflow-gpu==2.0.0-beta1) (1.11.2)
Requirement already satisfied: astor>=0.6.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow-gpu==2.0.0-beta1) (0.8.0)
Requirement already satisfied: setuptools in /usr/local/lib/python3.6/dist-packages (from protobuf>=3.6.1->tensorflow-gpu==2.0.0-beta1) (41.0.1)
Requirement already satisfied: markdown>=2.6.8 in /usr/local/lib/python3.6/dist-packages (from tb-nightly<1.14.0a20190604,>=1.14.0a20190603->tensorflow-gpu==2.0.0-beta1) (3.1.1)
Requirement already satisfied: werkzeug>=0.11.15 in /usr/local/lib/python3.6/dist-packages (from tb-nightly<1.14.0a20190604,>=1.14.0a20190603->tensorflow-gpu==2.0.0-beta1) (0.15.4)
Requirement already satisfied: h5py in /usr/local/lib/python3.6/dist-packages (from keras-applications>=1.0.6->tensorflow-gpu==2.0.0-beta1) (2.8.0)
In [3]:
import tensorflow as tf
import pprint
import logging
import time
import numpy as np

from sklearn.metrics import classification_report, f1_score

print("TensorFlow Version", tf.__version__)
print('GPU Enabled:', tf.test.is_gpu_available())
TensorFlow Version 2.0.0-beta1
GPU Enabled: True
In [0]:
def get_vocab(vocab_path):
  word2idx = {}
  with open(vocab_path) as f:
    for i, line in enumerate(f):
      line = line.rstrip()
      word2idx[line] = i
  return word2idx
In [0]:
def data_generator(f_path, params):
  print('Reading', f_path)
  with open(f_path) as f:
    for line in f:
      line = line.rstrip()
      text, slot_intent = line.split('\t')
      words = text.split()[1:-1]
      slot_intent = slot_intent.split()
      slots, intent = slot_intent[1:-1], slot_intent[-1]
      assert len(words) == len(slots)
      
      words = [params['word2idx'].get(w, len(params['word2idx'])) for w in words]
      intent = params['intent2idx'].get(intent, len(params['intent2idx']))
      slots = [params['slot2idx'].get(s, len(params['slot2idx'])) for s in slots]
      
      yield (words, (intent, slots))
In [0]:
def dataset(is_training, params):
  _shapes = ([None], ((), [None]))
  _types = (tf.int32, (tf.int32, tf.int32))
  _pads = (0, (-1, 0))
  
  if is_training:
    ds = tf.data.Dataset.from_generator(
      lambda: data_generator(params['train_path'], params),
      output_shapes = _shapes,
      output_types = _types,)
    ds = ds.shuffle(params['num_samples'])
    ds = ds.padded_batch(params['batch_size'], _shapes, _pads)
    ds = ds.prefetch(tf.data.experimental.AUTOTUNE)
  else:
    ds = tf.data.Dataset.from_generator(
      lambda: data_generator(params['test_path'], params),
      output_shapes = _shapes,
      output_types = _types,)
    ds = ds.padded_batch(params['batch_size'], _shapes, _pads)
    ds = ds.prefetch(tf.data.experimental.AUTOTUNE)
  
  return ds
In [0]:
def get_timing_signal_1d(length,
                         channels,
                         min_timescale=1.0,
                         max_timescale=1.0e4,
                         start_index=0):
  to_float = lambda x: tf.cast(x, tf.float32)
  position = to_float(tf.range(length) + start_index)
  num_timescales = channels // 2
  log_timescale_increment = (
      tf.math.log(float(max_timescale) / float(min_timescale)) /
      tf.maximum(to_float(num_timescales) - 1, 1))
  inv_timescales = min_timescale * tf.exp(
      to_float(tf.range(num_timescales)) * -log_timescale_increment)
  scaled_time = tf.expand_dims(position, 1) * tf.expand_dims(inv_timescales, 0)
  signal = tf.concat([tf.sin(scaled_time), tf.cos(scaled_time)], axis=1)
  signal = tf.pad(signal, [[0, 0], [0, tf.compat.v1.mod(channels, 2)]])
  signal = tf.reshape(signal, [1, length, channels])
  return signal
In [0]:
class LayerNorm(tf.keras.layers.Layer):
  def __init__(self, params):
    super().__init__()
    self._epsilon = params['epsilon']
    self._hidden_units = params['global_units']
  
  def build(self, input_shape):
    self.scale = self.add_weight(name='scale',
                                 shape=[self._hidden_units],
                                 initializer=tf.ones_initializer(),
                                 trainable=True)
    self.bias = self.add_weight(name='bias',
                                shape=[self._hidden_units],
                                initializer=tf.zeros_initializer(),
                                trainable=True)
    super().build(input_shape)
  
  def call(self, inputs):
    mean, variance = tf.nn.moments(inputs, [-1], keepdims=True)
    norm_x = (inputs - mean) * tf.math.rsqrt(variance + self._epsilon)
    return norm_x * self.scale + self.bias
  
  def compute_output_shape(self, input_shape):
    return input_shape


class EncoderBlock(tf.keras.Model):
  def __init__(self, SubModel, params, name):
    super().__init__(name = name)
    self.layer_norm = LayerNorm(params)
    self.sub_model = SubModel(params)
    self.dropout = tf.keras.layers.Dropout(params['dropout_rate'])
  
  def call(self, inputs, training=False):
    inputs, masks = inputs
    x = self.layer_norm(inputs)
    x = self.sub_model((x, masks), training=training)
    x = self.dropout(x, training=training)
    x += inputs
    return x


class MultiheadSelfAttention(tf.keras.Model):
  def __init__(self, params):
    super().__init__()
    self.qkv_linear = tf.keras.layers.Dense(3*params['global_units'], tf.nn.elu, name='qkv_linear')
    self.dropout = tf.keras.layers.Dropout(params['dropout_rate'])
    self.out_linear = tf.keras.layers.Dense(params['global_units'], tf.nn.elu, name='out_linear')
    self._num_heads = params['num_heads']
    self._is_bidirectional = params['is_bidirectional']
  
  def call(self, inputs, training=False):
    x, masks = inputs
    timesteps = tf.shape(x)[1]
    
    q_k_v = self.qkv_linear(x)
    q, k, v = tf.split(q_k_v, 3, axis=-1)
    
    if self._num_heads > 1:
      q = tf.concat(tf.split(q, self._num_heads, axis=2), axis=0)                        
      k = tf.concat(tf.split(k, self._num_heads, axis=2), axis=0)                        
      v = tf.concat(tf.split(v, self._num_heads, axis=2), axis=0)
    
    align = tf.matmul(q, k, transpose_b=True)
    align *= tf.math.rsqrt(tf.cast(k.shape[-1], tf.float32))
    
    if (masks is not None) or (not self._is_bidirectional):
      paddings = tf.fill(tf.shape(align), float('-inf'))
    
    if masks is not None:
      c_masks = tf.tile(masks, [params['num_heads'], 1])
      c_masks = tf.tile(tf.expand_dims(c_masks, 1), [1, timesteps, 1])
      align = tf.where(tf.equal(c_masks, 0), paddings, align)
    
    if not self._is_bidirectional:
      lower_tri = tf.ones((timesteps, timesteps))                                       
      lower_tri = tf.linalg.LinearOperatorLowerTriangular(lower_tri).to_dense()      
      t_masks = tf.tile(tf.expand_dims(lower_tri, 0), [tf.shape(align)[0], 1, 1])     
      align = tf.where(tf.equal(t_masks, 0), paddings, align)
    
    align = tf.nn.softmax(align)
    align = self.dropout(align, training=training)
    
    if masks is not None:
      q_masks = tf.tile(masks, [params['num_heads'], 1])
      q_masks = tf.tile(tf.expand_dims(q_masks, 2), [1, 1, timesteps])
      align *= tf.cast(q_masks, tf.float32)
    
    x = tf.matmul(align, v)
    if self._num_heads > 1:
      x = tf.concat(tf.split(x, self._num_heads, axis=0), axis=2)
    x = self.out_linear(x)
    
    return x
  

class PointwiseFFN(tf.keras.Model):
  def __init__(self, params):
    super().__init__()
    self.dense_1 = tf.keras.layers.Dense(params['multiplier']*params['global_units'], tf.nn.elu, name='fc')
    self.dropout = tf.keras.layers.Dropout(params['dropout_rate'])
    self.dense_2 = tf.keras.layers.Dense(params['global_units'], name='linear')
  
  def call(self, inputs, training=False):
    x, masks = inputs
    return self.dense_2(self.dropout(self.dense_1(x), training=training))
In [0]:
class Model(tf.keras.Model):
  def __init__(self, params: dict):
    super().__init__()
    self.embedding = tf.Variable(np.load(params['vocab_path']),
                                 dtype=tf.float32,
                                 name='pretrained_embedding')
    self.input_dropout = tf.keras.layers.Dropout(params['dropout_rate'])
    
    self.blocks = []
    for i in range(params['num_layers']):
      self.blocks.append(EncoderBlock(
          MultiheadSelfAttention, params, name='layer{}.1'.format(i+1)))
      self.blocks.append(EncoderBlock(
          PointwiseFFN, params, name='layer{}.2'.format(i+1)))
    
    self.out_linear_intent = tf.keras.layers.Dense(params['intent_size'], name='output_intent')
    self.out_linear_slot = tf.keras.layers.Dense(params['slot_size'], name='output_slot')
    
  
  def call(self, inputs, training=None):
    if inputs.dtype != tf.int32:
      inputs = tf.cast(inputs, tf.int32)
    masks = tf.sign(inputs)
    
    x = tf.nn.embedding_lookup(self.embedding, inputs)
    if params['is_embedding_scaled']:
      x *= tf.sqrt(tf.cast(params['global_units'], tf.float32))
    x += get_timing_signal_1d(tf.shape(x)[1], params['global_units'])
    x = self.input_dropout(x, training=training)
    
    for block in self.blocks:
      x = block((x, masks), training=training)
    
    x_intent = tf.reduce_max(x, 1)
    x_intent = self.out_linear_intent(x_intent)
    x_slot = self.out_linear_slot(x)
    return (x_intent, x_slot)
In [0]:
params = {
  'train_path': '../data/atis.train.w-intent.iob',
  'test_path': '../data/atis.test.w-intent.iob',
  'word_path': '../vocab/word.txt',
  'vocab_path': '../vocab/word.npy',
  'intent_path': '../vocab/intent.txt',
  'slot_path': '../vocab/slot.txt',
  'batch_size': 16,
  'num_samples': 4978,
  'num_layers': 2,
  'global_units': 300,
  'num_heads': 6,
  'multiplier': 2,
  'dropout_rate': 0.1,
  'epsilon': 1e-6,
  'is_bidirectional': True,
  'is_embedding_scaled': False,
  'lr': 3e-4,
  'num_patience': 3,
}
In [0]:
params['word2idx'] = get_vocab(params['word_path'])
params['intent2idx'] = get_vocab(params['intent_path'])
params['slot2idx'] = get_vocab(params['slot_path'])

params['word_size'] = len(params['word2idx']) + 1
params['intent_size'] = len(params['intent2idx']) + 1
params['slot_size'] = len(params['slot2idx']) + 1
In [0]:
def is_descending(history: list):
  history = history[-(params['num_patience']+1):]
  for i in range(1, len(history)):
    if history[i-1] <= history[i]:
      return False
  return True    
In [13]:
model = Model(params)
model.build(input_shape=(None, None))
pprint.pprint([(v.name, v.shape) for v in model.trainable_variables])

decay_lr = tf.optimizers.schedules.ExponentialDecay(params['lr'], 1000, 0.9)
optim = tf.optimizers.Adam(params['lr'])
global_step = 0

history_f1 = []
best_f1 = .0

t0 = time.time()
logger = logging.getLogger('tensorflow')
logger.setLevel(logging.INFO)

while True:
  # TRAINING
  for (words, (intent, slots)) in dataset(is_training=True, params=params):
    with tf.GradientTape() as tape:
      y_intent, y_slots = model(words, training=True)
      loss_intent = tf.nn.sparse_softmax_cross_entropy_with_logits(labels=intent, logits=y_intent)
      loss_intent = tf.reduce_mean(loss_intent)
      # weighted cross entropy (weight of 'O' is set to be small)
      loss_slots = tf.nn.sparse_softmax_cross_entropy_with_logits(labels=slots, logits=y_slots)
      weights = tf.cast(tf.sign(slots), tf.float32)
      padding = tf.constant(1e-2, tf.float32, weights.shape)
      weights = tf.where(tf.equal(weights, 0.), padding, weights)
      loss_slots = tf.reduce_mean(loss_slots * weights)
      # joint loss
      loss = loss_intent + loss_slots
    
    optim.lr.assign(decay_lr(global_step))
    grads = tape.gradient(loss, model.trainable_variables)
    optim.apply_gradients(zip(grads, model.trainable_variables))

    if global_step % 50 == 0:
      logger.info("Step {} | Loss: {:.4f} | Loss_intent: {:.4f} | Loss_slots: {:.4f} | Spent: {:.1f} secs | LR: {:.6f}".format(
          global_step, loss.numpy().item(), loss_intent.numpy().item(), loss_slots.numpy().item(), time.time()-t0, optim.lr.numpy().item()))
      t0 = time.time()
    global_step += 1
    
  # EVALUATION
  intent_true = []
  intent_pred = []
  slot_true = []
  slot_pred = []

  for (words, (intent, slots)) in dataset(is_training=False, params=params):
    y_intent, y_slots = model(words, training=False)
    y_intent = tf.argmax(y_intent, -1)
    y_slots = tf.argmax(y_slots, -1)
    
    intent_true += intent.numpy().flatten().tolist()
    intent_pred += y_intent.numpy().flatten().tolist()
    slot_true += slots.numpy().flatten().tolist()
    slot_pred += y_slots.numpy().flatten().tolist()
    
  f1_slots = f1_score(y_true = slot_true,
                      y_pred = slot_pred,
                      labels = list(params['slot2idx'].values()),
                      sample_weight = np.sign(slot_true),
                      average='micro',)
  history_f1.append(f1_slots)
  
  logger.info('\n'+classification_report(y_true = intent_true,
                                         y_pred = intent_pred,
                                         labels = list(params['intent2idx'].values()),
                                         target_names = list(params['intent2idx'].keys()),
                                         digits=3))
  logger.info('\n'+classification_report(y_true = slot_true,
                                         y_pred = slot_pred,
                                         labels = list(params['slot2idx'].values()),
                                         target_names = list(params['slot2idx'].keys()),
                                         sample_weight = np.sign(slot_true),
                                         digits=3))
  
  if f1_slots > best_f1:
    best_f1 = f1_slots
    # you can save model here
  logger.info("Best Slot F1: {:.3f}".format(best_f1))
  
  if len(history_f1) > params['num_patience'] and is_descending(history_f1):
    logger.info("Testing Slot Accuracy not improved over {} epochs, Early Stop".format(params['num_patience']))
    break
WARNING: Logging before flag parsing goes to stderr.
W0717 06:50:27.405614 140695199004544 deprecation.py:323] From /usr/local/lib/python3.6/dist-packages/tensorflow/python/data/ops/dataset_ops.py:505: py_func (from tensorflow.python.ops.script_ops) is deprecated and will be removed in a future version.
Instructions for updating:
tf.py_func is deprecated in TF V2. Instead, there are two
    options available in V2.
    - tf.py_function takes a python function which manipulates tf eager
    tensors instead of numpy arrays. It's easy to convert a tf eager tensor to
    an ndarray (just call tensor.numpy()) but having access to eager tensors
    means `tf.py_function`s can use accelerators such as GPUs as well as
    being differentiable using a gradient tape.
    - tf.numpy_function maintains the semantics of the deprecated tf.py_func
    (it is not differentiable, and manipulates numpy arrays). It drops the
    stateful argument making all functions stateful.
    
[('layer1.1/layer_norm/scale:0', TensorShape([300])),
 ('layer1.1/layer_norm/bias:0', TensorShape([300])),
 ('layer1.1/multihead_self_attention/qkv_linear/kernel:0',
  TensorShape([300, 900])),
 ('layer1.1/multihead_self_attention/qkv_linear/bias:0', TensorShape([900])),
 ('layer1.1/multihead_self_attention/out_linear/kernel:0',
  TensorShape([300, 300])),
 ('layer1.1/multihead_self_attention/out_linear/bias:0', TensorShape([300])),
 ('layer1.2/layer_norm_1/scale:0', TensorShape([300])),
 ('layer1.2/layer_norm_1/bias:0', TensorShape([300])),
 ('layer1.2/pointwise_ffn/fc/kernel:0', TensorShape([300, 600])),
 ('layer1.2/pointwise_ffn/fc/bias:0', TensorShape([600])),
 ('layer1.2/pointwise_ffn/linear/kernel:0', TensorShape([600, 300])),
 ('layer1.2/pointwise_ffn/linear/bias:0', TensorShape([300])),
 ('layer2.1/layer_norm_2/scale:0', TensorShape([300])),
 ('layer2.1/layer_norm_2/bias:0', TensorShape([300])),
 ('layer2.1/multihead_self_attention_1/qkv_linear/kernel:0',
  TensorShape([300, 900])),
 ('layer2.1/multihead_self_attention_1/qkv_linear/bias:0', TensorShape([900])),
 ('layer2.1/multihead_self_attention_1/out_linear/kernel:0',
  TensorShape([300, 300])),
 ('layer2.1/multihead_self_attention_1/out_linear/bias:0', TensorShape([300])),
 ('layer2.2/layer_norm_3/scale:0', TensorShape([300])),
 ('layer2.2/layer_norm_3/bias:0', TensorShape([300])),
 ('layer2.2/pointwise_ffn_1/fc/kernel:0', TensorShape([300, 600])),
 ('layer2.2/pointwise_ffn_1/fc/bias:0', TensorShape([600])),
 ('layer2.2/pointwise_ffn_1/linear/kernel:0', TensorShape([600, 300])),
 ('layer2.2/pointwise_ffn_1/linear/bias:0', TensorShape([300])),
 ('output_intent/kernel:0', TensorShape([300, 23])),
 ('output_intent/bias:0', TensorShape([23])),
 ('output_slot/kernel:0', TensorShape([300, 122])),
 ('output_slot/bias:0', TensorShape([122])),
 ('pretrained_embedding:0', TensorShape([750, 300]))]
Reading ../data/atis.train.w-intent.iob
I0717 06:50:29.023662 140695199004544 interactiveshell.py:2882] Step 0 | Loss: 8.5169 | Loss_intent: 7.1070 | Loss_slots: 1.4099 | Spent: 1.6 secs | LR: 0.000300
I0717 06:50:33.159497 140695199004544 interactiveshell.py:2882] Step 50 | Loss: 1.8417 | Loss_intent: 1.3645 | Loss_slots: 0.4772 | Spent: 4.1 secs | LR: 0.000298
I0717 06:50:37.297963 140695199004544 interactiveshell.py:2882] Step 100 | Loss: 0.8494 | Loss_intent: 0.3138 | Loss_slots: 0.5356 | Spent: 4.1 secs | LR: 0.000297
I0717 06:50:41.410565 140695199004544 interactiveshell.py:2882] Step 150 | Loss: 0.5292 | Loss_intent: 0.1183 | Loss_slots: 0.4109 | Spent: 4.1 secs | LR: 0.000295
I0717 06:50:45.531688 140695199004544 interactiveshell.py:2882] Step 200 | Loss: 0.4071 | Loss_intent: 0.0215 | Loss_slots: 0.3856 | Spent: 4.1 secs | LR: 0.000294
I0717 06:50:49.671260 140695199004544 interactiveshell.py:2882] Step 250 | Loss: 0.8820 | Loss_intent: 0.5222 | Loss_slots: 0.3598 | Spent: 4.1 secs | LR: 0.000292
I0717 06:50:53.792594 140695199004544 interactiveshell.py:2882] Step 300 | Loss: 0.9190 | Loss_intent: 0.7110 | Loss_slots: 0.2080 | Spent: 4.1 secs | LR: 0.000291
Reading ../data/atis.test.w-intent.iob
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.
  'precision', 'predicted', average, warn_for)
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1439: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples.
  'recall', 'true', average, warn_for)
I0717 06:50:56.848233 140695199004544 interactiveshell.py:2882] 
                                          precision    recall  f1-score   support

                             atis_flight      0.973     0.919     0.945       632
                            atis_airfare      0.887     0.979     0.931        48
                     atis_ground_service      0.900     1.000     0.947        36
                            atis_airline      0.966     0.737     0.836        38
                       atis_abbreviation      0.717     1.000     0.835        33
                           atis_aircraft      0.562     1.000     0.720         9
                        atis_flight_time      0.014     1.000     0.029         1
                           atis_quantity      0.000     0.000     0.000         3
                atis_flight#atis_airfare      0.800     0.333     0.471        12
                            atis_airport      0.696     0.889     0.780        18
                           atis_distance      0.000     0.000     0.000        10
                               atis_city      0.000     0.000     0.000         6
                        atis_ground_fare      1.000     0.286     0.444         7
                           atis_capacity      1.000     0.571     0.727        21
                          atis_flight_no      0.000     0.000     0.000         8
                               atis_meal      0.000     0.000     0.000         6
                        atis_restriction      0.000     0.000     0.000         0
             atis_airline#atis_flight_no      0.000     0.000     0.000         0
    atis_ground_service#atis_ground_fare      0.000     0.000     0.000         0
           atis_airfare#atis_flight_time      0.000     0.000     0.000         0
                           atis_cheapest      0.000     0.000     0.000         0
atis_aircraft#atis_flight#atis_flight_no      0.000     0.000     0.000         0

                               micro avg      0.861     0.866     0.864       888
                               macro avg      0.387     0.396     0.348       888
                            weighted avg      0.907     0.866     0.879       888

I0717 06:50:56.891137 140695199004544 interactiveshell.py:2882] 
                              precision    recall  f1-score   support

                           O      0.000     0.000     0.000       0.0
           B-toloc.city_name      0.613     0.902     0.730     716.0
         B-fromloc.city_name      0.767     0.753     0.760     704.0
           I-toloc.city_name      0.689     0.502     0.581     265.0
      B-depart_date.day_name      0.917     0.995     0.955     212.0
              B-airline_name      0.804     0.772     0.788     101.0
         I-fromloc.city_name      0.687     0.322     0.438     177.0
 B-depart_time.period_of_day      0.899     0.823     0.859     130.0
              I-airline_name      0.984     0.923     0.952      65.0
    B-depart_date.day_number      0.714     0.909     0.800      55.0
    B-depart_date.month_name      0.900     0.964     0.931      56.0
          B-depart_time.time      0.356     0.912     0.512      57.0
                B-round_trip      0.939     0.849     0.892      73.0
             B-cost_relative      1.000     0.919     0.958      37.0
                I-round_trip      0.912     0.873     0.892      71.0
                B-flight_mod      0.308     0.833     0.449      24.0
 B-depart_time.time_relative      0.588     0.923     0.719      65.0
          I-depart_time.time      0.437     1.000     0.608      52.0
         B-stoploc.city_name      0.000     0.000     0.000      20.0
                 B-city_name      0.325     0.439     0.373      57.0
                B-class_type      0.606     0.833     0.702      24.0
          B-arrive_time.time      0.250     0.029     0.053      34.0
 B-arrive_time.time_relative      0.455     0.161     0.238      31.0
                I-class_type      0.944     1.000     0.971      17.0
               B-flight_stop      0.913     1.000     0.955      21.0
          I-arrive_time.time      0.714     0.143     0.238      35.0
              B-airline_code      0.667     0.412     0.509      34.0
    I-depart_date.day_number      0.000     0.000     0.000      15.0
      I-fromloc.airport_name      0.121     0.533     0.198      15.0
      B-fromloc.airport_name      0.000     0.000     0.000      12.0
      B-arrive_date.day_name      0.000     0.000     0.000      11.0
          B-toloc.state_code      0.667     0.444     0.533      18.0
B-depart_date.today_relative      0.000     0.000     0.000       9.0
             B-flight_number      0.000     0.000     0.000      11.0
 B-depart_date.date_relative      0.750     0.353     0.480      17.0
          B-toloc.state_name      0.000     0.000     0.000      28.0
           B-fare_basis_code      0.485     0.941     0.640      17.0
               B-flight_time      0.000     0.000     0.000       1.0
                        B-or      0.250     1.000     0.400       3.0
 B-arrive_time.period_of_day      0.000     0.000     0.000       6.0
          B-meal_description      0.750     0.600     0.667      10.0
             I-cost_relative      0.500     0.667     0.571       3.0
              I-airport_name      0.000     0.000     0.000      29.0
               B-fare_amount      0.500     0.500     0.500       2.0
               I-fare_amount      0.500     0.500     0.500       2.0
                 I-city_name      0.250     0.033     0.059      30.0
        I-toloc.airport_name      0.000     0.000     0.000       3.0
            B-transport_type      0.500     0.100     0.167      10.0
    B-arrive_date.month_name      0.000     0.000     0.000       6.0
    B-arrive_date.day_number      0.000     0.000     0.000       6.0
         I-stoploc.city_name      0.000     0.000     0.000      10.0
                      B-meal      1.000     0.750     0.857      16.0
        B-fromloc.state_code      0.000     0.000     0.000      23.0
    B-depart_time.period_mod      0.000     0.000     0.000       5.0
                   B-connect      1.000     0.667     0.800       6.0
               B-flight_days      0.000     0.000     0.000      10.0
        B-toloc.airport_name      0.000     0.000     0.000       3.0
        B-fromloc.state_name      0.000     0.000     0.000      17.0
              B-airport_name      0.000     0.000     0.000      21.0
                   B-economy      0.000     0.000     0.000       6.0
               I-flight_time      0.000     0.000     0.000       1.0
             B-aircraft_code      0.000     0.000     0.000      33.0
                       B-mod      0.000     0.000     0.000       2.0
              B-airport_code      0.000     0.000     0.000       9.0
    B-depart_time.start_time      0.000     0.000     0.000       3.0
      B-depart_time.end_time      0.000     0.000     0.000       3.0
          B-depart_date.year      0.000     0.000     0.000       3.0
            I-transport_type      0.000     0.000     0.000       1.0
          B-restriction_code      0.000     0.000     0.000       4.0
    B-arrive_time.start_time      0.000     0.000     0.000       8.0
        B-toloc.airport_code      0.000     0.000     0.000       4.0
      B-arrive_time.end_time      0.000     0.000     0.000       8.0
      I-arrive_time.end_time      0.000     0.000     0.000       8.0
      I-depart_time.end_time      0.000     0.000     0.000       3.0
               I-flight_stop      0.000     0.000     0.000       0.0
      B-fromloc.airport_code      0.000     0.000     0.000       5.0
          I-restriction_code      1.000     1.000     1.000       3.0
    I-depart_time.start_time      0.000     0.000     0.000       1.0
          I-toloc.state_name      0.000     0.000     0.000       1.0
I-depart_date.today_relative      0.000     0.000     0.000       0.0
 B-arrive_date.date_relative      0.000     0.000     0.000       2.0
                I-flight_mod      0.000     0.000     0.000       6.0
                   I-economy      0.000     0.000     0.000       0.0
 B-return_date.date_relative      0.000     0.000     0.000       3.0
        I-fromloc.state_name      0.000     0.000     0.000       1.0
                B-state_code      0.000     0.000     0.000       1.0
    I-arrive_time.start_time      0.000     0.000     0.000       1.0
    I-arrive_date.day_number      0.000     0.000     0.000       0.0
                 B-meal_code      0.000     0.000     0.000       1.0
 I-depart_time.period_of_day      0.000     0.000     0.000       1.0
                  B-day_name      0.000     0.000     0.000       2.0
             B-period_of_day      0.000     0.000     0.000       4.0
        B-stoploc.state_code      0.000     0.000     0.000       0.0
    B-return_date.month_name      0.000     0.000     0.000       0.0
    B-return_date.day_number      0.000     0.000     0.000       0.0
    B-arrive_time.period_mod      0.000     0.000     0.000       0.0
                 I-meal_code      0.000     0.000     0.000       0.0
        B-toloc.country_name      0.000     0.000     0.000       1.0
                 B-days_code      0.000     0.000     0.000       1.0
 I-arrive_time.period_of_day      0.000     0.000     0.000       0.0
            I-today_relative      0.000     0.000     0.000       0.0
 B-return_time.period_of_day      0.000     0.000     0.000       0.0
                      B-time      0.000     0.000     0.000       0.0
           I-fare_basis_code      0.000     0.000     0.000       0.0
 I-arrive_time.time_relative      0.000     0.000     0.000       4.0
 I-depart_time.time_relative      0.000     0.000     0.000       1.0
            B-today_relative      0.000     0.000     0.000       0.0
                B-state_name      0.000     0.000     0.000       9.0
B-arrive_date.today_relative      0.000     0.000     0.000       0.0
    B-return_time.period_mod      0.000     0.000     0.000       0.0
                B-month_name      0.000     0.000     0.000       0.0
                B-day_number      0.000     0.000     0.000       0.0
 I-return_date.date_relative      0.000     0.000     0.000       3.0
I-return_date.today_relative      0.000     0.000     0.000       0.0
      B-stoploc.airport_name      0.000     0.000     0.000       0.0
             B-time_relative      0.000     0.000     0.000       0.0
                      I-time      0.000     0.000     0.000       0.0
    I-return_date.day_number      0.000     0.000     0.000       0.0
          I-meal_description      0.000     0.000     0.000       0.0
B-return_date.today_relative      0.000     0.000     0.000       0.0
      B-return_date.day_name      0.000     0.000     0.000       2.0

                   micro avg      0.668     0.669     0.669    3657.0
                   macro avg      0.212     0.217     0.200    3657.0
                weighted avg      0.632     0.669     0.629    3657.0

I0717 06:50:56.892061 140695199004544 interactiveshell.py:2882] Best Slot F1: 0.669
Reading ../data/atis.train.w-intent.iob
I0717 06:51:00.754630 140695199004544 interactiveshell.py:2882] Step 350 | Loss: 1.6240 | Loss_intent: 1.3071 | Loss_slots: 0.3169 | Spent: 7.0 secs | LR: 0.000289
I0717 06:51:04.889136 140695199004544 interactiveshell.py:2882] Step 400 | Loss: 0.2028 | Loss_intent: 0.0033 | Loss_slots: 0.1995 | Spent: 4.1 secs | LR: 0.000288
I0717 06:51:09.052312 140695199004544 interactiveshell.py:2882] Step 450 | Loss: 0.5439 | Loss_intent: 0.3387 | Loss_slots: 0.2052 | Spent: 4.2 secs | LR: 0.000286
I0717 06:51:13.262725 140695199004544 interactiveshell.py:2882] Step 500 | Loss: 0.8380 | Loss_intent: 0.5002 | Loss_slots: 0.3378 | Spent: 4.2 secs | LR: 0.000285
I0717 06:51:17.484139 140695199004544 interactiveshell.py:2882] Step 550 | Loss: 0.3473 | Loss_intent: 0.0570 | Loss_slots: 0.2902 | Spent: 4.2 secs | LR: 0.000283
I0717 06:51:21.689694 140695199004544 interactiveshell.py:2882] Step 600 | Loss: 0.1812 | Loss_intent: 0.0429 | Loss_slots: 0.1383 | Spent: 4.2 secs | LR: 0.000282
Reading ../data/atis.test.w-intent.iob
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.
  'precision', 'predicted', average, warn_for)
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1439: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples.
  'recall', 'true', average, warn_for)
I0717 06:51:25.312180 140695199004544 interactiveshell.py:2882] 
                                          precision    recall  f1-score   support

                             atis_flight      0.950     0.987     0.968       632
                            atis_airfare      0.940     0.979     0.959        48
                     atis_ground_service      0.878     1.000     0.935        36
                            atis_airline      0.905     1.000     0.950        38
                       atis_abbreviation      0.943     1.000     0.971        33
                           atis_aircraft      0.818     1.000     0.900         9
                        atis_flight_time      1.000     1.000     1.000         1
                           atis_quantity      0.000     0.000     0.000         3
                atis_flight#atis_airfare      0.000     0.000     0.000        12
                            atis_airport      0.900     1.000     0.947        18
                           atis_distance      1.000     0.800     0.889        10
                               atis_city      1.000     0.500     0.667         6
                        atis_ground_fare      0.000     0.000     0.000         7
                           atis_capacity      0.952     0.952     0.952        21
                          atis_flight_no      0.000     0.000     0.000         8
                               atis_meal      0.000     0.000     0.000         6
                        atis_restriction      0.000     0.000     0.000         0
             atis_airline#atis_flight_no      0.000     0.000     0.000         0
    atis_ground_service#atis_ground_fare      0.000     0.000     0.000         0
           atis_airfare#atis_flight_time      0.000     0.000     0.000         0
                           atis_cheapest      0.000     0.000     0.000         0
atis_aircraft#atis_flight#atis_flight_no      0.000     0.000     0.000         0

                               micro avg      0.937     0.943     0.940       888
                               macro avg      0.468     0.464     0.461       888
                            weighted avg      0.904     0.943     0.922       888

I0717 06:51:25.351607 140695199004544 interactiveshell.py:2882] 
                              precision    recall  f1-score   support

                           O      0.000     0.000     0.000       0.0
           B-toloc.city_name      0.843     0.885     0.864     716.0
         B-fromloc.city_name      0.862     0.903     0.882     704.0
           I-toloc.city_name      0.653     0.932     0.768     265.0
      B-depart_date.day_name      0.934     1.000     0.966     212.0
              B-airline_name      0.816     0.921     0.865     101.0
         I-fromloc.city_name      0.717     0.458     0.559     177.0
 B-depart_time.period_of_day      0.853     0.892     0.872     130.0
              I-airline_name      1.000     0.923     0.960      65.0
    B-depart_date.day_number      0.750     0.655     0.699      55.0
    B-depart_date.month_name      0.900     0.964     0.931      56.0
          B-depart_time.time      0.417     0.877     0.565      57.0
                B-round_trip      0.986     0.945     0.965      73.0
             B-cost_relative      0.878     0.973     0.923      37.0
                I-round_trip      0.958     0.972     0.965      71.0
                B-flight_mod      0.667     0.667     0.667      24.0
 B-depart_time.time_relative      0.881     0.800     0.839      65.0
          I-depart_time.time      0.510     0.981     0.671      52.0
         B-stoploc.city_name      0.062     0.050     0.056      20.0
                 B-city_name      0.411     0.404     0.407      57.0
                B-class_type      0.656     0.875     0.750      24.0
          B-arrive_time.time      0.333     0.029     0.054      34.0
 B-arrive_time.time_relative      0.595     0.710     0.647      31.0
                I-class_type      1.000     1.000     1.000      17.0
               B-flight_stop      1.000     1.000     1.000      21.0
          I-arrive_time.time      0.778     0.200     0.318      35.0
              B-airline_code      0.667     0.588     0.625      34.0
    I-depart_date.day_number      0.179     0.333     0.233      15.0
      I-fromloc.airport_name      0.263     0.667     0.377      15.0
      B-fromloc.airport_name      0.000     0.000     0.000      12.0
      B-arrive_date.day_name      0.000     0.000     0.000      11.0
          B-toloc.state_code      0.722     0.722     0.722      18.0
B-depart_date.today_relative      0.500     0.333     0.400       9.0
             B-flight_number      0.143     0.182     0.160      11.0
 B-depart_date.date_relative      0.800     0.706     0.750      17.0
          B-toloc.state_name      0.556     0.179     0.270      28.0
           B-fare_basis_code      0.667     0.824     0.737      17.0
               B-flight_time      1.000     1.000     1.000       1.0
                        B-or      1.000     1.000     1.000       3.0
 B-arrive_time.period_of_day      0.000     0.000     0.000       6.0
          B-meal_description      1.000     0.900     0.947      10.0
             I-cost_relative      1.000     0.667     0.800       3.0
              I-airport_name      0.300     0.103     0.154      29.0
               B-fare_amount      0.143     0.500     0.222       2.0
               I-fare_amount      0.667     1.000     0.800       2.0
                 I-city_name      0.333     0.067     0.111      30.0
        I-toloc.airport_name      0.000     0.000     0.000       3.0
            B-transport_type      1.000     0.500     0.667      10.0
    B-arrive_date.month_name      0.000     0.000     0.000       6.0
    B-arrive_date.day_number      0.000     0.000     0.000       6.0
         I-stoploc.city_name      0.000     0.000     0.000      10.0
                      B-meal      1.000     0.938     0.968      16.0
        B-fromloc.state_code      0.750     0.783     0.766      23.0
    B-depart_time.period_mod      0.800     0.800     0.800       5.0
                   B-connect      1.000     0.667     0.800       6.0
               B-flight_days      1.000     1.000     1.000      10.0
        B-toloc.airport_name      0.000     0.000     0.000       3.0
        B-fromloc.state_name      0.000     0.000     0.000      17.0
              B-airport_name      1.000     0.048     0.091      21.0
                   B-economy      1.000     1.000     1.000       6.0
               I-flight_time      1.000     1.000     1.000       1.0
             B-aircraft_code      0.944     0.515     0.667      33.0
                       B-mod      0.000     0.000     0.000       2.0
              B-airport_code      0.571     0.444     0.500       9.0
    B-depart_time.start_time      0.000     0.000     0.000       3.0
      B-depart_time.end_time      0.000     0.000     0.000       3.0
          B-depart_date.year      0.000     0.000     0.000       3.0
            I-transport_type      0.000     0.000     0.000       1.0
          B-restriction_code      0.333     0.250     0.286       4.0
    B-arrive_time.start_time      0.000     0.000     0.000       8.0
        B-toloc.airport_code      0.000     0.000     0.000       4.0
      B-arrive_time.end_time      0.000     0.000     0.000       8.0
      I-arrive_time.end_time      0.000     0.000     0.000       8.0
      I-depart_time.end_time      0.000     0.000     0.000       3.0
               I-flight_stop      0.000     0.000     0.000       0.0
      B-fromloc.airport_code      0.000     0.000     0.000       5.0
          I-restriction_code      0.600     1.000     0.750       3.0
    I-depart_time.start_time      0.000     0.000     0.000       1.0
          I-toloc.state_name      0.000     0.000     0.000       1.0
I-depart_date.today_relative      0.000     0.000     0.000       0.0
 B-arrive_date.date_relative      0.000     0.000     0.000       2.0
                I-flight_mod      0.000     0.000     0.000       6.0
                   I-economy      0.000     0.000     0.000       0.0
 B-return_date.date_relative      0.000     0.000     0.000       3.0
        I-fromloc.state_name      0.000     0.000     0.000       1.0
                B-state_code      0.000     0.000     0.000       1.0
    I-arrive_time.start_time      0.000     0.000     0.000       1.0
    I-arrive_date.day_number      0.000     0.000     0.000       0.0
                 B-meal_code      0.000     0.000     0.000       1.0
 I-depart_time.period_of_day      0.000     0.000     0.000       1.0
                  B-day_name      0.000     0.000     0.000       2.0
             B-period_of_day      0.000     0.000     0.000       4.0
        B-stoploc.state_code      0.000     0.000     0.000       0.0
    B-return_date.month_name      0.000     0.000     0.000       0.0
    B-return_date.day_number      0.000     0.000     0.000       0.0
    B-arrive_time.period_mod      0.000     0.000     0.000       0.0
                 I-meal_code      0.000     0.000     0.000       0.0
        B-toloc.country_name      0.000     0.000     0.000       1.0
                 B-days_code      0.000     0.000     0.000       1.0
 I-arrive_time.period_of_day      0.000     0.000     0.000       0.0
            I-today_relative      0.000     0.000     0.000       0.0
 B-return_time.period_of_day      0.000     0.000     0.000       0.0
                      B-time      0.000     0.000     0.000       0.0
           I-fare_basis_code      0.000     0.000     0.000       0.0
 I-arrive_time.time_relative      0.000     0.000     0.000       4.0
 I-depart_time.time_relative      0.000     0.000     0.000       1.0
            B-today_relative      0.000     0.000     0.000       0.0
                B-state_name      0.000     0.000     0.000       9.0
B-arrive_date.today_relative      0.000     0.000     0.000       0.0
    B-return_time.period_mod      0.000     0.000     0.000       0.0
                B-month_name      0.000     0.000     0.000       0.0
                B-day_number      0.000     0.000     0.000       0.0
 I-return_date.date_relative      0.000     0.000     0.000       3.0
I-return_date.today_relative      0.000     0.000     0.000       0.0
      B-stoploc.airport_name      0.000     0.000     0.000       0.0
             B-time_relative      0.000     0.000     0.000       0.0
                      I-time      0.000     0.000     0.000       0.0
    I-return_date.day_number      0.000     0.000     0.000       0.0
          I-meal_description      0.000     0.000     0.000       0.0
B-return_date.today_relative      0.000     0.000     0.000       0.0
      B-return_date.day_name      0.000     0.000     0.000       2.0

                   micro avg      0.770     0.771     0.771    3657.0
                   macro avg      0.326     0.312     0.304    3657.0
                weighted avg      0.751     0.771     0.746    3657.0

I0717 06:51:25.352374 140695199004544 interactiveshell.py:2882] Best Slot F1: 0.771
Reading ../data/atis.train.w-intent.iob
I0717 06:51:28.239351 140695199004544 interactiveshell.py:2882] Step 650 | Loss: 0.4039 | Loss_intent: 0.2406 | Loss_slots: 0.1633 | Spent: 6.5 secs | LR: 0.000280
I0717 06:51:32.371182 140695199004544 interactiveshell.py:2882] Step 700 | Loss: 0.5325 | Loss_intent: 0.3270 | Loss_slots: 0.2055 | Spent: 4.1 secs | LR: 0.000279
I0717 06:51:36.487001 140695199004544 interactiveshell.py:2882] Step 750 | Loss: 0.3430 | Loss_intent: 0.1663 | Loss_slots: 0.1766 | Spent: 4.1 secs | LR: 0.000277
I0717 06:51:40.914029 140695199004544 interactiveshell.py:2882] Step 800 | Loss: 1.2674 | Loss_intent: 1.0331 | Loss_slots: 0.2343 | Spent: 4.4 secs | LR: 0.000276
I0717 06:51:45.144248 140695199004544 interactiveshell.py:2882] Step 850 | Loss: 0.8099 | Loss_intent: 0.5640 | Loss_slots: 0.2459 | Spent: 4.2 secs | LR: 0.000274
I0717 06:51:49.268007 140695199004544 interactiveshell.py:2882] Step 900 | Loss: 1.0945 | Loss_intent: 0.9237 | Loss_slots: 0.1707 | Spent: 4.1 secs | LR: 0.000273
Reading ../data/atis.test.w-intent.iob
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.
  'precision', 'predicted', average, warn_for)
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1439: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples.
  'recall', 'true', average, warn_for)
I0717 06:51:53.864537 140695199004544 interactiveshell.py:2882] 
                                          precision    recall  f1-score   support

                             atis_flight      0.990     0.976     0.983       632
                            atis_airfare      0.814     1.000     0.897        48
                     atis_ground_service      1.000     0.972     0.986        36
                            atis_airline      0.927     1.000     0.962        38
                       atis_abbreviation      1.000     0.758     0.862        33
                           atis_aircraft      1.000     0.889     0.941         9
                        atis_flight_time      0.250     1.000     0.400         1
                           atis_quantity      0.375     1.000     0.545         3
                atis_flight#atis_airfare      0.667     0.333     0.444        12
                            atis_airport      0.643     1.000     0.783        18
                           atis_distance      1.000     0.800     0.889        10
                               atis_city      1.000     0.667     0.800         6
                        atis_ground_fare      0.833     0.714     0.769         7
                           atis_capacity      1.000     0.905     0.950        21
                          atis_flight_no      0.857     0.750     0.800         8
                               atis_meal      1.000     0.167     0.286         6
                        atis_restriction      0.000     0.000     0.000         0
             atis_airline#atis_flight_no      0.000     0.000     0.000         0
    atis_ground_service#atis_ground_fare      0.000     0.000     0.000         0
           atis_airfare#atis_flight_time      0.000     0.000     0.000         0
                           atis_cheapest      0.000     0.000     0.000         0
atis_aircraft#atis_flight#atis_flight_no      0.000     0.000     0.000         0

                               micro avg      0.941     0.946     0.943       888
                               macro avg      0.607     0.588     0.559       888
                            weighted avg      0.963     0.946     0.948       888

I0717 06:51:53.905481 140695199004544 interactiveshell.py:2882] 
                              precision    recall  f1-score   support

                           O      0.000     0.000     0.000       0.0
           B-toloc.city_name      0.875     0.919     0.896     716.0
         B-fromloc.city_name      0.880     0.923     0.901     704.0
           I-toloc.city_name      0.834     0.913     0.872     265.0
      B-depart_date.day_name      0.934     1.000     0.966     212.0
              B-airline_name      0.967     0.861     0.911     101.0
         I-fromloc.city_name      0.762     0.870     0.813     177.0
 B-depart_time.period_of_day      0.898     0.877     0.887     130.0
              I-airline_name      1.000     0.969     0.984      65.0
    B-depart_date.day_number      0.940     0.855     0.895      55.0
    B-depart_date.month_name      0.900     0.964     0.931      56.0
          B-depart_time.time      0.484     0.807     0.605      57.0
                B-round_trip      1.000     0.973     0.986      73.0
             B-cost_relative      0.923     0.973     0.947      37.0
                I-round_trip      1.000     1.000     1.000      71.0
                B-flight_mod      0.783     0.750     0.766      24.0
 B-depart_time.time_relative      0.920     0.708     0.800      65.0
          I-depart_time.time      0.529     0.692     0.600      52.0
         B-stoploc.city_name      0.000     0.000     0.000      20.0
                 B-city_name      0.426     0.456     0.441      57.0
                B-class_type      0.522     1.000     0.686      24.0
          B-arrive_time.time      0.677     0.618     0.646      34.0
 B-arrive_time.time_relative      0.532     0.806     0.641      31.0
                I-class_type      1.000     1.000     1.000      17.0
               B-flight_stop      1.000     1.000     1.000      21.0
          I-arrive_time.time      0.525     0.600     0.560      35.0
              B-airline_code      0.897     0.765     0.825      34.0
    I-depart_date.day_number      0.588     0.667     0.625      15.0
      I-fromloc.airport_name      0.393     0.733     0.512      15.0
      B-fromloc.airport_name      0.050     0.083     0.062      12.0
      B-arrive_date.day_name      0.000     0.000     0.000      11.0
          B-toloc.state_code      0.514     1.000     0.679      18.0
B-depart_date.today_relative      1.000     0.556     0.714       9.0
             B-flight_number      0.200     0.091     0.125      11.0
 B-depart_date.date_relative      0.850     1.000     0.919      17.0
          B-toloc.state_name      0.714     0.179     0.286      28.0
           B-fare_basis_code      0.739     1.000     0.850      17.0
               B-flight_time      0.500     1.000     0.667       1.0
                        B-or      1.000     1.000     1.000       3.0
 B-arrive_time.period_of_day      0.000     0.000     0.000       6.0
          B-meal_description      1.000     0.900     0.947      10.0
             I-cost_relative      1.000     0.667     0.800       3.0
              I-airport_name      0.778     0.241     0.368      29.0
               B-fare_amount      0.182     1.000     0.308       2.0
               I-fare_amount      1.000     1.000     1.000       2.0
                 I-city_name      0.462     0.200     0.279      30.0
        I-toloc.airport_name      0.500     0.333     0.400       3.0
            B-transport_type      0.643     0.900     0.750      10.0
    B-arrive_date.month_name      0.000     0.000     0.000       6.0
    B-arrive_date.day_number      0.000     0.000     0.000       6.0
         I-stoploc.city_name      0.000     0.000     0.000      10.0
                      B-meal      1.000     1.000     1.000      16.0
        B-fromloc.state_code      1.000     0.304     0.467      23.0
    B-depart_time.period_mod      0.800     0.800     0.800       5.0
                   B-connect      1.000     0.833     0.909       6.0
               B-flight_days      1.000     1.000     1.000      10.0
        B-toloc.airport_name      0.000     0.000     0.000       3.0
        B-fromloc.state_name      0.000     0.000     0.000      17.0
              B-airport_name      0.500     0.048     0.087      21.0
                   B-economy      1.000     1.000     1.000       6.0
               I-flight_time      0.000     0.000     0.000       1.0
             B-aircraft_code      1.000     0.818     0.900      33.0
                       B-mod      0.000     0.000     0.000       2.0
              B-airport_code      0.000     0.000     0.000       9.0
    B-depart_time.start_time      0.000     0.000     0.000       3.0
      B-depart_time.end_time      0.500     0.333     0.400       3.0
          B-depart_date.year      0.000     0.000     0.000       3.0
            I-transport_type      0.000     0.000     0.000       1.0
          B-restriction_code      0.571     1.000     0.727       4.0
    B-arrive_time.start_time      0.000     0.000     0.000       8.0
        B-toloc.airport_code      0.500     0.250     0.333       4.0
      B-arrive_time.end_time      0.000     0.000     0.000       8.0
      I-arrive_time.end_time      0.000     0.000     0.000       8.0
      I-depart_time.end_time      0.000     0.000     0.000       3.0
               I-flight_stop      0.000     0.000     0.000       0.0
      B-fromloc.airport_code      1.000     0.200     0.333       5.0
          I-restriction_code      0.000     0.000     0.000       3.0
    I-depart_time.start_time      0.000     0.000     0.000       1.0
          I-toloc.state_name      0.000     0.000     0.000       1.0
I-depart_date.today_relative      0.000     0.000     0.000       0.0
 B-arrive_date.date_relative      0.000     0.000     0.000       2.0
                I-flight_mod      0.000     0.000     0.000       6.0
                   I-economy      0.000     0.000     0.000       0.0
 B-return_date.date_relative      0.000     0.000     0.000       3.0
        I-fromloc.state_name      0.000     0.000     0.000       1.0
                B-state_code      0.000     0.000     0.000       1.0
    I-arrive_time.start_time      0.000     0.000     0.000       1.0
    I-arrive_date.day_number      0.000     0.000     0.000       0.0
                 B-meal_code      0.167     1.000     0.286       1.0
 I-depart_time.period_of_day      0.000     0.000     0.000       1.0
                  B-day_name      0.000     0.000     0.000       2.0
             B-period_of_day      0.000     0.000     0.000       4.0
        B-stoploc.state_code      0.000     0.000     0.000       0.0
    B-return_date.month_name      0.000     0.000     0.000       0.0
    B-return_date.day_number      0.000     0.000     0.000       0.0
    B-arrive_time.period_mod      0.000     0.000     0.000       0.0
                 I-meal_code      0.000     0.000     0.000       0.0
        B-toloc.country_name      0.000     0.000     0.000       1.0
                 B-days_code      0.000     0.000     0.000       1.0
 I-arrive_time.period_of_day      0.000     0.000     0.000       0.0
            I-today_relative      0.000     0.000     0.000       0.0
 B-return_time.period_of_day      0.000     0.000     0.000       0.0
                      B-time      0.000     0.000     0.000       0.0
           I-fare_basis_code      0.000     0.000     0.000       0.0
 I-arrive_time.time_relative      0.000     0.000     0.000       4.0
 I-depart_time.time_relative      0.000     0.000     0.000       1.0
            B-today_relative      0.000     0.000     0.000       0.0
                B-state_name      0.000     0.000     0.000       9.0
B-arrive_date.today_relative      0.000     0.000     0.000       0.0
    B-return_time.period_mod      0.000     0.000     0.000       0.0
                B-month_name      0.000     0.000     0.000       0.0
                B-day_number      0.000     0.000     0.000       0.0
 I-return_date.date_relative      0.000     0.000     0.000       3.0
I-return_date.today_relative      0.000     0.000     0.000       0.0
      B-stoploc.airport_name      0.000     0.000     0.000       0.0
             B-time_relative      0.000     0.000     0.000       0.0
                      I-time      0.000     0.000     0.000       0.0
    I-return_date.day_number      0.000     0.000     0.000       0.0
          I-meal_description      0.000     0.000     0.000       0.0
B-return_date.today_relative      0.000     0.000     0.000       0.0
      B-return_date.day_name      0.000     0.000     0.000       2.0

                   micro avg      0.818     0.820     0.819    3657.0
                   macro avg      0.350     0.351     0.331    3657.0
                weighted avg      0.799     0.820     0.800    3657.0

I0717 06:51:53.906309 140695199004544 interactiveshell.py:2882] Best Slot F1: 0.819
Reading ../data/atis.train.w-intent.iob
I0717 06:51:55.785865 140695199004544 interactiveshell.py:2882] Step 950 | Loss: 0.1438 | Loss_intent: 0.0016 | Loss_slots: 0.1422 | Spent: 6.5 secs | LR: 0.000271
I0717 06:51:59.925580 140695199004544 interactiveshell.py:2882] Step 1000 | Loss: 0.0851 | Loss_intent: 0.0112 | Loss_slots: 0.0739 | Spent: 4.1 secs | LR: 0.000270
I0717 06:52:04.055519 140695199004544 interactiveshell.py:2882] Step 1050 | Loss: 0.1428 | Loss_intent: 0.0182 | Loss_slots: 0.1247 | Spent: 4.1 secs | LR: 0.000269
I0717 06:52:08.194192 140695199004544 interactiveshell.py:2882] Step 1100 | Loss: 0.2755 | Loss_intent: 0.0316 | Loss_slots: 0.2439 | Spent: 4.1 secs | LR: 0.000267
I0717 06:52:12.335656 140695199004544 interactiveshell.py:2882] Step 1150 | Loss: 0.3042 | Loss_intent: 0.1610 | Loss_slots: 0.1432 | Spent: 4.1 secs | LR: 0.000266
I0717 06:52:16.486021 140695199004544 interactiveshell.py:2882] Step 1200 | Loss: 0.1034 | Loss_intent: 0.0008 | Loss_slots: 0.1026 | Spent: 4.1 secs | LR: 0.000264
Reading ../data/atis.test.w-intent.iob
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.
  'precision', 'predicted', average, warn_for)
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1439: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples.
  'recall', 'true', average, warn_for)
I0717 06:52:22.118044 140695199004544 interactiveshell.py:2882] 
                                          precision    recall  f1-score   support

                             atis_flight      0.979     0.978     0.979       632
                            atis_airfare      0.958     0.958     0.958        48
                     atis_ground_service      0.923     1.000     0.960        36
                            atis_airline      0.905     1.000     0.950        38
                       atis_abbreviation      0.943     1.000     0.971        33
                           atis_aircraft      0.692     1.000     0.818         9
                        atis_flight_time      1.000     1.000     1.000         1
                           atis_quantity      0.000     0.000     0.000         3
                atis_flight#atis_airfare      0.875     0.583     0.700        12
                            atis_airport      1.000     0.833     0.909        18
                           atis_distance      0.909     1.000     0.952        10
                               atis_city      0.571     0.667     0.615         6
                        atis_ground_fare      1.000     0.714     0.833         7
                           atis_capacity      0.840     1.000     0.913        21
                          atis_flight_no      0.875     0.875     0.875         8
                               atis_meal      1.000     0.167     0.286         6
                        atis_restriction      0.000     0.000     0.000         0
             atis_airline#atis_flight_no      0.000     0.000     0.000         0
    atis_ground_service#atis_ground_fare      0.000     0.000     0.000         0
           atis_airfare#atis_flight_time      0.000     0.000     0.000         0
                           atis_cheapest      0.000     0.000     0.000         0
atis_aircraft#atis_flight#atis_flight_no      0.000     0.000     0.000         0

                               micro avg      0.953     0.958     0.956       888
                               macro avg      0.612     0.581     0.578       888
                            weighted avg      0.957     0.958     0.954       888

I0717 06:52:22.159183 140695199004544 interactiveshell.py:2882] 
                              precision    recall  f1-score   support

                           O      0.000     0.000     0.000       0.0
           B-toloc.city_name      0.937     0.887     0.911     716.0
         B-fromloc.city_name      0.894     0.955     0.923     704.0
           I-toloc.city_name      0.894     0.955     0.923     265.0
      B-depart_date.day_name      0.946     1.000     0.972     212.0
              B-airline_name      0.860     0.970     0.912     101.0
         I-fromloc.city_name      0.943     0.836     0.886     177.0
 B-depart_time.period_of_day      0.908     0.838     0.872     130.0
              I-airline_name      1.000     0.938     0.968      65.0
    B-depart_date.day_number      0.810     0.927     0.864      55.0
    B-depart_date.month_name      0.900     0.964     0.931      56.0
          B-depart_time.time      0.485     0.877     0.625      57.0
                B-round_trip      0.973     0.973     0.973      73.0
             B-cost_relative      1.000     0.919     0.958      37.0
                I-round_trip      1.000     0.986     0.993      71.0
                B-flight_mod      0.690     0.833     0.755      24.0
 B-depart_time.time_relative      0.870     0.923     0.896      65.0
          I-depart_time.time      0.658     0.962     0.781      52.0
         B-stoploc.city_name      0.250     0.100     0.143      20.0
                 B-city_name      0.417     0.614     0.496      57.0
                B-class_type      0.880     0.917     0.898      24.0
          B-arrive_time.time      0.600     0.618     0.609      34.0
 B-arrive_time.time_relative      0.724     0.677     0.700      31.0
                I-class_type      1.000     1.000     1.000      17.0
               B-flight_stop      0.955     1.000     0.977      21.0
          I-arrive_time.time      0.706     0.686     0.696      35.0
              B-airline_code      0.769     0.882     0.822      34.0
    I-depart_date.day_number      0.778     0.467     0.583      15.0
      I-fromloc.airport_name      0.236     0.867     0.371      15.0
      B-fromloc.airport_name      0.200     0.417     0.270      12.0
      B-arrive_date.day_name      0.500     0.091     0.154      11.0
          B-toloc.state_code      0.667     1.000     0.800      18.0
B-depart_date.today_relative      1.000     0.667     0.800       9.0
             B-flight_number      0.182     0.182     0.182      11.0
 B-depart_date.date_relative      0.842     0.941     0.889      17.0
          B-toloc.state_name      0.590     0.821     0.687      28.0
           B-fare_basis_code      0.882     0.882     0.882      17.0
               B-flight_time      0.500     1.000     0.667       1.0
                        B-or      1.000     1.000     1.000       3.0
 B-arrive_time.period_of_day      1.000     0.167     0.286       6.0
          B-meal_description      1.000     0.800     0.889      10.0
             I-cost_relative      1.000     0.667     0.800       3.0
              I-airport_name      0.000     0.000     0.000      29.0
               B-fare_amount      0.250     0.500     0.333       2.0
               I-fare_amount      1.000     1.000     1.000       2.0
                 I-city_name      0.522     0.400     0.453      30.0
        I-toloc.airport_name      0.000     0.000     0.000       3.0
            B-transport_type      0.643     0.900     0.750      10.0
    B-arrive_date.month_name      0.000     0.000     0.000       6.0
    B-arrive_date.day_number      0.000     0.000     0.000       6.0
         I-stoploc.city_name      0.500     0.200     0.286      10.0
                      B-meal      1.000     1.000     1.000      16.0
        B-fromloc.state_code      1.000     0.609     0.757      23.0
    B-depart_time.period_mod      0.500     0.600     0.545       5.0
                   B-connect      1.000     1.000     1.000       6.0
               B-flight_days      0.833     1.000     0.909      10.0
        B-toloc.airport_name      0.000     0.000     0.000       3.0
        B-fromloc.state_name      0.500     0.059     0.105      17.0
              B-airport_name      1.000     0.048     0.091      21.0
                   B-economy      1.000     1.000     1.000       6.0
               I-flight_time      0.000     0.000     0.000       1.0
             B-aircraft_code      1.000     0.818     0.900      33.0
                       B-mod      0.000     0.000     0.000       2.0
              B-airport_code      0.000     0.000     0.000       9.0
    B-depart_time.start_time      0.000     0.000     0.000       3.0
      B-depart_time.end_time      1.000     0.333     0.500       3.0
          B-depart_date.year      0.000     0.000     0.000       3.0
            I-transport_type      0.000     0.000     0.000       1.0
          B-restriction_code      0.273     0.750     0.400       4.0
    B-arrive_time.start_time      0.000     0.000     0.000       8.0
        B-toloc.airport_code      0.500     0.250     0.333       4.0
      B-arrive_time.end_time      0.000     0.000     0.000       8.0
      I-arrive_time.end_time      0.000     0.000     0.000       8.0
      I-depart_time.end_time      0.000     0.000     0.000       3.0
               I-flight_stop      0.000     0.000     0.000       0.0
      B-fromloc.airport_code      0.000     0.000     0.000       5.0
          I-restriction_code      0.375     1.000     0.545       3.0
    I-depart_time.start_time      0.000     0.000     0.000       1.0
          I-toloc.state_name      0.000     0.000     0.000       1.0
I-depart_date.today_relative      0.000     0.000     0.000       0.0
 B-arrive_date.date_relative      0.000     0.000     0.000       2.0
                I-flight_mod      0.000     0.000     0.000       6.0
                   I-economy      0.000     0.000     0.000       0.0
 B-return_date.date_relative      0.000     0.000     0.000       3.0
        I-fromloc.state_name      0.000     0.000     0.000       1.0
                B-state_code      1.000     1.000     1.000       1.0
    I-arrive_time.start_time      0.000     0.000     0.000       1.0
    I-arrive_date.day_number      0.000     0.000     0.000       0.0
                 B-meal_code      0.000     0.000     0.000       1.0
 I-depart_time.period_of_day      0.000     0.000     0.000       1.0
                  B-day_name      0.500     0.500     0.500       2.0
             B-period_of_day      0.000     0.000     0.000       4.0
        B-stoploc.state_code      0.000     0.000     0.000       0.0
    B-return_date.month_name      0.000     0.000     0.000       0.0
    B-return_date.day_number      0.000     0.000     0.000       0.0
    B-arrive_time.period_mod      0.000     0.000     0.000       0.0
                 I-meal_code      0.000     0.000     0.000       0.0
        B-toloc.country_name      0.000     0.000     0.000       1.0
                 B-days_code      0.000     0.000     0.000       1.0
 I-arrive_time.period_of_day      0.000     0.000     0.000       0.0
            I-today_relative      0.000     0.000     0.000       0.0
 B-return_time.period_of_day      0.000     0.000     0.000       0.0
                      B-time      0.000     0.000     0.000       0.0
           I-fare_basis_code      0.000     0.000     0.000       0.0
 I-arrive_time.time_relative      0.000     0.000     0.000       4.0
 I-depart_time.time_relative      0.000     0.000     0.000       1.0
            B-today_relative      0.000     0.000     0.000       0.0
                B-state_name      0.000     0.000     0.000       9.0
B-arrive_date.today_relative      0.000     0.000     0.000       0.0
    B-return_time.period_mod      0.000     0.000     0.000       0.0
                B-month_name      0.000     0.000     0.000       0.0
                B-day_number      0.000     0.000     0.000       0.0
 I-return_date.date_relative      0.000     0.000     0.000       3.0
I-return_date.today_relative      0.000     0.000     0.000       0.0
      B-stoploc.airport_name      0.000     0.000     0.000       0.0
             B-time_relative      0.000     0.000     0.000       0.0
                      I-time      0.000     0.000     0.000       0.0
    I-return_date.day_number      0.000     0.000     0.000       0.0
          I-meal_description      0.000     0.000     0.000       0.0
B-return_date.today_relative      0.000     0.000     0.000       0.0
      B-return_date.day_name      0.000     0.000     0.000       2.0

                   micro avg      0.841     0.843     0.842    3657.0
                   macro avg      0.379     0.373     0.357    3657.0
                weighted avg      0.832     0.843     0.828    3657.0

I0717 06:52:22.160014 140695199004544 interactiveshell.py:2882] Best Slot F1: 0.842
Reading ../data/atis.train.w-intent.iob
I0717 06:52:23.045186 140695199004544 interactiveshell.py:2882] Step 1250 | Loss: 0.4887 | Loss_intent: 0.2979 | Loss_slots: 0.1908 | Spent: 6.6 secs | LR: 0.000263
I0717 06:52:27.204439 140695199004544 interactiveshell.py:2882] Step 1300 | Loss: 0.0824 | Loss_intent: 0.0024 | Loss_slots: 0.0800 | Spent: 4.2 secs | LR: 0.000262
I0717 06:52:31.350990 140695199004544 interactiveshell.py:2882] Step 1350 | Loss: 0.1277 | Loss_intent: 0.0138 | Loss_slots: 0.1139 | Spent: 4.1 secs | LR: 0.000260
I0717 06:52:35.576294 140695199004544 interactiveshell.py:2882] Step 1400 | Loss: 0.1035 | Loss_intent: 0.0202 | Loss_slots: 0.0834 | Spent: 4.2 secs | LR: 0.000259
I0717 06:52:39.785168 140695199004544 interactiveshell.py:2882] Step 1450 | Loss: 0.0929 | Loss_intent: 0.0007 | Loss_slots: 0.0923 | Spent: 4.2 secs | LR: 0.000257
I0717 06:52:43.967805 140695199004544 interactiveshell.py:2882] Step 1500 | Loss: 0.1687 | Loss_intent: 0.0852 | Loss_slots: 0.0835 | Spent: 4.2 secs | LR: 0.000256
I0717 06:52:48.107676 140695199004544 interactiveshell.py:2882] Step 1550 | Loss: 0.1053 | Loss_intent: 0.0036 | Loss_slots: 0.1017 | Spent: 4.1 secs | LR: 0.000255
Reading ../data/atis.test.w-intent.iob
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.
  'precision', 'predicted', average, warn_for)
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1439: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples.
  'recall', 'true', average, warn_for)
I0717 06:52:50.569650 140695199004544 interactiveshell.py:2882] 
                                          precision    recall  f1-score   support

                             atis_flight      0.978     0.986     0.982       632
                            atis_airfare      0.959     0.979     0.969        48
                     atis_ground_service      0.973     1.000     0.986        36
                            atis_airline      1.000     0.974     0.987        38
                       atis_abbreviation      0.892     1.000     0.943        33
                           atis_aircraft      0.889     0.889     0.889         9
                        atis_flight_time      0.333     1.000     0.500         1
                           atis_quantity      0.375     1.000     0.545         3
                atis_flight#atis_airfare      1.000     0.167     0.286        12
                            atis_airport      1.000     0.944     0.971        18
                           atis_distance      0.857     0.600     0.706        10
                               atis_city      0.571     0.667     0.615         6
                        atis_ground_fare      0.625     0.714     0.667         7
                           atis_capacity      1.000     1.000     1.000        21
                          atis_flight_no      0.875     0.875     0.875         8
                               atis_meal      1.000     0.833     0.909         6
                        atis_restriction      0.000     0.000     0.000         0
             atis_airline#atis_flight_no      0.000     0.000     0.000         0
    atis_ground_service#atis_ground_fare      0.000     0.000     0.000         0
           atis_airfare#atis_flight_time      0.000     0.000     0.000         0
                           atis_cheapest      0.000     0.000     0.000         0
atis_aircraft#atis_flight#atis_flight_no      0.000     0.000     0.000         0

                               micro avg      0.957     0.963     0.960       888
                               macro avg      0.606     0.619     0.583       888
                            weighted avg      0.964     0.963     0.958       888

I0717 06:52:50.610312 140695199004544 interactiveshell.py:2882] 
                              precision    recall  f1-score   support

                           O      0.000     0.000     0.000       0.0
           B-toloc.city_name      0.907     0.950     0.928     716.0
         B-fromloc.city_name      0.924     0.964     0.944     704.0
           I-toloc.city_name      0.870     0.981     0.922     265.0
      B-depart_date.day_name      0.951     1.000     0.975     212.0
              B-airline_name      0.933     0.970     0.951     101.0
         I-fromloc.city_name      0.945     0.881     0.912     177.0
 B-depart_time.period_of_day      0.871     0.885     0.878     130.0
              I-airline_name      1.000     0.938     0.968      65.0
    B-depart_date.day_number      0.939     0.836     0.885      55.0
    B-depart_date.month_name      0.915     0.964     0.939      56.0
          B-depart_time.time      0.593     0.947     0.730      57.0
                B-round_trip      0.972     0.959     0.966      73.0
             B-cost_relative      1.000     0.973     0.986      37.0
                I-round_trip      1.000     0.986     0.993      71.0
                B-flight_mod      0.704     0.792     0.745      24.0
 B-depart_time.time_relative      0.926     0.969     0.947      65.0
          I-depart_time.time      0.619     1.000     0.765      52.0
         B-stoploc.city_name      0.500     0.050     0.091      20.0
                 B-city_name      0.434     0.404     0.418      57.0
                B-class_type      0.550     0.917     0.687      24.0
          B-arrive_time.time      0.596     0.824     0.691      34.0
 B-arrive_time.time_relative      0.929     0.839     0.881      31.0
                I-class_type      1.000     1.000     1.000      17.0
               B-flight_stop      0.875     1.000     0.933      21.0
          I-arrive_time.time      0.909     0.571     0.702      35.0
              B-airline_code      1.000     0.735     0.847      34.0
    I-depart_date.day_number      0.684     0.867     0.765      15.0
      I-fromloc.airport_name      0.236     0.867     0.371      15.0
      B-fromloc.airport_name      0.190     0.333     0.242      12.0
      B-arrive_date.day_name      1.000     0.364     0.533      11.0
          B-toloc.state_code      0.667     1.000     0.800      18.0
B-depart_date.today_relative      1.000     0.444     0.615       9.0
             B-flight_number      0.571     0.727     0.640      11.0
 B-depart_date.date_relative      0.850     1.000     0.919      17.0
          B-toloc.state_name      0.833     0.357     0.500      28.0
           B-fare_basis_code      0.810     1.000     0.895      17.0
               B-flight_time      1.000     1.000     1.000       1.0
                        B-or      1.000     1.000     1.000       3.0
 B-arrive_time.period_of_day      0.000     0.000     0.000       6.0
          B-meal_description      1.000     0.800     0.889      10.0
             I-cost_relative      1.000     0.667     0.800       3.0
              I-airport_name      0.500     0.241     0.326      29.0
               B-fare_amount      0.333     0.500     0.400       2.0
               I-fare_amount      1.000     1.000     1.000       2.0
                 I-city_name      1.000     0.100     0.182      30.0
        I-toloc.airport_name      0.333     0.333     0.333       3.0
            B-transport_type      0.750     0.600     0.667      10.0
    B-arrive_date.month_name      0.333     0.167     0.222       6.0
    B-arrive_date.day_number      0.000     0.000     0.000       6.0
         I-stoploc.city_name      0.000     0.000     0.000      10.0
                      B-meal      1.000     1.000     1.000      16.0
        B-fromloc.state_code      1.000     0.565     0.722      23.0
    B-depart_time.period_mod      0.000     0.000     0.000       5.0
                   B-connect      1.000     1.000     1.000       6.0
               B-flight_days      1.000     1.000     1.000      10.0
        B-toloc.airport_name      0.000     0.000     0.000       3.0
        B-fromloc.state_name      0.700     0.412     0.519      17.0
              B-airport_name      1.000     0.048     0.091      21.0
                   B-economy      1.000     1.000     1.000       6.0
               I-flight_time      1.000     1.000     1.000       1.0
             B-aircraft_code      1.000     0.606     0.755      33.0
                       B-mod      0.000     0.000     0.000       2.0
              B-airport_code      0.000     0.000     0.000       9.0
    B-depart_time.start_time      0.000     0.000     0.000       3.0
      B-depart_time.end_time      0.000     0.000     0.000       3.0
          B-depart_date.year      0.000     0.000     0.000       3.0
            I-transport_type      0.000     0.000     0.000       1.0
          B-restriction_code      0.273     0.750     0.400       4.0
    B-arrive_time.start_time      0.000     0.000     0.000       8.0
        B-toloc.airport_code      0.500     0.750     0.600       4.0
      B-arrive_time.end_time      0.000     0.000     0.000       8.0
      I-arrive_time.end_time      1.000     0.250     0.400       8.0
      I-depart_time.end_time      0.000     0.000     0.000       3.0
               I-flight_stop      0.000     0.000     0.000       0.0
      B-fromloc.airport_code      1.000     0.200     0.333       5.0
          I-restriction_code      0.500     1.000     0.667       3.0
    I-depart_time.start_time      0.000     0.000     0.000       1.0
          I-toloc.state_name      0.500     1.000     0.667       1.0
I-depart_date.today_relative      0.000     0.000     0.000       0.0
 B-arrive_date.date_relative      0.000     0.000     0.000       2.0
                I-flight_mod      0.000     0.000     0.000       6.0
                   I-economy      0.000     0.000     0.000       0.0
 B-return_date.date_relative      0.000     0.000     0.000       3.0
        I-fromloc.state_name      0.000     0.000     0.000       1.0
                B-state_code      0.500     1.000     0.667       1.0
    I-arrive_time.start_time      0.000     0.000     0.000       1.0
    I-arrive_date.day_number      0.000     0.000     0.000       0.0
                 B-meal_code      1.000     1.000     1.000       1.0
 I-depart_time.period_of_day      0.000     0.000     0.000       1.0
                  B-day_name      1.000     0.500     0.667       2.0
             B-period_of_day      0.000     0.000     0.000       4.0
        B-stoploc.state_code      0.000     0.000     0.000       0.0
    B-return_date.month_name      0.000     0.000     0.000       0.0
    B-return_date.day_number      0.000     0.000     0.000       0.0
    B-arrive_time.period_mod      0.000     0.000     0.000       0.0
                 I-meal_code      0.000     0.000     0.000       0.0
        B-toloc.country_name      0.000     0.000     0.000       1.0
                 B-days_code      0.000     0.000     0.000       1.0
 I-arrive_time.period_of_day      0.000     0.000     0.000       0.0
            I-today_relative      0.000     0.000     0.000       0.0
 B-return_time.period_of_day      0.000     0.000     0.000       0.0
                      B-time      0.000     0.000     0.000       0.0
           I-fare_basis_code      0.000     0.000     0.000       0.0
 I-arrive_time.time_relative      0.000     0.000     0.000       4.0
 I-depart_time.time_relative      0.000     0.000     0.000       1.0
            B-today_relative      0.000     0.000     0.000       0.0
                B-state_name      0.000     0.000     0.000       9.0
B-arrive_date.today_relative      0.000     0.000     0.000       0.0
    B-return_time.period_mod      0.000     0.000     0.000       0.0
                B-month_name      0.000     0.000     0.000       0.0
                B-day_number      0.000     0.000     0.000       0.0
 I-return_date.date_relative      0.000     0.000     0.000       3.0
I-return_date.today_relative      0.000     0.000     0.000       0.0
      B-stoploc.airport_name      0.000     0.000     0.000       0.0
             B-time_relative      0.000     0.000     0.000       0.0
                      I-time      0.000     0.000     0.000       0.0
    I-return_date.day_number      0.000     0.000     0.000       0.0
          I-meal_description      0.000     0.000     0.000       0.0
B-return_date.today_relative      0.000     0.000     0.000       0.0
      B-return_date.day_name      0.000     0.000     0.000       2.0

                   micro avg      0.859     0.861     0.860    3657.0
                   macro avg      0.429     0.403     0.391    3657.0
                weighted avg      0.854     0.861     0.843    3657.0

I0717 06:52:50.611222 140695199004544 interactiveshell.py:2882] Best Slot F1: 0.860
Reading ../data/atis.train.w-intent.iob
I0717 06:52:54.653483 140695199004544 interactiveshell.py:2882] Step 1600 | Loss: 0.1009 | Loss_intent: 0.0033 | Loss_slots: 0.0976 | Spent: 6.5 secs | LR: 0.000253
I0717 06:52:58.793189 140695199004544 interactiveshell.py:2882] Step 1650 | Loss: 0.8860 | Loss_intent: 0.8129 | Loss_slots: 0.0731 | Spent: 4.1 secs | LR: 0.000252
I0717 06:53:02.940593 140695199004544 interactiveshell.py:2882] Step 1700 | Loss: 0.1215 | Loss_intent: 0.0261 | Loss_slots: 0.0954 | Spent: 4.1 secs | LR: 0.000251
I0717 06:53:07.078419 140695199004544 interactiveshell.py:2882] Step 1750 | Loss: 0.1495 | Loss_intent: 0.0233 | Loss_slots: 0.1262 | Spent: 4.1 secs | LR: 0.000249
I0717 06:53:11.236833 140695199004544 interactiveshell.py:2882] Step 1800 | Loss: 0.1034 | Loss_intent: 0.0007 | Loss_slots: 0.1027 | Spent: 4.2 secs | LR: 0.000248
I0717 06:53:15.402294 140695199004544 interactiveshell.py:2882] Step 1850 | Loss: 0.0714 | Loss_intent: 0.0040 | Loss_slots: 0.0673 | Spent: 4.2 secs | LR: 0.000247
Reading ../data/atis.test.w-intent.iob
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.
  'precision', 'predicted', average, warn_for)
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1439: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples.
  'recall', 'true', average, warn_for)
I0717 06:53:18.872637 140695199004544 interactiveshell.py:2882] 
                                          precision    recall  f1-score   support

                             atis_flight      0.983     0.986     0.984       632
                            atis_airfare      0.941     1.000     0.970        48
                     atis_ground_service      0.947     1.000     0.973        36
                            atis_airline      1.000     1.000     1.000        38
                       atis_abbreviation      0.971     1.000     0.985        33
                           atis_aircraft      1.000     0.889     0.941         9
                        atis_flight_time      1.000     1.000     1.000         1
                           atis_quantity      0.333     1.000     0.500         3
                atis_flight#atis_airfare      0.857     0.500     0.632        12
                            atis_airport      1.000     0.889     0.941        18
                           atis_distance      1.000     1.000     1.000        10
                               atis_city      0.667     1.000     0.800         6
                        atis_ground_fare      1.000     0.714     0.833         7
                           atis_capacity      1.000     0.857     0.923        21
                          atis_flight_no      0.875     0.875     0.875         8
                               atis_meal      0.833     0.833     0.833         6
                        atis_restriction      0.000     0.000     0.000         0
             atis_airline#atis_flight_no      0.000     0.000     0.000         0
    atis_ground_service#atis_ground_fare      0.000     0.000     0.000         0
           atis_airfare#atis_flight_time      0.000     0.000     0.000         0
                           atis_cheapest      0.000     0.000     0.000         0
atis_aircraft#atis_flight#atis_flight_no      0.000     0.000     0.000         0

                               micro avg      0.966     0.972     0.969       888
                               macro avg      0.655     0.661     0.645       888
                            weighted avg      0.973     0.972     0.970       888

I0717 06:53:18.913320 140695199004544 interactiveshell.py:2882] 
                              precision    recall  f1-score   support

                           O      0.000     0.000     0.000       0.0
           B-toloc.city_name      0.952     0.936     0.944     716.0
         B-fromloc.city_name      0.929     0.966     0.947     704.0
           I-toloc.city_name      0.895     0.970     0.931     265.0
      B-depart_date.day_name      0.990     0.901     0.943     212.0
              B-airline_name      0.933     0.960     0.946     101.0
         I-fromloc.city_name      0.940     0.887     0.913     177.0
 B-depart_time.period_of_day      0.991     0.838     0.908     130.0
              I-airline_name      1.000     0.969     0.984      65.0
    B-depart_date.day_number      0.962     0.909     0.935      55.0
    B-depart_date.month_name      0.965     0.982     0.973      56.0
          B-depart_time.time      0.736     0.930     0.822      57.0
                B-round_trip      0.986     0.959     0.972      73.0
             B-cost_relative      1.000     0.973     0.986      37.0
                I-round_trip      1.000     1.000     1.000      71.0
                B-flight_mod      0.815     0.917     0.863      24.0
 B-depart_time.time_relative      0.939     0.954     0.947      65.0
          I-depart_time.time      0.817     0.942     0.875      52.0
         B-stoploc.city_name      0.667     0.400     0.500      20.0
                 B-city_name      0.478     0.561     0.516      57.0
                B-class_type      0.786     0.917     0.846      24.0
          B-arrive_time.time      0.604     0.941     0.736      34.0
 B-arrive_time.time_relative      0.789     0.968     0.870      31.0
                I-class_type      1.000     1.000     1.000      17.0
               B-flight_stop      0.955     1.000     0.977      21.0
          I-arrive_time.time      0.717     0.943     0.815      35.0
              B-airline_code      0.969     0.912     0.939      34.0
    I-depart_date.day_number      0.857     0.800     0.828      15.0
      I-fromloc.airport_name      0.233     0.933     0.373      15.0
      B-fromloc.airport_name      0.211     0.333     0.258      12.0
      B-arrive_date.day_name      0.333     1.000     0.500      11.0
          B-toloc.state_code      0.818     1.000     0.900      18.0
B-depart_date.today_relative      1.000     0.889     0.941       9.0
             B-flight_number      0.636     0.636     0.636      11.0
 B-depart_date.date_relative      0.850     1.000     0.919      17.0
          B-toloc.state_name      0.767     0.821     0.793      28.0
           B-fare_basis_code      0.810     1.000     0.895      17.0
               B-flight_time      0.500     1.000     0.667       1.0
                        B-or      1.000     1.000     1.000       3.0
 B-arrive_time.period_of_day      0.375     1.000     0.545       6.0
          B-meal_description      1.000     0.900     0.947      10.0
             I-cost_relative      1.000     0.667     0.800       3.0
              I-airport_name      0.571     0.138     0.222      29.0
               B-fare_amount      0.200     1.000     0.333       2.0
               I-fare_amount      0.667     1.000     0.800       2.0
                 I-city_name      0.577     0.500     0.536      30.0
        I-toloc.airport_name      0.000     0.000     0.000       3.0
            B-transport_type      0.769     1.000     0.870      10.0
    B-arrive_date.month_name      0.800     0.667     0.727       6.0
    B-arrive_date.day_number      0.667     0.333     0.444       6.0
         I-stoploc.city_name      1.000     0.200     0.333      10.0
                      B-meal      1.000     1.000     1.000      16.0
        B-fromloc.state_code      1.000     0.783     0.878      23.0
    B-depart_time.period_mod      0.833     1.000     0.909       5.0
                   B-connect      1.000     1.000     1.000       6.0
               B-flight_days      1.000     1.000     1.000      10.0
        B-toloc.airport_name      0.000     0.000     0.000       3.0
        B-fromloc.state_name      0.833     0.294     0.435      17.0
              B-airport_name      1.000     0.048     0.091      21.0
                   B-economy      1.000     1.000     1.000       6.0
               I-flight_time      0.000     0.000     0.000       1.0
             B-aircraft_code      1.000     0.788     0.881      33.0
                       B-mod      0.000     0.000     0.000       2.0
              B-airport_code      0.571     0.444     0.500       9.0
    B-depart_time.start_time      0.000     0.000     0.000       3.0
      B-depart_time.end_time      0.000     0.000     0.000       3.0
          B-depart_date.year      0.000     0.000     0.000       3.0
            I-transport_type      0.000     0.000     0.000       1.0
          B-restriction_code      0.250     0.250     0.250       4.0
    B-arrive_time.start_time      0.000     0.000     0.000       8.0
        B-toloc.airport_code      0.750     0.750     0.750       4.0
      B-arrive_time.end_time      1.000     0.125     0.222       8.0
      I-arrive_time.end_time      0.000     0.000     0.000       8.0
      I-depart_time.end_time      0.000     0.000     0.000       3.0
               I-flight_stop      0.000     0.000     0.000       0.0
      B-fromloc.airport_code      1.000     0.400     0.571       5.0
          I-restriction_code      0.750     1.000     0.857       3.0
    I-depart_time.start_time      0.000     0.000     0.000       1.0
          I-toloc.state_name      0.500     1.000     0.667       1.0
I-depart_date.today_relative      0.000     0.000     0.000       0.0
 B-arrive_date.date_relative      0.000     0.000     0.000       2.0
                I-flight_mod      0.000     0.000     0.000       6.0
                   I-economy      0.000     0.000     0.000       0.0
 B-return_date.date_relative      0.000     0.000     0.000       3.0
        I-fromloc.state_name      0.000     0.000     0.000       1.0
                B-state_code      0.500     1.000     0.667       1.0
    I-arrive_time.start_time      0.000     0.000     0.000       1.0
    I-arrive_date.day_number      0.000     0.000     0.000       0.0
                 B-meal_code      1.000     1.000     1.000       1.0
 I-depart_time.period_of_day      0.000     0.000     0.000       1.0
                  B-day_name      1.000     0.500     0.667       2.0
             B-period_of_day      1.000     0.750     0.857       4.0
        B-stoploc.state_code      0.000     0.000     0.000       0.0
    B-return_date.month_name      0.000     0.000     0.000       0.0
    B-return_date.day_number      0.000     0.000     0.000       0.0
    B-arrive_time.period_mod      0.000     0.000     0.000       0.0
                 I-meal_code      0.000     0.000     0.000       0.0
        B-toloc.country_name      0.000     0.000     0.000       1.0
                 B-days_code      0.333     1.000     0.500       1.0
 I-arrive_time.period_of_day      0.000     0.000     0.000       0.0
            I-today_relative      0.000     0.000     0.000       0.0
 B-return_time.period_of_day      0.000     0.000     0.000       0.0
                      B-time      0.000     0.000     0.000       0.0
           I-fare_basis_code      0.000     0.000     0.000       0.0
 I-arrive_time.time_relative      0.000     0.000     0.000       4.0
 I-depart_time.time_relative      0.000     0.000     0.000       1.0
            B-today_relative      0.000     0.000     0.000       0.0
                B-state_name      0.000     0.000     0.000       9.0
B-arrive_date.today_relative      0.000     0.000     0.000       0.0
    B-return_time.period_mod      0.000     0.000     0.000       0.0
                B-month_name      0.000     0.000     0.000       0.0
                B-day_number      0.000     0.000     0.000       0.0
 I-return_date.date_relative      0.000     0.000     0.000       3.0
I-return_date.today_relative      0.000     0.000     0.000       0.0
      B-stoploc.airport_name      0.000     0.000     0.000       0.0
             B-time_relative      0.000     0.000     0.000       0.0
                      I-time      0.000     0.000     0.000       0.0
    I-return_date.day_number      0.000     0.000     0.000       0.0
          I-meal_description      0.000     0.000     0.000       0.0
B-return_date.today_relative      0.000     0.000     0.000       0.0
      B-return_date.day_name      0.000     0.000     0.000       2.0

                   micro avg      0.881     0.882     0.881    3657.0
                   macro avg      0.461     0.468     0.439    3657.0
                weighted avg      0.885     0.882     0.874    3657.0

I0717 06:53:18.914206 140695199004544 interactiveshell.py:2882] Best Slot F1: 0.881
Reading ../data/atis.train.w-intent.iob
I0717 06:53:21.952603 140695199004544 interactiveshell.py:2882] Step 1900 | Loss: 0.2018 | Loss_intent: 0.1430 | Loss_slots: 0.0588 | Spent: 6.5 secs | LR: 0.000246
I0717 06:53:26.107702 140695199004544 interactiveshell.py:2882] Step 1950 | Loss: 0.0995 | Loss_intent: 0.0044 | Loss_slots: 0.0951 | Spent: 4.2 secs | LR: 0.000244
I0717 06:53:30.256761 140695199004544 interactiveshell.py:2882] Step 2000 | Loss: 0.1151 | Loss_intent: 0.0037 | Loss_slots: 0.1115 | Spent: 4.1 secs | LR: 0.000243
I0717 06:53:34.409721 140695199004544 interactiveshell.py:2882] Step 2050 | Loss: 0.0977 | Loss_intent: 0.0223 | Loss_slots: 0.0755 | Spent: 4.2 secs | LR: 0.000242
I0717 06:53:38.604329 140695199004544 interactiveshell.py:2882] Step 2100 | Loss: 0.1461 | Loss_intent: 0.0013 | Loss_slots: 0.1448 | Spent: 4.2 secs | LR: 0.000240
I0717 06:53:42.746676 140695199004544 interactiveshell.py:2882] Step 2150 | Loss: 0.1511 | Loss_intent: 0.0068 | Loss_slots: 0.1442 | Spent: 4.1 secs | LR: 0.000239
Reading ../data/atis.test.w-intent.iob
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.
  'precision', 'predicted', average, warn_for)
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1439: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples.
  'recall', 'true', average, warn_for)
I0717 06:53:47.197358 140695199004544 interactiveshell.py:2882] 
                                          precision    recall  f1-score   support

                             atis_flight      0.978     0.989     0.983       632
                            atis_airfare      0.738     1.000     0.850        48
                     atis_ground_service      1.000     1.000     1.000        36
                            atis_airline      1.000     1.000     1.000        38
                       atis_abbreviation      1.000     0.697     0.821        33
                           atis_aircraft      1.000     1.000     1.000         9
                        atis_flight_time      0.167     1.000     0.286         1
                           atis_quantity      0.300     1.000     0.462         3
                atis_flight#atis_airfare      0.833     0.417     0.556        12
                            atis_airport      1.000     0.889     0.941        18
                           atis_distance      1.000     1.000     1.000        10
                               atis_city      1.000     0.667     0.800         6
                        atis_ground_fare      1.000     0.714     0.833         7
                           atis_capacity      1.000     0.905     0.950        21
                          atis_flight_no      0.000     0.000     0.000         8
                               atis_meal      0.833     0.833     0.833         6
                        atis_restriction      0.000     0.000     0.000         0
             atis_airline#atis_flight_no      0.000     0.000     0.000         0
    atis_ground_service#atis_ground_fare      0.000     0.000     0.000         0
           atis_airfare#atis_flight_time      0.000     0.000     0.000         0
                           atis_cheapest      0.000     0.000     0.000         0
atis_aircraft#atis_flight#atis_flight_no      0.000     0.000     0.000         0

                               micro avg      0.948     0.954     0.951       888
                               macro avg      0.584     0.596     0.560       888
                            weighted avg      0.955     0.954     0.950       888

I0717 06:53:47.237815 140695199004544 interactiveshell.py:2882] 
                              precision    recall  f1-score   support

                           O      0.000     0.000     0.000       0.0
           B-toloc.city_name      0.940     0.962     0.951     716.0
         B-fromloc.city_name      0.938     0.972     0.955     704.0
           I-toloc.city_name      0.888     0.985     0.934     265.0
      B-depart_date.day_name      0.990     0.967     0.979     212.0
              B-airline_name      0.951     0.970     0.961     101.0
         I-fromloc.city_name      0.940     0.887     0.913     177.0
 B-depart_time.period_of_day      0.974     0.869     0.919     130.0
              I-airline_name      1.000     0.969     0.984      65.0
    B-depart_date.day_number      0.927     0.927     0.927      55.0
    B-depart_date.month_name      0.965     0.982     0.973      56.0
          B-depart_time.time      0.786     0.965     0.866      57.0
                B-round_trip      0.960     0.986     0.973      73.0
             B-cost_relative      1.000     0.973     0.986      37.0
                I-round_trip      0.986     1.000     0.993      71.0
                B-flight_mod      0.733     0.917     0.815      24.0
 B-depart_time.time_relative      0.938     0.938     0.938      65.0
          I-depart_time.time      0.785     0.981     0.872      52.0
         B-stoploc.city_name      0.750     0.600     0.667      20.0
                 B-city_name      0.655     0.333     0.442      57.0
                B-class_type      0.917     0.917     0.917      24.0
          B-arrive_time.time      0.615     0.941     0.744      34.0
 B-arrive_time.time_relative      0.778     0.903     0.836      31.0
                I-class_type      1.000     1.000     1.000      17.0
               B-flight_stop      0.913     1.000     0.955      21.0
          I-arrive_time.time      0.750     0.943     0.835      35.0
              B-airline_code      0.889     0.706     0.787      34.0
    I-depart_date.day_number      0.867     0.867     0.867      15.0
      I-fromloc.airport_name      0.304     0.933     0.459      15.0
      B-fromloc.airport_name      0.129     0.333     0.186      12.0
      B-arrive_date.day_name      0.579     1.000     0.733      11.0
          B-toloc.state_code      0.900     1.000     0.947      18.0
B-depart_date.today_relative      1.000     0.889     0.941       9.0
             B-flight_number      0.381     0.727     0.500      11.0
 B-depart_date.date_relative      0.850     1.000     0.919      17.0
          B-toloc.state_name      0.852     0.821     0.836      28.0
           B-fare_basis_code      0.917     0.647     0.759      17.0
               B-flight_time      1.000     1.000     1.000       1.0
                        B-or      1.000     1.000     1.000       3.0
 B-arrive_time.period_of_day      0.714     0.833     0.769       6.0
          B-meal_description      1.000     0.900     0.947      10.0
             I-cost_relative      1.000     0.667     0.800       3.0
              I-airport_name      0.692     0.310     0.429      29.0
               B-fare_amount      0.333     1.000     0.500       2.0
               I-fare_amount      1.000     1.000     1.000       2.0
                 I-city_name      0.923     0.400     0.558      30.0
        I-toloc.airport_name      0.000     0.000     0.000       3.0
            B-transport_type      0.889     0.800     0.842      10.0
    B-arrive_date.month_name      0.800     0.667     0.727       6.0
    B-arrive_date.day_number      0.667     0.333     0.444       6.0
         I-stoploc.city_name      0.667     0.400     0.500      10.0
                      B-meal      1.000     1.000     1.000      16.0
        B-fromloc.state_code      1.000     0.913     0.955      23.0
    B-depart_time.period_mod      0.667     0.800     0.727       5.0
                   B-connect      1.000     1.000     1.000       6.0
               B-flight_days      1.000     1.000     1.000      10.0
        B-toloc.airport_name      1.000     0.333     0.500       3.0
        B-fromloc.state_name      0.889     0.471     0.615      17.0
              B-airport_name      0.400     0.095     0.154      21.0
                   B-economy      1.000     1.000     1.000       6.0
               I-flight_time      1.000     1.000     1.000       1.0
             B-aircraft_code      1.000     0.818     0.900      33.0
                       B-mod      0.000     0.000     0.000       2.0
              B-airport_code      0.000     0.000     0.000       9.0
    B-depart_time.start_time      0.333     0.333     0.333       3.0
      B-depart_time.end_time      1.000     0.333     0.500       3.0
          B-depart_date.year      0.000     0.000     0.000       3.0
            I-transport_type      0.000     0.000     0.000       1.0
          B-restriction_code      0.500     0.250     0.333       4.0
    B-arrive_time.start_time      0.000     0.000     0.000       8.0
        B-toloc.airport_code      0.400     0.500     0.444       4.0
      B-arrive_time.end_time      0.000     0.000     0.000       8.0
      I-arrive_time.end_time      0.000     0.000     0.000       8.0
      I-depart_time.end_time      0.000     0.000     0.000       3.0
               I-flight_stop      0.000     0.000     0.000       0.0
      B-fromloc.airport_code      1.000     0.200     0.333       5.0
          I-restriction_code      0.200     1.000     0.333       3.0
    I-depart_time.start_time      0.000     0.000     0.000       1.0
          I-toloc.state_name      0.500     1.000     0.667       1.0
I-depart_date.today_relative      0.000     0.000     0.000       0.0
 B-arrive_date.date_relative      1.000     1.000     1.000       2.0
                I-flight_mod      0.000     0.000     0.000       6.0
                   I-economy      0.000     0.000     0.000       0.0
 B-return_date.date_relative      0.500     0.333     0.400       3.0
        I-fromloc.state_name      0.000     0.000     0.000       1.0
                B-state_code      1.000     1.000     1.000       1.0
    I-arrive_time.start_time      0.000     0.000     0.000       1.0
    I-arrive_date.day_number      0.000     0.000     0.000       0.0
                 B-meal_code      0.000     0.000     0.000       1.0
 I-depart_time.period_of_day      0.000     0.000     0.000       1.0
                  B-day_name      1.000     0.500     0.667       2.0
             B-period_of_day      1.000     0.500     0.667       4.0
        B-stoploc.state_code      0.000     0.000     0.000       0.0
    B-return_date.month_name      0.000     0.000     0.000       0.0
    B-return_date.day_number      0.000     0.000     0.000       0.0
    B-arrive_time.period_mod      0.000     0.000     0.000       0.0
                 I-meal_code      0.000     0.000     0.000       0.0
        B-toloc.country_name      0.000     0.000     0.000       1.0
                 B-days_code      0.000     0.000     0.000       1.0
 I-arrive_time.period_of_day      0.000     0.000     0.000       0.0
            I-today_relative      0.000     0.000     0.000       0.0
 B-return_time.period_of_day      0.000     0.000     0.000       0.0
                      B-time      0.000     0.000     0.000       0.0
           I-fare_basis_code      0.000     0.000     0.000       0.0
 I-arrive_time.time_relative      0.000     0.000     0.000       4.0
 I-depart_time.time_relative      0.000     0.000     0.000       1.0
            B-today_relative      0.000     0.000     0.000       0.0
                B-state_name      0.000     0.000     0.000       9.0
B-arrive_date.today_relative      0.000     0.000     0.000       0.0
    B-return_time.period_mod      0.000     0.000     0.000       0.0
                B-month_name      0.000     0.000     0.000       0.0
                B-day_number      0.000     0.000     0.000       0.0
 I-return_date.date_relative      0.000     0.000     0.000       3.0
I-return_date.today_relative      0.000     0.000     0.000       0.0
      B-stoploc.airport_name      0.000     0.000     0.000       0.0
             B-time_relative      0.000     0.000     0.000       0.0
                      I-time      0.000     0.000     0.000       0.0
    I-return_date.day_number      0.000     0.000     0.000       0.0
          I-meal_description      0.000     0.000     0.000       0.0
B-return_date.today_relative      0.000     0.000     0.000       0.0
      B-return_date.day_name      0.000     0.000     0.000       2.0

                   micro avg      0.890     0.892     0.891    3657.0
                   macro avg      0.486     0.467     0.457    3657.0
                weighted avg      0.887     0.892     0.882    3657.0

I0717 06:53:47.238620 140695199004544 interactiveshell.py:2882] Best Slot F1: 0.891
Reading ../data/atis.train.w-intent.iob
I0717 06:53:49.288956 140695199004544 interactiveshell.py:2882] Step 2200 | Loss: 0.0608 | Loss_intent: 0.0093 | Loss_slots: 0.0515 | Spent: 6.5 secs | LR: 0.000238
I0717 06:53:53.490375 140695199004544 interactiveshell.py:2882] Step 2250 | Loss: 0.0676 | Loss_intent: 0.0069 | Loss_slots: 0.0607 | Spent: 4.2 secs | LR: 0.000237
I0717 06:53:57.716103 140695199004544 interactiveshell.py:2882] Step 2300 | Loss: 0.5240 | Loss_intent: 0.4084 | Loss_slots: 0.1156 | Spent: 4.2 secs | LR: 0.000235
I0717 06:54:01.944693 140695199004544 interactiveshell.py:2882] Step 2350 | Loss: 0.0791 | Loss_intent: 0.0080 | Loss_slots: 0.0711 | Spent: 4.2 secs | LR: 0.000234
I0717 06:54:06.089469 140695199004544 interactiveshell.py:2882] Step 2400 | Loss: 0.0952 | Loss_intent: 0.0027 | Loss_slots: 0.0925 | Spent: 4.1 secs | LR: 0.000233
I0717 06:54:10.247859 140695199004544 interactiveshell.py:2882] Step 2450 | Loss: 0.1110 | Loss_intent: 0.0031 | Loss_slots: 0.1080 | Spent: 4.2 secs | LR: 0.000232
Reading ../data/atis.test.w-intent.iob
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.
  'precision', 'predicted', average, warn_for)
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1439: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples.
  'recall', 'true', average, warn_for)
I0717 06:54:15.701805 140695199004544 interactiveshell.py:2882] 
                                          precision    recall  f1-score   support

                             atis_flight      0.984     0.987     0.986       632
                            atis_airfare      0.906     1.000     0.950        48
                     atis_ground_service      1.000     1.000     1.000        36
                            atis_airline      1.000     1.000     1.000        38
                       atis_abbreviation      1.000     0.939     0.969        33
                           atis_aircraft      0.727     0.889     0.800         9
                        atis_flight_time      1.000     1.000     1.000         1
                           atis_quantity      0.375     1.000     0.545         3
                atis_flight#atis_airfare      0.857     0.500     0.632        12
                            atis_airport      0.947     1.000     0.973        18
                           atis_distance      1.000     1.000     1.000        10
                               atis_city      1.000     0.667     0.800         6
                        atis_ground_fare      1.000     0.714     0.833         7
                           atis_capacity      1.000     0.952     0.976        21
                          atis_flight_no      0.889     1.000     0.941         8
                               atis_meal      1.000     0.833     0.909         6
                        atis_restriction      0.000     0.000     0.000         0
             atis_airline#atis_flight_no      0.000     0.000     0.000         0
    atis_ground_service#atis_ground_fare      0.000     0.000     0.000         0
           atis_airfare#atis_flight_time      0.000     0.000     0.000         0
                           atis_cheapest      0.000     0.000     0.000         0
atis_aircraft#atis_flight#atis_flight_no      0.000     0.000     0.000         0

                               micro avg      0.969     0.974     0.971       888
                               macro avg      0.668     0.658     0.651       888
                            weighted avg      0.975     0.974     0.973       888

I0717 06:54:15.755053 140695199004544 interactiveshell.py:2882] 
                              precision    recall  f1-score   support

                           O      0.000     0.000     0.000       0.0
           B-toloc.city_name      0.945     0.958     0.951     716.0
         B-fromloc.city_name      0.932     0.976     0.954     704.0
           I-toloc.city_name      0.927     0.958     0.942     265.0
      B-depart_date.day_name      0.986     0.981     0.983     212.0
              B-airline_name      0.952     0.980     0.966     101.0
         I-fromloc.city_name      0.913     0.944     0.928     177.0
 B-depart_time.period_of_day      0.942     0.877     0.908     130.0
              I-airline_name      1.000     0.969     0.984      65.0
    B-depart_date.day_number      0.927     0.927     0.927      55.0
    B-depart_date.month_name      0.965     0.982     0.973      56.0
          B-depart_time.time      0.754     0.912     0.825      57.0
                B-round_trip      0.986     0.986     0.986      73.0
             B-cost_relative      1.000     0.973     0.986      37.0
                I-round_trip      0.986     1.000     0.993      71.0
                B-flight_mod      0.731     0.792     0.760      24.0
 B-depart_time.time_relative      0.939     0.954     0.947      65.0
          I-depart_time.time      0.836     0.981     0.903      52.0
         B-stoploc.city_name      0.769     0.500     0.606      20.0
                 B-city_name      0.644     0.509     0.569      57.0
                B-class_type      0.917     0.917     0.917      24.0
          B-arrive_time.time      0.673     0.971     0.795      34.0
 B-arrive_time.time_relative      0.933     0.903     0.918      31.0
                I-class_type      1.000     1.000     1.000      17.0
               B-flight_stop      0.913     1.000     0.955      21.0
          I-arrive_time.time      0.810     0.971     0.883      35.0
              B-airline_code      0.926     0.735     0.820      34.0
    I-depart_date.day_number      0.933     0.933     0.933      15.0
      I-fromloc.airport_name      0.350     0.933     0.509      15.0
      B-fromloc.airport_name      0.333     0.417     0.370      12.0
      B-arrive_date.day_name      0.688     1.000     0.815      11.0
          B-toloc.state_code      0.947     1.000     0.973      18.0
B-depart_date.today_relative      1.000     0.778     0.875       9.0
             B-flight_number      0.320     0.727     0.444      11.0
 B-depart_date.date_relative      0.944     1.000     0.971      17.0
          B-toloc.state_name      0.857     0.857     0.857      28.0
           B-fare_basis_code      0.909     0.588     0.714      17.0
               B-flight_time      1.000     1.000     1.000       1.0
                        B-or      1.000     1.000     1.000       3.0
 B-arrive_time.period_of_day      0.800     0.667     0.727       6.0
          B-meal_description      1.000     0.900     0.947      10.0
             I-cost_relative      1.000     0.667     0.800       3.0
              I-airport_name      0.889     0.276     0.421      29.0
               B-fare_amount      0.333     1.000     0.500       2.0
               I-fare_amount      1.000     1.000     1.000       2.0
                 I-city_name      0.800     0.533     0.640      30.0
        I-toloc.airport_name      1.000     0.667     0.800       3.0
            B-transport_type      1.000     0.800     0.889      10.0
    B-arrive_date.month_name      0.800     0.667     0.727       6.0
    B-arrive_date.day_number      0.833     0.833     0.833       6.0
         I-stoploc.city_name      0.667     0.400     0.500      10.0
                      B-meal      1.000     1.000     1.000      16.0
        B-fromloc.state_code      1.000     0.913     0.955      23.0
    B-depart_time.period_mod      0.571     0.800     0.667       5.0
                   B-connect      1.000     1.000     1.000       6.0
               B-flight_days      1.000     1.000     1.000      10.0
        B-toloc.airport_name      0.500     0.333     0.400       3.0
        B-fromloc.state_name      0.800     0.706     0.750      17.0
              B-airport_name      0.400     0.095     0.154      21.0
                   B-economy      1.000     1.000     1.000       6.0
               I-flight_time      0.500     1.000     0.667       1.0
             B-aircraft_code      0.964     0.818     0.885      33.0
                       B-mod      0.000     0.000     0.000       2.0
              B-airport_code      0.000     0.000     0.000       9.0
    B-depart_time.start_time      0.400     0.667     0.500       3.0
      B-depart_time.end_time      0.500     0.333     0.400       3.0
          B-depart_date.year      0.000     0.000     0.000       3.0
            I-transport_type      0.000     0.000     0.000       1.0
          B-restriction_code      0.300     0.750     0.429       4.0
    B-arrive_time.start_time      0.000     0.000     0.000       8.0
        B-toloc.airport_code      0.500     0.250     0.333       4.0
      B-arrive_time.end_time      1.000     0.125     0.222       8.0
      I-arrive_time.end_time      0.500     0.125     0.200       8.0
      I-depart_time.end_time      0.000     0.000     0.000       3.0
               I-flight_stop      0.000     0.000     0.000       0.0
      B-fromloc.airport_code      1.000     0.400     0.571       5.0
          I-restriction_code      0.600     1.000     0.750       3.0
    I-depart_time.start_time      0.000     0.000     0.000       1.0
          I-toloc.state_name      0.500     1.000     0.667       1.0
I-depart_date.today_relative      0.000     0.000     0.000       0.0
 B-arrive_date.date_relative      1.000     1.000     1.000       2.0
                I-flight_mod      0.000     0.000     0.000       6.0
                   I-economy      0.000     0.000     0.000       0.0
 B-return_date.date_relative      0.000     0.000     0.000       3.0
        I-fromloc.state_name      0.000     0.000     0.000       1.0
                B-state_code      0.500     1.000     0.667       1.0
    I-arrive_time.start_time      0.000     0.000     0.000       1.0
    I-arrive_date.day_number      0.000     0.000     0.000       0.0
                 B-meal_code      1.000     1.000     1.000       1.0
 I-depart_time.period_of_day      0.000     0.000     0.000       1.0
                  B-day_name      1.000     0.500     0.667       2.0
             B-period_of_day      1.000     0.500     0.667       4.0
        B-stoploc.state_code      0.000     0.000     0.000       0.0
    B-return_date.month_name      0.000     0.000     0.000       0.0
    B-return_date.day_number      0.000     0.000     0.000       0.0
    B-arrive_time.period_mod      0.000     0.000     0.000       0.0
                 I-meal_code      0.000     0.000     0.000       0.0
        B-toloc.country_name      0.000     0.000     0.000       1.0
                 B-days_code      0.000     0.000     0.000       1.0
 I-arrive_time.period_of_day      0.000     0.000     0.000       0.0
            I-today_relative      0.000     0.000     0.000       0.0
 B-return_time.period_of_day      0.000     0.000     0.000       0.0
                      B-time      0.000     0.000     0.000       0.0
           I-fare_basis_code      0.000     0.000     0.000       0.0
 I-arrive_time.time_relative      0.000     0.000     0.000       4.0
 I-depart_time.time_relative      0.000     0.000     0.000       1.0
            B-today_relative      0.000     0.000     0.000       0.0
                B-state_name      0.000     0.000     0.000       9.0
B-arrive_date.today_relative      0.000     0.000     0.000       0.0
    B-return_time.period_mod      0.000     0.000     0.000       0.0
                B-month_name      0.000     0.000     0.000       0.0
                B-day_number      0.000     0.000     0.000       0.0
 I-return_date.date_relative      0.000     0.000     0.000       3.0
I-return_date.today_relative      0.000     0.000     0.000       0.0
      B-stoploc.airport_name      0.000     0.000     0.000       0.0
             B-time_relative      0.000     0.000     0.000       0.0
                      I-time      0.000     0.000     0.000       0.0
    I-return_date.day_number      0.000     0.000     0.000       0.0
          I-meal_description      0.000     0.000     0.000       0.0
B-return_date.today_relative      0.000     0.000     0.000       0.0
      B-return_date.day_name      0.000     0.000     0.000       2.0

                   micro avg      0.899     0.900     0.899    3657.0
                   macro avg      0.506     0.493     0.481    3657.0
                weighted avg      0.896     0.900     0.892    3657.0

I0717 06:54:15.755986 140695199004544 interactiveshell.py:2882] Best Slot F1: 0.899
Reading ../data/atis.train.w-intent.iob
I0717 06:54:16.815707 140695199004544 interactiveshell.py:2882] Step 2500 | Loss: 0.0871 | Loss_intent: 0.0328 | Loss_slots: 0.0543 | Spent: 6.6 secs | LR: 0.000231
I0717 06:54:21.450364 140695199004544 interactiveshell.py:2882] Step 2550 | Loss: 0.0688 | Loss_intent: 0.0002 | Loss_slots: 0.0686 | Spent: 4.6 secs | LR: 0.000229
I0717 06:54:26.177942 140695199004544 interactiveshell.py:2882] Step 2600 | Loss: 0.0401 | Loss_intent: 0.0009 | Loss_slots: 0.0392 | Spent: 4.7 secs | LR: 0.000228
I0717 06:54:30.567509 140695199004544 interactiveshell.py:2882] Step 2650 | Loss: 0.0579 | Loss_intent: 0.0002 | Loss_slots: 0.0577 | Spent: 4.4 secs | LR: 0.000227
I0717 06:54:34.716868 140695199004544 interactiveshell.py:2882] Step 2700 | Loss: 0.0503 | Loss_intent: 0.0020 | Loss_slots: 0.0483 | Spent: 4.1 secs | LR: 0.000226
I0717 06:54:38.866876 140695199004544 interactiveshell.py:2882] Step 2750 | Loss: 0.0674 | Loss_intent: 0.0060 | Loss_slots: 0.0614 | Spent: 4.1 secs | LR: 0.000225
I0717 06:54:43.015546 140695199004544 interactiveshell.py:2882] Step 2800 | Loss: 0.1411 | Loss_intent: 0.0835 | Loss_slots: 0.0576 | Spent: 4.1 secs | LR: 0.000223
Reading ../data/atis.test.w-intent.iob
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.
  'precision', 'predicted', average, warn_for)
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1439: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples.
  'recall', 'true', average, warn_for)
I0717 06:54:45.328923 140695199004544 interactiveshell.py:2882] 
                                          precision    recall  f1-score   support

                             atis_flight      0.983     0.987     0.985       632
                            atis_airfare      0.873     1.000     0.932        48
                     atis_ground_service      1.000     1.000     1.000        36
                            atis_airline      1.000     1.000     1.000        38
                       atis_abbreviation      1.000     1.000     1.000        33
                           atis_aircraft      1.000     0.889     0.941         9
                        atis_flight_time      1.000     1.000     1.000         1
                           atis_quantity      0.167     0.333     0.222         3
                atis_flight#atis_airfare      0.833     0.417     0.556        12
                            atis_airport      1.000     1.000     1.000        18
                           atis_distance      1.000     0.800     0.889        10
                               atis_city      0.667     1.000     0.800         6
                        atis_ground_fare      1.000     0.714     0.833         7
                           atis_capacity      0.913     1.000     0.955        21
                          atis_flight_no      0.875     0.875     0.875         8
                               atis_meal      1.000     0.667     0.800         6
                        atis_restriction      0.000     0.000     0.000         0
             atis_airline#atis_flight_no      0.000     0.000     0.000         0
    atis_ground_service#atis_ground_fare      0.000     0.000     0.000         0
           atis_airfare#atis_flight_time      0.000     0.000     0.000         0
                           atis_cheapest      0.000     0.000     0.000         0
atis_aircraft#atis_flight#atis_flight_no      0.000     0.000     0.000         0

                               micro avg      0.966     0.972     0.969       888
                               macro avg      0.650     0.622     0.627       888
                            weighted avg      0.970     0.972     0.969       888

I0717 06:54:45.370683 140695199004544 interactiveshell.py:2882] 
                              precision    recall  f1-score   support

                           O      0.000     0.000     0.000       0.0
           B-toloc.city_name      0.949     0.958     0.953     716.0
         B-fromloc.city_name      0.942     0.973     0.957     704.0
           I-toloc.city_name      0.907     0.989     0.946     265.0
      B-depart_date.day_name      0.977     0.991     0.984     212.0
              B-airline_name      0.971     0.980     0.975     101.0
         I-fromloc.city_name      0.965     0.932     0.948     177.0
 B-depart_time.period_of_day      0.974     0.877     0.923     130.0
              I-airline_name      0.984     0.938     0.961      65.0
    B-depart_date.day_number      0.962     0.909     0.935      55.0
    B-depart_date.month_name      0.982     0.982     0.982      56.0
          B-depart_time.time      0.786     0.965     0.866      57.0
                B-round_trip      0.973     0.986     0.980      73.0
             B-cost_relative      1.000     0.973     0.986      37.0
                I-round_trip      1.000     1.000     1.000      71.0
                B-flight_mod      0.733     0.917     0.815      24.0
 B-depart_time.time_relative      0.967     0.908     0.937      65.0
          I-depart_time.time      0.850     0.981     0.911      52.0
         B-stoploc.city_name      0.900     0.450     0.600      20.0
                 B-city_name      0.667     0.632     0.649      57.0
                B-class_type      0.793     0.958     0.868      24.0
          B-arrive_time.time      0.767     0.971     0.857      34.0
 B-arrive_time.time_relative      0.769     0.968     0.857      31.0
                I-class_type      1.000     1.000     1.000      17.0
               B-flight_stop      1.000     1.000     1.000      21.0
          I-arrive_time.time      0.791     0.971     0.872      35.0
              B-airline_code      0.838     0.912     0.873      34.0
    I-depart_date.day_number      0.938     1.000     0.968      15.0
      I-fromloc.airport_name      0.341     0.933     0.500      15.0
      B-fromloc.airport_name      0.312     0.417     0.357      12.0
      B-arrive_date.day_name      0.727     0.727     0.727      11.0
          B-toloc.state_code      0.895     0.944     0.919      18.0
B-depart_date.today_relative      1.000     0.889     0.941       9.0
             B-flight_number      0.348     0.727     0.471      11.0
 B-depart_date.date_relative      0.850     1.000     0.919      17.0
          B-toloc.state_name      0.750     0.964     0.844      28.0
           B-fare_basis_code      0.944     1.000     0.971      17.0
               B-flight_time      1.000     1.000     1.000       1.0
                        B-or      1.000     1.000     1.000       3.0
 B-arrive_time.period_of_day      0.857     1.000     0.923       6.0
          B-meal_description      1.000     0.900     0.947      10.0
             I-cost_relative      1.000     0.667     0.800       3.0
              I-airport_name      0.750     0.103     0.182      29.0
               B-fare_amount      0.250     1.000     0.400       2.0
               I-fare_amount      1.000     1.000     1.000       2.0
                 I-city_name      0.789     0.500     0.612      30.0
        I-toloc.airport_name      0.000     0.000     0.000       3.0
            B-transport_type      1.000     0.700     0.824      10.0
    B-arrive_date.month_name      0.833     0.833     0.833       6.0
    B-arrive_date.day_number      0.833     0.833     0.833       6.0
         I-stoploc.city_name      0.800     0.400     0.533      10.0
                      B-meal      1.000     1.000     1.000      16.0
        B-fromloc.state_code      0.955     0.913     0.933      23.0
    B-depart_time.period_mod      0.667     0.800     0.727       5.0
                   B-connect      1.000     1.000     1.000       6.0
               B-flight_days      1.000     1.000     1.000      10.0
        B-toloc.airport_name      1.000     0.333     0.500       3.0
        B-fromloc.state_name      0.812     0.765     0.788      17.0
              B-airport_name      0.667     0.095     0.167      21.0
                   B-economy      1.000     1.000     1.000       6.0
               I-flight_time      1.000     1.000     1.000       1.0
             B-aircraft_code      1.000     0.758     0.862      33.0
                       B-mod      0.000     0.000     0.000       2.0
              B-airport_code      0.000     0.000     0.000       9.0
    B-depart_time.start_time      0.333     0.333     0.333       3.0
      B-depart_time.end_time      1.000     0.333     0.500       3.0
          B-depart_date.year      0.000     0.000     0.000       3.0
            I-transport_type      0.000     0.000     0.000       1.0
          B-restriction_code      0.333     0.250     0.286       4.0
    B-arrive_time.start_time      0.500     0.125     0.200       8.0
        B-toloc.airport_code      0.400     0.500     0.444       4.0
      B-arrive_time.end_time      0.500     0.500     0.500       8.0
      I-arrive_time.end_time      1.000     0.250     0.400       8.0
      I-depart_time.end_time      0.000     0.000     0.000       3.0
               I-flight_stop      0.000     0.000     0.000       0.0
      B-fromloc.airport_code      1.000     0.400     0.571       5.0
          I-restriction_code      0.750     1.000     0.857       3.0
    I-depart_time.start_time      0.000     0.000     0.000       1.0
          I-toloc.state_name      1.000     1.000     1.000       1.0
I-depart_date.today_relative      0.000     0.000     0.000       0.0
 B-arrive_date.date_relative      1.000     0.500     0.667       2.0
                I-flight_mod      0.000     0.000     0.000       6.0
                   I-economy      0.000     0.000     0.000       0.0
 B-return_date.date_relative      0.000     0.000     0.000       3.0
        I-fromloc.state_name      0.000     0.000     0.000       1.0
                B-state_code      1.000     1.000     1.000       1.0
    I-arrive_time.start_time      0.000     0.000     0.000       1.0
    I-arrive_date.day_number      0.000     0.000     0.000       0.0
                 B-meal_code      1.000     1.000     1.000       1.0
 I-depart_time.period_of_day      0.000     0.000     0.000       1.0
                  B-day_name      1.000     0.500     0.667       2.0
             B-period_of_day      1.000     0.250     0.400       4.0
        B-stoploc.state_code      0.000     0.000     0.000       0.0
    B-return_date.month_name      0.000     0.000     0.000       0.0
    B-return_date.day_number      0.000     0.000     0.000       0.0
    B-arrive_time.period_mod      0.000     0.000     0.000       0.0
                 I-meal_code      0.000     0.000     0.000       0.0
        B-toloc.country_name      1.000     1.000     1.000       1.0
                 B-days_code      1.000     1.000     1.000       1.0
 I-arrive_time.period_of_day      0.000     0.000     0.000       0.0
            I-today_relative      0.000     0.000     0.000       0.0
 B-return_time.period_of_day      0.000     0.000     0.000       0.0
                      B-time      0.000     0.000     0.000       0.0
           I-fare_basis_code      0.000     0.000     0.000       0.0
 I-arrive_time.time_relative      0.000     0.000     0.000       4.0
 I-depart_time.time_relative      0.000     0.000     0.000       1.0
            B-today_relative      0.000     0.000     0.000       0.0
                B-state_name      0.000     0.000     0.000       9.0
B-arrive_date.today_relative      0.000     0.000     0.000       0.0
    B-return_time.period_mod      0.000     0.000     0.000       0.0
                B-month_name      0.000     0.000     0.000       0.0
                B-day_number      0.000     0.000     0.000       0.0
 I-return_date.date_relative      0.000     0.000     0.000       3.0
I-return_date.today_relative      0.000     0.000     0.000       0.0
      B-stoploc.airport_name      0.000     0.000     0.000       0.0
             B-time_relative      0.000     0.000     0.000       0.0
                      I-time      0.000     0.000     0.000       0.0
    I-return_date.day_number      0.000     0.000     0.000       0.0
          I-meal_description      0.000     0.000     0.000       0.0
B-return_date.today_relative      0.000     0.000     0.000       0.0
      B-return_date.day_name      0.000     0.000     0.000       2.0

                   micro avg      0.906     0.907     0.906    3657.0
                   macro avg      0.542     0.506     0.504    3657.0
                weighted avg      0.904     0.907     0.898    3657.0

I0717 06:54:45.371623 140695199004544 interactiveshell.py:2882] Best Slot F1: 0.906
Reading ../data/atis.train.w-intent.iob
I0717 06:54:49.593463 140695199004544 interactiveshell.py:2882] Step 2850 | Loss: 0.0792 | Loss_intent: 0.0007 | Loss_slots: 0.0785 | Spent: 6.6 secs | LR: 0.000222
I0717 06:54:53.757027 140695199004544 interactiveshell.py:2882] Step 2900 | Loss: 0.0923 | Loss_intent: 0.0000 | Loss_slots: 0.0923 | Spent: 4.2 secs | LR: 0.000221
I0717 06:54:57.930918 140695199004544 interactiveshell.py:2882] Step 2950 | Loss: 0.9255 | Loss_intent: 0.8774 | Loss_slots: 0.0480 | Spent: 4.2 secs | LR: 0.000220
I0717 06:55:02.094811 140695199004544 interactiveshell.py:2882] Step 3000 | Loss: 0.3441 | Loss_intent: 0.2465 | Loss_slots: 0.0976 | Spent: 4.2 secs | LR: 0.000219
I0717 06:55:06.250625 140695199004544 interactiveshell.py:2882] Step 3050 | Loss: 0.1376 | Loss_intent: 0.0101 | Loss_slots: 0.1275 | Spent: 4.2 secs | LR: 0.000218
I0717 06:55:10.415692 140695199004544 interactiveshell.py:2882] Step 3100 | Loss: 0.0663 | Loss_intent: 0.0006 | Loss_slots: 0.0656 | Spent: 4.2 secs | LR: 0.000216
Reading ../data/atis.test.w-intent.iob
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.
  'precision', 'predicted', average, warn_for)
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1439: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples.
  'recall', 'true', average, warn_for)
I0717 06:55:13.772472 140695199004544 interactiveshell.py:2882] 
                                          precision    recall  f1-score   support

                             atis_flight      0.977     0.992     0.984       632
                            atis_airfare      0.941     1.000     0.970        48
                     atis_ground_service      1.000     1.000     1.000        36
                            atis_airline      1.000     1.000     1.000        38
                       atis_abbreviation      1.000     1.000     1.000        33
                           atis_aircraft      1.000     0.889     0.941         9
                        atis_flight_time      1.000     1.000     1.000         1
                           atis_quantity      0.429     1.000     0.600         3
                atis_flight#atis_airfare      0.800     0.333     0.471        12
                            atis_airport      1.000     1.000     1.000        18
                           atis_distance      1.000     1.000     1.000        10
                               atis_city      1.000     1.000     1.000         6
                        atis_ground_fare      1.000     0.714     0.833         7
                           atis_capacity      1.000     0.952     0.976        21
                          atis_flight_no      0.875     0.875     0.875         8
                               atis_meal      1.000     0.667     0.800         6
                        atis_restriction      0.000     0.000     0.000         0
             atis_airline#atis_flight_no      0.000     0.000     0.000         0
    atis_ground_service#atis_ground_fare      0.000     0.000     0.000         0
           atis_airfare#atis_flight_time      0.000     0.000     0.000         0
                           atis_cheapest      0.000     0.000     0.000         0
atis_aircraft#atis_flight#atis_flight_no      0.000     0.000     0.000         0

                               micro avg      0.972     0.977     0.975       888
                               macro avg      0.683     0.656     0.657       888
                            weighted avg      0.974     0.977     0.974       888

I0717 06:55:13.812173 140695199004544 interactiveshell.py:2882] 
                              precision    recall  f1-score   support

                           O      0.000     0.000     0.000       0.0
           B-toloc.city_name      0.962     0.950     0.956     716.0
         B-fromloc.city_name      0.940     0.976     0.957     704.0
           I-toloc.city_name      0.952     0.970     0.961     265.0
      B-depart_date.day_name      0.981     0.976     0.979     212.0
              B-airline_name      0.970     0.970     0.970     101.0
         I-fromloc.city_name      0.938     0.938     0.938     177.0
 B-depart_time.period_of_day      0.975     0.885     0.927     130.0
              I-airline_name      1.000     0.985     0.992      65.0
    B-depart_date.day_number      0.945     0.945     0.945      55.0
    B-depart_date.month_name      0.964     0.946     0.955      56.0
          B-depart_time.time      0.820     0.877     0.847      57.0
                B-round_trip      1.000     0.986     0.993      73.0
             B-cost_relative      1.000     0.973     0.986      37.0
                I-round_trip      1.000     1.000     1.000      71.0
                B-flight_mod      0.917     0.917     0.917      24.0
 B-depart_time.time_relative      0.983     0.908     0.944      65.0
          I-depart_time.time      0.847     0.962     0.901      52.0
         B-stoploc.city_name      0.559     0.950     0.704      20.0
                 B-city_name      0.653     0.561     0.604      57.0
                B-class_type      0.917     0.917     0.917      24.0
          B-arrive_time.time      0.688     0.971     0.805      34.0
 B-arrive_time.time_relative      0.784     0.935     0.853      31.0
                I-class_type      1.000     1.000     1.000      17.0
               B-flight_stop      0.913     1.000     0.955      21.0
          I-arrive_time.time      0.810     0.971     0.883      35.0
              B-airline_code      0.857     0.706     0.774      34.0
    I-depart_date.day_number      1.000     0.867     0.929      15.0
      I-fromloc.airport_name      0.333     0.867     0.481      15.0
      B-fromloc.airport_name      0.222     0.333     0.267      12.0
      B-arrive_date.day_name      0.600     0.818     0.692      11.0
          B-toloc.state_code      0.900     1.000     0.947      18.0
B-depart_date.today_relative      1.000     0.889     0.941       9.0
             B-flight_number      0.333     0.727     0.457      11.0
 B-depart_date.date_relative      0.850     1.000     0.919      17.0
          B-toloc.state_name      0.839     0.929     0.881      28.0
           B-fare_basis_code      0.941     0.941     0.941      17.0
               B-flight_time      1.000     1.000     1.000       1.0
                        B-or      1.000     1.000     1.000       3.0
 B-arrive_time.period_of_day      0.857     1.000     0.923       6.0
          B-meal_description      1.000     0.900     0.947      10.0
             I-cost_relative      1.000     0.667     0.800       3.0
              I-airport_name      0.333     0.034     0.062      29.0
               B-fare_amount      0.286     1.000     0.444       2.0
               I-fare_amount      1.000     1.000     1.000       2.0
                 I-city_name      0.833     0.500     0.625      30.0
        I-toloc.airport_name      1.000     1.000     1.000       3.0
            B-transport_type      0.769     1.000     0.870      10.0
    B-arrive_date.month_name      0.571     0.667     0.615       6.0
    B-arrive_date.day_number      0.625     0.833     0.714       6.0
         I-stoploc.city_name      0.444     0.800     0.571      10.0
                      B-meal      1.000     1.000     1.000      16.0
        B-fromloc.state_code      1.000     0.913     0.955      23.0
    B-depart_time.period_mod      0.667     0.800     0.727       5.0
                   B-connect      1.000     1.000     1.000       6.0
               B-flight_days      1.000     1.000     1.000      10.0
        B-toloc.airport_name      0.500     0.333     0.400       3.0
        B-fromloc.state_name      0.929     0.765     0.839      17.0
              B-airport_name      0.667     0.095     0.167      21.0
                   B-economy      1.000     1.000     1.000       6.0
               I-flight_time      1.000     1.000     1.000       1.0
             B-aircraft_code      1.000     0.818     0.900      33.0
                       B-mod      0.000     0.000     0.000       2.0
              B-airport_code      0.333     0.111     0.167       9.0
    B-depart_time.start_time      0.333     0.333     0.333       3.0
      B-depart_time.end_time      1.000     0.333     0.500       3.0
          B-depart_date.year      0.000     0.000     0.000       3.0
            I-transport_type      0.000     0.000     0.000       1.0
          B-restriction_code      0.143     0.250     0.182       4.0
    B-arrive_time.start_time      1.000     0.375     0.545       8.0
        B-toloc.airport_code      0.500     0.250     0.333       4.0
      B-arrive_time.end_time      0.750     0.375     0.500       8.0
      I-arrive_time.end_time      0.750     0.375     0.500       8.0
      I-depart_time.end_time      1.000     0.333     0.500       3.0
               I-flight_stop      0.000     0.000     0.000       0.0
      B-fromloc.airport_code      0.750     0.600     0.667       5.0
          I-restriction_code      0.750     1.000     0.857       3.0
    I-depart_time.start_time      0.500     1.000     0.667       1.0
          I-toloc.state_name      0.500     1.000     0.667       1.0
I-depart_date.today_relative      0.000     0.000     0.000       0.0
 B-arrive_date.date_relative      0.000     0.000     0.000       2.0
                I-flight_mod      0.000     0.000     0.000       6.0
                   I-economy      0.000     0.000     0.000       0.0
 B-return_date.date_relative      0.500     0.333     0.400       3.0
        I-fromloc.state_name      0.000     0.000     0.000       1.0
                B-state_code      1.000     1.000     1.000       1.0
    I-arrive_time.start_time      0.000     0.000     0.000       1.0
    I-arrive_date.day_number      0.000     0.000     0.000       0.0
                 B-meal_code      0.000     0.000     0.000       1.0
 I-depart_time.period_of_day      0.000     0.000     0.000       1.0
                  B-day_name      1.000     0.500     0.667       2.0
             B-period_of_day      1.000     0.250     0.400       4.0
        B-stoploc.state_code      0.000     0.000     0.000       0.0
    B-return_date.month_name      0.000     0.000     0.000       0.0
    B-return_date.day_number      0.000     0.000     0.000       0.0
    B-arrive_time.period_mod      0.000     0.000     0.000       0.0
                 I-meal_code      0.000     0.000     0.000       0.0
        B-toloc.country_name      1.000     1.000     1.000       1.0
                 B-days_code      0.000     0.000     0.000       1.0
 I-arrive_time.period_of_day      0.000     0.000     0.000       0.0
            I-today_relative      0.000     0.000     0.000       0.0
 B-return_time.period_of_day      0.000     0.000     0.000       0.0
                      B-time      0.000     0.000     0.000       0.0
           I-fare_basis_code      0.000     0.000     0.000       0.0
 I-arrive_time.time_relative      0.000     0.000     0.000       4.0
 I-depart_time.time_relative      0.000     0.000     0.000       1.0
            B-today_relative      0.000     0.000     0.000       0.0
                B-state_name      0.000     0.000     0.000       9.0
B-arrive_date.today_relative      0.000     0.000     0.000       0.0
    B-return_time.period_mod      0.000     0.000     0.000       0.0
                B-month_name      0.000     0.000     0.000       0.0
                B-day_number      0.000     0.000     0.000       0.0
 I-return_date.date_relative      0.500     0.333     0.400       3.0
I-return_date.today_relative      0.000     0.000     0.000       0.0
      B-stoploc.airport_name      0.000     0.000     0.000       0.0
             B-time_relative      0.000     0.000     0.000       0.0
                      I-time      0.000     0.000     0.000       0.0
    I-return_date.day_number      0.000     0.000     0.000       0.0
          I-meal_description      0.000     0.000     0.000       0.0
B-return_date.today_relative      0.000     0.000     0.000       0.0
      B-return_date.day_name      0.000     0.000     0.000       2.0

                   micro avg      0.903     0.905     0.904    3657.0
                   macro avg      0.530     0.516     0.504    3657.0
                weighted avg      0.907     0.905     0.899    3657.0

I0717 06:55:13.813001 140695199004544 interactiveshell.py:2882] Best Slot F1: 0.906
Reading ../data/atis.train.w-intent.iob
I0717 06:55:17.098927 140695199004544 interactiveshell.py:2882] Step 3150 | Loss: 0.0471 | Loss_intent: 0.0211 | Loss_slots: 0.0260 | Spent: 6.7 secs | LR: 0.000215
I0717 06:55:21.358289 140695199004544 interactiveshell.py:2882] Step 3200 | Loss: 0.2075 | Loss_intent: 0.1187 | Loss_slots: 0.0888 | Spent: 4.3 secs | LR: 0.000214
I0717 06:55:25.526474 140695199004544 interactiveshell.py:2882] Step 3250 | Loss: 0.0315 | Loss_intent: 0.0009 | Loss_slots: 0.0307 | Spent: 4.2 secs | LR: 0.000213
I0717 06:55:29.687028 140695199004544 interactiveshell.py:2882] Step 3300 | Loss: 0.1539 | Loss_intent: 0.0183 | Loss_slots: 0.1356 | Spent: 4.2 secs | LR: 0.000212
I0717 06:55:33.873471 140695199004544 interactiveshell.py:2882] Step 3350 | Loss: 0.0714 | Loss_intent: 0.0000 | Loss_slots: 0.0713 | Spent: 4.2 secs | LR: 0.000211
I0717 06:55:38.030883 140695199004544 interactiveshell.py:2882] Step 3400 | Loss: 0.0660 | Loss_intent: 0.0019 | Loss_slots: 0.0642 | Spent: 4.2 secs | LR: 0.000210
Reading ../data/atis.test.w-intent.iob
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.
  'precision', 'predicted', average, warn_for)
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1439: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples.
  'recall', 'true', average, warn_for)
I0717 06:55:42.331586 140695199004544 interactiveshell.py:2882] 
                                          precision    recall  f1-score   support

                             atis_flight      0.986     0.981     0.983       632
                            atis_airfare      0.941     1.000     0.970        48
                     atis_ground_service      1.000     1.000     1.000        36
                            atis_airline      1.000     0.921     0.959        38
                       atis_abbreviation      1.000     0.970     0.985        33
                           atis_aircraft      0.800     0.889     0.842         9
                        atis_flight_time      0.333     1.000     0.500         1
                           atis_quantity      0.300     1.000     0.462         3
                atis_flight#atis_airfare      0.857     0.500     0.632        12
                            atis_airport      0.900     1.000     0.947        18
                           atis_distance      1.000     1.000     1.000        10
                               atis_city      0.600     1.000     0.750         6
                        atis_ground_fare      1.000     0.714     0.833         7
                           atis_capacity      1.000     0.952     0.976        21
                          atis_flight_no      0.889     1.000     0.941         8
                               atis_meal      0.833     0.833     0.833         6
                        atis_restriction      0.000     0.000     0.000         0
             atis_airline#atis_flight_no      0.000     0.000     0.000         0
    atis_ground_service#atis_ground_fare      0.000     0.000     0.000         0
           atis_airfare#atis_flight_time      0.000     0.000     0.000         0
                           atis_cheapest      0.000     0.000     0.000         0
atis_aircraft#atis_flight#atis_flight_no      0.000     0.000     0.000         0

                               micro avg      0.964     0.970     0.967       888
                               macro avg      0.611     0.671     0.619       888
                            weighted avg      0.973     0.970     0.969       888

I0717 06:55:42.373023 140695199004544 interactiveshell.py:2882] 
                              precision    recall  f1-score   support

                           O      0.000     0.000     0.000       0.0
           B-toloc.city_name      0.952     0.969     0.961     716.0
         B-fromloc.city_name      0.955     0.973     0.964     704.0
           I-toloc.city_name      0.897     0.981     0.937     265.0
      B-depart_date.day_name      0.990     0.976     0.983     212.0
              B-airline_name      0.943     0.980     0.961     101.0
         I-fromloc.city_name      0.939     0.955     0.947     177.0
 B-depart_time.period_of_day      0.991     0.838     0.908     130.0
              I-airline_name      1.000     0.954     0.976      65.0
    B-depart_date.day_number      0.962     0.909     0.935      55.0
    B-depart_date.month_name      0.981     0.946     0.964      56.0
          B-depart_time.time      0.779     0.930     0.848      57.0
                B-round_trip      1.000     0.973     0.986      73.0
             B-cost_relative      1.000     0.973     0.986      37.0
                I-round_trip      1.000     1.000     1.000      71.0
                B-flight_mod      0.852     0.958     0.902      24.0
 B-depart_time.time_relative      0.967     0.892     0.928      65.0
          I-depart_time.time      0.806     0.962     0.877      52.0
         B-stoploc.city_name      1.000     0.500     0.667      20.0
                 B-city_name      0.583     0.614     0.598      57.0
                B-class_type      0.885     0.958     0.920      24.0
          B-arrive_time.time      0.623     0.971     0.759      34.0
 B-arrive_time.time_relative      0.750     0.968     0.845      31.0
                I-class_type      1.000     1.000     1.000      17.0
               B-flight_stop      1.000     1.000     1.000      21.0
          I-arrive_time.time      0.739     0.971     0.840      35.0
              B-airline_code      0.931     0.794     0.857      34.0
    I-depart_date.day_number      0.875     0.933     0.903      15.0
      I-fromloc.airport_name      0.359     0.933     0.519      15.0
      B-fromloc.airport_name      0.273     0.500     0.353      12.0
      B-arrive_date.day_name      0.647     1.000     0.786      11.0
          B-toloc.state_code      0.947     1.000     0.973      18.0
B-depart_date.today_relative      1.000     0.889     0.941       9.0
             B-flight_number      0.381     0.727     0.500      11.0
 B-depart_date.date_relative      0.810     1.000     0.895      17.0
          B-toloc.state_name      0.900     0.964     0.931      28.0
           B-fare_basis_code      0.944     1.000     0.971      17.0
               B-flight_time      1.000     1.000     1.000       1.0
                        B-or      1.000     1.000     1.000       3.0
 B-arrive_time.period_of_day      0.429     1.000     0.600       6.0
          B-meal_description      1.000     0.900     0.947      10.0
             I-cost_relative      1.000     0.667     0.800       3.0
              I-airport_name      0.800     0.276     0.410      29.0
               B-fare_amount      0.250     1.000     0.400       2.0
               I-fare_amount      0.667     1.000     0.800       2.0
                 I-city_name      0.714     0.333     0.455      30.0
        I-toloc.airport_name      0.600     1.000     0.750       3.0
            B-transport_type      1.000     0.700     0.824      10.0
    B-arrive_date.month_name      0.625     0.833     0.714       6.0
    B-arrive_date.day_number      0.714     0.833     0.769       6.0
         I-stoploc.city_name      1.000     0.100     0.182      10.0
                      B-meal      1.000     1.000     1.000      16.0
        B-fromloc.state_code      1.000     0.913     0.955      23.0
    B-depart_time.period_mod      0.833     1.000     0.909       5.0
                   B-connect      1.000     1.000     1.000       6.0
               B-flight_days      1.000     1.000     1.000      10.0
        B-toloc.airport_name      0.500     0.333     0.400       3.0
        B-fromloc.state_name      0.933     0.824     0.875      17.0
              B-airport_name      0.667     0.190     0.296      21.0
                   B-economy      1.000     1.000     1.000       6.0
               I-flight_time      1.000     1.000     1.000       1.0
             B-aircraft_code      1.000     0.727     0.842      33.0
                       B-mod      0.000     0.000     0.000       2.0
              B-airport_code      0.000     0.000     0.000       9.0
    B-depart_time.start_time      0.333     0.333     0.333       3.0
      B-depart_time.end_time      1.000     0.333     0.500       3.0
          B-depart_date.year      0.000     0.000     0.000       3.0
            I-transport_type      0.000     0.000     0.000       1.0
          B-restriction_code      1.000     0.250     0.400       4.0
    B-arrive_time.start_time      0.000     0.000     0.000       8.0
        B-toloc.airport_code      0.667     0.500     0.571       4.0
      B-arrive_time.end_time      0.000     0.000     0.000       8.0
      I-arrive_time.end_time      0.000     0.000     0.000       8.0
      I-depart_time.end_time      0.000     0.000     0.000       3.0
               I-flight_stop      0.000     0.000     0.000       0.0
      B-fromloc.airport_code      1.000     0.200     0.333       5.0
          I-restriction_code      0.500     1.000     0.667       3.0
    I-depart_time.start_time      0.000     0.000     0.000       1.0
          I-toloc.state_name      0.500     1.000     0.667       1.0
I-depart_date.today_relative      0.000     0.000     0.000       0.0
 B-arrive_date.date_relative      0.000     0.000     0.000       2.0
                I-flight_mod      0.000     0.000     0.000       6.0
                   I-economy      0.000     0.000     0.000       0.0
 B-return_date.date_relative      0.500     0.333     0.400       3.0
        I-fromloc.state_name      0.000     0.000     0.000       1.0
                B-state_code      0.500     1.000     0.667       1.0
    I-arrive_time.start_time      0.000     0.000     0.000       1.0
    I-arrive_date.day_number      0.000     0.000     0.000       0.0
                 B-meal_code      1.000     1.000     1.000       1.0
 I-depart_time.period_of_day      0.000     0.000     0.000       1.0
                  B-day_name      0.500     0.500     0.500       2.0
             B-period_of_day      1.000     0.500     0.667       4.0
        B-stoploc.state_code      0.000     0.000     0.000       0.0
    B-return_date.month_name      0.000     0.000     0.000       0.0
    B-return_date.day_number      0.000     0.000     0.000       0.0
    B-arrive_time.period_mod      0.000     0.000     0.000       0.0
                 I-meal_code      0.000     0.000     0.000       0.0
        B-toloc.country_name      1.000     1.000     1.000       1.0
                 B-days_code      1.000     1.000     1.000       1.0
 I-arrive_time.period_of_day      0.000     0.000     0.000       0.0
            I-today_relative      0.000     0.000     0.000       0.0
 B-return_time.period_of_day      0.000     0.000     0.000       0.0
                      B-time      0.000     0.000     0.000       0.0
           I-fare_basis_code      0.000     0.000     0.000       0.0
 I-arrive_time.time_relative      0.000     0.000     0.000       4.0
 I-depart_time.time_relative      0.000     0.000     0.000       1.0
            B-today_relative      0.000     0.000     0.000       0.0
                B-state_name      0.000     0.000     0.000       9.0
B-arrive_date.today_relative      0.000     0.000     0.000       0.0
    B-return_time.period_mod      0.000     0.000     0.000       0.0
                B-month_name      0.000     0.000     0.000       0.0
                B-day_number      0.000     0.000     0.000       0.0
 I-return_date.date_relative      1.000     0.333     0.500       3.0
I-return_date.today_relative      0.000     0.000     0.000       0.0
      B-stoploc.airport_name      0.000     0.000     0.000       0.0
             B-time_relative      0.000     0.000     0.000       0.0
                      I-time      0.000     0.000     0.000       0.0
    I-return_date.day_number      0.000     0.000     0.000       0.0
          I-meal_description      0.000     0.000     0.000       0.0
B-return_date.today_relative      0.000     0.000     0.000       0.0
      B-return_date.day_name      0.000     0.000     0.000       2.0

                   micro avg      0.904     0.905     0.905    3657.0
                   macro avg      0.520     0.511     0.491    3657.0
                weighted avg      0.903     0.905     0.896    3657.0

I0717 06:55:42.374053 140695199004544 interactiveshell.py:2882] Best Slot F1: 0.906
Reading ../data/atis.train.w-intent.iob
I0717 06:55:44.602623 140695199004544 interactiveshell.py:2882] Step 3450 | Loss: 0.0595 | Loss_intent: 0.0160 | Loss_slots: 0.0435 | Spent: 6.6 secs | LR: 0.000209
I0717 06:55:48.767048 140695199004544 interactiveshell.py:2882] Step 3500 | Loss: 0.0741 | Loss_intent: 0.0222 | Loss_slots: 0.0519 | Spent: 4.2 secs | LR: 0.000207
I0717 06:55:52.937320 140695199004544 interactiveshell.py:2882] Step 3550 | Loss: 0.1541 | Loss_intent: 0.0007 | Loss_slots: 0.1534 | Spent: 4.2 secs | LR: 0.000206
I0717 06:55:57.110106 140695199004544 interactiveshell.py:2882] Step 3600 | Loss: 0.0646 | Loss_intent: 0.0000 | Loss_slots: 0.0646 | Spent: 4.2 secs | LR: 0.000205
I0717 06:56:01.259827 140695199004544 interactiveshell.py:2882] Step 3650 | Loss: 0.1090 | Loss_intent: 0.0197 | Loss_slots: 0.0893 | Spent: 4.1 secs | LR: 0.000204
I0717 06:56:05.524262 140695199004544 interactiveshell.py:2882] Step 3700 | Loss: 0.0656 | Loss_intent: 0.0195 | Loss_slots: 0.0461 | Spent: 4.3 secs | LR: 0.000203
Reading ../data/atis.test.w-intent.iob
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.
  'precision', 'predicted', average, warn_for)
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1439: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples.
  'recall', 'true', average, warn_for)
I0717 06:56:10.811572 140695199004544 interactiveshell.py:2882] 
                                          precision    recall  f1-score   support

                             atis_flight      0.990     0.984     0.987       632
                            atis_airfare      0.923     1.000     0.960        48
                     atis_ground_service      1.000     1.000     1.000        36
                            atis_airline      0.974     1.000     0.987        38
                       atis_abbreviation      0.941     0.970     0.955        33
                           atis_aircraft      0.800     0.889     0.842         9
                        atis_flight_time      1.000     1.000     1.000         1
                           atis_quantity      0.375     1.000     0.545         3
                atis_flight#atis_airfare      0.875     0.583     0.700        12
                            atis_airport      0.947     1.000     0.973        18
                           atis_distance      1.000     1.000     1.000        10
                               atis_city      0.571     0.667     0.615         6
                        atis_ground_fare      1.000     0.714     0.833         7
                           atis_capacity      1.000     1.000     1.000        21
                          atis_flight_no      0.889     1.000     0.941         8
                               atis_meal      1.000     0.833     0.909         6
                        atis_restriction      0.000     0.000     0.000         0
             atis_airline#atis_flight_no      0.000     0.000     0.000         0
    atis_ground_service#atis_ground_fare      0.000     0.000     0.000         0
           atis_airfare#atis_flight_time      0.000     0.000     0.000         0
                           atis_cheapest      0.000     0.000     0.000         0
atis_aircraft#atis_flight#atis_flight_no      0.000     0.000     0.000         0

                               micro avg      0.970     0.975     0.972       888
                               macro avg      0.649     0.665     0.648       888
                            weighted avg      0.975     0.975     0.974       888

I0717 06:56:10.852433 140695199004544 interactiveshell.py:2882] 
                              precision    recall  f1-score   support

                           O      0.000     0.000     0.000       0.0
           B-toloc.city_name      0.965     0.973     0.969     716.0
         B-fromloc.city_name      0.957     0.977     0.967     704.0
           I-toloc.city_name      0.949     0.974     0.961     265.0
      B-depart_date.day_name      0.990     0.967     0.979     212.0
              B-airline_name      0.980     0.970     0.975     101.0
         I-fromloc.city_name      0.939     0.960     0.950     177.0
 B-depart_time.period_of_day      0.982     0.862     0.918     130.0
              I-airline_name      1.000     0.985     0.992      65.0
    B-depart_date.day_number      0.964     0.964     0.964      55.0
    B-depart_date.month_name      0.981     0.946     0.964      56.0
          B-depart_time.time      0.828     0.930     0.876      57.0
                B-round_trip      0.986     0.986     0.986      73.0
             B-cost_relative      1.000     0.973     0.986      37.0
                I-round_trip      1.000     1.000     1.000      71.0
                B-flight_mod      0.917     0.917     0.917      24.0
 B-depart_time.time_relative      0.968     0.923     0.945      65.0
          I-depart_time.time      0.855     0.904     0.879      52.0
         B-stoploc.city_name      0.792     0.950     0.864      20.0
                 B-city_name      0.732     0.526     0.612      57.0
                B-class_type      0.920     0.958     0.939      24.0
          B-arrive_time.time      0.786     0.971     0.868      34.0
 B-arrive_time.time_relative      0.833     0.968     0.896      31.0
                I-class_type      1.000     1.000     1.000      17.0
               B-flight_stop      1.000     1.000     1.000      21.0
          I-arrive_time.time      0.773     0.971     0.861      35.0
              B-airline_code      1.000     0.706     0.828      34.0
    I-depart_date.day_number      1.000     1.000     1.000      15.0
      I-fromloc.airport_name      0.424     0.933     0.583      15.0
      B-fromloc.airport_name      0.238     0.417     0.303      12.0
      B-arrive_date.day_name      0.579     1.000     0.733      11.0
          B-toloc.state_code      0.857     1.000     0.923      18.0
B-depart_date.today_relative      1.000     0.889     0.941       9.0
             B-flight_number      0.333     0.727     0.457      11.0
 B-depart_date.date_relative      0.895     1.000     0.944      17.0
          B-toloc.state_name      0.897     0.929     0.912      28.0
           B-fare_basis_code      0.895     1.000     0.944      17.0
               B-flight_time      1.000     1.000     1.000       1.0
                        B-or      1.000     1.000     1.000       3.0
 B-arrive_time.period_of_day      0.462     1.000     0.632       6.0
          B-meal_description      1.000     0.900     0.947      10.0
             I-cost_relative      1.000     0.667     0.800       3.0
              I-airport_name      0.800     0.276     0.410      29.0
               B-fare_amount      0.250     1.000     0.400       2.0
               I-fare_amount      1.000     1.000     1.000       2.0
                 I-city_name      0.789     0.500     0.612      30.0
        I-toloc.airport_name      1.000     1.000     1.000       3.0
            B-transport_type      1.000     0.700     0.824      10.0
    B-arrive_date.month_name      0.625     0.833     0.714       6.0
    B-arrive_date.day_number      0.833     0.833     0.833       6.0
         I-stoploc.city_name      0.583     0.700     0.636      10.0
                      B-meal      1.000     1.000     1.000      16.0
        B-fromloc.state_code      1.000     0.870     0.930      23.0
    B-depart_time.period_mod      0.833     1.000     0.909       5.0
                   B-connect      1.000     1.000     1.000       6.0
               B-flight_days      1.000     1.000     1.000      10.0
        B-toloc.airport_name      0.500     0.333     0.400       3.0
        B-fromloc.state_name      0.933     0.824     0.875      17.0
              B-airport_name      0.467     0.333     0.389      21.0
                   B-economy      1.000     1.000     1.000       6.0
               I-flight_time      1.000     1.000     1.000       1.0
             B-aircraft_code      0.926     0.758     0.833      33.0
                       B-mod      0.000     0.000     0.000       2.0
              B-airport_code      0.000     0.000     0.000       9.0
    B-depart_time.start_time      0.500     0.333     0.400       3.0
      B-depart_time.end_time      1.000     0.667     0.800       3.0
          B-depart_date.year      0.000     0.000     0.000       3.0
            I-transport_type      0.000     0.000     0.000       1.0
          B-restriction_code      1.000     0.750     0.857       4.0
    B-arrive_time.start_time      1.000     0.375     0.545       8.0
        B-toloc.airport_code      1.000     0.500     0.667       4.0
      B-arrive_time.end_time      0.700     0.875     0.778       8.0
      I-arrive_time.end_time      0.800     0.500     0.615       8.0
      I-depart_time.end_time      1.000     0.333     0.500       3.0
               I-flight_stop      0.000     0.000     0.000       0.0
      B-fromloc.airport_code      1.000     0.800     0.889       5.0
          I-restriction_code      0.429     1.000     0.600       3.0
    I-depart_time.start_time      0.500     1.000     0.667       1.0
          I-toloc.state_name      0.500     1.000     0.667       1.0
I-depart_date.today_relative      0.000     0.000     0.000       0.0
 B-arrive_date.date_relative      0.500     1.000     0.667       2.0
                I-flight_mod      0.000     0.000     0.000       6.0
                   I-economy      0.000     0.000     0.000       0.0
 B-return_date.date_relative      0.500     0.333     0.400       3.0
        I-fromloc.state_name      0.000     0.000     0.000       1.0
                B-state_code      1.000     1.000     1.000       1.0
    I-arrive_time.start_time      0.000     0.000     0.000       1.0
    I-arrive_date.day_number      0.000     0.000     0.000       0.0
                 B-meal_code      1.000     1.000     1.000       1.0
 I-depart_time.period_of_day      0.000     0.000     0.000       1.0
                  B-day_name      1.000     0.500     0.667       2.0
             B-period_of_day      1.000     0.250     0.400       4.0
        B-stoploc.state_code      0.000     0.000     0.000       0.0
    B-return_date.month_name      0.000     0.000     0.000       0.0
    B-return_date.day_number      0.000     0.000     0.000       0.0
    B-arrive_time.period_mod      0.000     0.000     0.000       0.0
                 I-meal_code      0.000     0.000     0.000       0.0
        B-toloc.country_name      1.000     1.000     1.000       1.0
                 B-days_code      1.000     1.000     1.000       1.0
 I-arrive_time.period_of_day      0.000     0.000     0.000       0.0
            I-today_relative      0.000     0.000     0.000       0.0
 B-return_time.period_of_day      0.000     0.000     0.000       0.0
                      B-time      0.000     0.000     0.000       0.0
           I-fare_basis_code      0.000     0.000     0.000       0.0
 I-arrive_time.time_relative      0.000     0.000     0.000       4.0
 I-depart_time.time_relative      0.000     0.000     0.000       1.0
            B-today_relative      0.000     0.000     0.000       0.0
                B-state_name      0.000     0.000     0.000       9.0
B-arrive_date.today_relative      0.000     0.000     0.000       0.0
    B-return_time.period_mod      0.000     0.000     0.000       0.0
                B-month_name      0.000     0.000     0.000       0.0
                B-day_number      0.000     0.000     0.000       0.0
 I-return_date.date_relative      0.500     0.333     0.400       3.0
I-return_date.today_relative      0.000     0.000     0.000       0.0
      B-stoploc.airport_name      0.000     0.000     0.000       0.0
             B-time_relative      0.000     0.000     0.000       0.0
                      I-time      0.000     0.000     0.000       0.0
    I-return_date.day_number      0.000     0.000     0.000       0.0
          I-meal_description      0.000     0.000     0.000       0.0
B-return_date.today_relative      0.000     0.000     0.000       0.0
      B-return_date.day_name      0.000     0.000     0.000       2.0

                   micro avg      0.916     0.918     0.917    3657.0
                   macro avg      0.571     0.564     0.550    3657.0
                weighted avg      0.922     0.918     0.915    3657.0

I0717 06:56:10.853375 140695199004544 interactiveshell.py:2882] Best Slot F1: 0.917
Reading ../data/atis.train.w-intent.iob
I0717 06:56:12.086880 140695199004544 interactiveshell.py:2882] Step 3750 | Loss: 0.3266 | Loss_intent: 0.2711 | Loss_slots: 0.0556 | Spent: 6.6 secs | LR: 0.000202
I0717 06:56:16.294014 140695199004544 interactiveshell.py:2882] Step 3800 | Loss: 0.0620 | Loss_intent: 0.0001 | Loss_slots: 0.0619 | Spent: 4.2 secs | LR: 0.000201
I0717 06:56:20.477173 140695199004544 interactiveshell.py:2882] Step 3850 | Loss: 0.0758 | Loss_intent: 0.0042 | Loss_slots: 0.0716 | Spent: 4.2 secs | LR: 0.000200
I0717 06:56:24.684440 140695199004544 interactiveshell.py:2882] Step 3900 | Loss: 0.0692 | Loss_intent: 0.0001 | Loss_slots: 0.0691 | Spent: 4.2 secs | LR: 0.000199
I0717 06:56:28.849399 140695199004544 interactiveshell.py:2882] Step 3950 | Loss: 0.0945 | Loss_intent: 0.0008 | Loss_slots: 0.0937 | Spent: 4.2 secs | LR: 0.000198
I0717 06:56:33.044356 140695199004544 interactiveshell.py:2882] Step 4000 | Loss: 0.1345 | Loss_intent: 0.0775 | Loss_slots: 0.0570 | Spent: 4.2 secs | LR: 0.000197
I0717 06:56:37.300602 140695199004544 interactiveshell.py:2882] Step 4050 | Loss: 0.0499 | Loss_intent: 0.0005 | Loss_slots: 0.0494 | Spent: 4.3 secs | LR: 0.000196
Reading ../data/atis.test.w-intent.iob
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.
  'precision', 'predicted', average, warn_for)
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1439: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples.
  'recall', 'true', average, warn_for)
I0717 06:56:39.465694 140695199004544 interactiveshell.py:2882] 
                                          precision    recall  f1-score   support

                             atis_flight      0.972     0.987     0.980       632
                            atis_airfare      0.870     0.979     0.922        48
                     atis_ground_service      1.000     1.000     1.000        36
                            atis_airline      1.000     0.974     0.987        38
                       atis_abbreviation      1.000     1.000     1.000        33
                           atis_aircraft      1.000     0.889     0.941         9
                        atis_flight_time      1.000     1.000     1.000         1
                           atis_quantity      0.375     1.000     0.545         3
                atis_flight#atis_airfare      0.750     0.250     0.375        12
                            atis_airport      1.000     0.889     0.941        18
                           atis_distance      1.000     0.700     0.824        10
                               atis_city      0.571     0.667     0.615         6
                        atis_ground_fare      1.000     0.714     0.833         7
                           atis_capacity      1.000     1.000     1.000        21
                          atis_flight_no      0.889     1.000     0.941         8
                               atis_meal      1.000     0.833     0.909         6
                        atis_restriction      0.000     0.000     0.000         0
             atis_airline#atis_flight_no      0.000     0.000     0.000         0
    atis_ground_service#atis_ground_fare      0.000     0.000     0.000         0
           atis_airfare#atis_flight_time      0.000     0.000     0.000         0
                           atis_cheapest      0.000     0.000     0.000         0
atis_aircraft#atis_flight#atis_flight_no      0.000     0.000     0.000         0

                               micro avg      0.961     0.966     0.964       888
                               macro avg      0.656     0.631     0.628       888
                            weighted avg      0.964     0.966     0.962       888

I0717 06:56:39.514253 140695199004544 interactiveshell.py:2882] 
                              precision    recall  f1-score   support

                           O      0.000     0.000     0.000       0.0
           B-toloc.city_name      0.956     0.976     0.966     716.0
         B-fromloc.city_name      0.961     0.977     0.969     704.0
           I-toloc.city_name      0.926     0.989     0.956     265.0
      B-depart_date.day_name      0.990     0.972     0.981     212.0
              B-airline_name      0.961     0.980     0.971     101.0
         I-fromloc.city_name      0.955     0.955     0.955     177.0
 B-depart_time.period_of_day      0.991     0.854     0.917     130.0
              I-airline_name      0.985     0.985     0.985      65.0
    B-depart_date.day_number      0.964     0.964     0.964      55.0
    B-depart_date.month_name      0.981     0.946     0.964      56.0
          B-depart_time.time      0.823     0.895     0.857      57.0
                B-round_trip      1.000     0.986     0.993      73.0
             B-cost_relative      1.000     0.973     0.986      37.0
                I-round_trip      1.000     1.000     1.000      71.0
                B-flight_mod      0.958     0.958     0.958      24.0
 B-depart_time.time_relative      0.968     0.923     0.945      65.0
          I-depart_time.time      0.857     0.923     0.889      52.0
         B-stoploc.city_name      0.875     0.700     0.778      20.0
                 B-city_name      0.714     0.526     0.606      57.0
                B-class_type      0.923     1.000     0.960      24.0
          B-arrive_time.time      0.569     0.971     0.717      34.0
 B-arrive_time.time_relative      0.789     0.968     0.870      31.0
                I-class_type      1.000     1.000     1.000      17.0
               B-flight_stop      1.000     1.000     1.000      21.0
          I-arrive_time.time      0.708     0.971     0.819      35.0
              B-airline_code      0.879     0.853     0.866      34.0
    I-depart_date.day_number      1.000     0.933     0.966      15.0
      I-fromloc.airport_name      0.452     0.933     0.609      15.0
      B-fromloc.airport_name      0.250     0.500     0.333      12.0
      B-arrive_date.day_name      0.611     1.000     0.759      11.0
          B-toloc.state_code      0.818     1.000     0.900      18.0
B-depart_date.today_relative      1.000     0.889     0.941       9.0
             B-flight_number      0.381     0.727     0.500      11.0
 B-depart_date.date_relative      0.850     1.000     0.919      17.0
          B-toloc.state_name      0.900     0.964     0.931      28.0
           B-fare_basis_code      0.900     0.529     0.667      17.0
               B-flight_time      1.000     1.000     1.000       1.0
                        B-or      1.000     1.000     1.000       3.0
 B-arrive_time.period_of_day      0.750     1.000     0.857       6.0
          B-meal_description      1.000     0.900     0.947      10.0
             I-cost_relative      1.000     0.667     0.800       3.0
              I-airport_name      0.900     0.310     0.462      29.0
               B-fare_amount      0.333     0.500     0.400       2.0
               I-fare_amount      1.000     1.000     1.000       2.0
                 I-city_name      0.867     0.433     0.578      30.0
        I-toloc.airport_name      1.000     1.000     1.000       3.0
            B-transport_type      0.769     1.000     0.870      10.0
    B-arrive_date.month_name      0.625     0.833     0.714       6.0
    B-arrive_date.day_number      0.714     0.833     0.769       6.0
         I-stoploc.city_name      0.625     0.500     0.556      10.0
                      B-meal      1.000     1.000     1.000      16.0
        B-fromloc.state_code      1.000     0.826     0.905      23.0
    B-depart_time.period_mod      0.556     1.000     0.714       5.0
                   B-connect      1.000     1.000     1.000       6.0
               B-flight_days      1.000     1.000     1.000      10.0
        B-toloc.airport_name      0.500     0.333     0.400       3.0
        B-fromloc.state_name      0.938     0.882     0.909      17.0
              B-airport_name      0.667     0.286     0.400      21.0
                   B-economy      1.000     1.000     1.000       6.0
               I-flight_time      1.000     1.000     1.000       1.0
             B-aircraft_code      0.929     0.788     0.852      33.0
                       B-mod      0.000     0.000     0.000       2.0
              B-airport_code      0.000     0.000     0.000       9.0
    B-depart_time.start_time      0.333     0.333     0.333       3.0
      B-depart_time.end_time      1.000     0.667     0.800       3.0
          B-depart_date.year      0.000     0.000     0.000       3.0
            I-transport_type      0.000     0.000     0.000       1.0
          B-restriction_code      0.500     0.250     0.333       4.0
    B-arrive_time.start_time      0.000     0.000     0.000       8.0
        B-toloc.airport_code      0.750     0.750     0.750       4.0
      B-arrive_time.end_time      1.000     0.375     0.545       8.0
      I-arrive_time.end_time      0.000     0.000     0.000       8.0
      I-depart_time.end_time      1.000     0.333     0.500       3.0
               I-flight_stop      0.000     0.000     0.000       0.0
      B-fromloc.airport_code      1.000     0.800     0.889       5.0
          I-restriction_code      0.167     1.000     0.286       3.0
    I-depart_time.start_time      0.500     1.000     0.667       1.0
          I-toloc.state_name      0.500     1.000     0.667       1.0
I-depart_date.today_relative      0.000     0.000     0.000       0.0
 B-arrive_date.date_relative      1.000     0.500     0.667       2.0
                I-flight_mod      0.000     0.000     0.000       6.0
                   I-economy      0.000     0.000     0.000       0.0
 B-return_date.date_relative      1.000     0.333     0.500       3.0
        I-fromloc.state_name      0.000     0.000     0.000       1.0
                B-state_code      1.000     1.000     1.000       1.0
    I-arrive_time.start_time      0.000     0.000     0.000       1.0
    I-arrive_date.day_number      0.000     0.000     0.000       0.0
                 B-meal_code      0.000     0.000     0.000       1.0
 I-depart_time.period_of_day      0.000     0.000     0.000       1.0
                  B-day_name      0.500     0.500     0.500       2.0
             B-period_of_day      1.000     0.500     0.667       4.0
        B-stoploc.state_code      0.000     0.000     0.000       0.0
    B-return_date.month_name      0.000     0.000     0.000       0.0
    B-return_date.day_number      0.000     0.000     0.000       0.0
    B-arrive_time.period_mod      0.000     0.000     0.000       0.0
                 I-meal_code      0.000     0.000     0.000       0.0
        B-toloc.country_name      1.000     1.000     1.000       1.0
                 B-days_code      1.000     1.000     1.000       1.0
 I-arrive_time.period_of_day      0.000     0.000     0.000       0.0
            I-today_relative      0.000     0.000     0.000       0.0
 B-return_time.period_of_day      0.000     0.000     0.000       0.0
                      B-time      0.000     0.000     0.000       0.0
           I-fare_basis_code      0.000     0.000     0.000       0.0
 I-arrive_time.time_relative      0.000     0.000     0.000       4.0
 I-depart_time.time_relative      0.000     0.000     0.000       1.0
            B-today_relative      0.000     0.000     0.000       0.0
                B-state_name      0.000     0.000     0.000       9.0
B-arrive_date.today_relative      0.000     0.000     0.000       0.0
    B-return_time.period_mod      0.000     0.000     0.000       0.0
                B-month_name      0.000     0.000     0.000       0.0
                B-day_number      0.000     0.000     0.000       0.0
 I-return_date.date_relative      1.000     0.667     0.800       3.0
I-return_date.today_relative      0.000     0.000     0.000       0.0
      B-stoploc.airport_name      0.000     0.000     0.000       0.0
             B-time_relative      0.000     0.000     0.000       0.0
                      I-time      0.000     0.000     0.000       0.0
    I-return_date.day_number      0.000     0.000     0.000       0.0
          I-meal_description      0.000     0.000     0.000       0.0
B-return_date.today_relative      0.000     0.000     0.000       0.0
      B-return_date.day_name      0.000     0.000     0.000       2.0

                   micro avg      0.913     0.914     0.914    3657.0
                   macro avg      0.548     0.536     0.524    3657.0
                weighted avg      0.914     0.914     0.908    3657.0

I0717 06:56:39.515149 140695199004544 interactiveshell.py:2882] Best Slot F1: 0.917
Reading ../data/atis.train.w-intent.iob
I0717 06:56:44.104252 140695199004544 interactiveshell.py:2882] Step 4100 | Loss: 0.0326 | Loss_intent: 0.0079 | Loss_slots: 0.0247 | Spent: 6.8 secs | LR: 0.000195
I0717 06:56:48.536284 140695199004544 interactiveshell.py:2882] Step 4150 | Loss: 0.0594 | Loss_intent: 0.0021 | Loss_slots: 0.0573 | Spent: 4.4 secs | LR: 0.000194
I0717 06:56:52.697677 140695199004544 interactiveshell.py:2882] Step 4200 | Loss: 0.0778 | Loss_intent: 0.0004 | Loss_slots: 0.0774 | Spent: 4.2 secs | LR: 0.000193
I0717 06:56:56.863821 140695199004544 interactiveshell.py:2882] Step 4250 | Loss: 0.0824 | Loss_intent: 0.0430 | Loss_slots: 0.0394 | Spent: 4.2 secs | LR: 0.000192
I0717 06:57:01.032398 140695199004544 interactiveshell.py:2882] Step 4300 | Loss: 0.1296 | Loss_intent: 0.0002 | Loss_slots: 0.1293 | Spent: 4.2 secs | LR: 0.000191
I0717 06:57:05.208501 140695199004544 interactiveshell.py:2882] Step 4350 | Loss: 0.4374 | Loss_intent: 0.4022 | Loss_slots: 0.0352 | Spent: 4.2 secs | LR: 0.000190
Reading ../data/atis.test.w-intent.iob
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.
  'precision', 'predicted', average, warn_for)
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1439: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples.
  'recall', 'true', average, warn_for)
I0717 06:57:08.342306 140695199004544 interactiveshell.py:2882] 
                                          precision    recall  f1-score   support

                             atis_flight      0.969     0.986     0.977       632
                            atis_airfare      0.922     0.979     0.949        48
                     atis_ground_service      1.000     1.000     1.000        36
                            atis_airline      1.000     0.974     0.987        38
                       atis_abbreviation      1.000     0.970     0.985        33
                           atis_aircraft      0.727     0.889     0.800         9
                        atis_flight_time      1.000     1.000     1.000         1
                           atis_quantity      0.250     0.667     0.364         3
                atis_flight#atis_airfare      0.000     0.000     0.000        12
                            atis_airport      1.000     0.889     0.941        18
                           atis_distance      1.000     0.900     0.947        10
                               atis_city      0.571     0.667     0.615         6
                        atis_ground_fare      1.000     0.714     0.833         7
                           atis_capacity      0.955     1.000     0.977        21
                          atis_flight_no      0.889     1.000     0.941         8
                               atis_meal      1.000     0.833     0.909         6
                        atis_restriction      0.000     0.000     0.000         0
             atis_airline#atis_flight_no      0.000     0.000     0.000         0
    atis_ground_service#atis_ground_fare      0.000     0.000     0.000         0
           atis_airfare#atis_flight_time      0.000     0.000     0.000         0
                           atis_cheapest      0.000     0.000     0.000         0
atis_aircraft#atis_flight#atis_flight_no      0.000     0.000     0.000         0

                               micro avg      0.956     0.962     0.959       888
                               macro avg      0.604     0.612     0.601       888
                            weighted avg      0.950     0.962     0.955       888

I0717 06:57:08.383948 140695199004544 interactiveshell.py:2882] 
                              precision    recall  f1-score   support

                           O      0.000     0.000     0.000       0.0
           B-toloc.city_name      0.962     0.979     0.970     716.0
         B-fromloc.city_name      0.954     0.979     0.966     704.0
           I-toloc.city_name      0.936     0.989     0.961     265.0
      B-depart_date.day_name      0.990     0.981     0.986     212.0
              B-airline_name      0.980     0.970     0.975     101.0
         I-fromloc.city_name      0.971     0.955     0.963     177.0
 B-depart_time.period_of_day      0.974     0.869     0.919     130.0
              I-airline_name      0.985     1.000     0.992      65.0
    B-depart_date.day_number      0.981     0.927     0.953      55.0
    B-depart_date.month_name      0.982     0.964     0.973      56.0
          B-depart_time.time      0.750     0.895     0.816      57.0
                B-round_trip      1.000     0.986     0.993      73.0
             B-cost_relative      1.000     0.973     0.986      37.0
                I-round_trip      1.000     1.000     1.000      71.0
                B-flight_mod      0.957     0.917     0.936      24.0
 B-depart_time.time_relative      0.954     0.954     0.954      65.0
          I-depart_time.time      0.847     0.962     0.901      52.0
         B-stoploc.city_name      0.857     0.900     0.878      20.0
                 B-city_name      0.806     0.509     0.624      57.0
                B-class_type      0.857     1.000     0.923      24.0
          B-arrive_time.time      0.868     0.971     0.917      34.0
 B-arrive_time.time_relative      0.833     0.968     0.896      31.0
                I-class_type      1.000     1.000     1.000      17.0
               B-flight_stop      1.000     1.000     1.000      21.0
          I-arrive_time.time      0.850     0.971     0.907      35.0
              B-airline_code      1.000     0.706     0.828      34.0
    I-depart_date.day_number      1.000     1.000     1.000      15.0
      I-fromloc.airport_name      0.349     1.000     0.517      15.0
      B-fromloc.airport_name      0.208     0.417     0.278      12.0
      B-arrive_date.day_name      0.688     1.000     0.815      11.0
          B-toloc.state_code      0.947     1.000     0.973      18.0
B-depart_date.today_relative      1.000     0.889     0.941       9.0
             B-flight_number      0.321     0.818     0.462      11.0
 B-depart_date.date_relative      0.895     1.000     0.944      17.0
          B-toloc.state_name      0.871     0.964     0.915      28.0
           B-fare_basis_code      0.917     0.647     0.759      17.0
               B-flight_time      1.000     1.000     1.000       1.0
                        B-or      1.000     1.000     1.000       3.0
 B-arrive_time.period_of_day      0.545     1.000     0.706       6.0
          B-meal_description      1.000     0.900     0.947      10.0
             I-cost_relative      1.000     0.667     0.800       3.0
              I-airport_name      0.750     0.207     0.324      29.0
               B-fare_amount      0.400     1.000     0.571       2.0
               I-fare_amount      1.000     1.000     1.000       2.0
                 I-city_name      1.000     0.500     0.667      30.0
        I-toloc.airport_name      1.000     0.667     0.800       3.0
            B-transport_type      0.889     0.800     0.842      10.0
    B-arrive_date.month_name      0.714     0.833     0.769       6.0
    B-arrive_date.day_number      0.714     0.833     0.769       6.0
         I-stoploc.city_name      0.500     0.500     0.500      10.0
                      B-meal      1.000     1.000     1.000      16.0
        B-fromloc.state_code      1.000     0.957     0.978      23.0
    B-depart_time.period_mod      0.750     0.600     0.667       5.0
                   B-connect      1.000     1.000     1.000       6.0
               B-flight_days      1.000     1.000     1.000      10.0
        B-toloc.airport_name      0.500     0.333     0.400       3.0
        B-fromloc.state_name      0.789     0.882     0.833      17.0
              B-airport_name      0.625     0.238     0.345      21.0
                   B-economy      1.000     1.000     1.000       6.0
               I-flight_time      1.000     1.000     1.000       1.0
             B-aircraft_code      0.926     0.758     0.833      33.0
                       B-mod      0.000     0.000     0.000       2.0
              B-airport_code      0.000     0.000     0.000       9.0
    B-depart_time.start_time      0.500     0.333     0.400       3.0
      B-depart_time.end_time      1.000     0.667     0.800       3.0
          B-depart_date.year      1.000     0.667     0.800       3.0
            I-transport_type      0.000     0.000     0.000       1.0
          B-restriction_code      1.000     0.750     0.857       4.0
    B-arrive_time.start_time      0.600     0.750     0.667       8.0
        B-toloc.airport_code      1.000     0.500     0.667       4.0
      B-arrive_time.end_time      1.000     0.500     0.667       8.0
      I-arrive_time.end_time      0.833     0.625     0.714       8.0
      I-depart_time.end_time      1.000     0.333     0.500       3.0
               I-flight_stop      0.000     0.000     0.000       0.0
      B-fromloc.airport_code      1.000     0.600     0.750       5.0
          I-restriction_code      0.429     1.000     0.600       3.0
    I-depart_time.start_time      0.500     1.000     0.667       1.0
          I-toloc.state_name      0.500     1.000     0.667       1.0
I-depart_date.today_relative      0.000     0.000     0.000       0.0
 B-arrive_date.date_relative      1.000     0.500     0.667       2.0
                I-flight_mod      0.000     0.000     0.000       6.0
                   I-economy      0.000     0.000     0.000       0.0
 B-return_date.date_relative      0.500     0.333     0.400       3.0
        I-fromloc.state_name      0.000     0.000     0.000       1.0
                B-state_code      1.000     1.000     1.000       1.0
    I-arrive_time.start_time      0.000     0.000     0.000       1.0
    I-arrive_date.day_number      0.000     0.000     0.000       0.0
                 B-meal_code      0.000     0.000     0.000       1.0
 I-depart_time.period_of_day      0.000     0.000     0.000       1.0
                  B-day_name      1.000     0.500     0.667       2.0
             B-period_of_day      1.000     0.750     0.857       4.0
        B-stoploc.state_code      0.000     0.000     0.000       0.0
    B-return_date.month_name      0.000     0.000     0.000       0.0
    B-return_date.day_number      0.000     0.000     0.000       0.0
    B-arrive_time.period_mod      0.000     0.000     0.000       0.0
                 I-meal_code      0.000     0.000     0.000       0.0
        B-toloc.country_name      0.000     0.000     0.000       1.0
                 B-days_code      1.000     1.000     1.000       1.0
 I-arrive_time.period_of_day      0.000     0.000     0.000       0.0
            I-today_relative      0.000     0.000     0.000       0.0
 B-return_time.period_of_day      0.000     0.000     0.000       0.0
                      B-time      0.000     0.000     0.000       0.0
           I-fare_basis_code      0.000     0.000     0.000       0.0
 I-arrive_time.time_relative      0.000     0.000     0.000       4.0
 I-depart_time.time_relative      0.000     0.000     0.000       1.0
            B-today_relative      0.000     0.000     0.000       0.0
                B-state_name      0.000     0.000     0.000       9.0
B-arrive_date.today_relative      0.000     0.000     0.000       0.0
    B-return_time.period_mod      0.000     0.000     0.000       0.0
                B-month_name      0.000     0.000     0.000       0.0
                B-day_number      0.000     0.000     0.000       0.0
 I-return_date.date_relative      1.000     0.333     0.500       3.0
I-return_date.today_relative      0.000     0.000     0.000       0.0
      B-stoploc.airport_name      0.000     0.000     0.000       0.0
             B-time_relative      0.000     0.000     0.000       0.0
                      I-time      0.000     0.000     0.000       0.0
    I-return_date.day_number      0.000     0.000     0.000       0.0
          I-meal_description      0.000     0.000     0.000       0.0
B-return_date.today_relative      0.000     0.000     0.000       0.0
      B-return_date.day_name      0.000     0.000     0.000       2.0

                   micro avg      0.918     0.920     0.919    3657.0
                   macro avg      0.574     0.544     0.541    3657.0
                weighted avg      0.925     0.920     0.916    3657.0

I0717 06:57:08.384878 140695199004544 interactiveshell.py:2882] Best Slot F1: 0.919
Reading ../data/atis.train.w-intent.iob
I0717 06:57:11.788401 140695199004544 interactiveshell.py:2882] Step 4400 | Loss: 0.0173 | Loss_intent: 0.0004 | Loss_slots: 0.0169 | Spent: 6.6 secs | LR: 0.000189
I0717 06:57:15.961598 140695199004544 interactiveshell.py:2882] Step 4450 | Loss: 0.0235 | Loss_intent: 0.0002 | Loss_slots: 0.0233 | Spent: 4.2 secs | LR: 0.000188
I0717 06:57:20.144734 140695199004544 interactiveshell.py:2882] Step 4500 | Loss: 0.0137 | Loss_intent: 0.0019 | Loss_slots: 0.0117 | Spent: 4.2 secs | LR: 0.000187
I0717 06:57:24.343442 140695199004544 interactiveshell.py:2882] Step 4550 | Loss: 0.0584 | Loss_intent: 0.0000 | Loss_slots: 0.0584 | Spent: 4.2 secs | LR: 0.000186
I0717 06:57:28.505428 140695199004544 interactiveshell.py:2882] Step 4600 | Loss: 0.0572 | Loss_intent: 0.0216 | Loss_slots: 0.0356 | Spent: 4.2 secs | LR: 0.000185
I0717 06:57:32.697961 140695199004544 interactiveshell.py:2882] Step 4650 | Loss: 0.0816 | Loss_intent: 0.0084 | Loss_slots: 0.0732 | Spent: 4.2 secs | LR: 0.000184
Reading ../data/atis.test.w-intent.iob
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.
  'precision', 'predicted', average, warn_for)
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1439: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples.
  'recall', 'true', average, warn_for)
I0717 06:57:36.834648 140695199004544 interactiveshell.py:2882] 
                                          precision    recall  f1-score   support

                             atis_flight      0.990     0.981     0.986       632
                            atis_airfare      0.857     1.000     0.923        48
                     atis_ground_service      0.923     1.000     0.960        36
                            atis_airline      1.000     0.974     0.987        38
                       atis_abbreviation      0.917     1.000     0.957        33
                           atis_aircraft      0.889     0.889     0.889         9
                        atis_flight_time      0.500     1.000     0.667         1
                           atis_quantity      0.250     0.667     0.364         3
                atis_flight#atis_airfare      0.857     0.500     0.632        12
                            atis_airport      1.000     0.722     0.839        18
                           atis_distance      1.000     0.900     0.947        10
                               atis_city      0.400     0.667     0.500         6
                        atis_ground_fare      1.000     0.714     0.833         7
                           atis_capacity      0.955     1.000     0.977        21
                          atis_flight_no      0.889     1.000     0.941         8
                               atis_meal      1.000     0.833     0.909         6
                        atis_restriction      0.000     0.000     0.000         0
             atis_airline#atis_flight_no      0.000     0.000     0.000         0
    atis_ground_service#atis_ground_fare      0.000     0.000     0.000         0
           atis_airfare#atis_flight_time      0.000     0.000     0.000         0
                           atis_cheapest      0.000     0.000     0.000         0
atis_aircraft#atis_flight#atis_flight_no      0.000     0.000     0.000         0

                               micro avg      0.959     0.964     0.961       888
                               macro avg      0.610     0.629     0.605       888
                            weighted avg      0.967     0.964     0.963       888

I0717 06:57:36.875488 140695199004544 interactiveshell.py:2882] 
                              precision    recall  f1-score   support

                           O      0.000     0.000     0.000       0.0
           B-toloc.city_name      0.964     0.983     0.974     716.0
         B-fromloc.city_name      0.966     0.977     0.972     704.0
           I-toloc.city_name      0.904     0.992     0.946     265.0
      B-depart_date.day_name      0.981     0.991     0.986     212.0
              B-airline_name      0.971     0.980     0.975     101.0
         I-fromloc.city_name      0.964     0.915     0.939     177.0
 B-depart_time.period_of_day      0.991     0.877     0.931     130.0
              I-airline_name      1.000     0.985     0.992      65.0
    B-depart_date.day_number      0.963     0.945     0.954      55.0
    B-depart_date.month_name      0.981     0.946     0.964      56.0
          B-depart_time.time      0.828     0.930     0.876      57.0
                B-round_trip      0.986     0.973     0.979      73.0
             B-cost_relative      1.000     0.973     0.986      37.0
                I-round_trip      1.000     1.000     1.000      71.0
                B-flight_mod      1.000     0.917     0.957      24.0
 B-depart_time.time_relative      0.968     0.938     0.953      65.0
          I-depart_time.time      0.842     0.923     0.881      52.0
         B-stoploc.city_name      0.900     0.900     0.900      20.0
                 B-city_name      0.814     0.614     0.700      57.0
                B-class_type      0.923     1.000     0.960      24.0
          B-arrive_time.time      0.805     0.971     0.880      34.0
 B-arrive_time.time_relative      0.857     0.968     0.909      31.0
                I-class_type      1.000     1.000     1.000      17.0
               B-flight_stop      0.955     1.000     0.977      21.0
          I-arrive_time.time      0.756     0.971     0.850      35.0
              B-airline_code      1.000     0.853     0.921      34.0
    I-depart_date.day_number      1.000     0.933     0.966      15.0
      I-fromloc.airport_name      0.455     1.000     0.625      15.0
      B-fromloc.airport_name      0.350     0.583     0.438      12.0
      B-arrive_date.day_name      0.750     0.818     0.783      11.0
          B-toloc.state_code      1.000     1.000     1.000      18.0
B-depart_date.today_relative      1.000     0.889     0.941       9.0
             B-flight_number      0.300     0.818     0.439      11.0
 B-depart_date.date_relative      0.895     1.000     0.944      17.0
          B-toloc.state_name      0.871     0.964     0.915      28.0
           B-fare_basis_code      0.944     1.000     0.971      17.0
               B-flight_time      1.000     1.000     1.000       1.0
                        B-or      1.000     1.000     1.000       3.0
 B-arrive_time.period_of_day      0.600     1.000     0.750       6.0
          B-meal_description      1.000     0.900     0.947      10.0
             I-cost_relative      1.000     0.667     0.800       3.0
              I-airport_name      0.818     0.310     0.450      29.0
               B-fare_amount      0.333     1.000     0.500       2.0
               I-fare_amount      1.000     1.000     1.000       2.0
                 I-city_name      0.889     0.533     0.667      30.0
        I-toloc.airport_name      1.000     1.000     1.000       3.0
            B-transport_type      0.769     1.000     0.870      10.0
    B-arrive_date.month_name      0.625     0.833     0.714       6.0
    B-arrive_date.day_number      0.625     0.833     0.714       6.0
         I-stoploc.city_name      0.500     0.300     0.375      10.0
                      B-meal      1.000     1.000     1.000      16.0
        B-fromloc.state_code      1.000     0.957     0.978      23.0
    B-depart_time.period_mod      0.714     1.000     0.833       5.0
                   B-connect      1.000     1.000     1.000       6.0
               B-flight_days      1.000     1.000     1.000      10.0
        B-toloc.airport_name      0.500     0.333     0.400       3.0
        B-fromloc.state_name      0.800     0.941     0.865      17.0
              B-airport_name      0.875     0.333     0.483      21.0
                   B-economy      1.000     1.000     1.000       6.0
               I-flight_time      1.000     1.000     1.000       1.0
             B-aircraft_code      0.871     0.818     0.844      33.0
                       B-mod      0.000     0.000     0.000       2.0
              B-airport_code      0.000     0.000     0.000       9.0
    B-depart_time.start_time      0.500     0.333     0.400       3.0
      B-depart_time.end_time      1.000     0.667     0.800       3.0
          B-depart_date.year      1.000     0.667     0.800       3.0
            I-transport_type      0.000     0.000     0.000       1.0
          B-restriction_code      0.600     0.750     0.667       4.0
    B-arrive_time.start_time      0.750     0.375     0.500       8.0
        B-toloc.airport_code      1.000     0.500     0.667       4.0
      B-arrive_time.end_time      0.833     0.625     0.714       8.0
      I-arrive_time.end_time      1.000     0.375     0.545       8.0
      I-depart_time.end_time      1.000     0.333     0.500       3.0
               I-flight_stop      0.000     0.000     0.000       0.0
      B-fromloc.airport_code      1.000     1.000     1.000       5.0
          I-restriction_code      0.750     1.000     0.857       3.0
    I-depart_time.start_time      0.000     0.000     0.000       1.0
          I-toloc.state_name      1.000     1.000     1.000       1.0
I-depart_date.today_relative      0.000     0.000     0.000       0.0
 B-arrive_date.date_relative      1.000     1.000     1.000       2.0
                I-flight_mod      0.000     0.000     0.000       6.0
                   I-economy      0.000     0.000     0.000       0.0
 B-return_date.date_relative      0.500     0.333     0.400       3.0
        I-fromloc.state_name      1.000     1.000     1.000       1.0
                B-state_code      0.500     1.000     0.667       1.0
    I-arrive_time.start_time      0.000     0.000     0.000       1.0
    I-arrive_date.day_number      0.000     0.000     0.000       0.0
                 B-meal_code      1.000     1.000     1.000       1.0
 I-depart_time.period_of_day      0.000     0.000     0.000       1.0
                  B-day_name      1.000     0.500     0.667       2.0
             B-period_of_day      1.000     0.500     0.667       4.0
        B-stoploc.state_code      0.000     0.000     0.000       0.0
    B-return_date.month_name      0.000     0.000     0.000       0.0
    B-return_date.day_number      0.000     0.000     0.000       0.0
    B-arrive_time.period_mod      0.000     0.000     0.000       0.0
                 I-meal_code      0.000     0.000     0.000       0.0
        B-toloc.country_name      1.000     1.000     1.000       1.0
                 B-days_code      1.000     1.000     1.000       1.0
 I-arrive_time.period_of_day      0.000     0.000     0.000       0.0
            I-today_relative      0.000     0.000     0.000       0.0
 B-return_time.period_of_day      0.000     0.000     0.000       0.0
                      B-time      0.000     0.000     0.000       0.0
           I-fare_basis_code      0.000     0.000     0.000       0.0
 I-arrive_time.time_relative      1.000     0.500     0.667       4.0
 I-depart_time.time_relative      0.000     0.000     0.000       1.0
            B-today_relative      0.000     0.000     0.000       0.0
                B-state_name      0.000     0.000     0.000       9.0
B-arrive_date.today_relative      0.000     0.000     0.000       0.0
    B-return_time.period_mod      0.000     0.000     0.000       0.0
                B-month_name      0.000     0.000     0.000       0.0
                B-day_number      0.000     0.000     0.000       0.0
 I-return_date.date_relative      1.000     0.333     0.500       3.0
I-return_date.today_relative      0.000     0.000     0.000       0.0
      B-stoploc.airport_name      0.000     0.000     0.000       0.0
             B-time_relative      0.000     0.000     0.000       0.0
                      I-time      0.000     0.000     0.000       0.0
    I-return_date.day_number      0.000     0.000     0.000       0.0
          I-meal_description      0.000     0.000     0.000       0.0
B-return_date.today_relative      0.000     0.000     0.000       0.0
      B-return_date.day_name      0.000     0.000     0.000       2.0

                   micro avg      0.926     0.927     0.927    3657.0
                   macro avg      0.605     0.579     0.575    3657.0
                weighted avg      0.929     0.927     0.923    3657.0

I0717 06:57:36.876364 140695199004544 interactiveshell.py:2882] Best Slot F1: 0.927
Reading ../data/atis.train.w-intent.iob
I0717 06:57:39.273775 140695199004544 interactiveshell.py:2882] Step 4700 | Loss: 0.0243 | Loss_intent: 0.0001 | Loss_slots: 0.0242 | Spent: 6.6 secs | LR: 0.000183
I0717 06:57:43.441688 140695199004544 interactiveshell.py:2882] Step 4750 | Loss: 0.0351 | Loss_intent: 0.0001 | Loss_slots: 0.0350 | Spent: 4.2 secs | LR: 0.000182
I0717 06:57:47.624697 140695199004544 interactiveshell.py:2882] Step 4800 | Loss: 0.1354 | Loss_intent: 0.0761 | Loss_slots: 0.0593 | Spent: 4.2 secs | LR: 0.000181
I0717 06:57:51.799190 140695199004544 interactiveshell.py:2882] Step 4850 | Loss: 0.0463 | Loss_intent: 0.0000 | Loss_slots: 0.0463 | Spent: 4.2 secs | LR: 0.000180
I0717 06:57:56.044586 140695199004544 interactiveshell.py:2882] Step 4900 | Loss: 0.0486 | Loss_intent: 0.0001 | Loss_slots: 0.0485 | Spent: 4.2 secs | LR: 0.000179
I0717 06:58:00.296652 140695199004544 interactiveshell.py:2882] Step 4950 | Loss: 0.1113 | Loss_intent: 0.0009 | Loss_slots: 0.1105 | Spent: 4.3 secs | LR: 0.000178
Reading ../data/atis.test.w-intent.iob
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.
  'precision', 'predicted', average, warn_for)
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1439: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples.
  'recall', 'true', average, warn_for)
I0717 06:58:05.472481 140695199004544 interactiveshell.py:2882] 
                                          precision    recall  f1-score   support

                             atis_flight      0.979     0.981     0.980       632
                            atis_airfare      0.960     1.000     0.980        48
                     atis_ground_service      0.947     1.000     0.973        36
                            atis_airline      0.974     1.000     0.987        38
                       atis_abbreviation      1.000     0.970     0.985        33
                           atis_aircraft      0.800     0.889     0.842         9
                        atis_flight_time      0.333     1.000     0.500         1
                           atis_quantity      0.375     1.000     0.545         3
                atis_flight#atis_airfare      0.667     0.167     0.267        12
                            atis_airport      1.000     0.944     0.971        18
                           atis_distance      1.000     1.000     1.000        10
                               atis_city      0.444     0.667     0.533         6
                        atis_ground_fare      1.000     0.714     0.833         7
                           atis_capacity      1.000     1.000     1.000        21
                          atis_flight_no      0.889     1.000     0.941         8
                               atis_meal      1.000     0.833     0.909         6
                        atis_restriction      0.000     0.000     0.000         0
             atis_airline#atis_flight_no      0.000     0.000     0.000         0
    atis_ground_service#atis_ground_fare      0.000     0.000     0.000         0
           atis_airfare#atis_flight_time      0.000     0.000     0.000         0
                           atis_cheapest      0.000     0.000     0.000         0
atis_aircraft#atis_flight#atis_flight_no      0.000     0.000     0.000         0

                               micro avg      0.961     0.966     0.964       888
                               macro avg      0.608     0.644     0.602       888
                            weighted avg      0.966     0.966     0.963       888

I0717 06:58:05.513206 140695199004544 interactiveshell.py:2882] 
                              precision    recall  f1-score   support

                           O      0.000     0.000     0.000       0.0
           B-toloc.city_name      0.958     0.992     0.975     716.0
         B-fromloc.city_name      0.980     0.976     0.978     704.0
           I-toloc.city_name      0.923     0.992     0.956     265.0
      B-depart_date.day_name      0.990     0.962     0.976     212.0
              B-airline_name      0.980     0.980     0.980     101.0
         I-fromloc.city_name      0.982     0.910     0.944     177.0
 B-depart_time.period_of_day      0.991     0.869     0.926     130.0
              I-airline_name      1.000     1.000     1.000      65.0
    B-depart_date.day_number      0.962     0.909     0.935      55.0
    B-depart_date.month_name      0.981     0.929     0.954      56.0
          B-depart_time.time      0.754     0.912     0.825      57.0
                B-round_trip      1.000     0.945     0.972      73.0
             B-cost_relative      1.000     0.973     0.986      37.0
                I-round_trip      1.000     1.000     1.000      71.0
                B-flight_mod      1.000     0.917     0.957      24.0
 B-depart_time.time_relative      0.967     0.908     0.937      65.0
          I-depart_time.time      0.857     0.923     0.889      52.0
         B-stoploc.city_name      1.000     0.850     0.919      20.0
                 B-city_name      0.756     0.596     0.667      57.0
                B-class_type      0.923     1.000     0.960      24.0
          B-arrive_time.time      0.767     0.971     0.857      34.0
 B-arrive_time.time_relative      0.750     0.968     0.845      31.0
                I-class_type      1.000     1.000     1.000      17.0
               B-flight_stop      0.875     1.000     0.933      21.0
          I-arrive_time.time      0.756     0.971     0.850      35.0
              B-airline_code      1.000     0.824     0.903      34.0
    I-depart_date.day_number      1.000     0.867     0.929      15.0
      I-fromloc.airport_name      0.441     1.000     0.612      15.0
      B-fromloc.airport_name      0.435     0.833     0.571      12.0
      B-arrive_date.day_name      0.550     1.000     0.710      11.0
          B-toloc.state_code      1.000     1.000     1.000      18.0
B-depart_date.today_relative      1.000     0.889     0.941       9.0
             B-flight_number      0.538     0.636     0.583      11.0
 B-depart_date.date_relative      0.944     1.000     0.971      17.0
          B-toloc.state_name      0.931     0.964     0.947      28.0
           B-fare_basis_code      0.944     1.000     0.971      17.0
               B-flight_time      1.000     1.000     1.000       1.0
                        B-or      1.000     1.000     1.000       3.0
 B-arrive_time.period_of_day      0.667     1.000     0.800       6.0
          B-meal_description      1.000     0.900     0.947      10.0
             I-cost_relative      1.000     0.667     0.800       3.0
              I-airport_name      0.579     0.379     0.458      29.0
               B-fare_amount      0.400     1.000     0.571       2.0
               I-fare_amount      1.000     1.000     1.000       2.0
                 I-city_name      1.000     0.433     0.605      30.0
        I-toloc.airport_name      1.000     1.000     1.000       3.0
            B-transport_type      0.769     1.000     0.870      10.0
    B-arrive_date.month_name      0.556     0.833     0.667       6.0
    B-arrive_date.day_number      0.455     0.833     0.588       6.0
         I-stoploc.city_name      0.462     0.600     0.522      10.0
                      B-meal      1.000     1.000     1.000      16.0
        B-fromloc.state_code      1.000     0.957     0.978      23.0
    B-depart_time.period_mod      0.833     1.000     0.909       5.0
                   B-connect      1.000     1.000     1.000       6.0
               B-flight_days      1.000     1.000     1.000      10.0
        B-toloc.airport_name      0.500     0.333     0.400       3.0
        B-fromloc.state_name      0.882     0.882     0.882      17.0
              B-airport_name      0.800     0.381     0.516      21.0
                   B-economy      1.000     1.000     1.000       6.0
               I-flight_time      1.000     1.000     1.000       1.0
             B-aircraft_code      0.903     0.848     0.875      33.0
                       B-mod      0.000     0.000     0.000       2.0
              B-airport_code      0.571     0.444     0.500       9.0
    B-depart_time.start_time      0.500     0.333     0.400       3.0
      B-depart_time.end_time      1.000     0.333     0.500       3.0
          B-depart_date.year      1.000     0.667     0.800       3.0
            I-transport_type      0.000     0.000     0.000       1.0
          B-restriction_code      0.500     0.500     0.500       4.0
    B-arrive_time.start_time      0.800     0.500     0.615       8.0
        B-toloc.airport_code      1.000     0.500     0.667       4.0
      B-arrive_time.end_time      0.750     0.750     0.750       8.0
      I-arrive_time.end_time      0.667     0.250     0.364       8.0
      I-depart_time.end_time      1.000     0.333     0.500       3.0
               I-flight_stop      0.000     0.000     0.000       0.0
      B-fromloc.airport_code      1.000     1.000     1.000       5.0
          I-restriction_code      0.750     1.000     0.857       3.0
    I-depart_time.start_time      0.500     1.000     0.667       1.0
          I-toloc.state_name      0.500     1.000     0.667       1.0
I-depart_date.today_relative      0.000     0.000     0.000       0.0
 B-arrive_date.date_relative      1.000     1.000     1.000       2.0
                I-flight_mod      0.000     0.000     0.000       6.0
                   I-economy      0.000     0.000     0.000       0.0
 B-return_date.date_relative      0.500     0.333     0.400       3.0
        I-fromloc.state_name      0.000     0.000     0.000       1.0
                B-state_code      0.500     1.000     0.667       1.0
    I-arrive_time.start_time      0.000     0.000     0.000       1.0
    I-arrive_date.day_number      0.000     0.000     0.000       0.0
                 B-meal_code      1.000     1.000     1.000       1.0
 I-depart_time.period_of_day      0.000     0.000     0.000       1.0
                  B-day_name      1.000     0.500     0.667       2.0
             B-period_of_day      1.000     0.500     0.667       4.0
        B-stoploc.state_code      0.000     0.000     0.000       0.0
    B-return_date.month_name      0.000     0.000     0.000       0.0
    B-return_date.day_number      0.000     0.000     0.000       0.0
    B-arrive_time.period_mod      0.000     0.000     0.000       0.0
                 I-meal_code      0.000     0.000     0.000       0.0
        B-toloc.country_name      1.000     1.000     1.000       1.0
                 B-days_code      1.000     1.000     1.000       1.0
 I-arrive_time.period_of_day      0.000     0.000     0.000       0.0
            I-today_relative      0.000     0.000     0.000       0.0
 B-return_time.period_of_day      0.000     0.000     0.000       0.0
                      B-time      0.000     0.000     0.000       0.0
           I-fare_basis_code      0.000     0.000     0.000       0.0
 I-arrive_time.time_relative      0.000     0.000     0.000       4.0
 I-depart_time.time_relative      0.000     0.000     0.000       1.0
            B-today_relative      0.000     0.000     0.000       0.0
                B-state_name      0.000     0.000     0.000       9.0
B-arrive_date.today_relative      0.000     0.000     0.000       0.0
    B-return_time.period_mod      0.000     0.000     0.000       0.0
                B-month_name      0.000     0.000     0.000       0.0
                B-day_number      0.000     0.000     0.000       0.0
 I-return_date.date_relative      0.000     0.000     0.000       3.0
I-return_date.today_relative      0.000     0.000     0.000       0.0
      B-stoploc.airport_name      0.000     0.000     0.000       0.0
             B-time_relative      0.000     0.000     0.000       0.0
                      I-time      0.000     0.000     0.000       0.0
    I-return_date.day_number      0.000     0.000     0.000       0.0
          I-meal_description      0.000     0.000     0.000       0.0
B-return_date.today_relative      0.000     0.000     0.000       0.0
      B-return_date.day_name      0.000     0.000     0.000       2.0

                   micro avg      0.924     0.925     0.924    3657.0
                   macro avg      0.581     0.574     0.561    3657.0
                weighted avg      0.929     0.925     0.922    3657.0

I0717 06:58:05.514001 140695199004544 interactiveshell.py:2882] Best Slot F1: 0.927
Reading ../data/atis.train.w-intent.iob
I0717 06:58:06.915339 140695199004544 interactiveshell.py:2882] Step 5000 | Loss: 0.0477 | Loss_intent: 0.0020 | Loss_slots: 0.0457 | Spent: 6.6 secs | LR: 0.000177
I0717 06:58:11.095666 140695199004544 interactiveshell.py:2882] Step 5050 | Loss: 0.2041 | Loss_intent: 0.0001 | Loss_slots: 0.2040 | Spent: 4.2 secs | LR: 0.000176
I0717 06:58:15.292062 140695199004544 interactiveshell.py:2882] Step 5100 | Loss: 0.0307 | Loss_intent: 0.0163 | Loss_slots: 0.0144 | Spent: 4.2 secs | LR: 0.000175
I0717 06:58:19.467968 140695199004544 interactiveshell.py:2882] Step 5150 | Loss: 0.0117 | Loss_intent: 0.0000 | Loss_slots: 0.0117 | Spent: 4.2 secs | LR: 0.000174
I0717 06:58:23.676303 140695199004544 interactiveshell.py:2882] Step 5200 | Loss: 0.0419 | Loss_intent: 0.0000 | Loss_slots: 0.0418 | Spent: 4.2 secs | LR: 0.000173
I0717 06:58:27.850537 140695199004544 interactiveshell.py:2882] Step 5250 | Loss: 0.0328 | Loss_intent: 0.0000 | Loss_slots: 0.0327 | Spent: 4.2 secs | LR: 0.000173
I0717 06:58:32.023032 140695199004544 interactiveshell.py:2882] Step 5300 | Loss: 0.0236 | Loss_intent: 0.0013 | Loss_slots: 0.0223 | Spent: 4.2 secs | LR: 0.000172
Reading ../data/atis.test.w-intent.iob
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.
  'precision', 'predicted', average, warn_for)
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1439: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples.
  'recall', 'true', average, warn_for)
I0717 06:58:33.996889 140695199004544 interactiveshell.py:2882] 
                                          precision    recall  f1-score   support

                             atis_flight      0.981     0.979     0.980       632
                            atis_airfare      0.940     0.979     0.959        48
                     atis_ground_service      1.000     1.000     1.000        36
                            atis_airline      1.000     1.000     1.000        38
                       atis_abbreviation      0.917     1.000     0.957        33
                           atis_aircraft      0.727     0.889     0.800         9
                        atis_flight_time      1.000     1.000     1.000         1
                           atis_quantity      0.286     0.667     0.400         3
                atis_flight#atis_airfare      0.800     0.333     0.471        12
                            atis_airport      0.944     0.944     0.944        18
                           atis_distance      0.909     1.000     0.952        10
                               atis_city      0.444     0.667     0.533         6
                        atis_ground_fare      1.000     0.714     0.833         7
                           atis_capacity      0.955     1.000     0.977        21
                          atis_flight_no      0.889     1.000     0.941         8
                               atis_meal      1.000     0.667     0.800         6
                        atis_restriction      0.000     0.000     0.000         0
             atis_airline#atis_flight_no      0.000     0.000     0.000         0
    atis_ground_service#atis_ground_fare      0.000     0.000     0.000         0
           atis_airfare#atis_flight_time      0.000     0.000     0.000         0
                           atis_cheapest      0.000     0.000     0.000         0
atis_aircraft#atis_flight#atis_flight_no      0.000     0.000     0.000         0

                               micro avg      0.960     0.965     0.962       888
                               macro avg      0.627     0.629     0.616       888
                            weighted avg      0.964     0.965     0.962       888

I0717 06:58:34.037572 140695199004544 interactiveshell.py:2882] 
                              precision    recall  f1-score   support

                           O      0.000     0.000     0.000       0.0
           B-toloc.city_name      0.983     0.983     0.983     716.0
         B-fromloc.city_name      0.975     0.982     0.978     704.0
           I-toloc.city_name      0.959     0.966     0.962     265.0
      B-depart_date.day_name      0.990     0.972     0.981     212.0
              B-airline_name      0.980     0.970     0.975     101.0
         I-fromloc.city_name      0.982     0.915     0.947     177.0
 B-depart_time.period_of_day      0.966     0.885     0.924     130.0
              I-airline_name      1.000     1.000     1.000      65.0
    B-depart_date.day_number      0.981     0.927     0.953      55.0
    B-depart_date.month_name      0.981     0.946     0.964      56.0
          B-depart_time.time      0.828     0.930     0.876      57.0
                B-round_trip      0.973     0.986     0.980      73.0
             B-cost_relative      1.000     0.973     0.986      37.0
                I-round_trip      1.000     1.000     1.000      71.0
                B-flight_mod      1.000     0.917     0.957      24.0
 B-depart_time.time_relative      0.968     0.923     0.945      65.0
          I-depart_time.time      0.860     0.942     0.899      52.0
         B-stoploc.city_name      0.826     0.950     0.884      20.0
                 B-city_name      0.642     0.754     0.694      57.0
                B-class_type      0.857     1.000     0.923      24.0
          B-arrive_time.time      0.868     0.971     0.917      34.0
 B-arrive_time.time_relative      0.811     0.968     0.882      31.0
                I-class_type      1.000     1.000     1.000      17.0
               B-flight_stop      1.000     1.000     1.000      21.0
          I-arrive_time.time      0.850     0.971     0.907      35.0
              B-airline_code      1.000     0.824     0.903      34.0
    I-depart_date.day_number      1.000     1.000     1.000      15.0
      I-fromloc.airport_name      0.484     1.000     0.652      15.0
      B-fromloc.airport_name      0.400     0.500     0.444      12.0
      B-arrive_date.day_name      0.611     1.000     0.759      11.0
          B-toloc.state_code      0.947     1.000     0.973      18.0
B-depart_date.today_relative      1.000     0.889     0.941       9.0
             B-flight_number      0.450     0.818     0.581      11.0
 B-depart_date.date_relative      0.810     1.000     0.895      17.0
          B-toloc.state_name      0.900     0.964     0.931      28.0
           B-fare_basis_code      0.944     1.000     0.971      17.0
               B-flight_time      1.000     1.000     1.000       1.0
                        B-or      1.000     1.000     1.000       3.0
 B-arrive_time.period_of_day      0.857     1.000     0.923       6.0
          B-meal_description      1.000     0.900     0.947      10.0
             I-cost_relative      1.000     0.667     0.800       3.0
              I-airport_name      0.857     0.414     0.558      29.0
               B-fare_amount      0.286     1.000     0.444       2.0
               I-fare_amount      1.000     1.000     1.000       2.0
                 I-city_name      0.586     0.567     0.576      30.0
        I-toloc.airport_name      1.000     1.000     1.000       3.0
            B-transport_type      0.875     0.700     0.778      10.0
    B-arrive_date.month_name      0.625     0.833     0.714       6.0
    B-arrive_date.day_number      0.714     0.833     0.769       6.0
         I-stoploc.city_name      0.455     1.000     0.625      10.0
                      B-meal      1.000     1.000     1.000      16.0
        B-fromloc.state_code      1.000     0.957     0.978      23.0
    B-depart_time.period_mod      1.000     0.400     0.571       5.0
                   B-connect      1.000     1.000     1.000       6.0
               B-flight_days      1.000     1.000     1.000      10.0
        B-toloc.airport_name      1.000     0.333     0.500       3.0
        B-fromloc.state_name      0.895     1.000     0.944      17.0
              B-airport_name      0.857     0.286     0.429      21.0
                   B-economy      1.000     1.000     1.000       6.0
               I-flight_time      1.000     1.000     1.000       1.0
             B-aircraft_code      0.900     0.818     0.857      33.0
                       B-mod      0.000     0.000     0.000       2.0
              B-airport_code      0.000     0.000     0.000       9.0
    B-depart_time.start_time      0.333     0.333     0.333       3.0
      B-depart_time.end_time      1.000     0.667     0.800       3.0
          B-depart_date.year      1.000     0.667     0.800       3.0
            I-transport_type      0.000     0.000     0.000       1.0
          B-restriction_code      0.273     0.750     0.400       4.0
    B-arrive_time.start_time      0.833     0.625     0.714       8.0
        B-toloc.airport_code      1.000     0.500     0.667       4.0
      B-arrive_time.end_time      0.857     0.750     0.800       8.0
      I-arrive_time.end_time      0.714     0.625     0.667       8.0
      I-depart_time.end_time      1.000     0.333     0.500       3.0
               I-flight_stop      0.000     0.000     0.000       0.0
      B-fromloc.airport_code      1.000     0.800     0.889       5.0
          I-restriction_code      0.600     1.000     0.750       3.0
    I-depart_time.start_time      0.500     1.000     0.667       1.0
          I-toloc.state_name      0.500     1.000     0.667       1.0
I-depart_date.today_relative      0.000     0.000     0.000       0.0
 B-arrive_date.date_relative      0.000     0.000     0.000       2.0
                I-flight_mod      0.000     0.000     0.000       6.0
                   I-economy      0.000     0.000     0.000       0.0
 B-return_date.date_relative      1.000     0.667     0.800       3.0
        I-fromloc.state_name      0.000     0.000     0.000       1.0
                B-state_code      1.000     1.000     1.000       1.0
    I-arrive_time.start_time      0.000     0.000     0.000       1.0
    I-arrive_date.day_number      0.000     0.000     0.000       0.0
                 B-meal_code      1.000     1.000     1.000       1.0
 I-depart_time.period_of_day      0.000     0.000     0.000       1.0
                  B-day_name      1.000     0.500     0.667       2.0
             B-period_of_day      1.000     0.750     0.857       4.0
        B-stoploc.state_code      0.000     0.000     0.000       0.0
    B-return_date.month_name      0.000     0.000     0.000       0.0
    B-return_date.day_number      0.000     0.000     0.000       0.0
    B-arrive_time.period_mod      0.000     0.000     0.000       0.0
                 I-meal_code      0.000     0.000     0.000       0.0
        B-toloc.country_name      1.000     1.000     1.000       1.0
                 B-days_code      0.333     1.000     0.500       1.0
 I-arrive_time.period_of_day      0.000     0.000     0.000       0.0
            I-today_relative      0.000     0.000     0.000       0.0
 B-return_time.period_of_day      0.000     0.000     0.000       0.0
                      B-time      0.000     0.000     0.000       0.0
           I-fare_basis_code      0.000     0.000     0.000       0.0
 I-arrive_time.time_relative      0.000     0.000     0.000       4.0
 I-depart_time.time_relative      0.000     0.000     0.000       1.0
            B-today_relative      0.000     0.000     0.000       0.0
                B-state_name      0.000     0.000     0.000       9.0
B-arrive_date.today_relative      0.000     0.000     0.000       0.0
    B-return_time.period_mod      0.000     0.000     0.000       0.0
                B-month_name      0.000     0.000     0.000       0.0
                B-day_number      0.000     0.000     0.000       0.0
 I-return_date.date_relative      1.000     0.667     0.800       3.0
I-return_date.today_relative      0.000     0.000     0.000       0.0
      B-stoploc.airport_name      0.000     0.000     0.000       0.0
             B-time_relative      0.000     0.000     0.000       0.0
                      I-time      0.000     0.000     0.000       0.0
    I-return_date.day_number      0.000     0.000     0.000       0.0
          I-meal_description      0.000     0.000     0.000       0.0
B-return_date.today_relative      0.000     0.000     0.000       0.0
      B-return_date.day_name      0.000     0.000     0.000       2.0

                   micro avg      0.929     0.930     0.930    3657.0
                   macro avg      0.584     0.580     0.564    3657.0
                weighted avg      0.934     0.930     0.928    3657.0

I0717 06:58:34.038406 140695199004544 interactiveshell.py:2882] Best Slot F1: 0.930
Reading ../data/atis.train.w-intent.iob
I0717 06:58:38.619829 140695199004544 interactiveshell.py:2882] Step 5350 | Loss: 0.0759 | Loss_intent: 0.0224 | Loss_slots: 0.0535 | Spent: 6.6 secs | LR: 0.000171
I0717 06:58:42.809086 140695199004544 interactiveshell.py:2882] Step 5400 | Loss: 0.0601 | Loss_intent: 0.0002 | Loss_slots: 0.0598 | Spent: 4.2 secs | LR: 0.000170
I0717 06:58:47.017429 140695199004544 interactiveshell.py:2882] Step 5450 | Loss: 0.0366 | Loss_intent: 0.0041 | Loss_slots: 0.0325 | Spent: 4.2 secs | LR: 0.000169
I0717 06:58:51.202558 140695199004544 interactiveshell.py:2882] Step 5500 | Loss: 0.0102 | Loss_intent: 0.0005 | Loss_slots: 0.0096 | Spent: 4.2 secs | LR: 0.000168
I0717 06:58:55.388010 140695199004544 interactiveshell.py:2882] Step 5550 | Loss: 0.0488 | Loss_intent: 0.0000 | Loss_slots: 0.0487 | Spent: 4.2 secs | LR: 0.000167
I0717 06:58:59.569452 140695199004544 interactiveshell.py:2882] Step 5600 | Loss: 0.0437 | Loss_intent: 0.0000 | Loss_slots: 0.0437 | Spent: 4.2 secs | LR: 0.000166
Reading ../data/atis.test.w-intent.iob
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.
  'precision', 'predicted', average, warn_for)
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1439: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples.
  'recall', 'true', average, warn_for)
I0717 06:59:02.551051 140695199004544 interactiveshell.py:2882] 
                                          precision    recall  f1-score   support

                             atis_flight      0.983     0.986     0.984       632
                            atis_airfare      0.923     1.000     0.960        48
                     atis_ground_service      1.000     1.000     1.000        36
                            atis_airline      0.974     1.000     0.987        38
                       atis_abbreviation      1.000     0.939     0.969        33
                           atis_aircraft      0.800     0.889     0.842         9
                        atis_flight_time      1.000     1.000     1.000         1
                           atis_quantity      0.375     1.000     0.545         3
                atis_flight#atis_airfare      0.800     0.333     0.471        12
                            atis_airport      0.947     1.000     0.973        18
                           atis_distance      1.000     1.000     1.000        10
                               atis_city      0.667     0.667     0.667         6
                        atis_ground_fare      1.000     0.714     0.833         7
                           atis_capacity      1.000     1.000     1.000        21
                          atis_flight_no      0.889     1.000     0.941         8
                               atis_meal      1.000     0.833     0.909         6
                        atis_restriction      0.000     0.000     0.000         0
             atis_airline#atis_flight_no      0.000     0.000     0.000         0
    atis_ground_service#atis_ground_fare      0.000     0.000     0.000         0
           atis_airfare#atis_flight_time      0.000     0.000     0.000         0
                           atis_cheapest      0.000     0.000     0.000         0
atis_aircraft#atis_flight#atis_flight_no      0.000     0.000     0.000         0

                               micro avg      0.966     0.972     0.969       888
                               macro avg      0.653     0.653     0.640       888
                            weighted avg      0.971     0.972     0.969       888

I0717 06:59:02.591288 140695199004544 interactiveshell.py:2882] 
                              precision    recall  f1-score   support

                           O      0.000     0.000     0.000       0.0
           B-toloc.city_name      0.969     0.997     0.983     716.0
         B-fromloc.city_name      0.976     0.984     0.980     704.0
           I-toloc.city_name      0.929     0.992     0.960     265.0
      B-depart_date.day_name      0.990     0.972     0.981     212.0
              B-airline_name      0.971     0.980     0.975     101.0
         I-fromloc.city_name      0.959     0.932     0.946     177.0
 B-depart_time.period_of_day      0.983     0.869     0.922     130.0
              I-airline_name      1.000     0.985     0.992      65.0
    B-depart_date.day_number      0.962     0.927     0.944      55.0
    B-depart_date.month_name      0.981     0.946     0.964      56.0
          B-depart_time.time      0.828     0.930     0.876      57.0
                B-round_trip      1.000     0.986     0.993      73.0
             B-cost_relative      1.000     0.973     0.986      37.0
                I-round_trip      1.000     1.000     1.000      71.0
                B-flight_mod      1.000     0.958     0.979      24.0
 B-depart_time.time_relative      0.968     0.923     0.945      65.0
          I-depart_time.time      0.857     0.923     0.889      52.0
         B-stoploc.city_name      1.000     0.900     0.947      20.0
                 B-city_name      0.694     0.596     0.642      57.0
                B-class_type      0.923     1.000     0.960      24.0
          B-arrive_time.time      0.750     0.971     0.846      34.0
 B-arrive_time.time_relative      0.789     0.968     0.870      31.0
                I-class_type      1.000     1.000     1.000      17.0
               B-flight_stop      1.000     1.000     1.000      21.0
          I-arrive_time.time      0.829     0.971     0.895      35.0
              B-airline_code      1.000     0.912     0.954      34.0
    I-depart_date.day_number      1.000     1.000     1.000      15.0
      I-fromloc.airport_name      0.455     1.000     0.625      15.0
      B-fromloc.airport_name      0.300     0.500     0.375      12.0
      B-arrive_date.day_name      0.611     1.000     0.759      11.0
          B-toloc.state_code      0.947     1.000     0.973      18.0
B-depart_date.today_relative      1.000     1.000     1.000       9.0
             B-flight_number      0.600     0.818     0.692      11.0
 B-depart_date.date_relative      0.944     1.000     0.971      17.0
          B-toloc.state_name      0.929     0.929     0.929      28.0
           B-fare_basis_code      0.944     1.000     0.971      17.0
               B-flight_time      1.000     1.000     1.000       1.0
                        B-or      1.000     1.000     1.000       3.0
 B-arrive_time.period_of_day      0.667     1.000     0.800       6.0
          B-meal_description      1.000     0.900     0.947      10.0
             I-cost_relative      1.000     0.667     0.800       3.0
              I-airport_name      0.917     0.379     0.537      29.0
               B-fare_amount      0.500     1.000     0.667       2.0
               I-fare_amount      1.000     1.000     1.000       2.0
                 I-city_name      1.000     0.433     0.605      30.0
        I-toloc.airport_name      0.600     1.000     0.750       3.0
            B-transport_type      0.833     1.000     0.909      10.0
    B-arrive_date.month_name      0.625     0.833     0.714       6.0
    B-arrive_date.day_number      0.625     0.833     0.714       6.0
         I-stoploc.city_name      0.700     0.700     0.700      10.0
                      B-meal      1.000     1.000     1.000      16.0
        B-fromloc.state_code      1.000     0.957     0.978      23.0
    B-depart_time.period_mod      0.800     0.800     0.800       5.0
                   B-connect      1.000     1.000     1.000       6.0
               B-flight_days      1.000     1.000     1.000      10.0
        B-toloc.airport_name      0.500     0.333     0.400       3.0
        B-fromloc.state_name      0.941     0.941     0.941      17.0
              B-airport_name      0.700     0.333     0.452      21.0
                   B-economy      1.000     1.000     1.000       6.0
               I-flight_time      1.000     1.000     1.000       1.0
             B-aircraft_code      0.900     0.818     0.857      33.0
                       B-mod      0.000     0.000     0.000       2.0
              B-airport_code      0.000     0.000     0.000       9.0
    B-depart_time.start_time      0.500     0.333     0.400       3.0
      B-depart_time.end_time      1.000     0.667     0.800       3.0
          B-depart_date.year      1.000     0.667     0.800       3.0
            I-transport_type      0.000     0.000     0.000       1.0
          B-restriction_code      0.400     1.000     0.571       4.0
    B-arrive_time.start_time      0.833     0.625     0.714       8.0
        B-toloc.airport_code      1.000     0.750     0.857       4.0
      B-arrive_time.end_time      0.833     0.625     0.714       8.0
      I-arrive_time.end_time      0.714     0.625     0.667       8.0
      I-depart_time.end_time      1.000     0.333     0.500       3.0
               I-flight_stop      0.000     0.000     0.000       0.0
      B-fromloc.airport_code      1.000     1.000     1.000       5.0
          I-restriction_code      0.750     1.000     0.857       3.0
    I-depart_time.start_time      0.500     1.000     0.667       1.0
          I-toloc.state_name      1.000     1.000     1.000       1.0
I-depart_date.today_relative      0.000     0.000     0.000       0.0
 B-arrive_date.date_relative      1.000     1.000     1.000       2.0
                I-flight_mod      0.000     0.000     0.000       6.0
                   I-economy      0.000     0.000     0.000       0.0
 B-return_date.date_relative      1.000     0.667     0.800       3.0
        I-fromloc.state_name      1.000     1.000     1.000       1.0
                B-state_code      1.000     1.000     1.000       1.0
    I-arrive_time.start_time      0.000     0.000     0.000       1.0
    I-arrive_date.day_number      0.000     0.000     0.000       0.0
                 B-meal_code      1.000     1.000     1.000       1.0
 I-depart_time.period_of_day      0.000     0.000     0.000       1.0
                  B-day_name      1.000     0.500     0.667       2.0
             B-period_of_day      1.000     0.500     0.667       4.0
        B-stoploc.state_code      0.000     0.000     0.000       0.0
    B-return_date.month_name      0.000     0.000     0.000       0.0
    B-return_date.day_number      0.000     0.000     0.000       0.0
    B-arrive_time.period_mod      0.000     0.000     0.000       0.0
                 I-meal_code      0.000     0.000     0.000       0.0
        B-toloc.country_name      1.000     1.000     1.000       1.0
                 B-days_code      1.000     1.000     1.000       1.0
 I-arrive_time.period_of_day      0.000     0.000     0.000       0.0
            I-today_relative      0.000     0.000     0.000       0.0
 B-return_time.period_of_day      0.000     0.000     0.000       0.0
                      B-time      0.000     0.000     0.000       0.0
           I-fare_basis_code      0.000     0.000     0.000       0.0
 I-arrive_time.time_relative      0.000     0.000     0.000       4.0
 I-depart_time.time_relative      0.000     0.000     0.000       1.0
            B-today_relative      0.000     0.000     0.000       0.0
                B-state_name      0.000     0.000     0.000       9.0
B-arrive_date.today_relative      0.000     0.000     0.000       0.0
    B-return_time.period_mod      0.000     0.000     0.000       0.0
                B-month_name      0.000     0.000     0.000       0.0
                B-day_number      0.000     0.000     0.000       0.0
 I-return_date.date_relative      1.000     0.667     0.800       3.0
I-return_date.today_relative      0.000     0.000     0.000       0.0
      B-stoploc.airport_name      0.000     0.000     0.000       0.0
             B-time_relative      0.000     0.000     0.000       0.0
                      I-time      0.000     0.000     0.000       0.0
    I-return_date.day_number      0.000     0.000     0.000       0.0
          I-meal_description      0.000     0.000     0.000       0.0
B-return_date.today_relative      0.000     0.000     0.000       0.0
      B-return_date.day_name      0.000     0.000     0.000       2.0

                   micro avg      0.933     0.934     0.933    3657.0
                   macro avg      0.611     0.601     0.593    3657.0
                weighted avg      0.934     0.934     0.930    3657.0

I0717 06:59:02.592113 140695199004544 interactiveshell.py:2882] Best Slot F1: 0.933
Reading ../data/atis.train.w-intent.iob
I0717 06:59:06.162539 140695199004544 interactiveshell.py:2882] Step 5650 | Loss: 0.0188 | Loss_intent: 0.0006 | Loss_slots: 0.0181 | Spent: 6.6 secs | LR: 0.000165
I0717 06:59:10.348567 140695199004544 interactiveshell.py:2882] Step 5700 | Loss: 0.0781 | Loss_intent: 0.0009 | Loss_slots: 0.0772 | Spent: 4.2 secs | LR: 0.000165
I0717 06:59:14.563547 140695199004544 interactiveshell.py:2882] Step 5750 | Loss: 0.1326 | Loss_intent: 0.0622 | Loss_slots: 0.0704 | Spent: 4.2 secs | LR: 0.000164
I0717 06:59:18.840049 140695199004544 interactiveshell.py:2882] Step 5800 | Loss: 0.0214 | Loss_intent: 0.0008 | Loss_slots: 0.0206 | Spent: 4.3 secs | LR: 0.000163
I0717 06:59:23.101279 140695199004544 interactiveshell.py:2882] Step 5850 | Loss: 0.1425 | Loss_intent: 0.0001 | Loss_slots: 0.1424 | Spent: 4.3 secs | LR: 0.000162
I0717 06:59:27.331644 140695199004544 interactiveshell.py:2882] Step 5900 | Loss: 0.0709 | Loss_intent: 0.0068 | Loss_slots: 0.0641 | Spent: 4.2 secs | LR: 0.000161
Reading ../data/atis.test.w-intent.iob
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.
  'precision', 'predicted', average, warn_for)
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1439: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples.
  'recall', 'true', average, warn_for)
I0717 06:59:31.645255 140695199004544 interactiveshell.py:2882] 
                                          precision    recall  f1-score   support

                             atis_flight      0.979     0.981     0.980       632
                            atis_airfare      0.941     1.000     0.970        48
                     atis_ground_service      1.000     1.000     1.000        36
                            atis_airline      1.000     1.000     1.000        38
                       atis_abbreviation      1.000     1.000     1.000        33
                           atis_aircraft      0.889     0.889     0.889         9
                        atis_flight_time      1.000     1.000     1.000         1
                           atis_quantity      0.375     1.000     0.545         3
                atis_flight#atis_airfare      0.833     0.417     0.556        12
                            atis_airport      1.000     0.944     0.971        18
                           atis_distance      1.000     1.000     1.000        10
                               atis_city      0.364     0.667     0.471         6
                        atis_ground_fare      1.000     0.714     0.833         7
                           atis_capacity      1.000     1.000     1.000        21
                          atis_flight_no      0.889     1.000     0.941         8
                               atis_meal      1.000     0.833     0.909         6
                        atis_restriction      0.000     0.000     0.000         0
             atis_airline#atis_flight_no      0.000     0.000     0.000         0
    atis_ground_service#atis_ground_fare      0.000     0.000     0.000         0
           atis_airfare#atis_flight_time      0.000     0.000     0.000         0
                           atis_cheapest      0.000     0.000     0.000         0
atis_aircraft#atis_flight#atis_flight_no      0.000     0.000     0.000         0

                               micro avg      0.965     0.971     0.968       888
                               macro avg      0.649     0.657     0.639       888
                            weighted avg      0.971     0.971     0.969       888

I0717 06:59:31.687366 140695199004544 interactiveshell.py:2882] 
                              precision    recall  f1-score   support

                           O      0.000     0.000     0.000       0.0
           B-toloc.city_name      0.970     0.985     0.977     716.0
         B-fromloc.city_name      0.963     0.987     0.975     704.0
           I-toloc.city_name      0.981     0.992     0.987     265.0
      B-depart_date.day_name      0.990     0.976     0.983     212.0
              B-airline_name      0.961     0.980     0.971     101.0
         I-fromloc.city_name      0.961     0.983     0.972     177.0
 B-depart_time.period_of_day      0.991     0.877     0.931     130.0
              I-airline_name      1.000     0.985     0.992      65.0
    B-depart_date.day_number      0.962     0.927     0.944      55.0
    B-depart_date.month_name      0.981     0.946     0.964      56.0
          B-depart_time.time      0.828     0.930     0.876      57.0
                B-round_trip      1.000     0.986     0.993      73.0
             B-cost_relative      1.000     0.973     0.986      37.0
                I-round_trip      1.000     1.000     1.000      71.0
                B-flight_mod      0.821     0.958     0.885      24.0
 B-depart_time.time_relative      0.969     0.954     0.961      65.0
          I-depart_time.time      0.847     0.962     0.901      52.0
         B-stoploc.city_name      0.947     0.900     0.923      20.0
                 B-city_name      0.775     0.544     0.639      57.0
                B-class_type      0.923     1.000     0.960      24.0
          B-arrive_time.time      0.786     0.971     0.868      34.0
 B-arrive_time.time_relative      0.833     0.968     0.896      31.0
                I-class_type      1.000     1.000     1.000      17.0
               B-flight_stop      0.913     1.000     0.955      21.0
          I-arrive_time.time      0.895     0.971     0.932      35.0
              B-airline_code      1.000     0.853     0.921      34.0
    I-depart_date.day_number      1.000     0.933     0.966      15.0
      I-fromloc.airport_name      0.455     1.000     0.625      15.0
      B-fromloc.airport_name      0.300     0.500     0.375      12.0
      B-arrive_date.day_name      0.647     1.000     0.786      11.0
          B-toloc.state_code      0.900     1.000     0.947      18.0
B-depart_date.today_relative      1.000     0.889     0.941       9.0
             B-flight_number      0.500     0.818     0.621      11.0
 B-depart_date.date_relative      0.850     1.000     0.919      17.0
          B-toloc.state_name      0.900     0.964     0.931      28.0
           B-fare_basis_code      0.941     0.941     0.941      17.0
               B-flight_time      1.000     1.000     1.000       1.0
                        B-or      1.000     1.000     1.000       3.0
 B-arrive_time.period_of_day      0.750     1.000     0.857       6.0
          B-meal_description      1.000     0.900     0.947      10.0
             I-cost_relative      1.000     0.667     0.800       3.0
              I-airport_name      0.917     0.379     0.537      29.0
               B-fare_amount      0.400     1.000     0.571       2.0
               I-fare_amount      1.000     1.000     1.000       2.0
                 I-city_name      1.000     0.533     0.696      30.0
        I-toloc.airport_name      0.750     1.000     0.857       3.0
            B-transport_type      0.875     0.700     0.778      10.0
    B-arrive_date.month_name      0.625     0.833     0.714       6.0
    B-arrive_date.day_number      0.556     0.833     0.667       6.0
         I-stoploc.city_name      0.714     1.000     0.833      10.0
                      B-meal      1.000     1.000     1.000      16.0
        B-fromloc.state_code      1.000     0.913     0.955      23.0
    B-depart_time.period_mod      0.833     1.000     0.909       5.0
                   B-connect      1.000     1.000     1.000       6.0
               B-flight_days      1.000     1.000     1.000      10.0
        B-toloc.airport_name      0.333     0.333     0.333       3.0
        B-fromloc.state_name      0.889     0.941     0.914      17.0
              B-airport_name      0.778     0.333     0.467      21.0
                   B-economy      1.000     1.000     1.000       6.0
               I-flight_time      1.000     1.000     1.000       1.0
             B-aircraft_code      0.963     0.788     0.867      33.0
                       B-mod      0.000     0.000     0.000       2.0
              B-airport_code      0.000     0.000     0.000       9.0
    B-depart_time.start_time      0.500     0.333     0.400       3.0
      B-depart_time.end_time      0.667     0.667     0.667       3.0
          B-depart_date.year      1.000     0.667     0.800       3.0
            I-transport_type      0.000     0.000     0.000       1.0
          B-restriction_code      0.364     1.000     0.533       4.0
    B-arrive_time.start_time      0.667     0.750     0.706       8.0
        B-toloc.airport_code      1.000     0.750     0.857       4.0
      B-arrive_time.end_time      0.833     0.625     0.714       8.0
      I-arrive_time.end_time      0.750     0.750     0.750       8.0
      I-depart_time.end_time      1.000     0.333     0.500       3.0
               I-flight_stop      0.000     0.000     0.000       0.0
      B-fromloc.airport_code      1.000     1.000     1.000       5.0
          I-restriction_code      0.750     1.000     0.857       3.0
    I-depart_time.start_time      0.500     1.000     0.667       1.0
          I-toloc.state_name      0.500     1.000     0.667       1.0
I-depart_date.today_relative      0.000     0.000     0.000       0.0
 B-arrive_date.date_relative      1.000     0.500     0.667       2.0
                I-flight_mod      0.000     0.000     0.000       6.0
                   I-economy      0.000     0.000     0.000       0.0
 B-return_date.date_relative      1.000     0.667     0.800       3.0
        I-fromloc.state_name      0.000     0.000     0.000       1.0
                B-state_code      1.000     1.000     1.000       1.0
    I-arrive_time.start_time      0.000     0.000     0.000       1.0
    I-arrive_date.day_number      0.000     0.000     0.000       0.0
                 B-meal_code      1.000     1.000     1.000       1.0
 I-depart_time.period_of_day      0.000     0.000     0.000       1.0
                  B-day_name      1.000     0.500     0.667       2.0
             B-period_of_day      1.000     0.500     0.667       4.0
        B-stoploc.state_code      0.000     0.000     0.000       0.0
    B-return_date.month_name      0.000     0.000     0.000       0.0
    B-return_date.day_number      0.000     0.000     0.000       0.0
    B-arrive_time.period_mod      0.000     0.000     0.000       0.0
                 I-meal_code      0.000     0.000     0.000       0.0
        B-toloc.country_name      1.000     1.000     1.000       1.0
                 B-days_code      1.000     1.000     1.000       1.0
 I-arrive_time.period_of_day      0.000     0.000     0.000       0.0
            I-today_relative      0.000     0.000     0.000       0.0
 B-return_time.period_of_day      0.000     0.000     0.000       0.0
                      B-time      0.000     0.000     0.000       0.0
           I-fare_basis_code      0.000     0.000     0.000       0.0
 I-arrive_time.time_relative      0.000     0.000     0.000       4.0
 I-depart_time.time_relative      0.000     0.000     0.000       1.0
            B-today_relative      0.000     0.000     0.000       0.0
                B-state_name      0.000     0.000     0.000       9.0
B-arrive_date.today_relative      0.000     0.000     0.000       0.0
    B-return_time.period_mod      0.000     0.000     0.000       0.0
                B-month_name      0.000     0.000     0.000       0.0
                B-day_number      0.000     0.000     0.000       0.0
 I-return_date.date_relative      1.000     0.667     0.800       3.0
I-return_date.today_relative      0.000     0.000     0.000       0.0
      B-stoploc.airport_name      0.000     0.000     0.000       0.0
             B-time_relative      0.000     0.000     0.000       0.0
                      I-time      0.000     0.000     0.000       0.0
    I-return_date.day_number      0.000     0.000     0.000       0.0
          I-meal_description      0.000     0.000     0.000       0.0
B-return_date.today_relative      0.000     0.000     0.000       0.0
      B-return_date.day_name      0.000     0.000     0.000       2.0

                   micro avg      0.933     0.935     0.934    3657.0
                   macro avg      0.593     0.591     0.577    3657.0
                weighted avg      0.935     0.935     0.930    3657.0

I0717 06:59:31.688281 140695199004544 interactiveshell.py:2882] Best Slot F1: 0.934
Reading ../data/atis.train.w-intent.iob
I0717 06:59:34.523195 140695199004544 interactiveshell.py:2882] Step 5950 | Loss: 0.0249 | Loss_intent: 0.0000 | Loss_slots: 0.0249 | Spent: 7.2 secs | LR: 0.000160
I0717 06:59:39.182661 140695199004544 interactiveshell.py:2882] Step 6000 | Loss: 0.0631 | Loss_intent: 0.0002 | Loss_slots: 0.0629 | Spent: 4.7 secs | LR: 0.000159
I0717 06:59:43.365317 140695199004544 interactiveshell.py:2882] Step 6050 | Loss: 0.0359 | Loss_intent: 0.0009 | Loss_slots: 0.0350 | Spent: 4.2 secs | LR: 0.000159
I0717 06:59:47.544400 140695199004544 interactiveshell.py:2882] Step 6100 | Loss: 0.0306 | Loss_intent: 0.0002 | Loss_slots: 0.0304 | Spent: 4.2 secs | LR: 0.000158
I0717 06:59:51.722891 140695199004544 interactiveshell.py:2882] Step 6150 | Loss: 0.0323 | Loss_intent: 0.0002 | Loss_slots: 0.0321 | Spent: 4.2 secs | LR: 0.000157
I0717 06:59:55.905741 140695199004544 interactiveshell.py:2882] Step 6200 | Loss: 0.0427 | Loss_intent: 0.0031 | Loss_slots: 0.0396 | Spent: 4.2 secs | LR: 0.000156
Reading ../data/atis.test.w-intent.iob
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.
  'precision', 'predicted', average, warn_for)
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1439: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples.
  'recall', 'true', average, warn_for)
I0717 07:00:00.888966 140695199004544 interactiveshell.py:2882] 
                                          precision    recall  f1-score   support

                             atis_flight      0.986     0.983     0.984       632
                            atis_airfare      0.906     1.000     0.950        48
                     atis_ground_service      1.000     1.000     1.000        36
                            atis_airline      0.974     1.000     0.987        38
                       atis_abbreviation      1.000     1.000     1.000        33
                           atis_aircraft      0.800     0.889     0.842         9
                        atis_flight_time      0.500     1.000     0.667         1
                           atis_quantity      0.333     0.667     0.444         3
                atis_flight#atis_airfare      0.875     0.583     0.700        12
                            atis_airport      1.000     0.833     0.909        18
                           atis_distance      1.000     1.000     1.000        10
                               atis_city      0.444     0.667     0.533         6
                        atis_ground_fare      1.000     0.714     0.833         7
                           atis_capacity      0.913     1.000     0.955        21
                          atis_flight_no      0.889     1.000     0.941         8
                               atis_meal      1.000     0.833     0.909         6
                        atis_restriction      0.000     0.000     0.000         0
             atis_airline#atis_flight_no      0.000     0.000     0.000         0
    atis_ground_service#atis_ground_fare      0.000     0.000     0.000         0
           atis_airfare#atis_flight_time      0.000     0.000     0.000         0
                           atis_cheapest      0.000     0.000     0.000         0
atis_aircraft#atis_flight#atis_flight_no      0.000     0.000     0.000         0

                               micro avg      0.965     0.971     0.968       888
                               macro avg      0.619     0.644     0.621       888
                            weighted avg      0.970     0.971     0.969       888

I0717 07:00:00.929362 140695199004544 interactiveshell.py:2882] 
                              precision    recall  f1-score   support

                           O      0.000     0.000     0.000       0.0
           B-toloc.city_name      0.965     0.990     0.977     716.0
         B-fromloc.city_name      0.979     0.983     0.981     704.0
           I-toloc.city_name      0.949     0.992     0.970     265.0
      B-depart_date.day_name      0.990     0.981     0.986     212.0
              B-airline_name      0.943     0.980     0.961     101.0
         I-fromloc.city_name      0.961     0.983     0.972     177.0
 B-depart_time.period_of_day      0.966     0.869     0.915     130.0
              I-airline_name      1.000     0.985     0.992      65.0
    B-depart_date.day_number      0.962     0.927     0.944      55.0
    B-depart_date.month_name      0.981     0.946     0.964      56.0
          B-depart_time.time      0.848     0.982     0.911      57.0
                B-round_trip      1.000     0.986     0.993      73.0
             B-cost_relative      1.000     0.973     0.986      37.0
                I-round_trip      1.000     1.000     1.000      71.0
                B-flight_mod      0.958     0.958     0.958      24.0
 B-depart_time.time_relative      0.968     0.938     0.953      65.0
          I-depart_time.time      0.862     0.962     0.909      52.0
         B-stoploc.city_name      1.000     0.900     0.947      20.0
                 B-city_name      0.805     0.579     0.673      57.0
                B-class_type      0.923     1.000     0.960      24.0
          B-arrive_time.time      0.868     0.971     0.917      34.0
 B-arrive_time.time_relative      0.857     0.968     0.909      31.0
                I-class_type      1.000     1.000     1.000      17.0
               B-flight_stop      1.000     1.000     1.000      21.0
          I-arrive_time.time      0.895     0.971     0.932      35.0
              B-airline_code      1.000     0.853     0.921      34.0
    I-depart_date.day_number      1.000     1.000     1.000      15.0
      I-fromloc.airport_name      0.469     1.000     0.638      15.0
      B-fromloc.airport_name      0.375     0.750     0.500      12.0
      B-arrive_date.day_name      0.688     1.000     0.815      11.0
          B-toloc.state_code      0.947     1.000     0.973      18.0
B-depart_date.today_relative      1.000     0.889     0.941       9.0
             B-flight_number      0.556     0.909     0.690      11.0
 B-depart_date.date_relative      0.810     1.000     0.895      17.0
          B-toloc.state_name      0.931     0.964     0.947      28.0
           B-fare_basis_code      0.944     1.000     0.971      17.0
               B-flight_time      1.000     1.000     1.000       1.0
                        B-or      1.000     1.000     1.000       3.0
 B-arrive_time.period_of_day      0.462     1.000     0.632       6.0
          B-meal_description      1.000     0.900     0.947      10.0
             I-cost_relative      1.000     0.667     0.800       3.0
              I-airport_name      0.857     0.414     0.558      29.0
               B-fare_amount      0.500     1.000     0.667       2.0
               I-fare_amount      1.000     1.000     1.000       2.0
                 I-city_name      1.000     0.467     0.636      30.0
        I-toloc.airport_name      1.000     1.000     1.000       3.0
            B-transport_type      0.909     1.000     0.952      10.0
    B-arrive_date.month_name      0.625     0.833     0.714       6.0
    B-arrive_date.day_number      0.625     0.833     0.714       6.0
         I-stoploc.city_name      0.700     0.700     0.700      10.0
                      B-meal      1.000     1.000     1.000      16.0
        B-fromloc.state_code      1.000     0.957     0.978      23.0
    B-depart_time.period_mod      0.750     0.600     0.667       5.0
                   B-connect      1.000     1.000     1.000       6.0
               B-flight_days      1.000     1.000     1.000      10.0
        B-toloc.airport_name      1.000     0.333     0.500       3.0
        B-fromloc.state_name      0.773     1.000     0.872      17.0
              B-airport_name      0.700     0.333     0.452      21.0
                   B-economy      1.000     1.000     1.000       6.0
               I-flight_time      1.000     1.000     1.000       1.0
             B-aircraft_code      0.833     0.758     0.794      33.0
                       B-mod      0.000     0.000     0.000       2.0
              B-airport_code      0.000     0.000     0.000       9.0
    B-depart_time.start_time      0.500     0.333     0.400       3.0
      B-depart_time.end_time      1.000     0.667     0.800       3.0
          B-depart_date.year      0.500     0.667     0.571       3.0
            I-transport_type      0.000     0.000     0.000       1.0
          B-restriction_code      1.000     0.750     0.857       4.0
    B-arrive_time.start_time      0.833     0.625     0.714       8.0
        B-toloc.airport_code      1.000     0.750     0.857       4.0
      B-arrive_time.end_time      0.833     0.625     0.714       8.0
      I-arrive_time.end_time      0.750     0.750     0.750       8.0
      I-depart_time.end_time      1.000     0.667     0.800       3.0
               I-flight_stop      0.000     0.000     0.000       0.0
      B-fromloc.airport_code      1.000     1.000     1.000       5.0
          I-restriction_code      0.273     1.000     0.429       3.0
    I-depart_time.start_time      1.000     1.000     1.000       1.0
          I-toloc.state_name      1.000     1.000     1.000       1.0
I-depart_date.today_relative      0.000     0.000     0.000       0.0
 B-arrive_date.date_relative      0.000     0.000     0.000       2.0
                I-flight_mod      0.000     0.000     0.000       6.0
                   I-economy      0.000     0.000     0.000       0.0
 B-return_date.date_relative      1.000     0.333     0.500       3.0
        I-fromloc.state_name      1.000     1.000     1.000       1.0
                B-state_code      1.000     1.000     1.000       1.0
    I-arrive_time.start_time      0.000     0.000     0.000       1.0
    I-arrive_date.day_number      0.000     0.000     0.000       0.0
                 B-meal_code      1.000     1.000     1.000       1.0
 I-depart_time.period_of_day      0.000     0.000     0.000       1.0
                  B-day_name      1.000     0.500     0.667       2.0
             B-period_of_day      1.000     0.500     0.667       4.0
        B-stoploc.state_code      0.000     0.000     0.000       0.0
    B-return_date.month_name      0.000     0.000     0.000       0.0
    B-return_date.day_number      0.000     0.000     0.000       0.0
    B-arrive_time.period_mod      0.000     0.000     0.000       0.0
                 I-meal_code      0.000     0.000     0.000       0.0
        B-toloc.country_name      1.000     1.000     1.000       1.0
                 B-days_code      1.000     1.000     1.000       1.0
 I-arrive_time.period_of_day      0.000     0.000     0.000       0.0
            I-today_relative      0.000     0.000     0.000       0.0
 B-return_time.period_of_day      0.000     0.000     0.000       0.0
                      B-time      0.000     0.000     0.000       0.0
           I-fare_basis_code      0.000     0.000     0.000       0.0
 I-arrive_time.time_relative      1.000     0.500     0.667       4.0
 I-depart_time.time_relative      0.000     0.000     0.000       1.0
            B-today_relative      0.000     0.000     0.000       0.0
                B-state_name      0.000     0.000     0.000       9.0
B-arrive_date.today_relative      0.000     0.000     0.000       0.0
    B-return_time.period_mod      0.000     0.000     0.000       0.0
                B-month_name      0.000     0.000     0.000       0.0
                B-day_number      0.000     0.000     0.000       0.0
 I-return_date.date_relative      0.667     0.667     0.667       3.0
I-return_date.today_relative      0.000     0.000     0.000       0.0
      B-stoploc.airport_name      0.000     0.000     0.000       0.0
             B-time_relative      0.000     0.000     0.000       0.0
                      I-time      0.000     0.000     0.000       0.0
    I-return_date.day_number      0.000     0.000     0.000       0.0
          I-meal_description      0.000     0.000     0.000       0.0
B-return_date.today_relative      0.000     0.000     0.000       0.0
      B-return_date.day_name      0.000     0.000     0.000       2.0

                   micro avg      0.936     0.937     0.937    3657.0
                   macro avg      0.616     0.598     0.592    3657.0
                weighted avg      0.937     0.937     0.933    3657.0

I0717 07:00:00.930122 140695199004544 interactiveshell.py:2882] Best Slot F1: 0.937
Reading ../data/atis.train.w-intent.iob
I0717 07:00:02.510297 140695199004544 interactiveshell.py:2882] Step 6250 | Loss: 0.0811 | Loss_intent: 0.0007 | Loss_slots: 0.0804 | Spent: 6.6 secs | LR: 0.000155
I0717 07:00:06.680770 140695199004544 interactiveshell.py:2882] Step 6300 | Loss: 0.0109 | Loss_intent: 0.0002 | Loss_slots: 0.0106 | Spent: 4.2 secs | LR: 0.000154
I0717 07:00:10.881352 140695199004544 interactiveshell.py:2882] Step 6350 | Loss: 0.0767 | Loss_intent: 0.0244 | Loss_slots: 0.0523 | Spent: 4.2 secs | LR: 0.000154
I0717 07:00:15.074076 140695199004544 interactiveshell.py:2882] Step 6400 | Loss: 0.0934 | Loss_intent: 0.0548 | Loss_slots: 0.0387 | Spent: 4.2 secs | LR: 0.000153
I0717 07:00:19.269033 140695199004544 interactiveshell.py:2882] Step 6450 | Loss: 0.0463 | Loss_intent: 0.0358 | Loss_slots: 0.0105 | Spent: 4.2 secs | LR: 0.000152
I0717 07:00:23.470960 140695199004544 interactiveshell.py:2882] Step 6500 | Loss: 0.0151 | Loss_intent: 0.0000 | Loss_slots: 0.0151 | Spent: 4.2 secs | LR: 0.000151
I0717 07:00:27.667104 140695199004544 interactiveshell.py:2882] Step 6550 | Loss: 0.0238 | Loss_intent: 0.0061 | Loss_slots: 0.0177 | Spent: 4.2 secs | LR: 0.000150
Reading ../data/atis.test.w-intent.iob
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.
  'precision', 'predicted', average, warn_for)
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1439: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples.
  'recall', 'true', average, warn_for)
I0717 07:00:29.477958 140695199004544 interactiveshell.py:2882] 
                                          precision    recall  f1-score   support

                             atis_flight      0.976     0.979     0.978       632
                            atis_airfare      0.980     1.000     0.990        48
                     atis_ground_service      1.000     1.000     1.000        36
                            atis_airline      1.000     0.974     0.987        38
                       atis_abbreviation      0.941     0.970     0.955        33
                           atis_aircraft      0.889     0.889     0.889         9
                        atis_flight_time      1.000     1.000     1.000         1
                           atis_quantity      0.429     1.000     0.600         3
                atis_flight#atis_airfare      1.000     0.083     0.154        12
                            atis_airport      1.000     0.778     0.875        18
                           atis_distance      1.000     1.000     1.000        10
                               atis_city      0.333     1.000     0.500         6
                        atis_ground_fare      1.000     0.857     0.923         7
                           atis_capacity      1.000     1.000     1.000        21
                          atis_flight_no      0.889     1.000     0.941         8
                               atis_meal      1.000     0.833     0.909         6
                        atis_restriction      0.000     0.000     0.000         0
             atis_airline#atis_flight_no      0.000     0.000     0.000         0
    atis_ground_service#atis_ground_fare      0.000     0.000     0.000         0
           atis_airfare#atis_flight_time      0.000     0.000     0.000         0
                           atis_cheapest      0.000     0.000     0.000         0
atis_aircraft#atis_flight#atis_flight_no      0.000     0.000     0.000         0

                               micro avg      0.957     0.963     0.960       888
                               macro avg      0.656     0.653     0.623       888
                            weighted avg      0.971     0.963     0.960       888

I0717 07:00:29.518279 140695199004544 interactiveshell.py:2882] 
                              precision    recall  f1-score   support

                           O      0.000     0.000     0.000       0.0
           B-toloc.city_name      0.975     0.987     0.981     716.0
         B-fromloc.city_name      0.982     0.984     0.983     704.0
           I-toloc.city_name      0.970     0.985     0.978     265.0
      B-depart_date.day_name      0.986     0.981     0.983     212.0
              B-airline_name      0.952     0.980     0.966     101.0
         I-fromloc.city_name      0.972     0.977     0.975     177.0
 B-depart_time.period_of_day      0.983     0.877     0.927     130.0
              I-airline_name      1.000     1.000     1.000      65.0
    B-depart_date.day_number      0.964     0.964     0.964      55.0
    B-depart_date.month_name      0.982     0.964     0.973      56.0
          B-depart_time.time      0.838     1.000     0.912      57.0
                B-round_trip      1.000     0.986     0.993      73.0
             B-cost_relative      1.000     0.973     0.986      37.0
                I-round_trip      1.000     1.000     1.000      71.0
                B-flight_mod      0.957     0.917     0.936      24.0
 B-depart_time.time_relative      0.970     0.985     0.977      65.0
          I-depart_time.time      0.864     0.981     0.919      52.0
         B-stoploc.city_name      0.947     0.900     0.923      20.0
                 B-city_name      0.720     0.632     0.673      57.0
                B-class_type      0.923     1.000     0.960      24.0
          B-arrive_time.time      0.846     0.971     0.904      34.0
 B-arrive_time.time_relative      0.938     0.968     0.952      31.0
                I-class_type      1.000     1.000     1.000      17.0
               B-flight_stop      1.000     1.000     1.000      21.0
          I-arrive_time.time      0.895     0.971     0.932      35.0
              B-airline_code      1.000     0.912     0.954      34.0
    I-depart_date.day_number      1.000     1.000     1.000      15.0
      I-fromloc.airport_name      0.455     1.000     0.625      15.0
      B-fromloc.airport_name      0.310     0.750     0.439      12.0
      B-arrive_date.day_name      0.667     0.909     0.769      11.0
          B-toloc.state_code      0.947     1.000     0.973      18.0
B-depart_date.today_relative      1.000     0.889     0.941       9.0
             B-flight_number      0.600     0.818     0.692      11.0
 B-depart_date.date_relative      0.850     1.000     0.919      17.0
          B-toloc.state_name      0.926     0.893     0.909      28.0
           B-fare_basis_code      0.941     0.941     0.941      17.0
               B-flight_time      1.000     1.000     1.000       1.0
                        B-or      1.000     1.000     1.000       3.0
 B-arrive_time.period_of_day      0.857     1.000     0.923       6.0
          B-meal_description      1.000     0.900     0.947      10.0
             I-cost_relative      1.000     0.667     0.800       3.0
              I-airport_name      0.846     0.379     0.524      29.0
               B-fare_amount      0.400     1.000     0.571       2.0
               I-fare_amount      1.000     1.000     1.000       2.0
                 I-city_name      0.870     0.667     0.755      30.0
        I-toloc.airport_name      1.000     1.000     1.000       3.0
            B-transport_type      1.000     0.700     0.824      10.0
    B-arrive_date.month_name      0.714     0.833     0.769       6.0
    B-arrive_date.day_number      0.833     0.833     0.833       6.0
         I-stoploc.city_name      0.727     0.800     0.762      10.0
                      B-meal      1.000     1.000     1.000      16.0
        B-fromloc.state_code      1.000     0.957     0.978      23.0
    B-depart_time.period_mod      1.000     0.800     0.889       5.0
                   B-connect      1.000     1.000     1.000       6.0
               B-flight_days      1.000     1.000     1.000      10.0
        B-toloc.airport_name      1.000     0.333     0.500       3.0
        B-fromloc.state_name      0.708     1.000     0.829      17.0
              B-airport_name      1.000     0.238     0.385      21.0
                   B-economy      1.000     1.000     1.000       6.0
               I-flight_time      1.000     1.000     1.000       1.0
             B-aircraft_code      0.889     0.727     0.800      33.0
                       B-mod      0.000     0.000     0.000       2.0
              B-airport_code      0.500     0.444     0.471       9.0
    B-depart_time.start_time      0.500     0.333     0.400       3.0
      B-depart_time.end_time      1.000     0.667     0.800       3.0
          B-depart_date.year      1.000     0.667     0.800       3.0
            I-transport_type      0.200     1.000     0.333       1.0
          B-restriction_code      0.500     0.500     0.500       4.0
    B-arrive_time.start_time      0.667     0.750     0.706       8.0
        B-toloc.airport_code      1.000     0.750     0.857       4.0
      B-arrive_time.end_time      0.750     0.375     0.500       8.0
      I-arrive_time.end_time      0.833     0.625     0.714       8.0
      I-depart_time.end_time      1.000     0.333     0.500       3.0
               I-flight_stop      0.000     0.000     0.000       0.0
      B-fromloc.airport_code      1.000     1.000     1.000       5.0
          I-restriction_code      0.750     1.000     0.857       3.0
    I-depart_time.start_time      0.500     1.000     0.667       1.0
          I-toloc.state_name      1.000     1.000     1.000       1.0
I-depart_date.today_relative      0.000     0.000     0.000       0.0
 B-arrive_date.date_relative      0.000     0.000     0.000       2.0
                I-flight_mod      1.000     0.167     0.286       6.0
                   I-economy      0.000     0.000     0.000       0.0
 B-return_date.date_relative      1.000     0.333     0.500       3.0
        I-fromloc.state_name      1.000     1.000     1.000       1.0
                B-state_code      1.000     1.000     1.000       1.0
    I-arrive_time.start_time      1.000     1.000     1.000       1.0
    I-arrive_date.day_number      0.000     0.000     0.000       0.0
                 B-meal_code      1.000     1.000     1.000       1.0
 I-depart_time.period_of_day      0.000     0.000     0.000       1.0
                  B-day_name      1.000     0.500     0.667       2.0
             B-period_of_day      1.000     0.750     0.857       4.0
        B-stoploc.state_code      0.000     0.000     0.000       0.0
    B-return_date.month_name      0.000     0.000     0.000       0.0
    B-return_date.day_number      0.000     0.000     0.000       0.0
    B-arrive_time.period_mod      0.000     0.000     0.000       0.0
                 I-meal_code      0.000     0.000     0.000       0.0
        B-toloc.country_name      1.000     1.000     1.000       1.0
                 B-days_code      0.333     1.000     0.500       1.0
 I-arrive_time.period_of_day      0.000     0.000     0.000       0.0
            I-today_relative      0.000     0.000     0.000       0.0
 B-return_time.period_of_day      0.000     0.000     0.000       0.0
                      B-time      0.000     0.000     0.000       0.0
           I-fare_basis_code      0.000     0.000     0.000       0.0
 I-arrive_time.time_relative      1.000     0.500     0.667       4.0
 I-depart_time.time_relative      0.000     0.000     0.000       1.0
            B-today_relative      0.000     0.000     0.000       0.0
                B-state_name      0.000     0.000     0.000       9.0
B-arrive_date.today_relative      0.000     0.000     0.000       0.0
    B-return_time.period_mod      0.000     0.000     0.000       0.0
                B-month_name      0.000     0.000     0.000       0.0
                B-day_number      0.000     0.000     0.000       0.0
 I-return_date.date_relative      0.667     0.667     0.667       3.0
I-return_date.today_relative      0.000     0.000     0.000       0.0
      B-stoploc.airport_name      0.000     0.000     0.000       0.0
             B-time_relative      0.000     0.000     0.000       0.0
                      I-time      0.000     0.000     0.000       0.0
    I-return_date.day_number      0.000     0.000     0.000       0.0
          I-meal_description      0.000     0.000     0.000       0.0
B-return_date.today_relative      0.000     0.000     0.000       0.0
      B-return_date.day_name      0.000     0.000     0.000       2.0

                   micro avg      0.939     0.940     0.939    3657.0
                   macro avg      0.640     0.614     0.606    3657.0
                weighted avg      0.946     0.940     0.938    3657.0

I0717 07:00:29.519020 140695199004544 interactiveshell.py:2882] Best Slot F1: 0.939
Reading ../data/atis.train.w-intent.iob
I0717 07:00:34.296253 140695199004544 interactiveshell.py:2882] Step 6600 | Loss: 0.0096 | Loss_intent: 0.0010 | Loss_slots: 0.0086 | Spent: 6.6 secs | LR: 0.000150
I0717 07:00:38.541229 140695199004544 interactiveshell.py:2882] Step 6650 | Loss: 0.0311 | Loss_intent: 0.0001 | Loss_slots: 0.0309 | Spent: 4.2 secs | LR: 0.000149
I0717 07:00:42.788902 140695199004544 interactiveshell.py:2882] Step 6700 | Loss: 0.0045 | Loss_intent: 0.0000 | Loss_slots: 0.0045 | Spent: 4.2 secs | LR: 0.000148
I0717 07:00:46.981677 140695199004544 interactiveshell.py:2882] Step 6750 | Loss: 0.0334 | Loss_intent: 0.0001 | Loss_slots: 0.0332 | Spent: 4.2 secs | LR: 0.000147
I0717 07:00:51.181647 140695199004544 interactiveshell.py:2882] Step 6800 | Loss: 0.0336 | Loss_intent: 0.0242 | Loss_slots: 0.0094 | Spent: 4.2 secs | LR: 0.000147
I0717 07:00:55.351765 140695199004544 interactiveshell.py:2882] Step 6850 | Loss: 0.0726 | Loss_intent: 0.0003 | Loss_slots: 0.0723 | Spent: 4.2 secs | LR: 0.000146
Reading ../data/atis.test.w-intent.iob
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.
  'precision', 'predicted', average, warn_for)
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1439: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples.
  'recall', 'true', average, warn_for)
I0717 07:00:58.165798 140695199004544 interactiveshell.py:2882] 
                                          precision    recall  f1-score   support

                             atis_flight      0.978     0.975     0.976       632
                            atis_airfare      0.941     1.000     0.970        48
                     atis_ground_service      1.000     1.000     1.000        36
                            atis_airline      0.974     1.000     0.987        38
                       atis_abbreviation      1.000     0.970     0.985        33
                           atis_aircraft      0.529     1.000     0.692         9
                        atis_flight_time      1.000     1.000     1.000         1
                           atis_quantity      0.333     1.000     0.500         3
                atis_flight#atis_airfare      0.833     0.417     0.556        12
                            atis_airport      0.941     0.889     0.914        18
                           atis_distance      1.000     1.000     1.000        10
                               atis_city      0.333     0.667     0.444         6
                        atis_ground_fare      1.000     0.714     0.833         7
                           atis_capacity      1.000     0.571     0.727        21
                          atis_flight_no      0.889     1.000     0.941         8
                               atis_meal      0.833     0.833     0.833         6
                        atis_restriction      0.000     0.000     0.000         0
             atis_airline#atis_flight_no      0.000     0.000     0.000         0
    atis_ground_service#atis_ground_fare      0.000     0.000     0.000         0
           atis_airfare#atis_flight_time      0.000     0.000     0.000         0
                           atis_cheapest      0.000     0.000     0.000         0
atis_aircraft#atis_flight#atis_flight_no      0.000     0.000     0.000         0

                               micro avg      0.950     0.955     0.952       888
                               macro avg      0.618     0.638     0.607       888
                            weighted avg      0.963     0.955     0.955       888

I0717 07:00:58.205767 140695199004544 interactiveshell.py:2882] 
                              precision    recall  f1-score   support

                           O      0.000     0.000     0.000       0.0
           B-toloc.city_name      0.967     0.996     0.981     716.0
         B-fromloc.city_name      0.975     0.984     0.980     704.0
           I-toloc.city_name      0.956     0.992     0.974     265.0
      B-depart_date.day_name      0.990     0.976     0.983     212.0
              B-airline_name      0.952     0.990     0.971     101.0
         I-fromloc.city_name      0.972     0.977     0.975     177.0
 B-depart_time.period_of_day      0.991     0.846     0.913     130.0
              I-airline_name      1.000     1.000     1.000      65.0
    B-depart_date.day_number      0.962     0.909     0.935      55.0
    B-depart_date.month_name      0.981     0.946     0.964      56.0
          B-depart_time.time      0.797     0.965     0.873      57.0
                B-round_trip      1.000     0.986     0.993      73.0
             B-cost_relative      1.000     0.973     0.986      37.0
                I-round_trip      1.000     1.000     1.000      71.0
                B-flight_mod      1.000     0.958     0.979      24.0
 B-depart_time.time_relative      0.968     0.938     0.953      65.0
          I-depart_time.time      0.891     0.942     0.916      52.0
         B-stoploc.city_name      1.000     0.900     0.947      20.0
                 B-city_name      0.786     0.579     0.667      57.0
                B-class_type      0.923     1.000     0.960      24.0
          B-arrive_time.time      0.917     0.971     0.943      34.0
 B-arrive_time.time_relative      0.811     0.968     0.882      31.0
                I-class_type      1.000     1.000     1.000      17.0
               B-flight_stop      1.000     1.000     1.000      21.0
          I-arrive_time.time      0.895     0.971     0.932      35.0
              B-airline_code      0.967     0.853     0.906      34.0
    I-depart_date.day_number      1.000     1.000     1.000      15.0
      I-fromloc.airport_name      0.441     1.000     0.612      15.0
      B-fromloc.airport_name      0.304     0.583     0.400      12.0
      B-arrive_date.day_name      0.688     1.000     0.815      11.0
          B-toloc.state_code      0.947     1.000     0.973      18.0
B-depart_date.today_relative      1.000     0.889     0.941       9.0
             B-flight_number      0.400     0.727     0.516      11.0
 B-depart_date.date_relative      0.850     1.000     0.919      17.0
          B-toloc.state_name      0.929     0.929     0.929      28.0
           B-fare_basis_code      0.917     0.647     0.759      17.0
               B-flight_time      1.000     1.000     1.000       1.0
                        B-or      1.000     1.000     1.000       3.0
 B-arrive_time.period_of_day      0.667     1.000     0.800       6.0
          B-meal_description      1.000     0.900     0.947      10.0
             I-cost_relative      1.000     0.667     0.800       3.0
              I-airport_name      0.846     0.379     0.524      29.0
               B-fare_amount      0.333     1.000     0.500       2.0
               I-fare_amount      1.000     1.000     1.000       2.0
                 I-city_name      1.000     0.467     0.636      30.0
        I-toloc.airport_name      1.000     1.000     1.000       3.0
            B-transport_type      0.889     0.800     0.842      10.0
    B-arrive_date.month_name      0.625     0.833     0.714       6.0
    B-arrive_date.day_number      0.625     0.833     0.714       6.0
         I-stoploc.city_name      0.750     0.900     0.818      10.0
                      B-meal      1.000     1.000     1.000      16.0
        B-fromloc.state_code      1.000     0.957     0.978      23.0
    B-depart_time.period_mod      0.500     1.000     0.667       5.0
                   B-connect      1.000     1.000     1.000       6.0
               B-flight_days      1.000     1.000     1.000      10.0
        B-toloc.airport_name      1.000     0.333     0.500       3.0
        B-fromloc.state_name      0.727     0.941     0.821      17.0
              B-airport_name      0.778     0.333     0.467      21.0
                   B-economy      1.000     1.000     1.000       6.0
               I-flight_time      1.000     1.000     1.000       1.0
             B-aircraft_code      0.923     0.727     0.814      33.0
                       B-mod      0.000     0.000     0.000       2.0
              B-airport_code      0.000     0.000     0.000       9.0
    B-depart_time.start_time      0.333     0.333     0.333       3.0
      B-depart_time.end_time      0.400     0.667     0.500       3.0
          B-depart_date.year      1.000     0.667     0.800       3.0
            I-transport_type      0.000     0.000     0.000       1.0
          B-restriction_code      0.571     1.000     0.727       4.0
    B-arrive_time.start_time      0.875     0.875     0.875       8.0
        B-toloc.airport_code      1.000     0.500     0.667       4.0
      B-arrive_time.end_time      0.875     0.875     0.875       8.0
      I-arrive_time.end_time      0.875     0.875     0.875       8.0
      I-depart_time.end_time      1.000     0.333     0.500       3.0
               I-flight_stop      0.000     0.000     0.000       0.0
      B-fromloc.airport_code      1.000     1.000     1.000       5.0
          I-restriction_code      0.375     1.000     0.545       3.0
    I-depart_time.start_time      0.500     1.000     0.667       1.0
          I-toloc.state_name      0.500     1.000     0.667       1.0
I-depart_date.today_relative      0.000     0.000     0.000       0.0
 B-arrive_date.date_relative      0.000     0.000     0.000       2.0
                I-flight_mod      1.000     0.167     0.286       6.0
                   I-economy      0.000     0.000     0.000       0.0
 B-return_date.date_relative      0.500     0.333     0.400       3.0
        I-fromloc.state_name      0.000     0.000     0.000       1.0
                B-state_code      1.000     1.000     1.000       1.0
    I-arrive_time.start_time      1.000     1.000     1.000       1.0
    I-arrive_date.day_number      0.000     0.000     0.000       0.0
                 B-meal_code      1.000     1.000     1.000       1.0
 I-depart_time.period_of_day      0.000     0.000     0.000       1.0
                  B-day_name      1.000     0.500     0.667       2.0
             B-period_of_day      1.000     0.250     0.400       4.0
        B-stoploc.state_code      0.000     0.000     0.000       0.0
    B-return_date.month_name      0.000     0.000     0.000       0.0
    B-return_date.day_number      0.000     0.000     0.000       0.0
    B-arrive_time.period_mod      0.000     0.000     0.000       0.0
                 I-meal_code      0.000     0.000     0.000       0.0
        B-toloc.country_name      1.000     1.000     1.000       1.0
                 B-days_code      1.000     1.000     1.000       1.0
 I-arrive_time.period_of_day      0.000     0.000     0.000       0.0
            I-today_relative      0.000     0.000     0.000       0.0
 B-return_time.period_of_day      0.000     0.000     0.000       0.0
                      B-time      0.000     0.000     0.000       0.0
           I-fare_basis_code      0.000     0.000     0.000       0.0
 I-arrive_time.time_relative      0.000     0.000     0.000       4.0
 I-depart_time.time_relative      0.000     0.000     0.000       1.0
            B-today_relative      0.000     0.000     0.000       0.0
                B-state_name      0.000     0.000     0.000       9.0
B-arrive_date.today_relative      0.000     0.000     0.000       0.0
    B-return_time.period_mod      0.000     0.000     0.000       0.0
                B-month_name      0.000     0.000     0.000       0.0
                B-day_number      0.000     0.000     0.000       0.0
 I-return_date.date_relative      0.667     0.667     0.667       3.0
I-return_date.today_relative      0.000     0.000     0.000       0.0
      B-stoploc.airport_name      0.000     0.000     0.000       0.0
             B-time_relative      0.000     0.000     0.000       0.0
                      I-time      0.000     0.000     0.000       0.0
    I-return_date.day_number      0.000     0.000     0.000       0.0
          I-meal_description      0.000     0.000     0.000       0.0
B-return_date.today_relative      0.000     0.000     0.000       0.0
      B-return_date.day_name      0.000     0.000     0.000       2.0

                   micro avg      0.933     0.934     0.934    3657.0
                   macro avg      0.598     0.591     0.574    3657.0
                weighted avg      0.938     0.934     0.931    3657.0

I0717 07:00:58.206540 140695199004544 interactiveshell.py:2882] Best Slot F1: 0.939
Reading ../data/atis.train.w-intent.iob
I0717 07:01:01.954938 140695199004544 interactiveshell.py:2882] Step 6900 | Loss: 0.0137 | Loss_intent: 0.0007 | Loss_slots: 0.0129 | Spent: 6.6 secs | LR: 0.000145
I0717 07:01:06.123615 140695199004544 interactiveshell.py:2882] Step 6950 | Loss: 0.0125 | Loss_intent: 0.0006 | Loss_slots: 0.0119 | Spent: 4.2 secs | LR: 0.000144
I0717 07:01:10.312309 140695199004544 interactiveshell.py:2882] Step 7000 | Loss: 0.0198 | Loss_intent: 0.0002 | Loss_slots: 0.0196 | Spent: 4.2 secs | LR: 0.000143
I0717 07:01:14.486074 140695199004544 interactiveshell.py:2882] Step 7050 | Loss: 0.0307 | Loss_intent: 0.0000 | Loss_slots: 0.0307 | Spent: 4.2 secs | LR: 0.000143
I0717 07:01:18.675596 140695199004544 interactiveshell.py:2882] Step 7100 | Loss: 0.0313 | Loss_intent: 0.0021 | Loss_slots: 0.0292 | Spent: 4.2 secs | LR: 0.000142
I0717 07:01:22.866818 140695199004544 interactiveshell.py:2882] Step 7150 | Loss: 0.0360 | Loss_intent: 0.0169 | Loss_slots: 0.0191 | Spent: 4.2 secs | LR: 0.000141
Reading ../data/atis.test.w-intent.iob
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.
  'precision', 'predicted', average, warn_for)
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1439: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples.
  'recall', 'true', average, warn_for)
I0717 07:01:26.668585 140695199004544 interactiveshell.py:2882] 
                                          precision    recall  f1-score   support

                             atis_flight      0.976     0.986     0.981       632
                            atis_airfare      0.941     1.000     0.970        48
                     atis_ground_service      1.000     1.000     1.000        36
                            atis_airline      1.000     1.000     1.000        38
                       atis_abbreviation      1.000     1.000     1.000        33
                           atis_aircraft      1.000     0.889     0.941         9
                        atis_flight_time      1.000     1.000     1.000         1
                           atis_quantity      0.429     1.000     0.600         3
                atis_flight#atis_airfare      0.800     0.333     0.471        12
                            atis_airport      0.944     0.944     0.944        18
                           atis_distance      1.000     1.000     1.000        10
                               atis_city      0.500     0.667     0.571         6
                        atis_ground_fare      1.000     0.714     0.833         7
                           atis_capacity      1.000     1.000     1.000        21
                          atis_flight_no      0.889     1.000     0.941         8
                               atis_meal      1.000     0.833     0.909         6
                        atis_restriction      0.000     0.000     0.000         0
             atis_airline#atis_flight_no      0.000     0.000     0.000         0
    atis_ground_service#atis_ground_fare      0.000     0.000     0.000         0
           atis_airfare#atis_flight_time      0.000     0.000     0.000         0
                           atis_cheapest      0.000     0.000     0.000         0
atis_aircraft#atis_flight#atis_flight_no      0.000     0.000     0.000         0

                               micro avg      0.968     0.973     0.970       888
                               macro avg      0.658     0.653     0.644       888
                            weighted avg      0.970     0.973     0.969       888

I0717 07:01:26.710507 140695199004544 interactiveshell.py:2882] 
                              precision    recall  f1-score   support

                           O      0.000     0.000     0.000       0.0
           B-toloc.city_name      0.970     0.996     0.983     716.0
         B-fromloc.city_name      0.971     0.993     0.982     704.0
           I-toloc.city_name      0.967     0.992     0.980     265.0
      B-depart_date.day_name      0.990     0.972     0.981     212.0
              B-airline_name      0.990     0.980     0.985     101.0
         I-fromloc.city_name      0.961     0.977     0.969     177.0
 B-depart_time.period_of_day      0.991     0.854     0.917     130.0
              I-airline_name      1.000     0.985     0.992      65.0
    B-depart_date.day_number      0.963     0.945     0.954      55.0
    B-depart_date.month_name      0.981     0.946     0.964      56.0
          B-depart_time.time      0.821     0.965     0.887      57.0
                B-round_trip      1.000     0.986     0.993      73.0
             B-cost_relative      1.000     0.973     0.986      37.0
                I-round_trip      1.000     1.000     1.000      71.0
                B-flight_mod      1.000     0.958     0.979      24.0
 B-depart_time.time_relative      0.983     0.908     0.944      65.0
          I-depart_time.time      0.875     0.942     0.907      52.0
         B-stoploc.city_name      1.000     0.900     0.947      20.0
                 B-city_name      0.903     0.491     0.636      57.0
                B-class_type      0.923     1.000     0.960      24.0
          B-arrive_time.time      0.846     0.971     0.904      34.0
 B-arrive_time.time_relative      0.833     0.968     0.896      31.0
                I-class_type      1.000     1.000     1.000      17.0
               B-flight_stop      1.000     1.000     1.000      21.0
          I-arrive_time.time      0.895     0.971     0.932      35.0
              B-airline_code      1.000     0.912     0.954      34.0
    I-depart_date.day_number      1.000     0.933     0.966      15.0
      I-fromloc.airport_name      0.441     1.000     0.612      15.0
      B-fromloc.airport_name      0.321     0.750     0.450      12.0
      B-arrive_date.day_name      0.611     1.000     0.759      11.0
          B-toloc.state_code      0.947     1.000     0.973      18.0
B-depart_date.today_relative      1.000     1.000     1.000       9.0
             B-flight_number      0.500     0.818     0.621      11.0
 B-depart_date.date_relative      0.895     1.000     0.944      17.0
          B-toloc.state_name      0.926     0.893     0.909      28.0
           B-fare_basis_code      0.944     1.000     0.971      17.0
               B-flight_time      1.000     1.000     1.000       1.0
                        B-or      1.000     1.000     1.000       3.0
 B-arrive_time.period_of_day      0.750     1.000     0.857       6.0
          B-meal_description      1.000     0.900     0.947      10.0
             I-cost_relative      1.000     0.667     0.800       3.0
              I-airport_name      0.846     0.379     0.524      29.0
               B-fare_amount      0.400     1.000     0.571       2.0
               I-fare_amount      1.000     1.000     1.000       2.0
                 I-city_name      1.000     0.500     0.667      30.0
        I-toloc.airport_name      0.750     1.000     0.857       3.0
            B-transport_type      1.000     0.700     0.824      10.0
    B-arrive_date.month_name      0.625     0.833     0.714       6.0
    B-arrive_date.day_number      0.625     0.833     0.714       6.0
         I-stoploc.city_name      0.769     1.000     0.870      10.0
                      B-meal      1.000     1.000     1.000      16.0
        B-fromloc.state_code      1.000     0.957     0.978      23.0
    B-depart_time.period_mod      0.500     1.000     0.667       5.0
                   B-connect      1.000     1.000     1.000       6.0
               B-flight_days      1.000     1.000     1.000      10.0
        B-toloc.airport_name      1.000     0.333     0.500       3.0
        B-fromloc.state_name      0.708     1.000     0.829      17.0
              B-airport_name      0.875     0.333     0.483      21.0
                   B-economy      1.000     1.000     1.000       6.0
               I-flight_time      1.000     1.000     1.000       1.0
             B-aircraft_code      0.926     0.758     0.833      33.0
                       B-mod      0.000     0.000     0.000       2.0
              B-airport_code      0.000     0.000     0.000       9.0
    B-depart_time.start_time      0.500     0.333     0.400       3.0
      B-depart_time.end_time      1.000     0.667     0.800       3.0
          B-depart_date.year      1.000     0.667     0.800       3.0
            I-transport_type      0.200     1.000     0.333       1.0
          B-restriction_code      0.600     0.750     0.667       4.0
    B-arrive_time.start_time      0.700     0.875     0.778       8.0
        B-toloc.airport_code      1.000     0.500     0.667       4.0
      B-arrive_time.end_time      0.833     0.625     0.714       8.0
      I-arrive_time.end_time      0.875     0.875     0.875       8.0
      I-depart_time.end_time      1.000     0.333     0.500       3.0
               I-flight_stop      0.000     0.000     0.000       0.0
      B-fromloc.airport_code      1.000     1.000     1.000       5.0
          I-restriction_code      0.600     1.000     0.750       3.0
    I-depart_time.start_time      0.500     1.000     0.667       1.0
          I-toloc.state_name      1.000     1.000     1.000       1.0
I-depart_date.today_relative      0.000     0.000     0.000       0.0
 B-arrive_date.date_relative      1.000     0.500     0.667       2.0
                I-flight_mod      1.000     0.167     0.286       6.0
                   I-economy      0.000     0.000     0.000       0.0
 B-return_date.date_relative      1.000     0.333     0.500       3.0
        I-fromloc.state_name      1.000     1.000     1.000       1.0
                B-state_code      1.000     1.000     1.000       1.0
    I-arrive_time.start_time      1.000     1.000     1.000       1.0
    I-arrive_date.day_number      0.000     0.000     0.000       0.0
                 B-meal_code      1.000     1.000     1.000       1.0
 I-depart_time.period_of_day      0.000     0.000     0.000       1.0
                  B-day_name      1.000     0.500     0.667       2.0
             B-period_of_day      1.000     0.500     0.667       4.0
        B-stoploc.state_code      0.000     0.000     0.000       0.0
    B-return_date.month_name      0.000     0.000     0.000       0.0
    B-return_date.day_number      0.000     0.000     0.000       0.0
    B-arrive_time.period_mod      0.000     0.000     0.000       0.0
                 I-meal_code      0.000     0.000     0.000       0.0
        B-toloc.country_name      1.000     1.000     1.000       1.0
                 B-days_code      0.200     1.000     0.333       1.0
 I-arrive_time.period_of_day      0.000     0.000     0.000       0.0
            I-today_relative      0.000     0.000     0.000       0.0
 B-return_time.period_of_day      0.000     0.000     0.000       0.0
                      B-time      0.000     0.000     0.000       0.0
           I-fare_basis_code      0.000     0.000     0.000       0.0
 I-arrive_time.time_relative      1.000     0.500     0.667       4.0
 I-depart_time.time_relative      0.000     0.000     0.000       1.0
            B-today_relative      0.000     0.000     0.000       0.0
                B-state_name      0.000     0.000     0.000       9.0
B-arrive_date.today_relative      0.000     0.000     0.000       0.0
    B-return_time.period_mod      0.000     0.000     0.000       0.0
                B-month_name      0.000     0.000     0.000       0.0
                B-day_number      0.000     0.000     0.000       0.0
 I-return_date.date_relative      0.667     0.667     0.667       3.0
I-return_date.today_relative      0.000     0.000     0.000       0.0
      B-stoploc.airport_name      0.000     0.000     0.000       0.0
             B-time_relative      0.000     0.000     0.000       0.0
                      I-time      0.000     0.000     0.000       0.0
    I-return_date.day_number      0.000     0.000     0.000       0.0
          I-meal_description      0.000     0.000     0.000       0.0
B-return_date.today_relative      0.000     0.000     0.000       0.0
      B-return_date.day_name      0.000     0.000     0.000       2.0

                   micro avg      0.937     0.938     0.938    3657.0
                   macro avg      0.636     0.620     0.602    3657.0
                weighted avg      0.945     0.938     0.935    3657.0

I0717 07:01:26.711287 140695199004544 interactiveshell.py:2882] Best Slot F1: 0.939
Reading ../data/atis.train.w-intent.iob
I0717 07:01:29.466181 140695199004544 interactiveshell.py:2882] Step 7200 | Loss: 0.0409 | Loss_intent: 0.0001 | Loss_slots: 0.0408 | Spent: 6.6 secs | LR: 0.000140
I0717 07:01:33.640654 140695199004544 interactiveshell.py:2882] Step 7250 | Loss: 0.0257 | Loss_intent: 0.0000 | Loss_slots: 0.0257 | Spent: 4.2 secs | LR: 0.000140
I0717 07:01:37.822069 140695199004544 interactiveshell.py:2882] Step 7300 | Loss: 0.0118 | Loss_intent: 0.0000 | Loss_slots: 0.0118 | Spent: 4.2 secs | LR: 0.000139
I0717 07:01:42.009785 140695199004544 interactiveshell.py:2882] Step 7350 | Loss: 0.0194 | Loss_intent: 0.0000 | Loss_slots: 0.0193 | Spent: 4.2 secs | LR: 0.000138
I0717 07:01:46.209594 140695199004544 interactiveshell.py:2882] Step 7400 | Loss: 0.0267 | Loss_intent: 0.0001 | Loss_slots: 0.0266 | Spent: 4.2 secs | LR: 0.000138
I0717 07:01:50.607231 140695199004544 interactiveshell.py:2882] Step 7450 | Loss: 0.0096 | Loss_intent: 0.0003 | Loss_slots: 0.0093 | Spent: 4.4 secs | LR: 0.000137
Reading ../data/atis.test.w-intent.iob
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.
  'precision', 'predicted', average, warn_for)
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1439: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples.
  'recall', 'true', average, warn_for)
I0717 07:01:55.645368 140695199004544 interactiveshell.py:2882] 
                                          precision    recall  f1-score   support

                             atis_flight      0.986     0.981     0.983       632
                            atis_airfare      0.923     1.000     0.960        48
                     atis_ground_service      1.000     1.000     1.000        36
                            atis_airline      1.000     0.974     0.987        38
                       atis_abbreviation      0.971     1.000     0.985        33
                           atis_aircraft      0.889     0.889     0.889         9
                        atis_flight_time      0.500     1.000     0.667         1
                           atis_quantity      0.273     1.000     0.429         3
                atis_flight#atis_airfare      0.800     0.333     0.471        12
                            atis_airport      0.895     0.944     0.919        18
                           atis_distance      1.000     1.000     1.000        10
                               atis_city      0.545     1.000     0.706         6
                        atis_ground_fare      1.000     0.714     0.833         7
                           atis_capacity      1.000     0.952     0.976        21
                          atis_flight_no      0.875     0.875     0.875         8
                               atis_meal      1.000     0.833     0.909         6
                        atis_restriction      0.000     0.000     0.000         0
             atis_airline#atis_flight_no      0.000     0.000     0.000         0
    atis_ground_service#atis_ground_fare      0.000     0.000     0.000         0
           atis_airfare#atis_flight_time      0.000     0.000     0.000         0
                           atis_cheapest      0.000     0.000     0.000         0
atis_aircraft#atis_flight#atis_flight_no      0.000     0.000     0.000         0

                               micro avg      0.963     0.968     0.966       888
                               macro avg      0.621     0.659     0.618       888
                            weighted avg      0.971     0.968     0.967       888

I0717 07:01:55.686382 140695199004544 interactiveshell.py:2882] 
                              precision    recall  f1-score   support

                           O      0.000     0.000     0.000       0.0
           B-toloc.city_name      0.970     0.993     0.981     716.0
         B-fromloc.city_name      0.972     0.987     0.980     704.0
           I-toloc.city_name      0.957     0.996     0.976     265.0
      B-depart_date.day_name      0.990     0.981     0.986     212.0
              B-airline_name      0.971     1.000     0.985     101.0
         I-fromloc.city_name      0.961     0.972     0.966     177.0
 B-depart_time.period_of_day      0.991     0.869     0.926     130.0
              I-airline_name      1.000     1.000     1.000      65.0
    B-depart_date.day_number      0.963     0.945     0.954      55.0
    B-depart_date.month_name      0.982     0.964     0.973      56.0
          B-depart_time.time      0.848     0.982     0.911      57.0
                B-round_trip      1.000     0.973     0.986      73.0
             B-cost_relative      1.000     0.973     0.986      37.0
                I-round_trip      1.000     1.000     1.000      71.0
                B-flight_mod      1.000     0.958     0.979      24.0
 B-depart_time.time_relative      0.969     0.954     0.961      65.0
          I-depart_time.time      0.881     1.000     0.937      52.0
         B-stoploc.city_name      1.000     0.900     0.947      20.0
                 B-city_name      0.846     0.579     0.688      57.0
                B-class_type      0.923     1.000     0.960      24.0
          B-arrive_time.time      0.892     0.971     0.930      34.0
 B-arrive_time.time_relative      0.882     0.968     0.923      31.0
                I-class_type      1.000     1.000     1.000      17.0
               B-flight_stop      0.955     1.000     0.977      21.0
          I-arrive_time.time      0.971     0.971     0.971      35.0
              B-airline_code      0.970     0.941     0.955      34.0
    I-depart_date.day_number      1.000     1.000     1.000      15.0
      I-fromloc.airport_name      0.441     1.000     0.612      15.0
      B-fromloc.airport_name      0.308     0.667     0.421      12.0
      B-arrive_date.day_name      0.688     1.000     0.815      11.0
          B-toloc.state_code      0.947     1.000     0.973      18.0
B-depart_date.today_relative      1.000     1.000     1.000       9.0
             B-flight_number      0.526     0.909     0.667      11.0
 B-depart_date.date_relative      0.895     1.000     0.944      17.0
          B-toloc.state_name      0.931     0.964     0.947      28.0
           B-fare_basis_code      0.889     0.941     0.914      17.0
               B-flight_time      1.000     1.000     1.000       1.0
                        B-or      1.000     1.000     1.000       3.0
 B-arrive_time.period_of_day      0.857     1.000     0.923       6.0
          B-meal_description      1.000     0.900     0.947      10.0
             I-cost_relative      1.000     0.667     0.800       3.0
              I-airport_name      0.833     0.345     0.488      29.0
               B-fare_amount      0.500     1.000     0.667       2.0
               I-fare_amount      1.000     1.000     1.000       2.0
                 I-city_name      1.000     0.500     0.667      30.0
        I-toloc.airport_name      0.750     1.000     0.857       3.0
            B-transport_type      1.000     0.700     0.824      10.0
    B-arrive_date.month_name      0.714     0.833     0.769       6.0
    B-arrive_date.day_number      0.833     0.833     0.833       6.0
         I-stoploc.city_name      0.778     0.700     0.737      10.0
                      B-meal      1.000     1.000     1.000      16.0
        B-fromloc.state_code      1.000     0.957     0.978      23.0
    B-depart_time.period_mod      0.556     1.000     0.714       5.0
                   B-connect      1.000     1.000     1.000       6.0
               B-flight_days      1.000     1.000     1.000      10.0
        B-toloc.airport_name      0.500     0.333     0.400       3.0
        B-fromloc.state_name      0.762     0.941     0.842      17.0
              B-airport_name      0.875     0.333     0.483      21.0
                   B-economy      1.000     1.000     1.000       6.0
               I-flight_time      1.000     1.000     1.000       1.0
             B-aircraft_code      0.960     0.727     0.828      33.0
                       B-mod      0.000     0.000     0.000       2.0
              B-airport_code      0.000     0.000     0.000       9.0
    B-depart_time.start_time      0.500     0.333     0.400       3.0
      B-depart_time.end_time      1.000     0.667     0.800       3.0
          B-depart_date.year      1.000     0.667     0.800       3.0
            I-transport_type      0.167     1.000     0.286       1.0
          B-restriction_code      0.364     1.000     0.533       4.0
    B-arrive_time.start_time      0.875     0.875     0.875       8.0
        B-toloc.airport_code      1.000     0.500     0.667       4.0
      B-arrive_time.end_time      0.875     0.875     0.875       8.0
      I-arrive_time.end_time      0.875     0.875     0.875       8.0
      I-depart_time.end_time      1.000     0.333     0.500       3.0
               I-flight_stop      0.000     0.000     0.000       0.0
      B-fromloc.airport_code      1.000     1.000     1.000       5.0
          I-restriction_code      0.750     1.000     0.857       3.0
    I-depart_time.start_time      0.500     1.000     0.667       1.0
          I-toloc.state_name      1.000     1.000     1.000       1.0
I-depart_date.today_relative      0.000     0.000     0.000       0.0
 B-arrive_date.date_relative      1.000     0.500     0.667       2.0
                I-flight_mod      1.000     0.167     0.286       6.0
                   I-economy      0.000     0.000     0.000       0.0
 B-return_date.date_relative      1.000     0.667     0.800       3.0
        I-fromloc.state_name      1.000     1.000     1.000       1.0
                B-state_code      1.000     1.000     1.000       1.0
    I-arrive_time.start_time      1.000     1.000     1.000       1.0
    I-arrive_date.day_number      0.000     0.000     0.000       0.0
                 B-meal_code      1.000     1.000     1.000       1.0
 I-depart_time.period_of_day      0.000     0.000     0.000       1.0
                  B-day_name      1.000     0.500     0.667       2.0
             B-period_of_day      1.000     0.750     0.857       4.0
        B-stoploc.state_code      0.000     0.000     0.000       0.0
    B-return_date.month_name      0.000     0.000     0.000       0.0
    B-return_date.day_number      0.000     0.000     0.000       0.0
    B-arrive_time.period_mod      0.000     0.000     0.000       0.0
                 I-meal_code      0.000     0.000     0.000       0.0
        B-toloc.country_name      1.000     1.000     1.000       1.0
                 B-days_code      1.000     1.000     1.000       1.0
 I-arrive_time.period_of_day      0.000     0.000     0.000       0.0
            I-today_relative      0.000     0.000     0.000       0.0
 B-return_time.period_of_day      0.000     0.000     0.000       0.0
                      B-time      0.000     0.000     0.000       0.0
           I-fare_basis_code      0.000     0.000     0.000       0.0
 I-arrive_time.time_relative      1.000     0.500     0.667       4.0
 I-depart_time.time_relative      0.000     0.000     0.000       1.0
            B-today_relative      0.000     0.000     0.000       0.0
                B-state_name      0.000     0.000     0.000       9.0
B-arrive_date.today_relative      0.000     0.000     0.000       0.0
    B-return_time.period_mod      0.000     0.000     0.000       0.0
                B-month_name      0.000     0.000     0.000       0.0
                B-day_number      0.000     0.000     0.000       0.0
 I-return_date.date_relative      1.000     0.667     0.800       3.0
I-return_date.today_relative      0.000     0.000     0.000       0.0
      B-stoploc.airport_name      0.000     0.000     0.000       0.0
             B-time_relative      0.000     0.000     0.000       0.0
                      I-time      0.000     0.000     0.000       0.0
    I-return_date.day_number      0.000     0.000     0.000       0.0
          I-meal_description      0.000     0.000     0.000       0.0
B-return_date.today_relative      0.000     0.000     0.000       0.0
      B-return_date.day_name      0.000     0.000     0.000       2.0

                   micro avg      0.941     0.943     0.942    3657.0
                   macro avg      0.647     0.628     0.618    3657.0
                weighted avg      0.947     0.943     0.939    3657.0

I0717 07:01:55.687317 140695199004544 interactiveshell.py:2882] Best Slot F1: 0.942
Reading ../data/atis.train.w-intent.iob
I0717 07:01:57.451738 140695199004544 interactiveshell.py:2882] Step 7500 | Loss: 0.0319 | Loss_intent: 0.0002 | Loss_slots: 0.0318 | Spent: 6.8 secs | LR: 0.000136
I0717 07:02:01.704334 140695199004544 interactiveshell.py:2882] Step 7550 | Loss: 0.0268 | Loss_intent: 0.0003 | Loss_slots: 0.0266 | Spent: 4.3 secs | LR: 0.000135
I0717 07:02:05.949633 140695199004544 interactiveshell.py:2882] Step 7600 | Loss: 0.1497 | Loss_intent: 0.1147 | Loss_slots: 0.0349 | Spent: 4.2 secs | LR: 0.000135
I0717 07:02:10.128586 140695199004544 interactiveshell.py:2882] Step 7650 | Loss: 0.1242 | Loss_intent: 0.0718 | Loss_slots: 0.0524 | Spent: 4.2 secs | LR: 0.000134
I0717 07:02:14.297695 140695199004544 interactiveshell.py:2882] Step 7700 | Loss: 0.0232 | Loss_intent: 0.0001 | Loss_slots: 0.0232 | Spent: 4.2 secs | LR: 0.000133
I0717 07:02:18.499069 140695199004544 interactiveshell.py:2882] Step 7750 | Loss: 0.0336 | Loss_intent: 0.0118 | Loss_slots: 0.0218 | Spent: 4.2 secs | LR: 0.000133
Reading ../data/atis.test.w-intent.iob
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.
  'precision', 'predicted', average, warn_for)
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1439: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples.
  'recall', 'true', average, warn_for)
I0717 07:02:24.326734 140695199004544 interactiveshell.py:2882] 
                                          precision    recall  f1-score   support

                             atis_flight      0.981     0.986     0.983       632
                            atis_airfare      0.959     0.979     0.969        48
                     atis_ground_service      1.000     1.000     1.000        36
                            atis_airline      0.974     1.000     0.987        38
                       atis_abbreviation      1.000     0.970     0.985        33
                           atis_aircraft      0.800     0.889     0.842         9
                        atis_flight_time      1.000     1.000     1.000         1
                           atis_quantity      0.429     1.000     0.600         3
                atis_flight#atis_airfare      0.857     0.500     0.632        12
                            atis_airport      0.895     0.944     0.919        18
                           atis_distance      1.000     1.000     1.000        10
                               atis_city      0.667     0.667     0.667         6
                        atis_ground_fare      1.000     0.857     0.923         7
                           atis_capacity      1.000     1.000     1.000        21
                          atis_flight_no      0.889     1.000     0.941         8
                               atis_meal      1.000     0.833     0.909         6
                        atis_restriction      0.000     0.000     0.000         0
             atis_airline#atis_flight_no      0.000     0.000     0.000         0
    atis_ground_service#atis_ground_fare      0.000     0.000     0.000         0
           atis_airfare#atis_flight_time      0.000     0.000     0.000         0
                           atis_cheapest      0.000     0.000     0.000         0
atis_aircraft#atis_flight#atis_flight_no      0.000     0.000     0.000         0

                               micro avg      0.969     0.974     0.971       888
                               macro avg      0.657     0.665     0.653       888
                            weighted avg      0.972     0.974     0.972       888

I0717 07:02:24.368108 140695199004544 interactiveshell.py:2882] 
                              precision    recall  f1-score   support

                           O      0.000     0.000     0.000       0.0
           B-toloc.city_name      0.971     0.990     0.981     716.0
         B-fromloc.city_name      0.983     0.976     0.979     704.0
           I-toloc.city_name      0.940     0.996     0.967     265.0
      B-depart_date.day_name      0.990     0.976     0.983     212.0
              B-airline_name      0.943     0.990     0.966     101.0
         I-fromloc.city_name      0.970     0.927     0.948     177.0
 B-depart_time.period_of_day      0.975     0.885     0.927     130.0
              I-airline_name      1.000     0.985     0.992      65.0
    B-depart_date.day_number      0.964     0.964     0.964      55.0
    B-depart_date.month_name      0.981     0.946     0.964      56.0
          B-depart_time.time      0.824     0.982     0.896      57.0
                B-round_trip      1.000     0.945     0.972      73.0
             B-cost_relative      1.000     0.973     0.986      37.0
                I-round_trip      1.000     1.000     1.000      71.0
                B-flight_mod      1.000     0.958     0.979      24.0
 B-depart_time.time_relative      0.984     0.938     0.961      65.0
          I-depart_time.time      0.879     0.981     0.927      52.0
         B-stoploc.city_name      1.000     0.900     0.947      20.0
                 B-city_name      0.720     0.632     0.673      57.0
                B-class_type      0.923     1.000     0.960      24.0
          B-arrive_time.time      0.943     0.971     0.957      34.0
 B-arrive_time.time_relative      0.882     0.968     0.923      31.0
                I-class_type      1.000     1.000     1.000      17.0
               B-flight_stop      0.875     1.000     0.933      21.0
          I-arrive_time.time      0.971     0.971     0.971      35.0
              B-airline_code      1.000     0.824     0.903      34.0
    I-depart_date.day_number      1.000     1.000     1.000      15.0
      I-fromloc.airport_name      0.500     1.000     0.667      15.0
      B-fromloc.airport_name      0.417     0.833     0.556      12.0
      B-arrive_date.day_name      0.647     1.000     0.786      11.0
          B-toloc.state_code      0.947     1.000     0.973      18.0
B-depart_date.today_relative      1.000     0.889     0.941       9.0
             B-flight_number      0.364     0.727     0.485      11.0
 B-depart_date.date_relative      0.895     1.000     0.944      17.0
          B-toloc.state_name      0.844     0.964     0.900      28.0
           B-fare_basis_code      0.909     0.588     0.714      17.0
               B-flight_time      1.000     1.000     1.000       1.0
                        B-or      1.000     1.000     1.000       3.0
 B-arrive_time.period_of_day      0.857     1.000     0.923       6.0
          B-meal_description      1.000     0.900     0.947      10.0
             I-cost_relative      1.000     0.667     0.800       3.0
              I-airport_name      0.812     0.448     0.578      29.0
               B-fare_amount      0.400     1.000     0.571       2.0
               I-fare_amount      1.000     1.000     1.000       2.0
                 I-city_name      0.810     0.567     0.667      30.0
        I-toloc.airport_name      1.000     1.000     1.000       3.0
            B-transport_type      1.000     0.700     0.824      10.0
    B-arrive_date.month_name      0.625     0.833     0.714       6.0
    B-arrive_date.day_number      0.833     0.833     0.833       6.0
         I-stoploc.city_name      0.889     0.800     0.842      10.0
                      B-meal      1.000     1.000     1.000      16.0
        B-fromloc.state_code      1.000     0.957     0.978      23.0
    B-depart_time.period_mod      0.750     0.600     0.667       5.0
                   B-connect      1.000     1.000     1.000       6.0
               B-flight_days      1.000     1.000     1.000      10.0
        B-toloc.airport_name      0.250     0.333     0.286       3.0
        B-fromloc.state_name      0.810     1.000     0.895      17.0
              B-airport_name      0.727     0.381     0.500      21.0
                   B-economy      1.000     1.000     1.000       6.0
               I-flight_time      1.000     1.000     1.000       1.0
             B-aircraft_code      0.893     0.758     0.820      33.0
                       B-mod      0.000     0.000     0.000       2.0
              B-airport_code      0.000     0.000     0.000       9.0
    B-depart_time.start_time      0.500     0.333     0.400       3.0
      B-depart_time.end_time      1.000     0.667     0.800       3.0
          B-depart_date.year      1.000     0.667     0.800       3.0
            I-transport_type      0.200     1.000     0.333       1.0
          B-restriction_code      0.333     1.000     0.500       4.0
    B-arrive_time.start_time      0.889     1.000     0.941       8.0
        B-toloc.airport_code      1.000     0.500     0.667       4.0
      B-arrive_time.end_time      0.889     1.000     0.941       8.0
      I-arrive_time.end_time      0.800     1.000     0.889       8.0
      I-depart_time.end_time      1.000     0.333     0.500       3.0
               I-flight_stop      0.000     0.000     0.000       0.0
      B-fromloc.airport_code      1.000     1.000     1.000       5.0
          I-restriction_code      0.750     1.000     0.857       3.0
    I-depart_time.start_time      0.500     1.000     0.667       1.0
          I-toloc.state_name      1.000     1.000     1.000       1.0
I-depart_date.today_relative      0.000     0.000     0.000       0.0
 B-arrive_date.date_relative      1.000     1.000     1.000       2.0
                I-flight_mod      1.000     0.167     0.286       6.0
                   I-economy      0.000     0.000     0.000       0.0
 B-return_date.date_relative      0.667     0.667     0.667       3.0
        I-fromloc.state_name      1.000     1.000     1.000       1.0
                B-state_code      1.000     1.000     1.000       1.0
    I-arrive_time.start_time      0.000     0.000     0.000       1.0
    I-arrive_date.day_number      0.000     0.000     0.000       0.0
                 B-meal_code      1.000     1.000     1.000       1.0
 I-depart_time.period_of_day      0.000     0.000     0.000       1.0
                  B-day_name      1.000     0.500     0.667       2.0
             B-period_of_day      1.000     0.750     0.857       4.0
        B-stoploc.state_code      0.000     0.000     0.000       0.0
    B-return_date.month_name      0.000     0.000     0.000       0.0
    B-return_date.day_number      0.000     0.000     0.000       0.0
    B-arrive_time.period_mod      0.000     0.000     0.000       0.0
                 I-meal_code      0.000     0.000     0.000       0.0
        B-toloc.country_name      1.000     1.000     1.000       1.0
                 B-days_code      1.000     1.000     1.000       1.0
 I-arrive_time.period_of_day      0.000     0.000     0.000       0.0
            I-today_relative      0.000     0.000     0.000       0.0
 B-return_time.period_of_day      0.000     0.000     0.000       0.0
                      B-time      0.000     0.000     0.000       0.0
           I-fare_basis_code      0.000     0.000     0.000       0.0
 I-arrive_time.time_relative      1.000     0.500     0.667       4.0
 I-depart_time.time_relative      0.000     0.000     0.000       1.0
            B-today_relative      0.000     0.000     0.000       0.0
                B-state_name      0.000     0.000     0.000       9.0
B-arrive_date.today_relative      0.000     0.000     0.000       0.0
    B-return_time.period_mod      0.000     0.000     0.000       0.0
                B-month_name      0.000     0.000     0.000       0.0
                B-day_number      0.000     0.000     0.000       0.0
 I-return_date.date_relative      1.000     0.333     0.500       3.0
I-return_date.today_relative      0.000     0.000     0.000       0.0
      B-stoploc.airport_name      0.000     0.000     0.000       0.0
             B-time_relative      0.000     0.000     0.000       0.0
                      I-time      0.000     0.000     0.000       0.0
    I-return_date.day_number      0.000     0.000     0.000       0.0
          I-meal_description      0.000     0.000     0.000       0.0
B-return_date.today_relative      0.000     0.000     0.000       0.0
      B-return_date.day_name      0.000     0.000     0.000       2.0

                   micro avg      0.935     0.937     0.936    3657.0
                   macro avg      0.632     0.619     0.607    3657.0
                weighted avg      0.941     0.937     0.934    3657.0

I0717 07:02:24.369020 140695199004544 interactiveshell.py:2882] Best Slot F1: 0.942
Reading ../data/atis.train.w-intent.iob
I0717 07:02:25.107725 140695199004544 interactiveshell.py:2882] Step 7800 | Loss: 0.0602 | Loss_intent: 0.0394 | Loss_slots: 0.0208 | Spent: 6.6 secs | LR: 0.000132
I0717 07:02:29.314383 140695199004544 interactiveshell.py:2882] Step 7850 | Loss: 0.0266 | Loss_intent: 0.0001 | Loss_slots: 0.0265 | Spent: 4.2 secs | LR: 0.000131
I0717 07:02:33.497777 140695199004544 interactiveshell.py:2882] Step 7900 | Loss: 0.0141 | Loss_intent: 0.0004 | Loss_slots: 0.0137 | Spent: 4.2 secs | LR: 0.000131
I0717 07:02:37.681205 140695199004544 interactiveshell.py:2882] Step 7950 | Loss: 0.0241 | Loss_intent: 0.0004 | Loss_slots: 0.0236 | Spent: 4.2 secs | LR: 0.000130
I0717 07:02:41.868563 140695199004544 interactiveshell.py:2882] Step 8000 | Loss: 0.0118 | Loss_intent: 0.0001 | Loss_slots: 0.0117 | Spent: 4.2 secs | LR: 0.000129
I0717 07:02:46.071013 140695199004544 interactiveshell.py:2882] Step 8050 | Loss: 0.0151 | Loss_intent: 0.0000 | Loss_slots: 0.0151 | Spent: 4.2 secs | LR: 0.000128
I0717 07:02:50.262097 140695199004544 interactiveshell.py:2882] Step 8100 | Loss: 0.0695 | Loss_intent: 0.0573 | Loss_slots: 0.0121 | Spent: 4.2 secs | LR: 0.000128
Reading ../data/atis.test.w-intent.iob
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.
  'precision', 'predicted', average, warn_for)
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1439: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples.
  'recall', 'true', average, warn_for)
I0717 07:02:52.912813 140695199004544 interactiveshell.py:2882] 
                                          precision    recall  f1-score   support

                             atis_flight      0.970     0.987     0.979       632
                            atis_airfare      0.940     0.979     0.959        48
                     atis_ground_service      1.000     1.000     1.000        36
                            atis_airline      1.000     0.974     0.987        38
                       atis_abbreviation      1.000     1.000     1.000        33
                           atis_aircraft      0.900     1.000     0.947         9
                        atis_flight_time      1.000     1.000     1.000         1
                           atis_quantity      0.333     0.667     0.444         3
                atis_flight#atis_airfare      0.800     0.333     0.471        12
                            atis_airport      1.000     0.833     0.909        18
                           atis_distance      1.000     1.000     1.000        10
                               atis_city      0.667     0.667     0.667         6
                        atis_ground_fare      1.000     0.714     0.833         7
                           atis_capacity      0.955     1.000     0.977        21
                          atis_flight_no      0.889     1.000     0.941         8
                               atis_meal      1.000     0.833     0.909         6
                        atis_restriction      0.000     0.000     0.000         0
             atis_airline#atis_flight_no      0.000     0.000     0.000         0
    atis_ground_service#atis_ground_fare      0.000     0.000     0.000         0
           atis_airfare#atis_flight_time      0.000     0.000     0.000         0
                           atis_cheapest      0.000     0.000     0.000         0
atis_aircraft#atis_flight#atis_flight_no      0.000     0.000     0.000         0

                               micro avg      0.964     0.970     0.967       888
                               macro avg      0.657     0.636     0.637       888
                            weighted avg      0.965     0.970     0.965       888

I0717 07:02:52.953917 140695199004544 interactiveshell.py:2882] 
                              precision    recall  f1-score   support

                           O      0.000     0.000     0.000       0.0
           B-toloc.city_name      0.967     0.992     0.979     716.0
         B-fromloc.city_name      0.977     0.987     0.982     704.0
           I-toloc.city_name      0.946     0.989     0.967     265.0
      B-depart_date.day_name      0.990     0.976     0.983     212.0
              B-airline_name      0.980     0.980     0.980     101.0
         I-fromloc.city_name      0.971     0.960     0.966     177.0
 B-depart_time.period_of_day      0.975     0.885     0.927     130.0
              I-airline_name      1.000     1.000     1.000      65.0
    B-depart_date.day_number      0.964     0.964     0.964      55.0
    B-depart_date.month_name      0.981     0.946     0.964      56.0
          B-depart_time.time      0.781     1.000     0.877      57.0
                B-round_trip      1.000     0.986     0.993      73.0
             B-cost_relative      1.000     0.973     0.986      37.0
                I-round_trip      1.000     1.000     1.000      71.0
                B-flight_mod      0.885     0.958     0.920      24.0
 B-depart_time.time_relative      0.984     0.954     0.969      65.0
          I-depart_time.time      0.897     1.000     0.945      52.0
         B-stoploc.city_name      0.947     0.900     0.923      20.0
                 B-city_name      0.846     0.579     0.688      57.0
                B-class_type      0.923     1.000     0.960      24.0
          B-arrive_time.time      0.917     0.971     0.943      34.0
 B-arrive_time.time_relative      0.882     0.968     0.923      31.0
                I-class_type      1.000     1.000     1.000      17.0
               B-flight_stop      1.000     1.000     1.000      21.0
          I-arrive_time.time      0.971     0.971     0.971      35.0
              B-airline_code      1.000     0.853     0.921      34.0
    I-depart_date.day_number      1.000     1.000     1.000      15.0
      I-fromloc.airport_name      0.441     1.000     0.612      15.0
      B-fromloc.airport_name      0.321     0.750     0.450      12.0
      B-arrive_date.day_name      0.647     1.000     0.786      11.0
          B-toloc.state_code      0.947     1.000     0.973      18.0
B-depart_date.today_relative      1.000     0.889     0.941       9.0
             B-flight_number      0.467     0.636     0.538      11.0
 B-depart_date.date_relative      0.895     1.000     0.944      17.0
          B-toloc.state_name      0.871     0.964     0.915      28.0
           B-fare_basis_code      0.941     0.941     0.941      17.0
               B-flight_time      1.000     1.000     1.000       1.0
                        B-or      1.000     1.000     1.000       3.0
 B-arrive_time.period_of_day      0.545     1.000     0.706       6.0
          B-meal_description      1.000     0.900     0.947      10.0
             I-cost_relative      1.000     0.667     0.800       3.0
              I-airport_name      0.833     0.345     0.488      29.0
               B-fare_amount      0.333     1.000     0.500       2.0
               I-fare_amount      1.000     1.000     1.000       2.0
                 I-city_name      1.000     0.467     0.636      30.0
        I-toloc.airport_name      1.000     1.000     1.000       3.0
            B-transport_type      1.000     0.700     0.824      10.0
    B-arrive_date.month_name      0.625     0.833     0.714       6.0
    B-arrive_date.day_number      0.833     0.833     0.833       6.0
         I-stoploc.city_name      0.643     0.900     0.750      10.0
                      B-meal      1.000     1.000     1.000      16.0
        B-fromloc.state_code      1.000     0.957     0.978      23.0
    B-depart_time.period_mod      0.750     0.600     0.667       5.0
                   B-connect      1.000     1.000     1.000       6.0
               B-flight_days      1.000     1.000     1.000      10.0
        B-toloc.airport_name      0.500     0.333     0.400       3.0
        B-fromloc.state_name      0.800     0.941     0.865      17.0
              B-airport_name      0.600     0.143     0.231      21.0
                   B-economy      1.000     1.000     1.000       6.0
               I-flight_time      1.000     1.000     1.000       1.0
             B-aircraft_code      0.929     0.788     0.852      33.0
                       B-mod      0.000     0.000     0.000       2.0
              B-airport_code      0.000     0.000     0.000       9.0
    B-depart_time.start_time      0.500     0.333     0.400       3.0
      B-depart_time.end_time      1.000     0.667     0.800       3.0
          B-depart_date.year      1.000     0.667     0.800       3.0
            I-transport_type      0.250     1.000     0.400       1.0
          B-restriction_code      0.571     1.000     0.727       4.0
    B-arrive_time.start_time      0.833     0.625     0.714       8.0
        B-toloc.airport_code      1.000     0.750     0.857       4.0
      B-arrive_time.end_time      0.727     1.000     0.842       8.0
      I-arrive_time.end_time      0.778     0.875     0.824       8.0
      I-depart_time.end_time      1.000     0.333     0.500       3.0
               I-flight_stop      0.000     0.000     0.000       0.0
      B-fromloc.airport_code      0.833     1.000     0.909       5.0
          I-restriction_code      0.375     1.000     0.545       3.0
    I-depart_time.start_time      0.500     1.000     0.667       1.0
          I-toloc.state_name      1.000     1.000     1.000       1.0
I-depart_date.today_relative      0.000     0.000     0.000       0.0
 B-arrive_date.date_relative      1.000     1.000     1.000       2.0
                I-flight_mod      1.000     0.167     0.286       6.0
                   I-economy      0.000     0.000     0.000       0.0
 B-return_date.date_relative      1.000     0.667     0.800       3.0
        I-fromloc.state_name      1.000     1.000     1.000       1.0
                B-state_code      1.000     1.000     1.000       1.0
    I-arrive_time.start_time      0.000     0.000     0.000       1.0
    I-arrive_date.day_number      0.000     0.000     0.000       0.0
                 B-meal_code      1.000     1.000     1.000       1.0
 I-depart_time.period_of_day      0.000     0.000     0.000       1.0
                  B-day_name      1.000     0.500     0.667       2.0
             B-period_of_day      1.000     0.500     0.667       4.0
        B-stoploc.state_code      0.000     0.000     0.000       0.0
    B-return_date.month_name      0.000     0.000     0.000       0.0
    B-return_date.day_number      0.000     0.000     0.000       0.0
    B-arrive_time.period_mod      0.000     0.000     0.000       0.0
                 I-meal_code      0.000     0.000     0.000       0.0
        B-toloc.country_name      1.000     1.000     1.000       1.0
                 B-days_code      1.000     1.000     1.000       1.0
 I-arrive_time.period_of_day      0.000     0.000     0.000       0.0
            I-today_relative      0.000     0.000     0.000       0.0
 B-return_time.period_of_day      0.000     0.000     0.000       0.0
                      B-time      0.000     0.000     0.000       0.0
           I-fare_basis_code      0.000     0.000     0.000       0.0
 I-arrive_time.time_relative      1.000     0.500     0.667       4.0
 I-depart_time.time_relative      0.000     0.000     0.000       1.0
            B-today_relative      0.000     0.000     0.000       0.0
                B-state_name      0.000     0.000     0.000       9.0
B-arrive_date.today_relative      0.000     0.000     0.000       0.0
    B-return_time.period_mod      0.000     0.000     0.000       0.0
                B-month_name      0.000     0.000     0.000       0.0
                B-day_number      0.000     0.000     0.000       0.0
 I-return_date.date_relative      1.000     0.667     0.800       3.0
I-return_date.today_relative      0.000     0.000     0.000       0.0
      B-stoploc.airport_name      0.000     0.000     0.000       0.0
             B-time_relative      0.000     0.000     0.000       0.0
                      I-time      0.000     0.000     0.000       0.0
    I-return_date.day_number      0.000     0.000     0.000       0.0
          I-meal_description      0.000     0.000     0.000       0.0
B-return_date.today_relative      0.000     0.000     0.000       0.0
      B-return_date.day_name      0.000     0.000     0.000       2.0

                   micro avg      0.937     0.939     0.938    3657.0
                   macro avg      0.629     0.617     0.602    3657.0
                weighted avg      0.942     0.939     0.934    3657.0

I0717 07:02:52.954813 140695199004544 interactiveshell.py:2882] Best Slot F1: 0.942
Reading ../data/atis.train.w-intent.iob
I0717 07:02:56.853669 140695199004544 interactiveshell.py:2882] Step 8150 | Loss: 0.0327 | Loss_intent: 0.0000 | Loss_slots: 0.0327 | Spent: 6.6 secs | LR: 0.000127
I0717 07:03:01.024183 140695199004544 interactiveshell.py:2882] Step 8200 | Loss: 0.0036 | Loss_intent: 0.0009 | Loss_slots: 0.0027 | Spent: 4.2 secs | LR: 0.000126
I0717 07:03:05.216946 140695199004544 interactiveshell.py:2882] Step 8250 | Loss: 0.0083 | Loss_intent: 0.0007 | Loss_slots: 0.0076 | Spent: 4.2 secs | LR: 0.000126
I0717 07:03:09.430090 140695199004544 interactiveshell.py:2882] Step 8300 | Loss: 0.0017 | Loss_intent: 0.0000 | Loss_slots: 0.0017 | Spent: 4.2 secs | LR: 0.000125
I0717 07:03:13.620672 140695199004544 interactiveshell.py:2882] Step 8350 | Loss: 0.0205 | Loss_intent: 0.0001 | Loss_slots: 0.0204 | Spent: 4.2 secs | LR: 0.000124
I0717 07:03:17.872405 140695199004544 interactiveshell.py:2882] Step 8400 | Loss: 0.0476 | Loss_intent: 0.0329 | Loss_slots: 0.0147 | Spent: 4.3 secs | LR: 0.000124
Reading ../data/atis.test.w-intent.iob
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.
  'precision', 'predicted', average, warn_for)
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1439: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples.
  'recall', 'true', average, warn_for)
I0717 07:03:21.551226 140695199004544 interactiveshell.py:2882] 
                                          precision    recall  f1-score   support

                             atis_flight      0.981     0.987     0.984       632
                            atis_airfare      0.940     0.979     0.959        48
                     atis_ground_service      1.000     1.000     1.000        36
                            atis_airline      1.000     1.000     1.000        38
                       atis_abbreviation      0.971     1.000     0.985        33
                           atis_aircraft      0.900     1.000     0.947         9
                        atis_flight_time      1.000     1.000     1.000         1
                           atis_quantity      0.429     1.000     0.600         3
                atis_flight#atis_airfare      0.833     0.417     0.556        12
                            atis_airport      1.000     0.944     0.971        18
                           atis_distance      1.000     1.000     1.000        10
                               atis_city      0.750     1.000     0.857         6
                        atis_ground_fare      1.000     0.714     0.833         7
                           atis_capacity      1.000     1.000     1.000        21
                          atis_flight_no      0.889     1.000     0.941         8
                               atis_meal      1.000     0.833     0.909         6
                        atis_restriction      0.000     0.000     0.000         0
             atis_airline#atis_flight_no      0.000     0.000     0.000         0
    atis_ground_service#atis_ground_fare      0.000     0.000     0.000         0
           atis_airfare#atis_flight_time      0.000     0.000     0.000         0
                           atis_cheapest      0.000     0.000     0.000         0
atis_aircraft#atis_flight#atis_flight_no      0.000     0.000     0.000         0

                               micro avg      0.972     0.977     0.975       888
                               macro avg      0.668     0.676     0.661       888
                            weighted avg      0.974     0.977     0.974       888

I0717 07:03:21.597088 140695199004544 interactiveshell.py:2882] 
                              precision    recall  f1-score   support

                           O      0.000     0.000     0.000       0.0
           B-toloc.city_name      0.971     0.993     0.982     716.0
         B-fromloc.city_name      0.987     0.993     0.990     704.0
           I-toloc.city_name      0.927     1.000     0.962     265.0
      B-depart_date.day_name      0.990     0.981     0.986     212.0
              B-airline_name      0.981     1.000     0.990     101.0
         I-fromloc.city_name      0.982     0.938     0.960     177.0
 B-depart_time.period_of_day      0.991     0.869     0.926     130.0
              I-airline_name      1.000     1.000     1.000      65.0
    B-depart_date.day_number      0.964     0.964     0.964      55.0
    B-depart_date.month_name      0.981     0.946     0.964      56.0
          B-depart_time.time      0.851     1.000     0.919      57.0
                B-round_trip      1.000     0.986     0.993      73.0
             B-cost_relative      1.000     0.973     0.986      37.0
                I-round_trip      1.000     1.000     1.000      71.0
                B-flight_mod      1.000     0.958     0.979      24.0
 B-depart_time.time_relative      0.969     0.954     0.961      65.0
          I-depart_time.time      0.877     0.962     0.917      52.0
         B-stoploc.city_name      1.000     0.900     0.947      20.0
                 B-city_name      0.889     0.561     0.688      57.0
                B-class_type      0.923     1.000     0.960      24.0
          B-arrive_time.time      0.943     0.971     0.957      34.0
 B-arrive_time.time_relative      0.882     0.968     0.923      31.0
                I-class_type      1.000     1.000     1.000      17.0
               B-flight_stop      0.955     1.000     0.977      21.0
          I-arrive_time.time      0.919     0.971     0.944      35.0
              B-airline_code      1.000     0.853     0.921      34.0
    I-depart_date.day_number      1.000     1.000     1.000      15.0
      I-fromloc.airport_name      0.429     1.000     0.600      15.0
      B-fromloc.airport_name      0.355     0.917     0.512      12.0
      B-arrive_date.day_name      0.688     1.000     0.815      11.0
          B-toloc.state_code      0.947     1.000     0.973      18.0
B-depart_date.today_relative      1.000     0.889     0.941       9.0
             B-flight_number      0.346     0.818     0.486      11.0
 B-depart_date.date_relative      0.850     1.000     0.919      17.0
          B-toloc.state_name      0.897     0.929     0.912      28.0
           B-fare_basis_code      0.941     0.941     0.941      17.0
               B-flight_time      1.000     1.000     1.000       1.0
                        B-or      1.000     1.000     1.000       3.0
 B-arrive_time.period_of_day      0.857     1.000     0.923       6.0
          B-meal_description      1.000     0.900     0.947      10.0
             I-cost_relative      1.000     0.667     0.800       3.0
              I-airport_name      0.846     0.379     0.524      29.0
               B-fare_amount      0.500     0.500     0.500       2.0
               I-fare_amount      1.000     1.000     1.000       2.0
                 I-city_name      1.000     0.467     0.636      30.0
        I-toloc.airport_name      0.750     1.000     0.857       3.0
            B-transport_type      1.000     0.700     0.824      10.0
    B-arrive_date.month_name      0.625     0.833     0.714       6.0
    B-arrive_date.day_number      0.833     0.833     0.833       6.0
         I-stoploc.city_name      0.818     0.900     0.857      10.0
                      B-meal      1.000     1.000     1.000      16.0
        B-fromloc.state_code      1.000     0.957     0.978      23.0
    B-depart_time.period_mod      0.556     1.000     0.714       5.0
                   B-connect      1.000     1.000     1.000       6.0
               B-flight_days      1.000     1.000     1.000      10.0
        B-toloc.airport_name      0.500     0.333     0.400       3.0
        B-fromloc.state_name      0.773     1.000     0.872      17.0
              B-airport_name      0.750     0.286     0.414      21.0
                   B-economy      1.000     1.000     1.000       6.0
               I-flight_time      1.000     1.000     1.000       1.0
             B-aircraft_code      0.903     0.848     0.875      33.0
                       B-mod      0.000     0.000     0.000       2.0
              B-airport_code      0.000     0.000     0.000       9.0
    B-depart_time.start_time      0.500     0.333     0.400       3.0
      B-depart_time.end_time      0.400     0.667     0.500       3.0
          B-depart_date.year      1.000     0.667     0.800       3.0
            I-transport_type      0.200     1.000     0.333       1.0
          B-restriction_code      0.667     1.000     0.800       4.0
    B-arrive_time.start_time      0.857     0.750     0.800       8.0
        B-toloc.airport_code      1.000     0.500     0.667       4.0
      B-arrive_time.end_time      0.875     0.875     0.875       8.0
      I-arrive_time.end_time      0.875     0.875     0.875       8.0
      I-depart_time.end_time      1.000     0.333     0.500       3.0
               I-flight_stop      0.000     0.000     0.000       0.0
      B-fromloc.airport_code      1.000     1.000     1.000       5.0
          I-restriction_code      1.000     1.000     1.000       3.0
    I-depart_time.start_time      0.500     1.000     0.667       1.0
          I-toloc.state_name      1.000     1.000     1.000       1.0
I-depart_date.today_relative      0.000     0.000     0.000       0.0
 B-arrive_date.date_relative      1.000     0.500     0.667       2.0
                I-flight_mod      1.000     0.167     0.286       6.0
                   I-economy      0.000     0.000     0.000       0.0
 B-return_date.date_relative      1.000     0.667     0.800       3.0
        I-fromloc.state_name      1.000     1.000     1.000       1.0
                B-state_code      1.000     1.000     1.000       1.0
    I-arrive_time.start_time      1.000     1.000     1.000       1.0
    I-arrive_date.day_number      0.000     0.000     0.000       0.0
                 B-meal_code      1.000     1.000     1.000       1.0
 I-depart_time.period_of_day      0.000     0.000     0.000       1.0
                  B-day_name      1.000     0.500     0.667       2.0
             B-period_of_day      1.000     0.750     0.857       4.0
        B-stoploc.state_code      0.000     0.000     0.000       0.0
    B-return_date.month_name      0.000     0.000     0.000       0.0
    B-return_date.day_number      0.000     0.000     0.000       0.0
    B-arrive_time.period_mod      0.000     0.000     0.000       0.0
                 I-meal_code      0.000     0.000     0.000       0.0
        B-toloc.country_name      1.000     1.000     1.000       1.0
                 B-days_code      1.000     1.000     1.000       1.0
 I-arrive_time.period_of_day      0.000     0.000     0.000       0.0
            I-today_relative      0.000     0.000     0.000       0.0
 B-return_time.period_of_day      0.000     0.000     0.000       0.0
                      B-time      0.000     0.000     0.000       0.0
           I-fare_basis_code      0.000     0.000     0.000       0.0
 I-arrive_time.time_relative      1.000     0.500     0.667       4.0
 I-depart_time.time_relative      0.000     0.000     0.000       1.0
            B-today_relative      0.000     0.000     0.000       0.0
                B-state_name      0.000     0.000     0.000       9.0
B-arrive_date.today_relative      0.000     0.000     0.000       0.0
    B-return_time.period_mod      0.000     0.000     0.000       0.0
                B-month_name      0.000     0.000     0.000       0.0
                B-day_number      0.000     0.000     0.000       0.0
 I-return_date.date_relative      1.000     0.667     0.800       3.0
I-return_date.today_relative      0.000     0.000     0.000       0.0
      B-stoploc.airport_name      0.000     0.000     0.000       0.0
             B-time_relative      0.000     0.000     0.000       0.0
                      I-time      0.000     0.000     0.000       0.0
    I-return_date.day_number      0.000     0.000     0.000       0.0
          I-meal_description      0.000     0.000     0.000       0.0
B-return_date.today_relative      0.000     0.000     0.000       0.0
      B-return_date.day_name      0.000     0.000     0.000       2.0

                   micro avg      0.941     0.942     0.942    3657.0
                   macro avg      0.645     0.625     0.616    3657.0
                weighted avg      0.948     0.942     0.939    3657.0

I0717 07:03:21.597997 140695199004544 interactiveshell.py:2882] Best Slot F1: 0.942
Reading ../data/atis.train.w-intent.iob
I0717 07:03:24.545400 140695199004544 interactiveshell.py:2882] Step 8450 | Loss: 0.0050 | Loss_intent: 0.0002 | Loss_slots: 0.0048 | Spent: 6.7 secs | LR: 0.000123
I0717 07:03:28.712290 140695199004544 interactiveshell.py:2882] Step 8500 | Loss: 0.0101 | Loss_intent: 0.0001 | Loss_slots: 0.0100 | Spent: 4.2 secs | LR: 0.000123
I0717 07:03:32.905836 140695199004544 interactiveshell.py:2882] Step 8550 | Loss: 0.0206 | Loss_intent: 0.0047 | Loss_slots: 0.0159 | Spent: 4.2 secs | LR: 0.000122
I0717 07:03:37.081303 140695199004544 interactiveshell.py:2882] Step 8600 | Loss: 0.0047 | Loss_intent: 0.0000 | Loss_slots: 0.0047 | Spent: 4.2 secs | LR: 0.000121
I0717 07:03:41.274714 140695199004544 interactiveshell.py:2882] Step 8650 | Loss: 0.0480 | Loss_intent: 0.0000 | Loss_slots: 0.0479 | Spent: 4.2 secs | LR: 0.000121
I0717 07:03:45.443771 140695199004544 interactiveshell.py:2882] Step 8700 | Loss: 0.0295 | Loss_intent: 0.0000 | Loss_slots: 0.0295 | Spent: 4.2 secs | LR: 0.000120
Reading ../data/atis.test.w-intent.iob
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.
  'precision', 'predicted', average, warn_for)
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1439: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples.
  'recall', 'true', average, warn_for)
I0717 07:03:50.117304 140695199004544 interactiveshell.py:2882] 
                                          precision    recall  f1-score   support

                             atis_flight      0.990     0.984     0.987       632
                            atis_airfare      0.923     1.000     0.960        48
                     atis_ground_service      1.000     1.000     1.000        36
                            atis_airline      0.974     1.000     0.987        38
                       atis_abbreviation      0.971     1.000     0.985        33
                           atis_aircraft      0.900     1.000     0.947         9
                        atis_flight_time      1.000     1.000     1.000         1
                           atis_quantity      0.429     1.000     0.600         3
                atis_flight#atis_airfare      0.875     0.583     0.700        12
                            atis_airport      0.947     1.000     0.973        18
                           atis_distance      1.000     1.000     1.000        10
                               atis_city      0.667     1.000     0.800         6
                        atis_ground_fare      1.000     0.714     0.833         7
                           atis_capacity      1.000     1.000     1.000        21
                          atis_flight_no      0.889     1.000     0.941         8
                               atis_meal      1.000     0.833     0.909         6
                        atis_restriction      0.000     0.000     0.000         0
             atis_airline#atis_flight_no      0.000     0.000     0.000         0
    atis_ground_service#atis_ground_fare      0.000     0.000     0.000         0
           atis_airfare#atis_flight_time      0.000     0.000     0.000         0
                           atis_cheapest      0.000     0.000     0.000         0
atis_aircraft#atis_flight#atis_flight_no      0.000     0.000     0.000         0

                               micro avg      0.974     0.980     0.977       888
                               macro avg      0.662     0.687     0.665       888
                            weighted avg      0.978     0.980     0.977       888

I0717 07:03:50.157897 140695199004544 interactiveshell.py:2882] 
                              precision    recall  f1-score   support

                           O      0.000     0.000     0.000       0.0
           B-toloc.city_name      0.969     0.994     0.981     716.0
         B-fromloc.city_name      0.990     0.979     0.984     704.0
           I-toloc.city_name      0.967     0.989     0.978     265.0
      B-depart_date.day_name      0.990     0.972     0.981     212.0
              B-airline_name      0.980     0.990     0.985     101.0
         I-fromloc.city_name      0.983     0.966     0.974     177.0
 B-depart_time.period_of_day      0.966     0.862     0.911     130.0
              I-airline_name      1.000     1.000     1.000      65.0
    B-depart_date.day_number      0.962     0.909     0.935      55.0
    B-depart_date.month_name      0.981     0.946     0.964      56.0
          B-depart_time.time      0.862     0.982     0.918      57.0
                B-round_trip      1.000     0.986     0.993      73.0
             B-cost_relative      1.000     0.973     0.986      37.0
                I-round_trip      1.000     1.000     1.000      71.0
                B-flight_mod      0.885     0.958     0.920      24.0
 B-depart_time.time_relative      0.984     0.954     0.969      65.0
          I-depart_time.time      0.864     0.981     0.919      52.0
         B-stoploc.city_name      1.000     0.950     0.974      20.0
                 B-city_name      0.692     0.632     0.661      57.0
                B-class_type      0.923     1.000     0.960      24.0
          B-arrive_time.time      0.892     0.971     0.930      34.0
 B-arrive_time.time_relative      0.882     0.968     0.923      31.0
                I-class_type      1.000     1.000     1.000      17.0
               B-flight_stop      1.000     1.000     1.000      21.0
          I-arrive_time.time      0.944     0.971     0.958      35.0
              B-airline_code      1.000     0.853     0.921      34.0
    I-depart_date.day_number      1.000     1.000     1.000      15.0
      I-fromloc.airport_name      0.417     1.000     0.588      15.0
      B-fromloc.airport_name      0.345     0.833     0.488      12.0
      B-arrive_date.day_name      0.611     1.000     0.759      11.0
          B-toloc.state_code      0.947     1.000     0.973      18.0
B-depart_date.today_relative      1.000     1.000     1.000       9.0
             B-flight_number      0.400     0.909     0.556      11.0
 B-depart_date.date_relative      0.895     1.000     0.944      17.0
          B-toloc.state_name      0.931     0.964     0.947      28.0
           B-fare_basis_code      0.909     0.588     0.714      17.0
               B-flight_time      1.000     1.000     1.000       1.0
                        B-or      1.000     1.000     1.000       3.0
 B-arrive_time.period_of_day      0.857     1.000     0.923       6.0
          B-meal_description      1.000     0.900     0.947      10.0
             I-cost_relative      1.000     0.667     0.800       3.0
              I-airport_name      0.846     0.379     0.524      29.0
               B-fare_amount      0.333     0.500     0.400       2.0
               I-fare_amount      1.000     1.000     1.000       2.0
                 I-city_name      0.889     0.533     0.667      30.0
        I-toloc.airport_name      0.600     1.000     0.750       3.0
            B-transport_type      1.000     0.700     0.824      10.0
    B-arrive_date.month_name      0.625     0.833     0.714       6.0
    B-arrive_date.day_number      0.556     0.833     0.667       6.0
         I-stoploc.city_name      0.562     0.900     0.692      10.0
                      B-meal      1.000     1.000     1.000      16.0
        B-fromloc.state_code      1.000     0.957     0.978      23.0
    B-depart_time.period_mod      1.000     0.400     0.571       5.0
                   B-connect      1.000     1.000     1.000       6.0
               B-flight_days      1.000     1.000     1.000      10.0
        B-toloc.airport_name      0.500     0.333     0.400       3.0
        B-fromloc.state_name      0.938     0.882     0.909      17.0
              B-airport_name      0.778     0.333     0.467      21.0
                   B-economy      1.000     1.000     1.000       6.0
               I-flight_time      1.000     1.000     1.000       1.0
             B-aircraft_code      0.900     0.818     0.857      33.0
                       B-mod      0.000     0.000     0.000       2.0
              B-airport_code      0.000     0.000     0.000       9.0
    B-depart_time.start_time      0.500     0.333     0.400       3.0
      B-depart_time.end_time      1.000     0.667     0.800       3.0
          B-depart_date.year      1.000     0.667     0.800       3.0
            I-transport_type      0.200     1.000     0.333       1.0
          B-restriction_code      0.364     1.000     0.533       4.0
    B-arrive_time.start_time      0.889     1.000     0.941       8.0
        B-toloc.airport_code      1.000     0.500     0.667       4.0
      B-arrive_time.end_time      0.889     1.000     0.941       8.0
      I-arrive_time.end_time      0.778     0.875     0.824       8.0
      I-depart_time.end_time      1.000     0.333     0.500       3.0
               I-flight_stop      0.000     0.000     0.000       0.0
      B-fromloc.airport_code      1.000     1.000     1.000       5.0
          I-restriction_code      0.750     1.000     0.857       3.0
    I-depart_time.start_time      0.500     1.000     0.667       1.0
          I-toloc.state_name      1.000     1.000     1.000       1.0
I-depart_date.today_relative      0.000     0.000     0.000       0.0
 B-arrive_date.date_relative      1.000     0.500     0.667       2.0
                I-flight_mod      1.000     0.167     0.286       6.0
                   I-economy      0.000     0.000     0.000       0.0
 B-return_date.date_relative      1.000     0.333     0.500       3.0
        I-fromloc.state_name      1.000     1.000     1.000       1.0
                B-state_code      1.000     1.000     1.000       1.0
    I-arrive_time.start_time      0.000     0.000     0.000       1.0
    I-arrive_date.day_number      0.000     0.000     0.000       0.0
                 B-meal_code      1.000     1.000     1.000       1.0
 I-depart_time.period_of_day      0.000     0.000     0.000       1.0
                  B-day_name      1.000     0.500     0.667       2.0
             B-period_of_day      1.000     0.750     0.857       4.0
        B-stoploc.state_code      0.000     0.000     0.000       0.0
    B-return_date.month_name      0.000     0.000     0.000       0.0
    B-return_date.day_number      0.000     0.000     0.000       0.0
    B-arrive_time.period_mod      0.000     0.000     0.000       0.0
                 I-meal_code      0.000     0.000     0.000       0.0
        B-toloc.country_name      1.000     1.000     1.000       1.0
                 B-days_code      1.000     1.000     1.000       1.0
 I-arrive_time.period_of_day      0.000     0.000     0.000       0.0
            I-today_relative      0.000     0.000     0.000       0.0
 B-return_time.period_of_day      0.000     0.000     0.000       0.0
                      B-time      0.000     0.000     0.000       0.0
           I-fare_basis_code      0.000     0.000     0.000       0.0
 I-arrive_time.time_relative      1.000     0.500     0.667       4.0
 I-depart_time.time_relative      0.000     0.000     0.000       1.0
            B-today_relative      0.000     0.000     0.000       0.0
                B-state_name      0.000     0.000     0.000       9.0
B-arrive_date.today_relative      0.000     0.000     0.000       0.0
    B-return_time.period_mod      0.000     0.000     0.000       0.0
                B-month_name      0.000     0.000     0.000       0.0
                B-day_number      0.000     0.000     0.000       0.0
 I-return_date.date_relative      1.000     0.667     0.800       3.0
I-return_date.today_relative      0.000     0.000     0.000       0.0
      B-stoploc.airport_name      0.000     0.000     0.000       0.0
             B-time_relative      0.000     0.000     0.000       0.0
                      I-time      0.000     0.000     0.000       0.0
    I-return_date.day_number      0.000     0.000     0.000       0.0
          I-meal_description      0.000     0.000     0.000       0.0
B-return_date.today_relative      0.000     0.000     0.000       0.0
      B-return_date.day_name      0.000     0.000     0.000       2.0

                   micro avg      0.937     0.938     0.937    3657.0
                   macro avg      0.632     0.610     0.599    3657.0
                weighted avg      0.945     0.938     0.936    3657.0

I0717 07:03:50.158637 140695199004544 interactiveshell.py:2882] Best Slot F1: 0.942
Reading ../data/atis.train.w-intent.iob
I0717 07:03:52.055950 140695199004544 interactiveshell.py:2882] Step 8750 | Loss: 0.0076 | Loss_intent: 0.0002 | Loss_slots: 0.0074 | Spent: 6.6 secs | LR: 0.000119
I0717 07:03:56.235776 140695199004544 interactiveshell.py:2882] Step 8800 | Loss: 0.0072 | Loss_intent: 0.0001 | Loss_slots: 0.0071 | Spent: 4.2 secs | LR: 0.000119
I0717 07:04:00.440772 140695199004544 interactiveshell.py:2882] Step 8850 | Loss: 0.0215 | Loss_intent: 0.0000 | Loss_slots: 0.0214 | Spent: 4.2 secs | LR: 0.000118
I0717 07:04:04.630397 140695199004544 interactiveshell.py:2882] Step 8900 | Loss: 0.0623 | Loss_intent: 0.0003 | Loss_slots: 0.0621 | Spent: 4.2 secs | LR: 0.000117
I0717 07:04:08.821554 140695199004544 interactiveshell.py:2882] Step 8950 | Loss: 0.0118 | Loss_intent: 0.0003 | Loss_slots: 0.0115 | Spent: 4.2 secs | LR: 0.000117
I0717 07:04:13.026412 140695199004544 interactiveshell.py:2882] Step 9000 | Loss: 0.0342 | Loss_intent: 0.0000 | Loss_slots: 0.0342 | Spent: 4.2 secs | LR: 0.000116
Reading ../data/atis.test.w-intent.iob
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.
  'precision', 'predicted', average, warn_for)
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1439: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples.
  'recall', 'true', average, warn_for)
I0717 07:04:18.670587 140695199004544 interactiveshell.py:2882] 
                                          precision    recall  f1-score   support

                             atis_flight      0.994     0.981     0.987       632
                            atis_airfare      0.857     1.000     0.923        48
                     atis_ground_service      1.000     1.000     1.000        36
                            atis_airline      0.974     1.000     0.987        38
                       atis_abbreviation      0.970     0.970     0.970        33
                           atis_aircraft      0.900     1.000     0.947         9
                        atis_flight_time      1.000     1.000     1.000         1
                           atis_quantity      0.429     1.000     0.600         3
                atis_flight#atis_airfare      0.833     0.417     0.556        12
                            atis_airport      0.947     1.000     0.973        18
                           atis_distance      1.000     1.000     1.000        10
                               atis_city      0.545     1.000     0.706         6
                        atis_ground_fare      1.000     0.714     0.833         7
                           atis_capacity      1.000     1.000     1.000        21
                          atis_flight_no      0.889     1.000     0.941         8
                               atis_meal      1.000     0.833     0.909         6
                        atis_restriction      0.000     0.000     0.000         0
             atis_airline#atis_flight_no      0.000     0.000     0.000         0
    atis_ground_service#atis_ground_fare      0.000     0.000     0.000         0
           atis_airfare#atis_flight_time      0.000     0.000     0.000         0
                           atis_cheapest      0.000     0.000     0.000         0
atis_aircraft#atis_flight#atis_flight_no      0.000     0.000     0.000         0

                               micro avg      0.969     0.974     0.971       888
                               macro avg      0.652     0.678     0.651       888
                            weighted avg      0.975     0.974     0.972       888

I0717 07:04:18.721522 140695199004544 interactiveshell.py:2882] 
                              precision    recall  f1-score   support

                           O      0.000     0.000     0.000       0.0
           B-toloc.city_name      0.973     0.992     0.982     716.0
         B-fromloc.city_name      0.979     0.990     0.984     704.0
           I-toloc.city_name      0.974     0.996     0.985     265.0
      B-depart_date.day_name      0.990     0.976     0.983     212.0
              B-airline_name      0.980     0.990     0.985     101.0
         I-fromloc.city_name      0.983     0.977     0.980     177.0
 B-depart_time.period_of_day      0.991     0.862     0.922     130.0
              I-airline_name      1.000     1.000     1.000      65.0
    B-depart_date.day_number      0.962     0.909     0.935      55.0
    B-depart_date.month_name      0.981     0.946     0.964      56.0
          B-depart_time.time      0.859     0.965     0.909      57.0
                B-round_trip      1.000     0.986     0.993      73.0
             B-cost_relative      1.000     0.973     0.986      37.0
                I-round_trip      1.000     1.000     1.000      71.0
                B-flight_mod      0.920     0.958     0.939      24.0
 B-depart_time.time_relative      0.968     0.923     0.945      65.0
          I-depart_time.time      0.864     0.981     0.919      52.0
         B-stoploc.city_name      0.950     0.950     0.950      20.0
                 B-city_name      0.773     0.596     0.673      57.0
                B-class_type      0.923     1.000     0.960      24.0
          B-arrive_time.time      0.892     0.971     0.930      34.0
 B-arrive_time.time_relative      0.833     0.968     0.896      31.0
                I-class_type      1.000     1.000     1.000      17.0
               B-flight_stop      0.955     1.000     0.977      21.0
          I-arrive_time.time      0.944     0.971     0.958      35.0
              B-airline_code      1.000     0.853     0.921      34.0
    I-depart_date.day_number      1.000     1.000     1.000      15.0
      I-fromloc.airport_name      0.441     1.000     0.612      15.0
      B-fromloc.airport_name      0.346     0.750     0.474      12.0
      B-arrive_date.day_name      0.647     1.000     0.786      11.0
          B-toloc.state_code      1.000     1.000     1.000      18.0
B-depart_date.today_relative      1.000     1.000     1.000       9.0
             B-flight_number      0.526     0.909     0.667      11.0
 B-depart_date.date_relative      0.944     1.000     0.971      17.0
          B-toloc.state_name      0.900     0.964     0.931      28.0
           B-fare_basis_code      0.941     0.941     0.941      17.0
               B-flight_time      1.000     1.000     1.000       1.0
                        B-or      1.000     1.000     1.000       3.0
 B-arrive_time.period_of_day      0.857     1.000     0.923       6.0
          B-meal_description      1.000     0.900     0.947      10.0
             I-cost_relative      1.000     0.667     0.800       3.0
              I-airport_name      0.846     0.379     0.524      29.0
               B-fare_amount      0.250     0.500     0.333       2.0
               I-fare_amount      1.000     1.000     1.000       2.0
                 I-city_name      1.000     0.567     0.723      30.0
        I-toloc.airport_name      0.750     1.000     0.857       3.0
            B-transport_type      1.000     0.800     0.889      10.0
    B-arrive_date.month_name      0.625     0.833     0.714       6.0
    B-arrive_date.day_number      0.556     0.833     0.667       6.0
         I-stoploc.city_name      0.643     0.900     0.750      10.0
                      B-meal      1.000     1.000     1.000      16.0
        B-fromloc.state_code      1.000     1.000     1.000      23.0
    B-depart_time.period_mod      0.714     1.000     0.833       5.0
                   B-connect      1.000     1.000     1.000       6.0
               B-flight_days      1.000     1.000     1.000      10.0
        B-toloc.airport_name      1.000     0.333     0.500       3.0
        B-fromloc.state_name      0.941     0.941     0.941      17.0
              B-airport_name      0.727     0.381     0.500      21.0
                   B-economy      1.000     1.000     1.000       6.0
               I-flight_time      1.000     1.000     1.000       1.0
             B-aircraft_code      0.848     0.848     0.848      33.0
                       B-mod      0.000     0.000     0.000       2.0
              B-airport_code      0.000     0.000     0.000       9.0
    B-depart_time.start_time      0.500     0.333     0.400       3.0
      B-depart_time.end_time      0.667     0.667     0.667       3.0
          B-depart_date.year      1.000     0.667     0.800       3.0
            I-transport_type      0.000     0.000     0.000       1.0
          B-restriction_code      0.364     1.000     0.533       4.0
    B-arrive_time.start_time      0.889     1.000     0.941       8.0
        B-toloc.airport_code      1.000     0.500     0.667       4.0
      B-arrive_time.end_time      0.857     0.750     0.800       8.0
      I-arrive_time.end_time      0.778     0.875     0.824       8.0
      I-depart_time.end_time      1.000     0.333     0.500       3.0
               I-flight_stop      0.000     0.000     0.000       0.0
      B-fromloc.airport_code      1.000     1.000     1.000       5.0
          I-restriction_code      0.750     1.000     0.857       3.0
    I-depart_time.start_time      0.500     1.000     0.667       1.0
          I-toloc.state_name      1.000     1.000     1.000       1.0
I-depart_date.today_relative      0.000     0.000     0.000       0.0
 B-arrive_date.date_relative      1.000     1.000     1.000       2.0
                I-flight_mod      1.000     0.167     0.286       6.0
                   I-economy      0.000     0.000     0.000       0.0
 B-return_date.date_relative      1.000     0.333     0.500       3.0
        I-fromloc.state_name      1.000     1.000     1.000       1.0
                B-state_code      1.000     1.000     1.000       1.0
    I-arrive_time.start_time      0.000     0.000     0.000       1.0
    I-arrive_date.day_number      0.000     0.000     0.000       0.0
                 B-meal_code      1.000     1.000     1.000       1.0
 I-depart_time.period_of_day      0.000     0.000     0.000       1.0
                  B-day_name      1.000     0.500     0.667       2.0
             B-period_of_day      1.000     0.750     0.857       4.0
        B-stoploc.state_code      0.000     0.000     0.000       0.0
    B-return_date.month_name      0.000     0.000     0.000       0.0
    B-return_date.day_number      0.000     0.000     0.000       0.0
    B-arrive_time.period_mod      0.000     0.000     0.000       0.0
                 I-meal_code      0.000     0.000     0.000       0.0
        B-toloc.country_name      1.000     1.000     1.000       1.0
                 B-days_code      1.000     1.000     1.000       1.0
 I-arrive_time.period_of_day      0.000     0.000     0.000       0.0
            I-today_relative      0.000     0.000     0.000       0.0
 B-return_time.period_of_day      0.000     0.000     0.000       0.0
                      B-time      0.000     0.000     0.000       0.0
           I-fare_basis_code      0.000     0.000     0.000       0.0
 I-arrive_time.time_relative      1.000     0.500     0.667       4.0
 I-depart_time.time_relative      0.000     0.000     0.000       1.0
            B-today_relative      0.000     0.000     0.000       0.0
                B-state_name      0.000     0.000     0.000       9.0
B-arrive_date.today_relative      0.000     0.000     0.000       0.0
    B-return_time.period_mod      0.000     0.000     0.000       0.0
                B-month_name      0.000     0.000     0.000       0.0
                B-day_number      0.000     0.000     0.000       0.0
 I-return_date.date_relative      0.667     0.667     0.667       3.0
I-return_date.today_relative      0.000     0.000     0.000       0.0
      B-stoploc.airport_name      0.000     0.000     0.000       0.0
             B-time_relative      0.000     0.000     0.000       0.0
                      I-time      0.000     0.000     0.000       0.0
    I-return_date.day_number      0.000     0.000     0.000       0.0
          I-meal_description      0.000     0.000     0.000       0.0
B-return_date.today_relative      0.000     0.000     0.000       0.0
      B-return_date.day_name      0.000     0.000     0.000       2.0

                   micro avg      0.942     0.943     0.942    3657.0
                   macro avg      0.630     0.613     0.604    3657.0
                weighted avg      0.947     0.943     0.940    3657.0

I0717 07:04:18.722627 140695199004544 interactiveshell.py:2882] Best Slot F1: 0.942
Reading ../data/atis.train.w-intent.iob
I0717 07:04:19.616852 140695199004544 interactiveshell.py:2882] Step 9050 | Loss: 0.0269 | Loss_intent: 0.0000 | Loss_slots: 0.0269 | Spent: 6.6 secs | LR: 0.000116
I0717 07:04:23.785882 140695199004544 interactiveshell.py:2882] Step 9100 | Loss: 0.0074 | Loss_intent: 0.0046 | Loss_slots: 0.0028 | Spent: 4.2 secs | LR: 0.000115
I0717 07:04:27.970923 140695199004544 interactiveshell.py:2882] Step 9150 | Loss: 0.0084 | Loss_intent: 0.0000 | Loss_slots: 0.0083 | Spent: 4.2 secs | LR: 0.000114
I0717 07:04:32.197667 140695199004544 interactiveshell.py:2882] Step 9200 | Loss: 0.0202 | Loss_intent: 0.0039 | Loss_slots: 0.0163 | Spent: 4.2 secs | LR: 0.000114
I0717 07:04:36.413265 140695199004544 interactiveshell.py:2882] Step 9250 | Loss: 0.0111 | Loss_intent: 0.0001 | Loss_slots: 0.0110 | Spent: 4.2 secs | LR: 0.000113
I0717 07:04:40.928836 140695199004544 interactiveshell.py:2882] Step 9300 | Loss: 0.0194 | Loss_intent: 0.0004 | Loss_slots: 0.0190 | Spent: 4.5 secs | LR: 0.000113
I0717 07:04:45.701622 140695199004544 interactiveshell.py:2882] Step 9350 | Loss: 0.0056 | Loss_intent: 0.0000 | Loss_slots: 0.0056 | Spent: 4.8 secs | LR: 0.000112
Reading ../data/atis.test.w-intent.iob
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.
  'precision', 'predicted', average, warn_for)
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1439: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples.
  'recall', 'true', average, warn_for)
I0717 07:04:48.414595 140695199004544 interactiveshell.py:2882] 
                                          precision    recall  f1-score   support

                             atis_flight      0.983     0.983     0.983       632
                            atis_airfare      0.904     0.979     0.940        48
                     atis_ground_service      1.000     1.000     1.000        36
                            atis_airline      0.974     1.000     0.987        38
                       atis_abbreviation      1.000     0.970     0.985        33
                           atis_aircraft      0.818     1.000     0.900         9
                        atis_flight_time      1.000     1.000     1.000         1
                           atis_quantity      0.429     1.000     0.600         3
                atis_flight#atis_airfare      0.833     0.417     0.556        12
                            atis_airport      0.947     1.000     0.973        18
                           atis_distance      1.000     1.000     1.000        10
                               atis_city      0.500     0.667     0.571         6
                        atis_ground_fare      1.000     0.714     0.833         7
                           atis_capacity      1.000     1.000     1.000        21
                          atis_flight_no      0.889     1.000     0.941         8
                               atis_meal      1.000     0.667     0.800         6
                        atis_restriction      0.000     0.000     0.000         0
             atis_airline#atis_flight_no      0.000     0.000     0.000         0
    atis_ground_service#atis_ground_fare      0.000     0.000     0.000         0
           atis_airfare#atis_flight_time      0.000     0.000     0.000         0
                           atis_cheapest      0.000     0.000     0.000         0
atis_aircraft#atis_flight#atis_flight_no      0.000     0.000     0.000         0

                               micro avg      0.965     0.971     0.968       888
                               macro avg      0.649     0.654     0.639       888
                            weighted avg      0.970     0.971     0.968       888

I0717 07:04:48.456040 140695199004544 interactiveshell.py:2882] 
                              precision    recall  f1-score   support

                           O      0.000     0.000     0.000       0.0
           B-toloc.city_name      0.971     0.994     0.983     716.0
         B-fromloc.city_name      0.982     0.990     0.986     704.0
           I-toloc.city_name      0.971     0.996     0.983     265.0
      B-depart_date.day_name      0.991     0.986     0.988     212.0
              B-airline_name      0.971     0.990     0.980     101.0
         I-fromloc.city_name      0.967     0.983     0.975     177.0
 B-depart_time.period_of_day      0.975     0.885     0.927     130.0
              I-airline_name      1.000     1.000     1.000      65.0
    B-depart_date.day_number      0.964     0.964     0.964      55.0
    B-depart_date.month_name      0.982     0.964     0.973      56.0
          B-depart_time.time      0.851     1.000     0.919      57.0
                B-round_trip      1.000     0.986     0.993      73.0
             B-cost_relative      1.000     0.973     0.986      37.0
                I-round_trip      1.000     1.000     1.000      71.0
                B-flight_mod      1.000     0.958     0.979      24.0
 B-depart_time.time_relative      0.985     0.985     0.985      65.0
          I-depart_time.time      0.881     1.000     0.937      52.0
         B-stoploc.city_name      1.000     0.950     0.974      20.0
                 B-city_name      0.756     0.596     0.667      57.0
                B-class_type      0.923     1.000     0.960      24.0
          B-arrive_time.time      0.971     0.971     0.971      34.0
 B-arrive_time.time_relative      0.938     0.968     0.952      31.0
                I-class_type      1.000     1.000     1.000      17.0
               B-flight_stop      0.955     1.000     0.977      21.0
          I-arrive_time.time      0.971     0.971     0.971      35.0
              B-airline_code      0.935     0.853     0.892      34.0
    I-depart_date.day_number      1.000     1.000     1.000      15.0
      I-fromloc.airport_name      0.469     1.000     0.638      15.0
      B-fromloc.airport_name      0.391     0.750     0.514      12.0
      B-arrive_date.day_name      0.733     1.000     0.846      11.0
          B-toloc.state_code      1.000     1.000     1.000      18.0
B-depart_date.today_relative      1.000     1.000     1.000       9.0
             B-flight_number      0.333     0.909     0.488      11.0
 B-depart_date.date_relative      0.895     1.000     0.944      17.0
          B-toloc.state_name      0.926     0.893     0.909      28.0
           B-fare_basis_code      0.909     0.588     0.714      17.0
               B-flight_time      1.000     1.000     1.000       1.0
                        B-or      1.000     1.000     1.000       3.0
 B-arrive_time.period_of_day      1.000     1.000     1.000       6.0
          B-meal_description      1.000     0.900     0.947      10.0
             I-cost_relative      1.000     0.667     0.800       3.0
              I-airport_name      0.857     0.414     0.558      29.0
               B-fare_amount      0.400     1.000     0.571       2.0
               I-fare_amount      1.000     1.000     1.000       2.0
                 I-city_name      1.000     0.567     0.723      30.0
        I-toloc.airport_name      1.000     1.000     1.000       3.0
            B-transport_type      1.000     1.000     1.000      10.0
    B-arrive_date.month_name      0.714     0.833     0.769       6.0
    B-arrive_date.day_number      0.833     0.833     0.833       6.0
         I-stoploc.city_name      0.818     0.900     0.857      10.0
                      B-meal      1.000     1.000     1.000      16.0
        B-fromloc.state_code      1.000     1.000     1.000      23.0
    B-depart_time.period_mod      0.600     0.600     0.600       5.0
                   B-connect      1.000     1.000     1.000       6.0
               B-flight_days      1.000     1.000     1.000      10.0
        B-toloc.airport_name      0.500     0.333     0.400       3.0
        B-fromloc.state_name      0.850     1.000     0.919      17.0
              B-airport_name      0.800     0.381     0.516      21.0
                   B-economy      1.000     1.000     1.000       6.0
               I-flight_time      1.000     1.000     1.000       1.0
             B-aircraft_code      0.900     0.818     0.857      33.0
                       B-mod      0.000     0.000     0.000       2.0
              B-airport_code      0.000     0.000     0.000       9.0
    B-depart_time.start_time      0.500     0.333     0.400       3.0
      B-depart_time.end_time      1.000     0.667     0.800       3.0
          B-depart_date.year      1.000     0.667     0.800       3.0
            I-transport_type      0.500     1.000     0.667       1.0
          B-restriction_code      0.500     0.500     0.500       4.0
    B-arrive_time.start_time      0.875     0.875     0.875       8.0
        B-toloc.airport_code      1.000     0.500     0.667       4.0
      B-arrive_time.end_time      0.800     1.000     0.889       8.0
      I-arrive_time.end_time      0.778     0.875     0.824       8.0
      I-depart_time.end_time      1.000     0.333     0.500       3.0
               I-flight_stop      0.000     0.000     0.000       0.0
      B-fromloc.airport_code      0.833     1.000     0.909       5.0
          I-restriction_code      1.000     1.000     1.000       3.0
    I-depart_time.start_time      0.500     1.000     0.667       1.0
          I-toloc.state_name      1.000     1.000     1.000       1.0
I-depart_date.today_relative      0.000     0.000     0.000       0.0
 B-arrive_date.date_relative      1.000     0.500     0.667       2.0
                I-flight_mod      1.000     0.167     0.286       6.0
                   I-economy      0.000     0.000     0.000       0.0
 B-return_date.date_relative      0.000     0.000     0.000       3.0
        I-fromloc.state_name      1.000     1.000     1.000       1.0
                B-state_code      1.000     1.000     1.000       1.0
    I-arrive_time.start_time      0.000     0.000     0.000       1.0
    I-arrive_date.day_number      0.000     0.000     0.000       0.0
                 B-meal_code      1.000     1.000     1.000       1.0
 I-depart_time.period_of_day      0.000     0.000     0.000       1.0
                  B-day_name      1.000     0.500     0.667       2.0
             B-period_of_day      1.000     0.750     0.857       4.0
        B-stoploc.state_code      0.000     0.000     0.000       0.0
    B-return_date.month_name      0.000     0.000     0.000       0.0
    B-return_date.day_number      0.000     0.000     0.000       0.0
    B-arrive_time.period_mod      0.000     0.000     0.000       0.0
                 I-meal_code      0.000     0.000     0.000       0.0
        B-toloc.country_name      1.000     1.000     1.000       1.0
                 B-days_code      1.000     1.000     1.000       1.0
 I-arrive_time.period_of_day      0.000     0.000     0.000       0.0
            I-today_relative      0.000     0.000     0.000       0.0
 B-return_time.period_of_day      0.000     0.000     0.000       0.0
                      B-time      0.000     0.000     0.000       0.0
           I-fare_basis_code      0.000     0.000     0.000       0.0
 I-arrive_time.time_relative      1.000     0.500     0.667       4.0
 I-depart_time.time_relative      0.000     0.000     0.000       1.0
            B-today_relative      0.000     0.000     0.000       0.0
                B-state_name      0.000     0.000     0.000       9.0
B-arrive_date.today_relative      0.000     0.000     0.000       0.0
    B-return_time.period_mod      0.000     0.000     0.000       0.0
                B-month_name      0.000     0.000     0.000       0.0
                B-day_number      0.000     0.000     0.000       0.0
 I-return_date.date_relative      0.500     0.667     0.571       3.0
I-return_date.today_relative      0.000     0.000     0.000       0.0
      B-stoploc.airport_name      0.000     0.000     0.000       0.0
             B-time_relative      0.000     0.000     0.000       0.0
                      I-time      0.000     0.000     0.000       0.0
    I-return_date.day_number      0.000     0.000     0.000       0.0
          I-meal_description      0.000     0.000     0.000       0.0
B-return_date.today_relative      0.000     0.000     0.000       0.0
      B-return_date.day_name      0.000     0.000     0.000       2.0

                   micro avg      0.945     0.946     0.945    3657.0
                   macro avg      0.633     0.613     0.608    3657.0
                weighted avg      0.948     0.946     0.943    3657.0

I0717 07:04:48.457007 140695199004544 interactiveshell.py:2882] Best Slot F1: 0.945
Reading ../data/atis.train.w-intent.iob
I0717 07:04:52.571242 140695199004544 interactiveshell.py:2882] Step 9400 | Loss: 0.0042 | Loss_intent: 0.0000 | Loss_slots: 0.0041 | Spent: 6.9 secs | LR: 0.000111
I0717 07:04:56.759433 140695199004544 interactiveshell.py:2882] Step 9450 | Loss: 0.0065 | Loss_intent: 0.0037 | Loss_slots: 0.0028 | Spent: 4.2 secs | LR: 0.000111
I0717 07:05:00.927234 140695199004544 interactiveshell.py:2882] Step 9500 | Loss: 0.0102 | Loss_intent: 0.0000 | Loss_slots: 0.0102 | Spent: 4.2 secs | LR: 0.000110
I0717 07:05:05.099976 140695199004544 interactiveshell.py:2882] Step 9550 | Loss: 0.0090 | Loss_intent: 0.0000 | Loss_slots: 0.0090 | Spent: 4.2 secs | LR: 0.000110
I0717 07:05:09.300403 140695199004544 interactiveshell.py:2882] Step 9600 | Loss: 0.0260 | Loss_intent: 0.0000 | Loss_slots: 0.0259 | Spent: 4.2 secs | LR: 0.000109
I0717 07:05:13.477851 140695199004544 interactiveshell.py:2882] Step 9650 | Loss: 0.0144 | Loss_intent: 0.0000 | Loss_slots: 0.0143 | Spent: 4.2 secs | LR: 0.000109
Reading ../data/atis.test.w-intent.iob
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.
  'precision', 'predicted', average, warn_for)
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1439: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples.
  'recall', 'true', average, warn_for)
I0717 07:05:16.971606 140695199004544 interactiveshell.py:2882] 
                                          precision    recall  f1-score   support

                             atis_flight      0.983     0.989     0.986       632
                            atis_airfare      0.904     0.979     0.940        48
                     atis_ground_service      1.000     1.000     1.000        36
                            atis_airline      0.950     1.000     0.974        38
                       atis_abbreviation      1.000     0.970     0.985        33
                           atis_aircraft      1.000     0.889     0.941         9
                        atis_flight_time      1.000     1.000     1.000         1
                           atis_quantity      0.429     1.000     0.600         3
                atis_flight#atis_airfare      0.833     0.417     0.556        12
                            atis_airport      1.000     1.000     1.000        18
                           atis_distance      1.000     1.000     1.000        10
                               atis_city      0.750     1.000     0.857         6
                        atis_ground_fare      1.000     0.714     0.833         7
                           atis_capacity      0.952     0.952     0.952        21
                          atis_flight_no      0.889     1.000     0.941         8
                               atis_meal      1.000     0.500     0.667         6
                        atis_restriction      0.000     0.000     0.000         0
             atis_airline#atis_flight_no      0.000     0.000     0.000         0
    atis_ground_service#atis_ground_fare      0.000     0.000     0.000         0
           atis_airfare#atis_flight_time      0.000     0.000     0.000         0
                           atis_cheapest      0.000     0.000     0.000         0
atis_aircraft#atis_flight#atis_flight_no      0.000     0.000     0.000         0

                               micro avg      0.969     0.974     0.971       888
                               macro avg      0.668     0.655     0.647       888
                            weighted avg      0.972     0.974     0.971       888

I0717 07:05:17.011124 140695199004544 interactiveshell.py:2882] 
                              precision    recall  f1-score   support

                           O      0.000     0.000     0.000       0.0
           B-toloc.city_name      0.970     0.994     0.982     716.0
         B-fromloc.city_name      0.986     0.994     0.990     704.0
           I-toloc.city_name      0.974     0.989     0.981     265.0
      B-depart_date.day_name      0.990     0.981     0.986     212.0
              B-airline_name      0.981     1.000     0.990     101.0
         I-fromloc.city_name      0.978     0.983     0.980     177.0
 B-depart_time.period_of_day      0.983     0.877     0.927     130.0
              I-airline_name      1.000     1.000     1.000      65.0
    B-depart_date.day_number      0.945     0.945     0.945      55.0
    B-depart_date.month_name      0.964     0.964     0.964      56.0
          B-depart_time.time      0.838     1.000     0.912      57.0
                B-round_trip      1.000     0.986     0.993      73.0
             B-cost_relative      1.000     0.973     0.986      37.0
                I-round_trip      1.000     1.000     1.000      71.0
                B-flight_mod      1.000     0.958     0.979      24.0
 B-depart_time.time_relative      0.968     0.938     0.953      65.0
          I-depart_time.time      0.881     1.000     0.937      52.0
         B-stoploc.city_name      1.000     0.950     0.974      20.0
                 B-city_name      0.868     0.579     0.695      57.0
                B-class_type      0.923     1.000     0.960      24.0
          B-arrive_time.time      0.892     0.971     0.930      34.0
 B-arrive_time.time_relative      0.857     0.968     0.909      31.0
                I-class_type      1.000     1.000     1.000      17.0
               B-flight_stop      0.955     1.000     0.977      21.0
          I-arrive_time.time      0.971     0.971     0.971      35.0
              B-airline_code      1.000     0.853     0.921      34.0
    I-depart_date.day_number      1.000     1.000     1.000      15.0
      I-fromloc.airport_name      0.405     1.000     0.577      15.0
      B-fromloc.airport_name      0.379     0.917     0.537      12.0
      B-arrive_date.day_name      0.733     1.000     0.846      11.0
          B-toloc.state_code      1.000     1.000     1.000      18.0
B-depart_date.today_relative      1.000     1.000     1.000       9.0
             B-flight_number      0.400     0.909     0.556      11.0
 B-depart_date.date_relative      0.850     1.000     0.919      17.0
          B-toloc.state_name      0.893     0.893     0.893      28.0
           B-fare_basis_code      0.909     0.588     0.714      17.0
               B-flight_time      1.000     1.000     1.000       1.0
                        B-or      1.000     1.000     1.000       3.0
 B-arrive_time.period_of_day      1.000     0.833     0.909       6.0
          B-meal_description      1.000     0.900     0.947      10.0
             I-cost_relative      1.000     0.667     0.800       3.0
              I-airport_name      0.909     0.345     0.500      29.0
               B-fare_amount      0.333     0.500     0.400       2.0
               I-fare_amount      1.000     1.000     1.000       2.0
                 I-city_name      1.000     0.533     0.696      30.0
        I-toloc.airport_name      0.750     1.000     0.857       3.0
            B-transport_type      1.000     0.800     0.889      10.0
    B-arrive_date.month_name      0.667     0.667     0.667       6.0
    B-arrive_date.day_number      0.667     0.667     0.667       6.0
         I-stoploc.city_name      0.714     1.000     0.833      10.0
                      B-meal      1.000     1.000     1.000      16.0
        B-fromloc.state_code      1.000     1.000     1.000      23.0
    B-depart_time.period_mod      0.625     1.000     0.769       5.0
                   B-connect      1.000     1.000     1.000       6.0
               B-flight_days      1.000     1.000     1.000      10.0
        B-toloc.airport_name      0.500     0.333     0.400       3.0
        B-fromloc.state_name      0.895     1.000     0.944      17.0
              B-airport_name      0.800     0.381     0.516      21.0
                   B-economy      1.000     1.000     1.000       6.0
               I-flight_time      1.000     1.000     1.000       1.0
             B-aircraft_code      0.875     0.848     0.862      33.0
                       B-mod      1.000     0.500     0.667       2.0
              B-airport_code      0.000     0.000     0.000       9.0
    B-depart_time.start_time      0.500     0.333     0.400       3.0
      B-depart_time.end_time      1.000     0.667     0.800       3.0
          B-depart_date.year      1.000     0.667     0.800       3.0
            I-transport_type      0.250     1.000     0.400       1.0
          B-restriction_code      0.364     1.000     0.533       4.0
    B-arrive_time.start_time      0.875     0.875     0.875       8.0
        B-toloc.airport_code      1.000     0.500     0.667       4.0
      B-arrive_time.end_time      0.857     0.750     0.800       8.0
      I-arrive_time.end_time      0.875     0.875     0.875       8.0
      I-depart_time.end_time      1.000     0.333     0.500       3.0
               I-flight_stop      0.000     0.000     0.000       0.0
      B-fromloc.airport_code      1.000     1.000     1.000       5.0
          I-restriction_code      0.750     1.000     0.857       3.0
    I-depart_time.start_time      0.500     1.000     0.667       1.0
          I-toloc.state_name      1.000     1.000     1.000       1.0
I-depart_date.today_relative      0.000     0.000     0.000       0.0
 B-arrive_date.date_relative      0.000     0.000     0.000       2.0
                I-flight_mod      1.000     0.167     0.286       6.0
                   I-economy      0.000     0.000     0.000       0.0
 B-return_date.date_relative      1.000     0.667     0.800       3.0
        I-fromloc.state_name      1.000     1.000     1.000       1.0
                B-state_code      1.000     1.000     1.000       1.0
    I-arrive_time.start_time      1.000     1.000     1.000       1.0
    I-arrive_date.day_number      0.000     0.000     0.000       0.0
                 B-meal_code      1.000     1.000     1.000       1.0
 I-depart_time.period_of_day      0.000     0.000     0.000       1.0
                  B-day_name      1.000     0.500     0.667       2.0
             B-period_of_day      1.000     0.750     0.857       4.0
        B-stoploc.state_code      0.000     0.000     0.000       0.0
    B-return_date.month_name      0.000     0.000     0.000       0.0
    B-return_date.day_number      0.000     0.000     0.000       0.0
    B-arrive_time.period_mod      0.000     0.000     0.000       0.0
                 I-meal_code      0.000     0.000     0.000       0.0
        B-toloc.country_name      1.000     1.000     1.000       1.0
                 B-days_code      1.000     1.000     1.000       1.0
 I-arrive_time.period_of_day      0.000     0.000     0.000       0.0
            I-today_relative      0.000     0.000     0.000       0.0
 B-return_time.period_of_day      0.000     0.000     0.000       0.0
                      B-time      0.000     0.000     0.000       0.0
           I-fare_basis_code      0.000     0.000     0.000       0.0
 I-arrive_time.time_relative      1.000     0.500     0.667       4.0
 I-depart_time.time_relative      0.000     0.000     0.000       1.0
            B-today_relative      0.000     0.000     0.000       0.0
                B-state_name      0.000     0.000     0.000       9.0
B-arrive_date.today_relative      0.000     0.000     0.000       0.0
    B-return_time.period_mod      0.000     0.000     0.000       0.0
                B-month_name      0.000     0.000     0.000       0.0
                B-day_number      0.000     0.000     0.000       0.0
 I-return_date.date_relative      1.000     0.667     0.800       3.0
I-return_date.today_relative      0.000     0.000     0.000       0.0
      B-stoploc.airport_name      0.000     0.000     0.000       0.0
             B-time_relative      0.000     0.000     0.000       0.0
                      I-time      0.000     0.000     0.000       0.0
    I-return_date.day_number      0.000     0.000     0.000       0.0
          I-meal_description      0.000     0.000     0.000       0.0
B-return_date.today_relative      0.000     0.000     0.000       0.0
      B-return_date.day_name      0.000     0.000     0.000       2.0

                   micro avg      0.943     0.945     0.944    3657.0
                   macro avg      0.646     0.623     0.615    3657.0
                weighted avg      0.950     0.945     0.942    3657.0

I0717 07:05:17.012001 140695199004544 interactiveshell.py:2882] Best Slot F1: 0.945
Reading ../data/atis.train.w-intent.iob
I0717 07:05:20.079523 140695199004544 interactiveshell.py:2882] Step 9700 | Loss: 0.0071 | Loss_intent: 0.0000 | Loss_slots: 0.0071 | Spent: 6.6 secs | LR: 0.000108
I0717 07:05:24.283446 140695199004544 interactiveshell.py:2882] Step 9750 | Loss: 0.1086 | Loss_intent: 0.1008 | Loss_slots: 0.0078 | Spent: 4.2 secs | LR: 0.000107
I0717 07:05:28.474312 140695199004544 interactiveshell.py:2882] Step 9800 | Loss: 0.0529 | Loss_intent: 0.0007 | Loss_slots: 0.0522 | Spent: 4.2 secs | LR: 0.000107
I0717 07:05:32.678003 140695199004544 interactiveshell.py:2882] Step 9850 | Loss: 0.0164 | Loss_intent: 0.0122 | Loss_slots: 0.0042 | Spent: 4.2 secs | LR: 0.000106
I0717 07:05:36.875582 140695199004544 interactiveshell.py:2882] Step 9900 | Loss: 0.0114 | Loss_intent: 0.0000 | Loss_slots: 0.0114 | Spent: 4.2 secs | LR: 0.000106
I0717 07:05:41.059658 140695199004544 interactiveshell.py:2882] Step 9950 | Loss: 0.0175 | Loss_intent: 0.0003 | Loss_slots: 0.0173 | Spent: 4.2 secs | LR: 0.000105
Reading ../data/atis.test.w-intent.iob
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.
  'precision', 'predicted', average, warn_for)
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1439: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples.
  'recall', 'true', average, warn_for)
I0717 07:05:45.543171 140695199004544 interactiveshell.py:2882] 
                                          precision    recall  f1-score   support

                             atis_flight      0.984     0.984     0.984       632
                            atis_airfare      0.922     0.979     0.949        48
                     atis_ground_service      1.000     1.000     1.000        36
                            atis_airline      0.950     1.000     0.974        38
                       atis_abbreviation      1.000     1.000     1.000        33
                           atis_aircraft      0.818     1.000     0.900         9
                        atis_flight_time      1.000     1.000     1.000         1
                           atis_quantity      0.429     1.000     0.600         3
                atis_flight#atis_airfare      0.833     0.417     0.556        12
                            atis_airport      1.000     1.000     1.000        18
                           atis_distance      1.000     1.000     1.000        10
                               atis_city      0.500     0.667     0.571         6
                        atis_ground_fare      1.000     0.714     0.833         7
                           atis_capacity      1.000     1.000     1.000        21
                          atis_flight_no      0.889     1.000     0.941         8
                               atis_meal      1.000     0.833     0.909         6
                        atis_restriction      0.000     0.000     0.000         0
             atis_airline#atis_flight_no      0.000     0.000     0.000         0
    atis_ground_service#atis_ground_fare      0.000     0.000     0.000         0
           atis_airfare#atis_flight_time      0.000     0.000     0.000         0
                           atis_cheapest      0.000     0.000     0.000         0
atis_aircraft#atis_flight#atis_flight_no      0.000     0.000     0.000         0

                               micro avg      0.969     0.974     0.971       888
                               macro avg      0.651     0.663     0.646       888
                            weighted avg      0.972     0.974     0.971       888

I0717 07:05:45.594762 140695199004544 interactiveshell.py:2882] 
                              precision    recall  f1-score   support

                           O      0.000     0.000     0.000       0.0
           B-toloc.city_name      0.971     0.992     0.981     716.0
         B-fromloc.city_name      0.986     0.987     0.987     704.0
           I-toloc.city_name      0.956     0.992     0.974     265.0
      B-depart_date.day_name      0.990     0.976     0.983     212.0
              B-airline_name      0.953     1.000     0.976     101.0
         I-fromloc.city_name      0.966     0.977     0.972     177.0
 B-depart_time.period_of_day      0.983     0.877     0.927     130.0
              I-airline_name      1.000     1.000     1.000      65.0
    B-depart_date.day_number      0.946     0.964     0.955      55.0
    B-depart_date.month_name      0.982     0.964     0.973      56.0
          B-depart_time.time      0.812     0.982     0.889      57.0
                B-round_trip      1.000     0.986     0.993      73.0
             B-cost_relative      1.000     0.973     0.986      37.0
                I-round_trip      1.000     1.000     1.000      71.0
                B-flight_mod      1.000     0.958     0.979      24.0
 B-depart_time.time_relative      0.984     0.954     0.969      65.0
          I-depart_time.time      0.867     1.000     0.929      52.0
         B-stoploc.city_name      0.947     0.900     0.923      20.0
                 B-city_name      0.805     0.579     0.673      57.0
                B-class_type      0.923     1.000     0.960      24.0
          B-arrive_time.time      0.971     0.971     0.971      34.0
 B-arrive_time.time_relative      0.882     0.968     0.923      31.0
                I-class_type      1.000     1.000     1.000      17.0
               B-flight_stop      1.000     1.000     1.000      21.0
          I-arrive_time.time      0.971     0.971     0.971      35.0
              B-airline_code      1.000     0.853     0.921      34.0
    I-depart_date.day_number      1.000     1.000     1.000      15.0
      I-fromloc.airport_name      0.385     1.000     0.556      15.0
      B-fromloc.airport_name      0.324     0.917     0.478      12.0
      B-arrive_date.day_name      0.647     1.000     0.786      11.0
          B-toloc.state_code      0.947     1.000     0.973      18.0
B-depart_date.today_relative      1.000     1.000     1.000       9.0
             B-flight_number      0.471     0.727     0.571      11.0
 B-depart_date.date_relative      0.941     0.941     0.941      17.0
          B-toloc.state_name      0.903     1.000     0.949      28.0
           B-fare_basis_code      0.941     0.941     0.941      17.0
               B-flight_time      1.000     1.000     1.000       1.0
                        B-or      1.000     1.000     1.000       3.0
 B-arrive_time.period_of_day      0.857     1.000     0.923       6.0
          B-meal_description      1.000     0.900     0.947      10.0
             I-cost_relative      1.000     0.667     0.800       3.0
              I-airport_name      0.857     0.207     0.333      29.0
               B-fare_amount      0.500     1.000     0.667       2.0
               I-fare_amount      1.000     1.000     1.000       2.0
                 I-city_name      1.000     0.533     0.696      30.0
        I-toloc.airport_name      0.750     1.000     0.857       3.0
            B-transport_type      1.000     0.800     0.889      10.0
    B-arrive_date.month_name      0.714     0.833     0.769       6.0
    B-arrive_date.day_number      0.800     0.667     0.727       6.0
         I-stoploc.city_name      0.900     0.900     0.900      10.0
                      B-meal      1.000     1.000     1.000      16.0
        B-fromloc.state_code      1.000     0.957     0.978      23.0
    B-depart_time.period_mod      0.571     0.800     0.667       5.0
                   B-connect      1.000     1.000     1.000       6.0
               B-flight_days      1.000     1.000     1.000      10.0
        B-toloc.airport_name      0.500     0.333     0.400       3.0
        B-fromloc.state_name      0.944     1.000     0.971      17.0
              B-airport_name      0.750     0.286     0.414      21.0
                   B-economy      1.000     1.000     1.000       6.0
               I-flight_time      1.000     1.000     1.000       1.0
             B-aircraft_code      0.862     0.758     0.806      33.0
                       B-mod      1.000     0.500     0.667       2.0
              B-airport_code      0.000     0.000     0.000       9.0
    B-depart_time.start_time      0.500     0.333     0.400       3.0
      B-depart_time.end_time      1.000     0.667     0.800       3.0
          B-depart_date.year      1.000     0.667     0.800       3.0
            I-transport_type      0.250     1.000     0.400       1.0
          B-restriction_code      0.364     1.000     0.533       4.0
    B-arrive_time.start_time      0.889     1.000     0.941       8.0
        B-toloc.airport_code      1.000     0.500     0.667       4.0
      B-arrive_time.end_time      0.875     0.875     0.875       8.0
      I-arrive_time.end_time      0.875     0.875     0.875       8.0
      I-depart_time.end_time      1.000     0.333     0.500       3.0
               I-flight_stop      0.000     0.000     0.000       0.0
      B-fromloc.airport_code      1.000     1.000     1.000       5.0
          I-restriction_code      0.750     1.000     0.857       3.0
    I-depart_time.start_time      0.500     1.000     0.667       1.0
          I-toloc.state_name      1.000     1.000     1.000       1.0
I-depart_date.today_relative      0.000     0.000     0.000       0.0
 B-arrive_date.date_relative      1.000     1.000     1.000       2.0
                I-flight_mod      1.000     0.167     0.286       6.0
                   I-economy      0.000     0.000     0.000       0.0
 B-return_date.date_relative      0.667     0.667     0.667       3.0
        I-fromloc.state_name      1.000     1.000     1.000       1.0
                B-state_code      1.000     1.000     1.000       1.0
    I-arrive_time.start_time      1.000     1.000     1.000       1.0
    I-arrive_date.day_number      0.000     0.000     0.000       0.0
                 B-meal_code      1.000     1.000     1.000       1.0
 I-depart_time.period_of_day      0.000     0.000     0.000       1.0
                  B-day_name      1.000     0.500     0.667       2.0
             B-period_of_day      1.000     0.500     0.667       4.0
        B-stoploc.state_code      0.000     0.000     0.000       0.0
    B-return_date.month_name      0.000     0.000     0.000       0.0
    B-return_date.day_number      0.000     0.000     0.000       0.0
    B-arrive_time.period_mod      0.000     0.000     0.000       0.0
                 I-meal_code      0.000     0.000     0.000       0.0
        B-toloc.country_name      1.000     1.000     1.000       1.0
                 B-days_code      1.000     1.000     1.000       1.0
 I-arrive_time.period_of_day      0.000     0.000     0.000       0.0
            I-today_relative      0.000     0.000     0.000       0.0
 B-return_time.period_of_day      0.000     0.000     0.000       0.0
                      B-time      0.000     0.000     0.000       0.0
           I-fare_basis_code      0.000     0.000     0.000       0.0
 I-arrive_time.time_relative      1.000     0.500     0.667       4.0
 I-depart_time.time_relative      0.000     0.000     0.000       1.0
            B-today_relative      0.000     0.000     0.000       0.0
                B-state_name      0.000     0.000     0.000       9.0
B-arrive_date.today_relative      0.000     0.000     0.000       0.0
    B-return_time.period_mod      0.000     0.000     0.000       0.0
                B-month_name      0.000     0.000     0.000       0.0
                B-day_number      0.000     0.000     0.000       0.0
 I-return_date.date_relative      1.000     0.667     0.800       3.0
I-return_date.today_relative      0.000     0.000     0.000       0.0
      B-stoploc.airport_name      0.000     0.000     0.000       0.0
             B-time_relative      0.000     0.000     0.000       0.0
                      I-time      0.000     0.000     0.000       0.0
    I-return_date.day_number      0.000     0.000     0.000       0.0
          I-meal_description      0.000     0.000     0.000       0.0
B-return_date.today_relative      0.000     0.000     0.000       0.0
      B-return_date.day_name      0.000     0.000     0.000       2.0

                   micro avg      0.941     0.943     0.942    3657.0
                   macro avg      0.654     0.634     0.624    3657.0
                weighted avg      0.948     0.943     0.939    3657.0

I0717 07:05:45.595604 140695199004544 interactiveshell.py:2882] Best Slot F1: 0.945
Reading ../data/atis.train.w-intent.iob
I0717 07:05:47.675866 140695199004544 interactiveshell.py:2882] Step 10000 | Loss: 0.0172 | Loss_intent: 0.0001 | Loss_slots: 0.0172 | Spent: 6.6 secs | LR: 0.000105
I0717 07:05:51.869201 140695199004544 interactiveshell.py:2882] Step 10050 | Loss: 0.0065 | Loss_intent: 0.0000 | Loss_slots: 0.0065 | Spent: 4.2 secs | LR: 0.000104
I0717 07:05:56.079030 140695199004544 interactiveshell.py:2882] Step 10100 | Loss: 0.0268 | Loss_intent: 0.0000 | Loss_slots: 0.0268 | Spent: 4.2 secs | LR: 0.000104
I0717 07:06:00.333833 140695199004544 interactiveshell.py:2882] Step 10150 | Loss: 0.0030 | Loss_intent: 0.0001 | Loss_slots: 0.0028 | Spent: 4.3 secs | LR: 0.000103
I0717 07:06:04.578803 140695199004544 interactiveshell.py:2882] Step 10200 | Loss: 0.0258 | Loss_intent: 0.0000 | Loss_slots: 0.0257 | Spent: 4.2 secs | LR: 0.000102
I0717 07:06:08.781899 140695199004544 interactiveshell.py:2882] Step 10250 | Loss: 0.0156 | Loss_intent: 0.0002 | Loss_slots: 0.0153 | Spent: 4.2 secs | LR: 0.000102
Reading ../data/atis.test.w-intent.iob
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.
  'precision', 'predicted', average, warn_for)
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1439: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples.
  'recall', 'true', average, warn_for)
I0717 07:06:14.258352 140695199004544 interactiveshell.py:2882] 
                                          precision    recall  f1-score   support

                             atis_flight      0.990     0.981     0.986       632
                            atis_airfare      0.889     1.000     0.941        48
                     atis_ground_service      1.000     1.000     1.000        36
                            atis_airline      1.000     0.974     0.987        38
                       atis_abbreviation      1.000     1.000     1.000        33
                           atis_aircraft      0.818     1.000     0.900         9
                        atis_flight_time      0.500     1.000     0.667         1
                           atis_quantity      0.333     1.000     0.500         3
                atis_flight#atis_airfare      0.833     0.417     0.556        12
                            atis_airport      0.947     1.000     0.973        18
                           atis_distance      1.000     1.000     1.000        10
                               atis_city      0.600     1.000     0.750         6
                        atis_ground_fare      1.000     0.714     0.833         7
                           atis_capacity      1.000     0.952     0.976        21
                          atis_flight_no      0.889     1.000     0.941         8
                               atis_meal      0.833     0.833     0.833         6
                        atis_restriction      0.000     0.000     0.000         0
             atis_airline#atis_flight_no      0.000     0.000     0.000         0
    atis_ground_service#atis_ground_fare      0.000     0.000     0.000         0
           atis_airfare#atis_flight_time      0.000     0.000     0.000         0
                           atis_cheapest      0.000     0.000     0.000         0
atis_aircraft#atis_flight#atis_flight_no      0.000     0.000     0.000         0

                               micro avg      0.968     0.973     0.970       888
                               macro avg      0.620     0.676     0.629       888
                            weighted avg      0.974     0.973     0.971       888

I0717 07:06:14.298982 140695199004544 interactiveshell.py:2882] 
                              precision    recall  f1-score   support

                           O      0.000     0.000     0.000       0.0
           B-toloc.city_name      0.971     0.993     0.982     716.0
         B-fromloc.city_name      0.986     0.990     0.988     704.0
           I-toloc.city_name      0.981     0.996     0.989     265.0
      B-depart_date.day_name      0.990     0.981     0.986     212.0
              B-airline_name      0.971     1.000     0.985     101.0
         I-fromloc.city_name      0.983     0.977     0.980     177.0
 B-depart_time.period_of_day      0.982     0.862     0.918     130.0
              I-airline_name      1.000     1.000     1.000      65.0
    B-depart_date.day_number      0.964     0.964     0.964      55.0
    B-depart_date.month_name      0.982     0.964     0.973      56.0
          B-depart_time.time      0.848     0.982     0.911      57.0
                B-round_trip      1.000     0.973     0.986      73.0
             B-cost_relative      1.000     0.973     0.986      37.0
                I-round_trip      1.000     1.000     1.000      71.0
                B-flight_mod      0.885     0.958     0.920      24.0
 B-depart_time.time_relative      0.984     0.954     0.969      65.0
          I-depart_time.time      0.881     1.000     0.937      52.0
         B-stoploc.city_name      0.950     0.950     0.950      20.0
                 B-city_name      0.829     0.596     0.694      57.0
                B-class_type      0.923     1.000     0.960      24.0
          B-arrive_time.time      0.971     0.971     0.971      34.0
 B-arrive_time.time_relative      0.882     0.968     0.923      31.0
                I-class_type      1.000     1.000     1.000      17.0
               B-flight_stop      0.955     1.000     0.977      21.0
          I-arrive_time.time      0.971     0.971     0.971      35.0
              B-airline_code      1.000     0.853     0.921      34.0
    I-depart_date.day_number      1.000     1.000     1.000      15.0
      I-fromloc.airport_name      0.417     1.000     0.588      15.0
      B-fromloc.airport_name      0.367     0.917     0.524      12.0
      B-arrive_date.day_name      0.688     1.000     0.815      11.0
          B-toloc.state_code      1.000     1.000     1.000      18.0
B-depart_date.today_relative      1.000     1.000     1.000       9.0
             B-flight_number      0.333     0.909     0.488      11.0
 B-depart_date.date_relative      0.944     1.000     0.971      17.0
          B-toloc.state_name      0.900     0.964     0.931      28.0
           B-fare_basis_code      0.909     0.588     0.714      17.0
               B-flight_time      1.000     1.000     1.000       1.0
                        B-or      1.000     1.000     1.000       3.0
 B-arrive_time.period_of_day      0.750     1.000     0.857       6.0
          B-meal_description      1.000     0.900     0.947      10.0
             I-cost_relative      1.000     0.667     0.800       3.0
              I-airport_name      0.833     0.345     0.488      29.0
               B-fare_amount      0.333     1.000     0.500       2.0
               I-fare_amount      1.000     1.000     1.000       2.0
                 I-city_name      1.000     0.633     0.776      30.0
        I-toloc.airport_name      0.600     1.000     0.750       3.0
            B-transport_type      1.000     0.600     0.750      10.0
    B-arrive_date.month_name      0.714     0.833     0.769       6.0
    B-arrive_date.day_number      0.833     0.833     0.833       6.0
         I-stoploc.city_name      0.667     1.000     0.800      10.0
                      B-meal      1.000     1.000     1.000      16.0
        B-fromloc.state_code      1.000     1.000     1.000      23.0
    B-depart_time.period_mod      0.556     1.000     0.714       5.0
                   B-connect      1.000     1.000     1.000       6.0
               B-flight_days      1.000     1.000     1.000      10.0
        B-toloc.airport_name      1.000     0.333     0.500       3.0
        B-fromloc.state_name      0.941     0.941     0.941      17.0
              B-airport_name      0.750     0.286     0.414      21.0
                   B-economy      1.000     1.000     1.000       6.0
               I-flight_time      1.000     1.000     1.000       1.0
             B-aircraft_code      0.897     0.788     0.839      33.0
                       B-mod      0.000     0.000     0.000       2.0
              B-airport_code      0.000     0.000     0.000       9.0
    B-depart_time.start_time      0.500     0.333     0.400       3.0
      B-depart_time.end_time      1.000     0.667     0.800       3.0
          B-depart_date.year      1.000     0.667     0.800       3.0
            I-transport_type      0.000     0.000     0.000       1.0
          B-restriction_code      0.500     0.750     0.600       4.0
    B-arrive_time.start_time      0.889     1.000     0.941       8.0
        B-toloc.airport_code      1.000     0.500     0.667       4.0
      B-arrive_time.end_time      0.889     1.000     0.941       8.0
      I-arrive_time.end_time      0.875     0.875     0.875       8.0
      I-depart_time.end_time      1.000     0.333     0.500       3.0
               I-flight_stop      0.000     0.000     0.000       0.0
      B-fromloc.airport_code      1.000     1.000     1.000       5.0
          I-restriction_code      0.750     1.000     0.857       3.0
    I-depart_time.start_time      0.500     1.000     0.667       1.0
          I-toloc.state_name      1.000     1.000     1.000       1.0
I-depart_date.today_relative      0.000     0.000     0.000       0.0
 B-arrive_date.date_relative      1.000     1.000     1.000       2.0
                I-flight_mod      1.000     0.167     0.286       6.0
                   I-economy      0.000     0.000     0.000       0.0
 B-return_date.date_relative      1.000     0.667     0.800       3.0
        I-fromloc.state_name      1.000     1.000     1.000       1.0
                B-state_code      1.000     1.000     1.000       1.0
    I-arrive_time.start_time      1.000     1.000     1.000       1.0
    I-arrive_date.day_number      0.000     0.000     0.000       0.0
                 B-meal_code      1.000     1.000     1.000       1.0
 I-depart_time.period_of_day      0.000     0.000     0.000       1.0
                  B-day_name      1.000     0.500     0.667       2.0
             B-period_of_day      1.000     0.250     0.400       4.0
        B-stoploc.state_code      0.000     0.000     0.000       0.0
    B-return_date.month_name      0.000     0.000     0.000       0.0
    B-return_date.day_number      0.000     0.000     0.000       0.0
    B-arrive_time.period_mod      0.000     0.000     0.000       0.0
                 I-meal_code      0.000     0.000     0.000       0.0
        B-toloc.country_name      1.000     1.000     1.000       1.0
                 B-days_code      1.000     1.000     1.000       1.0
 I-arrive_time.period_of_day      0.000     0.000     0.000       0.0
            I-today_relative      0.000     0.000     0.000       0.0
 B-return_time.period_of_day      0.000     0.000     0.000       0.0
                      B-time      0.000     0.000     0.000       0.0
           I-fare_basis_code      0.000     0.000     0.000       0.0
 I-arrive_time.time_relative      1.000     0.500     0.667       4.0
 I-depart_time.time_relative      0.000     0.000     0.000       1.0
            B-today_relative      0.000     0.000     0.000       0.0
                B-state_name      0.000     0.000     0.000       9.0
B-arrive_date.today_relative      0.000     0.000     0.000       0.0
    B-return_time.period_mod      0.000     0.000     0.000       0.0
                B-month_name      0.000     0.000     0.000       0.0
                B-day_number      0.000     0.000     0.000       0.0
 I-return_date.date_relative      1.000     0.667     0.800       3.0
I-return_date.today_relative      0.000     0.000     0.000       0.0
      B-stoploc.airport_name      0.000     0.000     0.000       0.0
             B-time_relative      0.000     0.000     0.000       0.0
                      I-time      0.000     0.000     0.000       0.0
    I-return_date.day_number      0.000     0.000     0.000       0.0
          I-meal_description      0.000     0.000     0.000       0.0
B-return_date.today_relative      0.000     0.000     0.000       0.0
      B-return_date.day_name      0.000     0.000     0.000       2.0

                   micro avg      0.943     0.944     0.943    3657.0
                   macro avg      0.646     0.622     0.613    3657.0
                weighted avg      0.951     0.944     0.942    3657.0

I0717 07:06:14.299841 140695199004544 interactiveshell.py:2882] Best Slot F1: 0.945
Reading ../data/atis.train.w-intent.iob
I0717 07:06:15.379957 140695199004544 interactiveshell.py:2882] Step 10300 | Loss: 0.0345 | Loss_intent: 0.0004 | Loss_slots: 0.0341 | Spent: 6.6 secs | LR: 0.000101
I0717 07:06:19.569094 140695199004544 interactiveshell.py:2882] Step 10350 | Loss: 0.0154 | Loss_intent: 0.0004 | Loss_slots: 0.0150 | Spent: 4.2 secs | LR: 0.000101
I0717 07:06:23.889396 140695199004544 interactiveshell.py:2882] Step 10400 | Loss: 0.0247 | Loss_intent: 0.0000 | Loss_slots: 0.0247 | Spent: 4.3 secs | LR: 0.000100
I0717 07:06:28.077777 140695199004544 interactiveshell.py:2882] Step 10450 | Loss: 0.0054 | Loss_intent: 0.0003 | Loss_slots: 0.0051 | Spent: 4.2 secs | LR: 0.000100
I0717 07:06:32.250928 140695199004544 interactiveshell.py:2882] Step 10500 | Loss: 0.0022 | Loss_intent: 0.0001 | Loss_slots: 0.0021 | Spent: 4.2 secs | LR: 0.000099
I0717 07:06:36.434932 140695199004544 interactiveshell.py:2882] Step 10550 | Loss: 0.0026 | Loss_intent: 0.0002 | Loss_slots: 0.0023 | Spent: 4.2 secs | LR: 0.000099
I0717 07:06:40.604643 140695199004544 interactiveshell.py:2882] Step 10600 | Loss: 0.0028 | Loss_intent: 0.0000 | Loss_slots: 0.0028 | Spent: 4.2 secs | LR: 0.000098
Reading ../data/atis.test.w-intent.iob
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.
  'precision', 'predicted', average, warn_for)
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1439: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples.
  'recall', 'true', average, warn_for)
I0717 07:06:42.908692 140695199004544 interactiveshell.py:2882] 
                                          precision    recall  f1-score   support

                             atis_flight      0.987     0.979     0.983       632
                            atis_airfare      0.923     1.000     0.960        48
                     atis_ground_service      1.000     1.000     1.000        36
                            atis_airline      1.000     1.000     1.000        38
                       atis_abbreviation      1.000     1.000     1.000        33
                           atis_aircraft      0.750     1.000     0.857         9
                        atis_flight_time      1.000     1.000     1.000         1
                           atis_quantity      0.429     1.000     0.600         3
                atis_flight#atis_airfare      0.857     0.500     0.632        12
                            atis_airport      0.947     1.000     0.973        18
                           atis_distance      1.000     1.000     1.000        10
                               atis_city      0.545     1.000     0.706         6
                        atis_ground_fare      1.000     0.714     0.833         7
                           atis_capacity      1.000     1.000     1.000        21
                          atis_flight_no      0.889     1.000     0.941         8
                               atis_meal      1.000     0.833     0.909         6
                        atis_restriction      0.000     0.000     0.000         0
             atis_airline#atis_flight_no      0.000     0.000     0.000         0
    atis_ground_service#atis_ground_fare      0.000     0.000     0.000         0
           atis_airfare#atis_flight_time      0.000     0.000     0.000         0
                           atis_cheapest      0.000     0.000     0.000         0
atis_aircraft#atis_flight#atis_flight_no      0.000     0.000     0.000         0

                               micro avg      0.970     0.975     0.972       888
                               macro avg      0.651     0.683     0.654       888
                            weighted avg      0.975     0.975     0.973       888

I0717 07:06:42.949668 140695199004544 interactiveshell.py:2882] 
                              precision    recall  f1-score   support

                           O      0.000     0.000     0.000       0.0
           B-toloc.city_name      0.973     0.994     0.983     716.0
         B-fromloc.city_name      0.987     0.993     0.990     704.0
           I-toloc.city_name      0.964     0.996     0.980     265.0
      B-depart_date.day_name      0.990     0.981     0.986     212.0
              B-airline_name      0.953     1.000     0.976     101.0
         I-fromloc.city_name      0.972     0.977     0.975     177.0
 B-depart_time.period_of_day      0.974     0.869     0.919     130.0
              I-airline_name      1.000     1.000     1.000      65.0
    B-depart_date.day_number      0.964     0.964     0.964      55.0
    B-depart_date.month_name      0.982     0.964     0.973      56.0
          B-depart_time.time      0.848     0.982     0.911      57.0
                B-round_trip      1.000     0.986     0.993      73.0
             B-cost_relative      1.000     0.973     0.986      37.0
                I-round_trip      1.000     1.000     1.000      71.0
                B-flight_mod      1.000     0.958     0.979      24.0
 B-depart_time.time_relative      0.984     0.954     0.969      65.0
          I-depart_time.time      0.867     1.000     0.929      52.0
         B-stoploc.city_name      1.000     0.950     0.974      20.0
                 B-city_name      0.825     0.579     0.680      57.0
                B-class_type      0.923     1.000     0.960      24.0
          B-arrive_time.time      0.971     0.971     0.971      34.0
 B-arrive_time.time_relative      0.882     0.968     0.923      31.0
                I-class_type      1.000     1.000     1.000      17.0
               B-flight_stop      0.955     1.000     0.977      21.0
          I-arrive_time.time      0.971     0.971     0.971      35.0
              B-airline_code      1.000     0.853     0.921      34.0
    I-depart_date.day_number      1.000     1.000     1.000      15.0
      I-fromloc.airport_name      0.417     1.000     0.588      15.0
      B-fromloc.airport_name      0.367     0.917     0.524      12.0
      B-arrive_date.day_name      0.688     1.000     0.815      11.0
          B-toloc.state_code      1.000     1.000     1.000      18.0
B-depart_date.today_relative      1.000     1.000     1.000       9.0
             B-flight_number      0.333     0.909     0.488      11.0
 B-depart_date.date_relative      0.850     1.000     0.919      17.0
          B-toloc.state_name      0.903     1.000     0.949      28.0
           B-fare_basis_code      0.909     0.588     0.714      17.0
               B-flight_time      1.000     1.000     1.000       1.0
                        B-or      1.000     1.000     1.000       3.0
 B-arrive_time.period_of_day      0.750     1.000     0.857       6.0
          B-meal_description      1.000     0.900     0.947      10.0
             I-cost_relative      1.000     0.667     0.800       3.0
              I-airport_name      0.833     0.345     0.488      29.0
               B-fare_amount      0.500     1.000     0.667       2.0
               I-fare_amount      1.000     1.000     1.000       2.0
                 I-city_name      1.000     0.567     0.723      30.0
        I-toloc.airport_name      1.000     1.000     1.000       3.0
            B-transport_type      1.000     0.700     0.824      10.0
    B-arrive_date.month_name      0.714     0.833     0.769       6.0
    B-arrive_date.day_number      0.833     0.833     0.833       6.0
         I-stoploc.city_name      1.000     0.900     0.947      10.0
                      B-meal      1.000     1.000     1.000      16.0
        B-fromloc.state_code      1.000     1.000     1.000      23.0
    B-depart_time.period_mod      0.500     0.600     0.545       5.0
                   B-connect      1.000     1.000     1.000       6.0
               B-flight_days      1.000     1.000     1.000      10.0
        B-toloc.airport_name      0.333     0.333     0.333       3.0
        B-fromloc.state_name      0.944     1.000     0.971      17.0
              B-airport_name      0.778     0.333     0.467      21.0
                   B-economy      1.000     1.000     1.000       6.0
               I-flight_time      1.000     1.000     1.000       1.0
             B-aircraft_code      0.833     0.758     0.794      33.0
                       B-mod      0.000     0.000     0.000       2.0
              B-airport_code      0.000     0.000     0.000       9.0
    B-depart_time.start_time      0.500     0.333     0.400       3.0
      B-depart_time.end_time      1.000     0.667     0.800       3.0
          B-depart_date.year      1.000     0.667     0.800       3.0
            I-transport_type      0.200     1.000     0.333       1.0
          B-restriction_code      0.500     0.750     0.600       4.0
    B-arrive_time.start_time      0.889     1.000     0.941       8.0
        B-toloc.airport_code      1.000     0.500     0.667       4.0
      B-arrive_time.end_time      0.875     0.875     0.875       8.0
      I-arrive_time.end_time      0.875     0.875     0.875       8.0
      I-depart_time.end_time      1.000     0.333     0.500       3.0
               I-flight_stop      0.000     0.000     0.000       0.0
      B-fromloc.airport_code      1.000     1.000     1.000       5.0
          I-restriction_code      0.750     1.000     0.857       3.0
    I-depart_time.start_time      0.500     1.000     0.667       1.0
          I-toloc.state_name      1.000     1.000     1.000       1.0
I-depart_date.today_relative      0.000     0.000     0.000       0.0
 B-arrive_date.date_relative      0.000     0.000     0.000       2.0
                I-flight_mod      1.000     0.167     0.286       6.0
                   I-economy      0.000     0.000     0.000       0.0
 B-return_date.date_relative      1.000     0.333     0.500       3.0
        I-fromloc.state_name      1.000     1.000     1.000       1.0
                B-state_code      1.000     1.000     1.000       1.0
    I-arrive_time.start_time      1.000     1.000     1.000       1.0
    I-arrive_date.day_number      0.000     0.000     0.000       0.0
                 B-meal_code      1.000     1.000     1.000       1.0
 I-depart_time.period_of_day      0.000     0.000     0.000       1.0
                  B-day_name      1.000     0.500     0.667       2.0
             B-period_of_day      1.000     0.500     0.667       4.0
        B-stoploc.state_code      0.000     0.000     0.000       0.0
    B-return_date.month_name      0.000     0.000     0.000       0.0
    B-return_date.day_number      0.000     0.000     0.000       0.0
    B-arrive_time.period_mod      0.000     0.000     0.000       0.0
                 I-meal_code      0.000     0.000     0.000       0.0
        B-toloc.country_name      1.000     1.000     1.000       1.0
                 B-days_code      1.000     1.000     1.000       1.0
 I-arrive_time.period_of_day      0.000     0.000     0.000       0.0
            I-today_relative      0.000     0.000     0.000       0.0
 B-return_time.period_of_day      0.000     0.000     0.000       0.0
                      B-time      0.000     0.000     0.000       0.0
           I-fare_basis_code      0.000     0.000     0.000       0.0
 I-arrive_time.time_relative      1.000     0.500     0.667       4.0
 I-depart_time.time_relative      0.000     0.000     0.000       1.0
            B-today_relative      0.000     0.000     0.000       0.0
                B-state_name      0.000     0.000     0.000       9.0
B-arrive_date.today_relative      0.000     0.000     0.000       0.0
    B-return_time.period_mod      0.000     0.000     0.000       0.0
                B-month_name      0.000     0.000     0.000       0.0
                B-day_number      0.000     0.000     0.000       0.0
 I-return_date.date_relative      0.667     0.667     0.667       3.0
I-return_date.today_relative      0.000     0.000     0.000       0.0
      B-stoploc.airport_name      0.000     0.000     0.000       0.0
             B-time_relative      0.000     0.000     0.000       0.0
                      I-time      0.000     0.000     0.000       0.0
    I-return_date.day_number      0.000     0.000     0.000       0.0
          I-meal_description      0.000     0.000     0.000       0.0
B-return_date.today_relative      0.000     0.000     0.000       0.0
      B-return_date.day_name      0.000     0.000     0.000       2.0

                   micro avg      0.943     0.944     0.943    3657.0
                   macro avg      0.638     0.618     0.608    3657.0
                weighted avg      0.949     0.944     0.941    3657.0

I0717 07:06:42.950424 140695199004544 interactiveshell.py:2882] Best Slot F1: 0.945
Reading ../data/atis.train.w-intent.iob
I0717 07:06:47.198855 140695199004544 interactiveshell.py:2882] Step 10650 | Loss: 0.0046 | Loss_intent: 0.0000 | Loss_slots: 0.0045 | Spent: 6.6 secs | LR: 0.000098
I0717 07:06:51.369285 140695199004544 interactiveshell.py:2882] Step 10700 | Loss: 0.0158 | Loss_intent: 0.0007 | Loss_slots: 0.0151 | Spent: 4.2 secs | LR: 0.000097
I0717 07:06:55.750157 140695199004544 interactiveshell.py:2882] Step 10750 | Loss: 0.0132 | Loss_intent: 0.0000 | Loss_slots: 0.0132 | Spent: 4.4 secs | LR: 0.000097
I0717 07:07:00.128576 140695199004544 interactiveshell.py:2882] Step 10800 | Loss: 0.0380 | Loss_intent: 0.0002 | Loss_slots: 0.0378 | Spent: 4.4 secs | LR: 0.000096
I0717 07:07:04.317603 140695199004544 interactiveshell.py:2882] Step 10850 | Loss: 0.0081 | Loss_intent: 0.0003 | Loss_slots: 0.0078 | Spent: 4.2 secs | LR: 0.000096
I0717 07:07:08.495505 140695199004544 interactiveshell.py:2882] Step 10900 | Loss: 0.0206 | Loss_intent: 0.0001 | Loss_slots: 0.0205 | Spent: 4.2 secs | LR: 0.000095
Reading ../data/atis.test.w-intent.iob
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.
  'precision', 'predicted', average, warn_for)
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1439: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples.
  'recall', 'true', average, warn_for)
I0717 07:07:11.815737 140695199004544 interactiveshell.py:2882] 
                                          precision    recall  f1-score   support

                             atis_flight      0.986     0.981     0.983       632
                            atis_airfare      0.923     1.000     0.960        48
                     atis_ground_service      1.000     1.000     1.000        36
                            atis_airline      1.000     1.000     1.000        38
                       atis_abbreviation      1.000     1.000     1.000        33
                           atis_aircraft      0.818     1.000     0.900         9
                        atis_flight_time      1.000     1.000     1.000         1
                           atis_quantity      0.429     1.000     0.600         3
                atis_flight#atis_airfare      0.833     0.417     0.556        12
                            atis_airport      0.947     1.000     0.973        18
                           atis_distance      1.000     1.000     1.000        10
                               atis_city      0.545     1.000     0.706         6
                        atis_ground_fare      1.000     0.714     0.833         7
                           atis_capacity      1.000     1.000     1.000        21
                          atis_flight_no      0.889     1.000     0.941         8
                               atis_meal      1.000     0.833     0.909         6
                        atis_restriction      0.000     0.000     0.000         0
             atis_airline#atis_flight_no      0.000     0.000     0.000         0
    atis_ground_service#atis_ground_fare      0.000     0.000     0.000         0
           atis_airfare#atis_flight_time      0.000     0.000     0.000         0
                           atis_cheapest      0.000     0.000     0.000         0
atis_aircraft#atis_flight#atis_flight_no      0.000     0.000     0.000         0

                               micro avg      0.970     0.975     0.972       888
                               macro avg      0.653     0.679     0.653       888
                            weighted avg      0.974     0.975     0.973       888

I0717 07:07:11.856092 140695199004544 interactiveshell.py:2882] 
                              precision    recall  f1-score   support

                           O      0.000     0.000     0.000       0.0
           B-toloc.city_name      0.974     0.993     0.983     716.0
         B-fromloc.city_name      0.982     0.987     0.984     704.0
           I-toloc.city_name      0.978     0.996     0.987     265.0
      B-depart_date.day_name      0.990     0.976     0.983     212.0
              B-airline_name      0.952     0.990     0.971     101.0
         I-fromloc.city_name      0.978     0.983     0.980     177.0
 B-depart_time.period_of_day      0.974     0.862     0.914     130.0
              I-airline_name      1.000     1.000     1.000      65.0
    B-depart_date.day_number      0.964     0.964     0.964      55.0
    B-depart_date.month_name      0.982     0.964     0.973      56.0
          B-depart_time.time      0.838     1.000     0.912      57.0
                B-round_trip      1.000     0.986     0.993      73.0
             B-cost_relative      1.000     0.973     0.986      37.0
                I-round_trip      1.000     1.000     1.000      71.0
                B-flight_mod      0.793     0.958     0.868      24.0
 B-depart_time.time_relative      0.984     0.954     0.969      65.0
          I-depart_time.time      0.867     1.000     0.929      52.0
         B-stoploc.city_name      0.952     1.000     0.976      20.0
                 B-city_name      0.791     0.596     0.680      57.0
                B-class_type      0.923     1.000     0.960      24.0
          B-arrive_time.time      0.971     0.971     0.971      34.0
 B-arrive_time.time_relative      0.882     0.968     0.923      31.0
                I-class_type      1.000     1.000     1.000      17.0
               B-flight_stop      0.955     1.000     0.977      21.0
          I-arrive_time.time      0.971     0.971     0.971      35.0
              B-airline_code      1.000     0.853     0.921      34.0
    I-depart_date.day_number      1.000     1.000     1.000      15.0
      I-fromloc.airport_name      0.429     1.000     0.600      15.0
      B-fromloc.airport_name      0.367     0.917     0.524      12.0
      B-arrive_date.day_name      0.647     1.000     0.786      11.0
          B-toloc.state_code      1.000     1.000     1.000      18.0
B-depart_date.today_relative      1.000     1.000     1.000       9.0
             B-flight_number      0.391     0.818     0.529      11.0
 B-depart_date.date_relative      0.944     1.000     0.971      17.0
          B-toloc.state_name      0.903     1.000     0.949      28.0
           B-fare_basis_code      0.944     1.000     0.971      17.0
               B-flight_time      1.000     1.000     1.000       1.0
                        B-or      1.000     1.000     1.000       3.0
 B-arrive_time.period_of_day      0.750     1.000     0.857       6.0
          B-meal_description      1.000     0.900     0.947      10.0
             I-cost_relative      1.000     0.667     0.800       3.0
              I-airport_name      0.846     0.379     0.524      29.0
               B-fare_amount      0.500     1.000     0.667       2.0
               I-fare_amount      1.000     1.000     1.000       2.0
                 I-city_name      1.000     0.567     0.723      30.0
        I-toloc.airport_name      0.750     1.000     0.857       3.0
            B-transport_type      1.000     0.700     0.824      10.0
    B-arrive_date.month_name      0.714     0.833     0.769       6.0
    B-arrive_date.day_number      0.833     0.833     0.833       6.0
         I-stoploc.city_name      0.769     1.000     0.870      10.0
                      B-meal      1.000     1.000     1.000      16.0
        B-fromloc.state_code      1.000     1.000     1.000      23.0
    B-depart_time.period_mod      0.600     0.600     0.600       5.0
                   B-connect      1.000     1.000     1.000       6.0
               B-flight_days      1.000     1.000     1.000      10.0
        B-toloc.airport_name      1.000     0.333     0.500       3.0
        B-fromloc.state_name      0.944     1.000     0.971      17.0
              B-airport_name      0.750     0.286     0.414      21.0
                   B-economy      1.000     1.000     1.000       6.0
               I-flight_time      1.000     1.000     1.000       1.0
             B-aircraft_code      0.926     0.758     0.833      33.0
                       B-mod      0.000     0.000     0.000       2.0
              B-airport_code      0.000     0.000     0.000       9.0
    B-depart_time.start_time      0.500     0.333     0.400       3.0
      B-depart_time.end_time      0.667     0.667     0.667       3.0
          B-depart_date.year      1.000     0.667     0.800       3.0
            I-transport_type      0.200     1.000     0.333       1.0
          B-restriction_code      1.000     0.750     0.857       4.0
    B-arrive_time.start_time      0.889     1.000     0.941       8.0
        B-toloc.airport_code      1.000     0.500     0.667       4.0
      B-arrive_time.end_time      0.889     1.000     0.941       8.0
      I-arrive_time.end_time      0.875     0.875     0.875       8.0
      I-depart_time.end_time      1.000     0.333     0.500       3.0
               I-flight_stop      0.000     0.000     0.000       0.0
      B-fromloc.airport_code      1.000     1.000     1.000       5.0
          I-restriction_code      0.500     1.000     0.667       3.0
    I-depart_time.start_time      0.500     1.000     0.667       1.0
          I-toloc.state_name      1.000     1.000     1.000       1.0
I-depart_date.today_relative      0.000     0.000     0.000       0.0
 B-arrive_date.date_relative      1.000     1.000     1.000       2.0
                I-flight_mod      1.000     0.167     0.286       6.0
                   I-economy      0.000     0.000     0.000       0.0
 B-return_date.date_relative      1.000     0.333     0.500       3.0
        I-fromloc.state_name      1.000     1.000     1.000       1.0
                B-state_code      1.000     1.000     1.000       1.0
    I-arrive_time.start_time      1.000     1.000     1.000       1.0
    I-arrive_date.day_number      0.000     0.000     0.000       0.0
                 B-meal_code      1.000     1.000     1.000       1.0
 I-depart_time.period_of_day      0.000     0.000     0.000       1.0
                  B-day_name      1.000     0.500     0.667       2.0
             B-period_of_day      1.000     0.500     0.667       4.0
        B-stoploc.state_code      0.000     0.000     0.000       0.0
    B-return_date.month_name      0.000     0.000     0.000       0.0
    B-return_date.day_number      0.000     0.000     0.000       0.0
    B-arrive_time.period_mod      0.000     0.000     0.000       0.0
                 I-meal_code      0.000     0.000     0.000       0.0
        B-toloc.country_name      1.000     1.000     1.000       1.0
                 B-days_code      1.000     1.000     1.000       1.0
 I-arrive_time.period_of_day      0.000     0.000     0.000       0.0
            I-today_relative      0.000     0.000     0.000       0.0
 B-return_time.period_of_day      0.000     0.000     0.000       0.0
                      B-time      0.000     0.000     0.000       0.0
           I-fare_basis_code      0.000     0.000     0.000       0.0
 I-arrive_time.time_relative      1.000     0.500     0.667       4.0
 I-depart_time.time_relative      0.000     0.000     0.000       1.0
            B-today_relative      0.000     0.000     0.000       0.0
                B-state_name      0.000     0.000     0.000       9.0
B-arrive_date.today_relative      0.000     0.000     0.000       0.0
    B-return_time.period_mod      0.000     0.000     0.000       0.0
                B-month_name      0.000     0.000     0.000       0.0
                B-day_number      0.000     0.000     0.000       0.0
 I-return_date.date_relative      0.667     0.667     0.667       3.0
I-return_date.today_relative      0.000     0.000     0.000       0.0
      B-stoploc.airport_name      0.000     0.000     0.000       0.0
             B-time_relative      0.000     0.000     0.000       0.0
                      I-time      0.000     0.000     0.000       0.0
    I-return_date.day_number      0.000     0.000     0.000       0.0
          I-meal_description      0.000     0.000     0.000       0.0
B-return_date.today_relative      0.000     0.000     0.000       0.0
      B-return_date.day_name      0.000     0.000     0.000       2.0

                   micro avg      0.944     0.946     0.945    3657.0
                   macro avg      0.648     0.631     0.619    3657.0
                weighted avg      0.949     0.946     0.942    3657.0

I0717 07:07:11.857053 140695199004544 interactiveshell.py:2882] Best Slot F1: 0.945
Reading ../data/atis.train.w-intent.iob
I0717 07:07:15.090802 140695199004544 interactiveshell.py:2882] Step 10950 | Loss: 0.0430 | Loss_intent: 0.0075 | Loss_slots: 0.0355 | Spent: 6.6 secs | LR: 0.000095
I0717 07:07:19.358304 140695199004544 interactiveshell.py:2882] Step 11000 | Loss: 0.0074 | Loss_intent: 0.0000 | Loss_slots: 0.0074 | Spent: 4.3 secs | LR: 0.000094
I0717 07:07:23.606685 140695199004544 interactiveshell.py:2882] Step 11050 | Loss: 0.0183 | Loss_intent: 0.0017 | Loss_slots: 0.0166 | Spent: 4.2 secs | LR: 0.000094
I0717 07:07:27.822830 140695199004544 interactiveshell.py:2882] Step 11100 | Loss: 0.0083 | Loss_intent: 0.0000 | Loss_slots: 0.0083 | Spent: 4.2 secs | LR: 0.000093
I0717 07:07:32.003520 140695199004544 interactiveshell.py:2882] Step 11150 | Loss: 0.0331 | Loss_intent: 0.0000 | Loss_slots: 0.0330 | Spent: 4.2 secs | LR: 0.000093
I0717 07:07:36.228013 140695199004544 interactiveshell.py:2882] Step 11200 | Loss: 0.0141 | Loss_intent: 0.0004 | Loss_slots: 0.0137 | Spent: 4.2 secs | LR: 0.000092
Reading ../data/atis.test.w-intent.iob
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.
  'precision', 'predicted', average, warn_for)
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1439: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples.
  'recall', 'true', average, warn_for)
I0717 07:07:40.548344 140695199004544 interactiveshell.py:2882] 
                                          precision    recall  f1-score   support

                             atis_flight      0.972     0.991     0.981       632
                            atis_airfare      0.922     0.979     0.949        48
                     atis_ground_service      1.000     1.000     1.000        36
                            atis_airline      1.000     0.974     0.987        38
                       atis_abbreviation      0.971     1.000     0.985        33
                           atis_aircraft      0.889     0.889     0.889         9
                        atis_flight_time      1.000     1.000     1.000         1
                           atis_quantity      0.300     1.000     0.462         3
                atis_flight#atis_airfare      0.500     0.083     0.143        12
                            atis_airport      1.000     1.000     1.000        18
                           atis_distance      1.000     1.000     1.000        10
                               atis_city      1.000     0.500     0.667         6
                        atis_ground_fare      1.000     0.714     0.833         7
                           atis_capacity      1.000     0.905     0.950        21
                          atis_flight_no      0.889     1.000     0.941         8
                               atis_meal      1.000     0.833     0.909         6
                        atis_restriction      0.000     0.000     0.000         0
             atis_airline#atis_flight_no      0.000     0.000     0.000         0
    atis_ground_service#atis_ground_fare      0.000     0.000     0.000         0
           atis_airfare#atis_flight_time      0.000     0.000     0.000         0
                           atis_cheapest      0.000     0.000     0.000         0
atis_aircraft#atis_flight#atis_flight_no      0.000     0.000     0.000         0

                               micro avg      0.963     0.968     0.966       888
                               macro avg      0.656     0.630     0.623       888
                            weighted avg      0.964     0.968     0.962       888

I0717 07:07:40.588826 140695199004544 interactiveshell.py:2882] 
                              precision    recall  f1-score   support

                           O      0.000     0.000     0.000       0.0
           B-toloc.city_name      0.970     0.993     0.981     716.0
         B-fromloc.city_name      0.980     0.994     0.987     704.0
           I-toloc.city_name      0.971     1.000     0.985     265.0
      B-depart_date.day_name      0.990     0.976     0.983     212.0
              B-airline_name      0.962     0.990     0.976     101.0
         I-fromloc.city_name      0.989     0.989     0.989     177.0
 B-depart_time.period_of_day      0.991     0.869     0.926     130.0
              I-airline_name      1.000     1.000     1.000      65.0
    B-depart_date.day_number      0.962     0.927     0.944      55.0
    B-depart_date.month_name      0.982     0.964     0.973      56.0
          B-depart_time.time      0.838     1.000     0.912      57.0
                B-round_trip      1.000     0.986     0.993      73.0
             B-cost_relative      1.000     0.973     0.986      37.0
                I-round_trip      1.000     1.000     1.000      71.0
                B-flight_mod      1.000     0.958     0.979      24.0
 B-depart_time.time_relative      0.969     0.954     0.961      65.0
          I-depart_time.time      0.881     1.000     0.937      52.0
         B-stoploc.city_name      0.952     1.000     0.976      20.0
                 B-city_name      0.810     0.596     0.687      57.0
                B-class_type      0.923     1.000     0.960      24.0
          B-arrive_time.time      0.971     0.971     0.971      34.0
 B-arrive_time.time_relative      0.882     0.968     0.923      31.0
                I-class_type      1.000     1.000     1.000      17.0
               B-flight_stop      1.000     1.000     1.000      21.0
          I-arrive_time.time      1.000     0.971     0.986      35.0
              B-airline_code      1.000     0.853     0.921      34.0
    I-depart_date.day_number      1.000     1.000     1.000      15.0
      I-fromloc.airport_name      0.429     1.000     0.600      15.0
      B-fromloc.airport_name      0.440     0.917     0.595      12.0
      B-arrive_date.day_name      0.647     1.000     0.786      11.0
          B-toloc.state_code      0.947     1.000     0.973      18.0
B-depart_date.today_relative      1.000     0.889     0.941       9.0
             B-flight_number      0.455     0.909     0.606      11.0
 B-depart_date.date_relative      0.944     1.000     0.971      17.0
          B-toloc.state_name      0.889     0.857     0.873      28.0
           B-fare_basis_code      0.944     1.000     0.971      17.0
               B-flight_time      1.000     1.000     1.000       1.0
                        B-or      1.000     1.000     1.000       3.0
 B-arrive_time.period_of_day      0.750     1.000     0.857       6.0
          B-meal_description      1.000     0.900     0.947      10.0
             I-cost_relative      1.000     0.667     0.800       3.0
              I-airport_name      0.846     0.379     0.524      29.0
               B-fare_amount      0.400     1.000     0.571       2.0
               I-fare_amount      1.000     1.000     1.000       2.0
                 I-city_name      1.000     0.500     0.667      30.0
        I-toloc.airport_name      0.750     1.000     0.857       3.0
            B-transport_type      1.000     0.700     0.824      10.0
    B-arrive_date.month_name      0.714     0.833     0.769       6.0
    B-arrive_date.day_number      0.625     0.833     0.714       6.0
         I-stoploc.city_name      0.833     1.000     0.909      10.0
                      B-meal      1.000     1.000     1.000      16.0
        B-fromloc.state_code      1.000     0.957     0.978      23.0
    B-depart_time.period_mod      0.625     1.000     0.769       5.0
                   B-connect      1.000     1.000     1.000       6.0
               B-flight_days      1.000     1.000     1.000      10.0
        B-toloc.airport_name      1.000     0.333     0.500       3.0
        B-fromloc.state_name      0.882     0.882     0.882      17.0
              B-airport_name      0.800     0.381     0.516      21.0
                   B-economy      1.000     1.000     1.000       6.0
               I-flight_time      1.000     1.000     1.000       1.0
             B-aircraft_code      0.926     0.758     0.833      33.0
                       B-mod      0.000     0.000     0.000       2.0
              B-airport_code      0.000     0.000     0.000       9.0
    B-depart_time.start_time      0.500     0.333     0.400       3.0
      B-depart_time.end_time      0.667     0.667     0.667       3.0
          B-depart_date.year      1.000     0.667     0.800       3.0
            I-transport_type      0.250     1.000     0.400       1.0
          B-restriction_code      1.000     0.750     0.857       4.0
    B-arrive_time.start_time      0.889     1.000     0.941       8.0
        B-toloc.airport_code      1.000     0.500     0.667       4.0
      B-arrive_time.end_time      0.889     1.000     0.941       8.0
      I-arrive_time.end_time      0.800     1.000     0.889       8.0
      I-depart_time.end_time      1.000     0.667     0.800       3.0
               I-flight_stop      0.000     0.000     0.000       0.0
      B-fromloc.airport_code      1.000     1.000     1.000       5.0
          I-restriction_code      0.429     1.000     0.600       3.0
    I-depart_time.start_time      1.000     1.000     1.000       1.0
          I-toloc.state_name      1.000     1.000     1.000       1.0
I-depart_date.today_relative      0.000     0.000     0.000       0.0
 B-arrive_date.date_relative      1.000     1.000     1.000       2.0
                I-flight_mod      0.000     0.000     0.000       6.0
                   I-economy      0.000     0.000     0.000       0.0
 B-return_date.date_relative      1.000     0.667     0.800       3.0
        I-fromloc.state_name      1.000     1.000     1.000       1.0
                B-state_code      1.000     1.000     1.000       1.0
    I-arrive_time.start_time      0.000     0.000     0.000       1.0
    I-arrive_date.day_number      0.000     0.000     0.000       0.0
                 B-meal_code      1.000     1.000     1.000       1.0
 I-depart_time.period_of_day      0.000     0.000     0.000       1.0
                  B-day_name      1.000     0.500     0.667       2.0
             B-period_of_day      1.000     0.500     0.667       4.0
        B-stoploc.state_code      0.000     0.000     0.000       0.0
    B-return_date.month_name      0.000     0.000     0.000       0.0
    B-return_date.day_number      0.000     0.000     0.000       0.0
    B-arrive_time.period_mod      0.000     0.000     0.000       0.0
                 I-meal_code      0.000     0.000     0.000       0.0
        B-toloc.country_name      1.000     1.000     1.000       1.0
                 B-days_code      1.000     1.000     1.000       1.0
 I-arrive_time.period_of_day      0.000     0.000     0.000       0.0
            I-today_relative      0.000     0.000     0.000       0.0
 B-return_time.period_of_day      0.000     0.000     0.000       0.0
                      B-time      0.000     0.000     0.000       0.0
           I-fare_basis_code      0.000     0.000     0.000       0.0
 I-arrive_time.time_relative      1.000     0.500     0.667       4.0
 I-depart_time.time_relative      0.000     0.000     0.000       1.0
            B-today_relative      0.000     0.000     0.000       0.0
                B-state_name      0.000     0.000     0.000       9.0
B-arrive_date.today_relative      0.000     0.000     0.000       0.0
    B-return_time.period_mod      0.000     0.000     0.000       0.0
                B-month_name      0.000     0.000     0.000       0.0
                B-day_number      0.000     0.000     0.000       0.0
 I-return_date.date_relative      1.000     0.667     0.800       3.0
I-return_date.today_relative      0.000     0.000     0.000       0.0
      B-stoploc.airport_name      0.000     0.000     0.000       0.0
             B-time_relative      0.000     0.000     0.000       0.0
                      I-time      0.000     0.000     0.000       0.0
    I-return_date.day_number      0.000     0.000     0.000       0.0
          I-meal_description      0.000     0.000     0.000       0.0
B-return_date.today_relative      0.000     0.000     0.000       0.0
      B-return_date.day_name      0.000     0.000     0.000       2.0

                   micro avg      0.945     0.946     0.946    3657.0
                   macro avg      0.639     0.628     0.618    3657.0
                weighted avg      0.949     0.946     0.943    3657.0

I0717 07:07:40.589619 140695199004544 interactiveshell.py:2882] Best Slot F1: 0.946
Reading ../data/atis.train.w-intent.iob
I0717 07:07:42.821547 140695199004544 interactiveshell.py:2882] Step 11250 | Loss: 0.0259 | Loss_intent: 0.0209 | Loss_slots: 0.0050 | Spent: 6.6 secs | LR: 0.000092
I0717 07:07:46.993544 140695199004544 interactiveshell.py:2882] Step 11300 | Loss: 0.0052 | Loss_intent: 0.0000 | Loss_slots: 0.0052 | Spent: 4.2 secs | LR: 0.000091
I0717 07:07:51.173329 140695199004544 interactiveshell.py:2882] Step 11350 | Loss: 0.0284 | Loss_intent: 0.0004 | Loss_slots: 0.0280 | Spent: 4.2 secs | LR: 0.000091
I0717 07:07:55.345927 140695199004544 interactiveshell.py:2882] Step 11400 | Loss: 0.0010 | Loss_intent: 0.0000 | Loss_slots: 0.0010 | Spent: 4.2 secs | LR: 0.000090
I0717 07:07:59.540080 140695199004544 interactiveshell.py:2882] Step 11450 | Loss: 0.0220 | Loss_intent: 0.0000 | Loss_slots: 0.0220 | Spent: 4.2 secs | LR: 0.000090
I0717 07:08:03.722238 140695199004544 interactiveshell.py:2882] Step 11500 | Loss: 0.0103 | Loss_intent: 0.0000 | Loss_slots: 0.0103 | Spent: 4.2 secs | LR: 0.000089
Reading ../data/atis.test.w-intent.iob
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.
  'precision', 'predicted', average, warn_for)
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1439: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples.
  'recall', 'true', average, warn_for)
I0717 07:08:09.027765 140695199004544 interactiveshell.py:2882] 
                                          precision    recall  f1-score   support

                             atis_flight      0.987     0.983     0.985       632
                            atis_airfare      0.906     1.000     0.950        48
                     atis_ground_service      1.000     1.000     1.000        36
                            atis_airline      0.974     0.974     0.974        38
                       atis_abbreviation      1.000     1.000     1.000        33
                           atis_aircraft      0.800     0.889     0.842         9
                        atis_flight_time      1.000     1.000     1.000         1
                           atis_quantity      0.333     1.000     0.500         3
                atis_flight#atis_airfare      0.833     0.417     0.556        12
                            atis_airport      1.000     1.000     1.000        18
                           atis_distance      1.000     1.000     1.000        10
                               atis_city      0.545     1.000     0.706         6
                        atis_ground_fare      1.000     0.714     0.833         7
                           atis_capacity      1.000     0.952     0.976        21
                          atis_flight_no      0.889     1.000     0.941         8
                               atis_meal      1.000     0.833     0.909         6
                        atis_restriction      0.000     0.000     0.000         0
             atis_airline#atis_flight_no      0.000     0.000     0.000         0
    atis_ground_service#atis_ground_fare      0.000     0.000     0.000         0
           atis_airfare#atis_flight_time      0.000     0.000     0.000         0
                           atis_cheapest      0.000     0.000     0.000         0
atis_aircraft#atis_flight#atis_flight_no      0.000     0.000     0.000         0

                               micro avg      0.968     0.973     0.970       888
                               macro avg      0.649     0.671     0.644       888
                            weighted avg      0.974     0.973     0.971       888

I0717 07:08:09.068533 140695199004544 interactiveshell.py:2882] 
                              precision    recall  f1-score   support

                           O      0.000     0.000     0.000       0.0
           B-toloc.city_name      0.971     0.996     0.983     716.0
         B-fromloc.city_name      0.987     0.994     0.991     704.0
           I-toloc.city_name      0.964     1.000     0.981     265.0
      B-depart_date.day_name      0.990     0.976     0.983     212.0
              B-airline_name      0.980     0.990     0.985     101.0
         I-fromloc.city_name      0.983     0.983     0.983     177.0
 B-depart_time.period_of_day      0.991     0.862     0.922     130.0
              I-airline_name      1.000     1.000     1.000      65.0
    B-depart_date.day_number      0.980     0.909     0.943      55.0
    B-depart_date.month_name      0.982     0.964     0.973      56.0
          B-depart_time.time      0.851     1.000     0.919      57.0
                B-round_trip      1.000     0.986     0.993      73.0
             B-cost_relative      1.000     0.973     0.986      37.0
                I-round_trip      1.000     1.000     1.000      71.0
                B-flight_mod      1.000     0.958     0.979      24.0
 B-depart_time.time_relative      0.984     0.954     0.969      65.0
          I-depart_time.time      0.867     1.000     0.929      52.0
         B-stoploc.city_name      1.000     1.000     1.000      20.0
                 B-city_name      0.825     0.579     0.680      57.0
                B-class_type      0.889     1.000     0.941      24.0
          B-arrive_time.time      0.971     0.971     0.971      34.0
 B-arrive_time.time_relative      0.882     0.968     0.923      31.0
                I-class_type      1.000     1.000     1.000      17.0
               B-flight_stop      0.955     1.000     0.977      21.0
          I-arrive_time.time      0.971     0.971     0.971      35.0
              B-airline_code      1.000     0.853     0.921      34.0
    I-depart_date.day_number      1.000     1.000     1.000      15.0
      I-fromloc.airport_name      0.429     1.000     0.600      15.0
      B-fromloc.airport_name      0.407     0.917     0.564      12.0
      B-arrive_date.day_name      0.647     1.000     0.786      11.0
          B-toloc.state_code      0.947     1.000     0.973      18.0
B-depart_date.today_relative      1.000     0.889     0.941       9.0
             B-flight_number      0.400     0.909     0.556      11.0
 B-depart_date.date_relative      0.895     1.000     0.944      17.0
          B-toloc.state_name      0.900     0.964     0.931      28.0
           B-fare_basis_code      0.944     1.000     0.971      17.0
               B-flight_time      1.000     1.000     1.000       1.0
                        B-or      1.000     1.000     1.000       3.0
 B-arrive_time.period_of_day      0.750     1.000     0.857       6.0
          B-meal_description      1.000     0.900     0.947      10.0
             I-cost_relative      1.000     0.667     0.800       3.0
              I-airport_name      0.917     0.379     0.537      29.0
               B-fare_amount      0.500     1.000     0.667       2.0
               I-fare_amount      1.000     1.000     1.000       2.0
                 I-city_name      1.000     0.567     0.723      30.0
        I-toloc.airport_name      0.600     1.000     0.750       3.0
            B-transport_type      1.000     0.800     0.889      10.0
    B-arrive_date.month_name      0.714     0.833     0.769       6.0
    B-arrive_date.day_number      0.556     0.833     0.667       6.0
         I-stoploc.city_name      1.000     1.000     1.000      10.0
                      B-meal      1.000     1.000     1.000      16.0
        B-fromloc.state_code      1.000     0.957     0.978      23.0
    B-depart_time.period_mod      0.625     1.000     0.769       5.0
                   B-connect      1.000     1.000     1.000       6.0
               B-flight_days      1.000     1.000     1.000      10.0
        B-toloc.airport_name      1.000     0.333     0.500       3.0
        B-fromloc.state_name      0.941     0.941     0.941      17.0
              B-airport_name      0.778     0.333     0.467      21.0
                   B-economy      1.000     1.000     1.000       6.0
               I-flight_time      1.000     1.000     1.000       1.0
             B-aircraft_code      0.931     0.818     0.871      33.0
                       B-mod      0.000     0.000     0.000       2.0
              B-airport_code      0.000     0.000     0.000       9.0
    B-depart_time.start_time      0.500     0.333     0.400       3.0
      B-depart_time.end_time      0.667     0.667     0.667       3.0
          B-depart_date.year      1.000     0.667     0.800       3.0
            I-transport_type      0.333     1.000     0.500       1.0
          B-restriction_code      0.600     0.750     0.667       4.0
    B-arrive_time.start_time      0.889     1.000     0.941       8.0
        B-toloc.airport_code      1.000     0.500     0.667       4.0
      B-arrive_time.end_time      0.889     1.000     0.941       8.0
      I-arrive_time.end_time      0.875     0.875     0.875       8.0
      I-depart_time.end_time      1.000     0.333     0.500       3.0
               I-flight_stop      0.000     0.000     0.000       0.0
      B-fromloc.airport_code      1.000     1.000     1.000       5.0
          I-restriction_code      0.750     1.000     0.857       3.0
    I-depart_time.start_time      0.500     1.000     0.667       1.0
          I-toloc.state_name      1.000     1.000     1.000       1.0
I-depart_date.today_relative      0.000     0.000     0.000       0.0
 B-arrive_date.date_relative      1.000     1.000     1.000       2.0
                I-flight_mod      1.000     0.167     0.286       6.0
                   I-economy      0.000     0.000     0.000       0.0
 B-return_date.date_relative      1.000     0.333     0.500       3.0
        I-fromloc.state_name      1.000     1.000     1.000       1.0
                B-state_code      1.000     1.000     1.000       1.0
    I-arrive_time.start_time      1.000     1.000     1.000       1.0
    I-arrive_date.day_number      0.000     0.000     0.000       0.0
                 B-meal_code      1.000     1.000     1.000       1.0
 I-depart_time.period_of_day      0.000     0.000     0.000       1.0
                  B-day_name      1.000     0.500     0.667       2.0
             B-period_of_day      1.000     0.500     0.667       4.0
        B-stoploc.state_code      0.000     0.000     0.000       0.0
    B-return_date.month_name      0.000     0.000     0.000       0.0
    B-return_date.day_number      0.000     0.000     0.000       0.0
    B-arrive_time.period_mod      0.000     0.000     0.000       0.0
                 I-meal_code      0.000     0.000     0.000       0.0
        B-toloc.country_name      1.000     1.000     1.000       1.0
                 B-days_code      1.000     1.000     1.000       1.0
 I-arrive_time.period_of_day      0.000     0.000     0.000       0.0
            I-today_relative      0.000     0.000     0.000       0.0
 B-return_time.period_of_day      0.000     0.000     0.000       0.0
                      B-time      0.000     0.000     0.000       0.0
           I-fare_basis_code      0.000     0.000     0.000       0.0
 I-arrive_time.time_relative      1.000     0.500     0.667       4.0
 I-depart_time.time_relative      0.000     0.000     0.000       1.0
            B-today_relative      0.000     0.000     0.000       0.0
                B-state_name      1.000     0.111     0.200       9.0
B-arrive_date.today_relative      0.000     0.000     0.000       0.0
    B-return_time.period_mod      0.000     0.000     0.000       0.0
                B-month_name      0.000     0.000     0.000       0.0
                B-day_number      0.000     0.000     0.000       0.0
 I-return_date.date_relative      0.667     0.667     0.667       3.0
I-return_date.today_relative      0.000     0.000     0.000       0.0
      B-stoploc.airport_name      0.000     0.000     0.000       0.0
             B-time_relative      0.000     0.000     0.000       0.0
                      I-time      0.000     0.000     0.000       0.0
    I-return_date.day_number      0.000     0.000     0.000       0.0
          I-meal_description      0.000     0.000     0.000       0.0
B-return_date.today_relative      0.000     0.000     0.000       0.0
      B-return_date.day_name      0.000     0.000     0.000       2.0

                   micro avg      0.946     0.948     0.947    3657.0
                   macro avg      0.658     0.635     0.623    3657.0
                weighted avg      0.955     0.948     0.945    3657.0

I0717 07:08:09.069435 140695199004544 interactiveshell.py:2882] Best Slot F1: 0.947
Reading ../data/atis.train.w-intent.iob
I0717 07:08:10.304952 140695199004544 interactiveshell.py:2882] Step 11550 | Loss: 0.0095 | Loss_intent: 0.0000 | Loss_slots: 0.0095 | Spent: 6.6 secs | LR: 0.000089
I0717 07:08:14.490918 140695199004544 interactiveshell.py:2882] Step 11600 | Loss: 0.0041 | Loss_intent: 0.0000 | Loss_slots: 0.0041 | Spent: 4.2 secs | LR: 0.000088
I0717 07:08:18.686771 140695199004544 interactiveshell.py:2882] Step 11650 | Loss: 0.0199 | Loss_intent: 0.0000 | Loss_slots: 0.0199 | Spent: 4.2 secs | LR: 0.000088
I0717 07:08:22.866694 140695199004544 interactiveshell.py:2882] Step 11700 | Loss: 0.0046 | Loss_intent: 0.0000 | Loss_slots: 0.0046 | Spent: 4.2 secs | LR: 0.000087
I0717 07:08:27.047321 140695199004544 interactiveshell.py:2882] Step 11750 | Loss: 0.0111 | Loss_intent: 0.0008 | Loss_slots: 0.0103 | Spent: 4.2 secs | LR: 0.000087
I0717 07:08:31.240342 140695199004544 interactiveshell.py:2882] Step 11800 | Loss: 0.0053 | Loss_intent: 0.0001 | Loss_slots: 0.0052 | Spent: 4.2 secs | LR: 0.000087
I0717 07:08:35.431953 140695199004544 interactiveshell.py:2882] Step 11850 | Loss: 0.0078 | Loss_intent: 0.0024 | Loss_slots: 0.0055 | Spent: 4.2 secs | LR: 0.000086
Reading ../data/atis.test.w-intent.iob
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.
  'precision', 'predicted', average, warn_for)
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1439: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples.
  'recall', 'true', average, warn_for)
I0717 07:08:37.598174 140695199004544 interactiveshell.py:2882] 
                                          precision    recall  f1-score   support

                             atis_flight      0.978     0.987     0.983       632
                            atis_airfare      0.906     1.000     0.950        48
                     atis_ground_service      1.000     1.000     1.000        36
                            atis_airline      1.000     0.974     0.987        38
                       atis_abbreviation      1.000     1.000     1.000        33
                           atis_aircraft      0.889     0.889     0.889         9
                        atis_flight_time      1.000     1.000     1.000         1
                           atis_quantity      0.333     1.000     0.500         3
                atis_flight#atis_airfare      0.800     0.333     0.471        12
                            atis_airport      1.000     1.000     1.000        18
                           atis_distance      1.000     1.000     1.000        10
                               atis_city      0.667     0.667     0.667         6
                        atis_ground_fare      1.000     0.714     0.833         7
                           atis_capacity      1.000     0.905     0.950        21
                          atis_flight_no      0.889     1.000     0.941         8
                               atis_meal      1.000     0.833     0.909         6
                        atis_restriction      0.000     0.000     0.000         0
             atis_airline#atis_flight_no      0.000     0.000     0.000         0
    atis_ground_service#atis_ground_fare      0.000     0.000     0.000         0
           atis_airfare#atis_flight_time      0.000     0.000     0.000         0
                           atis_cheapest      0.000     0.000     0.000         0
atis_aircraft#atis_flight#atis_flight_no      0.000     0.000     0.000         0

                               micro avg      0.966     0.972     0.969       888
                               macro avg      0.657     0.650     0.640       888
                            weighted avg      0.970     0.972     0.969       888

I0717 07:08:37.638886 140695199004544 interactiveshell.py:2882] 
                              precision    recall  f1-score   support

                           O      0.000     0.000     0.000       0.0
           B-toloc.city_name      0.971     0.994     0.983     716.0
         B-fromloc.city_name      0.983     0.994     0.989     704.0
           I-toloc.city_name      0.967     0.992     0.980     265.0
      B-depart_date.day_name      0.990     0.976     0.983     212.0
              B-airline_name      0.971     1.000     0.985     101.0
         I-fromloc.city_name      0.961     0.983     0.972     177.0
 B-depart_time.period_of_day      0.983     0.900     0.940     130.0
              I-airline_name      1.000     1.000     1.000      65.0
    B-depart_date.day_number      0.981     0.927     0.953      55.0
    B-depart_date.month_name      0.982     0.964     0.973      56.0
          B-depart_time.time      0.836     0.982     0.903      57.0
                B-round_trip      1.000     0.986     0.993      73.0
             B-cost_relative      1.000     0.973     0.986      37.0
                I-round_trip      1.000     1.000     1.000      71.0
                B-flight_mod      0.852     0.958     0.902      24.0
 B-depart_time.time_relative      0.984     0.954     0.969      65.0
          I-depart_time.time      0.929     1.000     0.963      52.0
         B-stoploc.city_name      1.000     1.000     1.000      20.0
                 B-city_name      0.825     0.579     0.680      57.0
                B-class_type      0.889     1.000     0.941      24.0
          B-arrive_time.time      0.943     0.971     0.957      34.0
 B-arrive_time.time_relative      0.882     0.968     0.923      31.0
                I-class_type      1.000     1.000     1.000      17.0
               B-flight_stop      1.000     1.000     1.000      21.0
          I-arrive_time.time      0.971     0.971     0.971      35.0
              B-airline_code      1.000     0.853     0.921      34.0
    I-depart_date.day_number      1.000     1.000     1.000      15.0
      I-fromloc.airport_name      0.455     1.000     0.625      15.0
      B-fromloc.airport_name      0.440     0.917     0.595      12.0
      B-arrive_date.day_name      0.647     1.000     0.786      11.0
          B-toloc.state_code      0.947     1.000     0.973      18.0
B-depart_date.today_relative      1.000     1.000     1.000       9.0
             B-flight_number      0.300     0.818     0.439      11.0
 B-depart_date.date_relative      0.944     1.000     0.971      17.0
          B-toloc.state_name      0.903     1.000     0.949      28.0
           B-fare_basis_code      0.909     0.588     0.714      17.0
               B-flight_time      1.000     1.000     1.000       1.0
                        B-or      1.000     1.000     1.000       3.0
 B-arrive_time.period_of_day      0.750     1.000     0.857       6.0
          B-meal_description      1.000     0.900     0.947      10.0
             I-cost_relative      1.000     0.667     0.800       3.0
              I-airport_name      0.846     0.379     0.524      29.0
               B-fare_amount      0.400     1.000     0.571       2.0
               I-fare_amount      1.000     1.000     1.000       2.0
                 I-city_name      1.000     0.533     0.696      30.0
        I-toloc.airport_name      0.750     1.000     0.857       3.0
            B-transport_type      1.000     0.700     0.824      10.0
    B-arrive_date.month_name      0.714     0.833     0.769       6.0
    B-arrive_date.day_number      0.714     0.833     0.769       6.0
         I-stoploc.city_name      0.909     1.000     0.952      10.0
                      B-meal      1.000     1.000     1.000      16.0
        B-fromloc.state_code      1.000     0.957     0.978      23.0
    B-depart_time.period_mod      0.667     0.800     0.727       5.0
                   B-connect      1.000     1.000     1.000       6.0
               B-flight_days      1.000     1.000     1.000      10.0
        B-toloc.airport_name      1.000     0.333     0.500       3.0
        B-fromloc.state_name      0.944     1.000     0.971      17.0
              B-airport_name      0.727     0.381     0.500      21.0
                   B-economy      1.000     1.000     1.000       6.0
               I-flight_time      1.000     1.000     1.000       1.0
             B-aircraft_code      0.929     0.788     0.852      33.0
                       B-mod      0.000     0.000     0.000       2.0
              B-airport_code      0.000     0.000     0.000       9.0
    B-depart_time.start_time      0.667     0.667     0.667       3.0
      B-depart_time.end_time      1.000     0.667     0.800       3.0
          B-depart_date.year      1.000     0.667     0.800       3.0
            I-transport_type      0.200     1.000     0.333       1.0
          B-restriction_code      0.500     0.750     0.600       4.0
    B-arrive_time.start_time      0.875     0.875     0.875       8.0
        B-toloc.airport_code      1.000     0.500     0.667       4.0
      B-arrive_time.end_time      0.889     1.000     0.941       8.0
      I-arrive_time.end_time      0.875     0.875     0.875       8.0
      I-depart_time.end_time      1.000     0.333     0.500       3.0
               I-flight_stop      0.000     0.000     0.000       0.0
      B-fromloc.airport_code      1.000     1.000     1.000       5.0
          I-restriction_code      0.600     1.000     0.750       3.0
    I-depart_time.start_time      0.500     1.000     0.667       1.0
          I-toloc.state_name      1.000     1.000     1.000       1.0
I-depart_date.today_relative      0.000     0.000     0.000       0.0
 B-arrive_date.date_relative      1.000     1.000     1.000       2.0
                I-flight_mod      1.000     0.167     0.286       6.0
                   I-economy      0.000     0.000     0.000       0.0
 B-return_date.date_relative      1.000     0.333     0.500       3.0
        I-fromloc.state_name      1.000     1.000     1.000       1.0
                B-state_code      1.000     1.000     1.000       1.0
    I-arrive_time.start_time      1.000     1.000     1.000       1.0
    I-arrive_date.day_number      0.000     0.000     0.000       0.0
                 B-meal_code      0.000     0.000     0.000       1.0
 I-depart_time.period_of_day      0.000     0.000     0.000       1.0
                  B-day_name      1.000     0.500     0.667       2.0
             B-period_of_day      1.000     0.500     0.667       4.0
        B-stoploc.state_code      0.000     0.000     0.000       0.0
    B-return_date.month_name      0.000     0.000     0.000       0.0
    B-return_date.day_number      0.000     0.000     0.000       0.0
    B-arrive_time.period_mod      0.000     0.000     0.000       0.0
                 I-meal_code      0.000     0.000     0.000       0.0
        B-toloc.country_name      1.000     1.000     1.000       1.0
                 B-days_code      1.000     1.000     1.000       1.0
 I-arrive_time.period_of_day      0.000     0.000     0.000       0.0
            I-today_relative      0.000     0.000     0.000       0.0
 B-return_time.period_of_day      0.000     0.000     0.000       0.0
                      B-time      0.000     0.000     0.000       0.0
           I-fare_basis_code      0.000     0.000     0.000       0.0
 I-arrive_time.time_relative      1.000     0.500     0.667       4.0
 I-depart_time.time_relative      0.000     0.000     0.000       1.0
            B-today_relative      0.000     0.000     0.000       0.0
                B-state_name      0.000     0.000     0.000       9.0
B-arrive_date.today_relative      0.000     0.000     0.000       0.0
    B-return_time.period_mod      0.000     0.000     0.000       0.0
                B-month_name      0.000     0.000     0.000       0.0
                B-day_number      0.000     0.000     0.000       0.0
 I-return_date.date_relative      0.667     0.667     0.667       3.0
I-return_date.today_relative      0.000     0.000     0.000       0.0
      B-stoploc.airport_name      0.000     0.000     0.000       0.0
             B-time_relative      0.000     0.000     0.000       0.0
                      I-time      0.000     0.000     0.000       0.0
    I-return_date.day_number      0.000     0.000     0.000       0.0
          I-meal_description      0.000     0.000     0.000       0.0
B-return_date.today_relative      0.000     0.000     0.000       0.0
      B-return_date.day_name      0.000     0.000     0.000       2.0

                   micro avg      0.945     0.946     0.945    3657.0
                   macro avg      0.641     0.623     0.611    3657.0
                weighted avg      0.950     0.946     0.942    3657.0

I0717 07:08:37.639951 140695199004544 interactiveshell.py:2882] Best Slot F1: 0.947
Reading ../data/atis.train.w-intent.iob
I0717 07:08:42.144435 140695199004544 interactiveshell.py:2882] Step 11900 | Loss: 0.0212 | Loss_intent: 0.0000 | Loss_slots: 0.0212 | Spent: 6.7 secs | LR: 0.000086
I0717 07:08:46.375545 140695199004544 interactiveshell.py:2882] Step 11950 | Loss: 0.0024 | Loss_intent: 0.0000 | Loss_slots: 0.0024 | Spent: 4.2 secs | LR: 0.000085
I0717 07:08:50.567497 140695199004544 interactiveshell.py:2882] Step 12000 | Loss: 0.0119 | Loss_intent: 0.0014 | Loss_slots: 0.0105 | Spent: 4.2 secs | LR: 0.000085
I0717 07:08:54.763113 140695199004544 interactiveshell.py:2882] Step 12050 | Loss: 0.0408 | Loss_intent: 0.0148 | Loss_slots: 0.0261 | Spent: 4.2 secs | LR: 0.000084
I0717 07:08:58.946954 140695199004544 interactiveshell.py:2882] Step 12100 | Loss: 0.0090 | Loss_intent: 0.0000 | Loss_slots: 0.0090 | Spent: 4.2 secs | LR: 0.000084
I0717 07:09:03.153627 140695199004544 interactiveshell.py:2882] Step 12150 | Loss: 0.0080 | Loss_intent: 0.0018 | Loss_slots: 0.0062 | Spent: 4.2 secs | LR: 0.000083
Reading ../data/atis.test.w-intent.iob
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.
  'precision', 'predicted', average, warn_for)
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1439: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples.
  'recall', 'true', average, warn_for)
I0717 07:09:06.312664 140695199004544 interactiveshell.py:2882] 
                                          precision    recall  f1-score   support

                             atis_flight      0.984     0.983     0.983       632
                            atis_airfare      0.906     1.000     0.950        48
                     atis_ground_service      1.000     1.000     1.000        36
                            atis_airline      0.950     1.000     0.974        38
                       atis_abbreviation      0.971     1.000     0.985        33
                           atis_aircraft      0.889     0.889     0.889         9
                        atis_flight_time      1.000     1.000     1.000         1
                           atis_quantity      0.375     1.000     0.545         3
                atis_flight#atis_airfare      0.833     0.417     0.556        12
                            atis_airport      1.000     1.000     1.000        18
                           atis_distance      1.000     1.000     1.000        10
                               atis_city      0.500     0.667     0.571         6
                        atis_ground_fare      1.000     0.714     0.833         7
                           atis_capacity      1.000     0.952     0.976        21
                          atis_flight_no      0.889     1.000     0.941         8
                               atis_meal      1.000     0.833     0.909         6
                        atis_restriction      0.000     0.000     0.000         0
             atis_airline#atis_flight_no      0.000     0.000     0.000         0
    atis_ground_service#atis_ground_fare      0.000     0.000     0.000         0
           atis_airfare#atis_flight_time      0.000     0.000     0.000         0
                           atis_cheapest      0.000     0.000     0.000         0
atis_aircraft#atis_flight#atis_flight_no      0.000     0.000     0.000         0

                               micro avg      0.966     0.972     0.969       888
                               macro avg      0.650     0.657     0.642       888
                            weighted avg      0.971     0.972     0.969       888

I0717 07:09:06.353625 140695199004544 interactiveshell.py:2882] 
                              precision    recall  f1-score   support

                           O      0.000     0.000     0.000       0.0
           B-toloc.city_name      0.973     0.993     0.983     716.0
         B-fromloc.city_name      0.985     0.994     0.989     704.0
           I-toloc.city_name      0.974     0.989     0.981     265.0
      B-depart_date.day_name      0.990     0.976     0.983     212.0
              B-airline_name      0.971     1.000     0.985     101.0
         I-fromloc.city_name      0.972     0.977     0.975     177.0
 B-depart_time.period_of_day      0.974     0.869     0.919     130.0
              I-airline_name      1.000     1.000     1.000      65.0
    B-depart_date.day_number      0.981     0.964     0.972      55.0
    B-depart_date.month_name      0.982     0.964     0.973      56.0
          B-depart_time.time      0.851     1.000     0.919      57.0
                B-round_trip      1.000     0.986     0.993      73.0
             B-cost_relative      1.000     0.973     0.986      37.0
                I-round_trip      1.000     1.000     1.000      71.0
                B-flight_mod      0.793     0.958     0.868      24.0
 B-depart_time.time_relative      0.984     0.969     0.977      65.0
          I-depart_time.time      0.881     1.000     0.937      52.0
         B-stoploc.city_name      0.952     1.000     0.976      20.0
                 B-city_name      0.825     0.579     0.680      57.0
                B-class_type      0.889     1.000     0.941      24.0
          B-arrive_time.time      0.971     0.971     0.971      34.0
 B-arrive_time.time_relative      0.909     0.968     0.937      31.0
                I-class_type      1.000     1.000     1.000      17.0
               B-flight_stop      0.955     1.000     0.977      21.0
          I-arrive_time.time      0.971     0.971     0.971      35.0
              B-airline_code      0.967     0.853     0.906      34.0
    I-depart_date.day_number      1.000     1.000     1.000      15.0
      I-fromloc.airport_name      0.429     1.000     0.600      15.0
      B-fromloc.airport_name      0.407     0.917     0.564      12.0
      B-arrive_date.day_name      0.688     1.000     0.815      11.0
          B-toloc.state_code      0.947     1.000     0.973      18.0
B-depart_date.today_relative      1.000     1.000     1.000       9.0
             B-flight_number      0.385     0.909     0.541      11.0
 B-depart_date.date_relative      0.941     0.941     0.941      17.0
          B-toloc.state_name      0.903     1.000     0.949      28.0
           B-fare_basis_code      0.941     0.941     0.941      17.0
               B-flight_time      1.000     1.000     1.000       1.0
                        B-or      1.000     1.000     1.000       3.0
 B-arrive_time.period_of_day      0.857     1.000     0.923       6.0
          B-meal_description      1.000     0.900     0.947      10.0
             I-cost_relative      1.000     0.667     0.800       3.0
              I-airport_name      0.917     0.379     0.537      29.0
               B-fare_amount      0.400     1.000     0.571       2.0
               I-fare_amount      1.000     1.000     1.000       2.0
                 I-city_name      1.000     0.567     0.723      30.0
        I-toloc.airport_name      0.750     1.000     0.857       3.0
            B-transport_type      1.000     0.700     0.824      10.0
    B-arrive_date.month_name      0.714     0.833     0.769       6.0
    B-arrive_date.day_number      0.833     0.833     0.833       6.0
         I-stoploc.city_name      0.714     1.000     0.833      10.0
                      B-meal      1.000     1.000     1.000      16.0
        B-fromloc.state_code      1.000     0.957     0.978      23.0
    B-depart_time.period_mod      0.600     0.600     0.600       5.0
                   B-connect      1.000     1.000     1.000       6.0
               B-flight_days      1.000     1.000     1.000      10.0
        B-toloc.airport_name      1.000     0.333     0.500       3.0
        B-fromloc.state_name      0.944     1.000     0.971      17.0
              B-airport_name      0.778     0.333     0.467      21.0
                   B-economy      1.000     1.000     1.000       6.0
               I-flight_time      1.000     1.000     1.000       1.0
             B-aircraft_code      0.926     0.758     0.833      33.0
                       B-mod      1.000     0.500     0.667       2.0
              B-airport_code      0.000     0.000     0.000       9.0
    B-depart_time.start_time      0.500     0.333     0.400       3.0
      B-depart_time.end_time      0.500     0.667     0.571       3.0
          B-depart_date.year      1.000     0.667     0.800       3.0
            I-transport_type      0.250     1.000     0.400       1.0
          B-restriction_code      0.500     0.750     0.600       4.0
    B-arrive_time.start_time      0.889     1.000     0.941       8.0
        B-toloc.airport_code      1.000     0.500     0.667       4.0
      B-arrive_time.end_time      0.875     0.875     0.875       8.0
      I-arrive_time.end_time      0.875     0.875     0.875       8.0
      I-depart_time.end_time      1.000     0.333     0.500       3.0
               I-flight_stop      0.000     0.000     0.000       0.0
      B-fromloc.airport_code      1.000     1.000     1.000       5.0
          I-restriction_code      0.600     1.000     0.750       3.0
    I-depart_time.start_time      0.500     1.000     0.667       1.0
          I-toloc.state_name      1.000     1.000     1.000       1.0
I-depart_date.today_relative      0.000     0.000     0.000       0.0
 B-arrive_date.date_relative      1.000     1.000     1.000       2.0
                I-flight_mod      1.000     0.167     0.286       6.0
                   I-economy      0.000     0.000     0.000       0.0
 B-return_date.date_relative      0.500     0.333     0.400       3.0
        I-fromloc.state_name      1.000     1.000     1.000       1.0
                B-state_code      1.000     1.000     1.000       1.0
    I-arrive_time.start_time      1.000     1.000     1.000       1.0
    I-arrive_date.day_number      0.000     0.000     0.000       0.0
                 B-meal_code      0.000     0.000     0.000       1.0
 I-depart_time.period_of_day      0.000     0.000     0.000       1.0
                  B-day_name      1.000     0.500     0.667       2.0
             B-period_of_day      1.000     0.500     0.667       4.0
        B-stoploc.state_code      0.000     0.000     0.000       0.0
    B-return_date.month_name      0.000     0.000     0.000       0.0
    B-return_date.day_number      0.000     0.000     0.000       0.0
    B-arrive_time.period_mod      0.000     0.000     0.000       0.0
                 I-meal_code      0.000     0.000     0.000       0.0
        B-toloc.country_name      1.000     1.000     1.000       1.0
                 B-days_code      1.000     1.000     1.000       1.0
 I-arrive_time.period_of_day      0.000     0.000     0.000       0.0
            I-today_relative      0.000     0.000     0.000       0.0
 B-return_time.period_of_day      0.000     0.000     0.000       0.0
                      B-time      0.000     0.000     0.000       0.0
           I-fare_basis_code      0.000     0.000     0.000       0.0
 I-arrive_time.time_relative      1.000     0.500     0.667       4.0
 I-depart_time.time_relative      0.000     0.000     0.000       1.0
            B-today_relative      0.000     0.000     0.000       0.0
                B-state_name      0.000     0.000     0.000       9.0
B-arrive_date.today_relative      0.000     0.000     0.000       0.0
    B-return_time.period_mod      0.000     0.000     0.000       0.0
                B-month_name      0.000     0.000     0.000       0.0
                B-day_number      0.000     0.000     0.000       0.0
 I-return_date.date_relative      0.667     0.667     0.667       3.0
I-return_date.today_relative      0.000     0.000     0.000       0.0
      B-stoploc.airport_name      0.000     0.000     0.000       0.0
             B-time_relative      0.000     0.000     0.000       0.0
                      I-time      0.000     0.000     0.000       0.0
    I-return_date.day_number      0.000     0.000     0.000       0.0
          I-meal_description      0.000     0.000     0.000       0.0
B-return_date.today_relative      0.000     0.000     0.000       0.0
      B-return_date.day_name      0.000     0.000     0.000       2.0

                   micro avg      0.945     0.946     0.946    3657.0
                   macro avg      0.640     0.626     0.613    3657.0
                weighted avg      0.950     0.946     0.943    3657.0

I0717 07:09:06.354908 140695199004544 interactiveshell.py:2882] Best Slot F1: 0.947
Reading ../data/atis.train.w-intent.iob
I0717 07:09:09.764081 140695199004544 interactiveshell.py:2882] Step 12200 | Loss: 0.0103 | Loss_intent: 0.0001 | Loss_slots: 0.0102 | Spent: 6.6 secs | LR: 0.000083
I0717 07:09:13.976946 140695199004544 interactiveshell.py:2882] Step 12250 | Loss: 0.0019 | Loss_intent: 0.0000 | Loss_slots: 0.0019 | Spent: 4.2 secs | LR: 0.000083
I0717 07:09:18.173135 140695199004544 interactiveshell.py:2882] Step 12300 | Loss: 0.0029 | Loss_intent: 0.0000 | Loss_slots: 0.0029 | Spent: 4.2 secs | LR: 0.000082
I0717 07:09:22.365713 140695199004544 interactiveshell.py:2882] Step 12350 | Loss: 0.0320 | Loss_intent: 0.0000 | Loss_slots: 0.0320 | Spent: 4.2 secs | LR: 0.000082
I0717 07:09:26.570758 140695199004544 interactiveshell.py:2882] Step 12400 | Loss: 0.0181 | Loss_intent: 0.0000 | Loss_slots: 0.0181 | Spent: 4.2 secs | LR: 0.000081
I0717 07:09:30.750061 140695199004544 interactiveshell.py:2882] Step 12450 | Loss: 0.0218 | Loss_intent: 0.0000 | Loss_slots: 0.0218 | Spent: 4.2 secs | LR: 0.000081
Reading ../data/atis.test.w-intent.iob
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.
  'precision', 'predicted', average, warn_for)
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1439: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples.
  'recall', 'true', average, warn_for)
I0717 07:09:34.902980 140695199004544 interactiveshell.py:2882] 
                                          precision    recall  f1-score   support

                             atis_flight      0.984     0.983     0.983       632
                            atis_airfare      0.941     1.000     0.970        48
                     atis_ground_service      1.000     1.000     1.000        36
                            atis_airline      0.974     1.000     0.987        38
                       atis_abbreviation      0.943     1.000     0.971        33
                           atis_aircraft      1.000     0.889     0.941         9
                        atis_flight_time      1.000     1.000     1.000         1
                           atis_quantity      0.167     0.333     0.222         3
                atis_flight#atis_airfare      0.833     0.417     0.556        12
                            atis_airport      0.947     1.000     0.973        18
                           atis_distance      1.000     1.000     1.000        10
                               atis_city      0.444     0.667     0.533         6
                        atis_ground_fare      1.000     0.714     0.833         7
                           atis_capacity      0.913     1.000     0.955        21
                          atis_flight_no      0.889     1.000     0.941         8
                               atis_meal      1.000     0.833     0.909         6
                        atis_restriction      0.000     0.000     0.000         0
             atis_airline#atis_flight_no      0.000     0.000     0.000         0
    atis_ground_service#atis_ground_fare      0.000     0.000     0.000         0
           atis_airfare#atis_flight_time      0.000     0.000     0.000         0
                           atis_cheapest      0.000     0.000     0.000         0
atis_aircraft#atis_flight#atis_flight_no      0.000     0.000     0.000         0

                               micro avg      0.965     0.971     0.968       888
                               macro avg      0.638     0.629     0.626       888
                            weighted avg      0.969     0.971     0.968       888

I0717 07:09:34.944885 140695199004544 interactiveshell.py:2882] 
                              precision    recall  f1-score   support

                           O      0.000     0.000     0.000       0.0
           B-toloc.city_name      0.973     0.992     0.982     716.0
         B-fromloc.city_name      0.986     0.991     0.989     704.0
           I-toloc.city_name      0.960     0.992     0.976     265.0
      B-depart_date.day_name      0.990     0.976     0.983     212.0
              B-airline_name      0.981     1.000     0.990     101.0
         I-fromloc.city_name      0.972     0.983     0.978     177.0
 B-depart_time.period_of_day      0.983     0.869     0.922     130.0
              I-airline_name      1.000     1.000     1.000      65.0
    B-depart_date.day_number      0.963     0.945     0.954      55.0
    B-depart_date.month_name      0.982     0.964     0.973      56.0
          B-depart_time.time      0.838     1.000     0.912      57.0
                B-round_trip      1.000     0.986     0.993      73.0
             B-cost_relative      1.000     0.973     0.986      37.0
                I-round_trip      1.000     1.000     1.000      71.0
                B-flight_mod      0.885     0.958     0.920      24.0
 B-depart_time.time_relative      0.984     0.969     0.977      65.0
          I-depart_time.time      0.897     1.000     0.945      52.0
         B-stoploc.city_name      0.952     1.000     0.976      20.0
                 B-city_name      0.846     0.579     0.688      57.0
                B-class_type      0.923     1.000     0.960      24.0
          B-arrive_time.time      0.943     0.971     0.957      34.0
 B-arrive_time.time_relative      0.909     0.968     0.937      31.0
                I-class_type      1.000     1.000     1.000      17.0
               B-flight_stop      0.955     1.000     0.977      21.0
          I-arrive_time.time      0.971     0.971     0.971      35.0
              B-airline_code      1.000     0.853     0.921      34.0
    I-depart_date.day_number      1.000     1.000     1.000      15.0
      I-fromloc.airport_name      0.429     1.000     0.600      15.0
      B-fromloc.airport_name      0.393     0.917     0.550      12.0
      B-arrive_date.day_name      0.647     1.000     0.786      11.0
          B-toloc.state_code      0.947     1.000     0.973      18.0
B-depart_date.today_relative      1.000     0.889     0.941       9.0
             B-flight_number      0.455     0.909     0.606      11.0
 B-depart_date.date_relative      0.895     1.000     0.944      17.0
          B-toloc.state_name      0.903     1.000     0.949      28.0
           B-fare_basis_code      0.944     1.000     0.971      17.0
               B-flight_time      1.000     1.000     1.000       1.0
                        B-or      1.000     1.000     1.000       3.0
 B-arrive_time.period_of_day      0.750     1.000     0.857       6.0
          B-meal_description      1.000     0.900     0.947      10.0
             I-cost_relative      1.000     0.667     0.800       3.0
              I-airport_name      0.846     0.379     0.524      29.0
               B-fare_amount      0.500     1.000     0.667       2.0
               I-fare_amount      1.000     1.000     1.000       2.0
                 I-city_name      1.000     0.467     0.636      30.0
        I-toloc.airport_name      0.750     1.000     0.857       3.0
            B-transport_type      1.000     0.700     0.824      10.0
    B-arrive_date.month_name      0.714     0.833     0.769       6.0
    B-arrive_date.day_number      0.714     0.833     0.769       6.0
         I-stoploc.city_name      0.909     1.000     0.952      10.0
                      B-meal      1.000     1.000     1.000      16.0
        B-fromloc.state_code      1.000     0.957     0.978      23.0
    B-depart_time.period_mod      1.000     0.800     0.889       5.0
                   B-connect      1.000     1.000     1.000       6.0
               B-flight_days      1.000     1.000     1.000      10.0
        B-toloc.airport_name      1.000     0.333     0.500       3.0
        B-fromloc.state_name      0.944     1.000     0.971      17.0
              B-airport_name      0.727     0.381     0.500      21.0
                   B-economy      1.000     1.000     1.000       6.0
               I-flight_time      1.000     1.000     1.000       1.0
             B-aircraft_code      0.933     0.848     0.889      33.0
                       B-mod      0.000     0.000     0.000       2.0
              B-airport_code      0.000     0.000     0.000       9.0
    B-depart_time.start_time      0.500     0.333     0.400       3.0
      B-depart_time.end_time      0.667     0.667     0.667       3.0
          B-depart_date.year      1.000     0.667     0.800       3.0
            I-transport_type      0.000     0.000     0.000       1.0
          B-restriction_code      0.333     0.750     0.462       4.0
    B-arrive_time.start_time      0.875     0.875     0.875       8.0
        B-toloc.airport_code      1.000     0.500     0.667       4.0
      B-arrive_time.end_time      0.889     1.000     0.941       8.0
      I-arrive_time.end_time      0.875     0.875     0.875       8.0
      I-depart_time.end_time      1.000     0.333     0.500       3.0
               I-flight_stop      0.000     0.000     0.000       0.0
      B-fromloc.airport_code      1.000     1.000     1.000       5.0
          I-restriction_code      0.600     1.000     0.750       3.0
    I-depart_time.start_time      0.500     1.000     0.667       1.0
          I-toloc.state_name      1.000     1.000     1.000       1.0
I-depart_date.today_relative      0.000     0.000     0.000       0.0
 B-arrive_date.date_relative      1.000     1.000     1.000       2.0
                I-flight_mod      1.000     0.167     0.286       6.0
                   I-economy      0.000     0.000     0.000       0.0
 B-return_date.date_relative      1.000     0.333     0.500       3.0
        I-fromloc.state_name      1.000     1.000     1.000       1.0
                B-state_code      1.000     1.000     1.000       1.0
    I-arrive_time.start_time      1.000     1.000     1.000       1.0
    I-arrive_date.day_number      0.000     0.000     0.000       0.0
                 B-meal_code      0.000     0.000     0.000       1.0
 I-depart_time.period_of_day      0.000     0.000     0.000       1.0
                  B-day_name      1.000     0.500     0.667       2.0
             B-period_of_day      1.000     0.500     0.667       4.0
        B-stoploc.state_code      0.000     0.000     0.000       0.0
    B-return_date.month_name      0.000     0.000     0.000       0.0
    B-return_date.day_number      0.000     0.000     0.000       0.0
    B-arrive_time.period_mod      0.000     0.000     0.000       0.0
                 I-meal_code      0.000     0.000     0.000       0.0
        B-toloc.country_name      1.000     1.000     1.000       1.0
                 B-days_code      1.000     1.000     1.000       1.0
 I-arrive_time.period_of_day      0.000     0.000     0.000       0.0
            I-today_relative      0.000     0.000     0.000       0.0
 B-return_time.period_of_day      0.000     0.000     0.000       0.0
                      B-time      0.000     0.000     0.000       0.0
           I-fare_basis_code      0.000     0.000     0.000       0.0
 I-arrive_time.time_relative      1.000     0.500     0.667       4.0
 I-depart_time.time_relative      0.000     0.000     0.000       1.0
            B-today_relative      0.000     0.000     0.000       0.0
                B-state_name      0.000     0.000     0.000       9.0
B-arrive_date.today_relative      0.000     0.000     0.000       0.0
    B-return_time.period_mod      0.000     0.000     0.000       0.0
                B-month_name      0.000     0.000     0.000       0.0
                B-day_number      0.000     0.000     0.000       0.0
 I-return_date.date_relative      0.750     1.000     0.857       3.0
I-return_date.today_relative      0.000     0.000     0.000       0.0
      B-stoploc.airport_name      0.000     0.000     0.000       0.0
             B-time_relative      0.000     0.000     0.000       0.0
                      I-time      0.000     0.000     0.000       0.0
    I-return_date.day_number      0.000     0.000     0.000       0.0
          I-meal_description      0.000     0.000     0.000       0.0
B-return_date.today_relative      0.000     0.000     0.000       0.0
      B-return_date.day_name      0.000     0.000     0.000       2.0

                   micro avg      0.945     0.946     0.946    3657.0
                   macro avg      0.639     0.618     0.610    3657.0
                weighted avg      0.951     0.946     0.943    3657.0

I0717 07:09:34.945782 140695199004544 interactiveshell.py:2882] Best Slot F1: 0.947
Reading ../data/atis.train.w-intent.iob
I0717 07:09:37.386673 140695199004544 interactiveshell.py:2882] Step 12500 | Loss: 0.0228 | Loss_intent: 0.0043 | Loss_slots: 0.0185 | Spent: 6.6 secs | LR: 0.000080
I0717 07:09:41.563000 140695199004544 interactiveshell.py:2882] Step 12550 | Loss: 0.0030 | Loss_intent: 0.0000 | Loss_slots: 0.0030 | Spent: 4.2 secs | LR: 0.000080
I0717 07:09:45.749756 140695199004544 interactiveshell.py:2882] Step 12600 | Loss: 0.0048 | Loss_intent: 0.0003 | Loss_slots: 0.0045 | Spent: 4.2 secs | LR: 0.000080
I0717 07:09:50.043678 140695199004544 interactiveshell.py:2882] Step 12650 | Loss: 0.0038 | Loss_intent: 0.0000 | Loss_slots: 0.0037 | Spent: 4.3 secs | LR: 0.000079
I0717 07:09:54.775331 140695199004544 interactiveshell.py:2882] Step 12700 | Loss: 0.0089 | Loss_intent: 0.0000 | Loss_slots: 0.0088 | Spent: 4.7 secs | LR: 0.000079
I0717 07:09:59.577277 140695199004544 interactiveshell.py:2882] Step 12750 | Loss: 0.0042 | Loss_intent: 0.0000 | Loss_slots: 0.0042 | Spent: 4.8 secs | LR: 0.000078
Reading ../data/atis.test.w-intent.iob
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.
  'precision', 'predicted', average, warn_for)
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1439: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples.
  'recall', 'true', average, warn_for)
I0717 07:10:04.791076 140695199004544 interactiveshell.py:2882] 
                                          precision    recall  f1-score   support

                             atis_flight      0.986     0.983     0.984       632
                            atis_airfare      0.923     1.000     0.960        48
                     atis_ground_service      1.000     1.000     1.000        36
                            atis_airline      0.974     1.000     0.987        38
                       atis_abbreviation      1.000     1.000     1.000        33
                           atis_aircraft      1.000     0.889     0.941         9
                        atis_flight_time      0.500     1.000     0.667         1
                           atis_quantity      0.286     0.667     0.400         3
                atis_flight#atis_airfare      0.833     0.417     0.556        12
                            atis_airport      1.000     1.000     1.000        18
                           atis_distance      1.000     1.000     1.000        10
                               atis_city      0.400     0.667     0.500         6
                        atis_ground_fare      1.000     0.714     0.833         7
                           atis_capacity      0.913     1.000     0.955        21
                          atis_flight_no      0.889     1.000     0.941         8
                               atis_meal      1.000     0.833     0.909         6
                        atis_restriction      0.000     0.000     0.000         0
             atis_airline#atis_flight_no      0.000     0.000     0.000         0
    atis_ground_service#atis_ground_fare      0.000     0.000     0.000         0
           atis_airfare#atis_flight_time      0.000     0.000     0.000         0
                           atis_cheapest      0.000     0.000     0.000         0
atis_aircraft#atis_flight#atis_flight_no      0.000     0.000     0.000         0

                               micro avg      0.966     0.972     0.969       888
                               macro avg      0.623     0.644     0.620       888
                            weighted avg      0.972     0.972     0.970       888

I0717 07:10:04.831817 140695199004544 interactiveshell.py:2882] 
                              precision    recall  f1-score   support

                           O      0.000     0.000     0.000       0.0
           B-toloc.city_name      0.973     0.994     0.983     716.0
         B-fromloc.city_name      0.989     0.991     0.990     704.0
           I-toloc.city_name      0.964     0.996     0.980     265.0
      B-depart_date.day_name      0.990     0.976     0.983     212.0
              B-airline_name      0.981     1.000     0.990     101.0
         I-fromloc.city_name      0.978     0.983     0.980     177.0
 B-depart_time.period_of_day      0.983     0.900     0.940     130.0
              I-airline_name      1.000     1.000     1.000      65.0
    B-depart_date.day_number      0.963     0.945     0.954      55.0
    B-depart_date.month_name      0.982     0.964     0.973      56.0
          B-depart_time.time      0.851     1.000     0.919      57.0
                B-round_trip      1.000     0.986     0.993      73.0
             B-cost_relative      1.000     0.973     0.986      37.0
                I-round_trip      1.000     1.000     1.000      71.0
                B-flight_mod      1.000     0.958     0.979      24.0
 B-depart_time.time_relative      0.984     0.954     0.969      65.0
          I-depart_time.time      0.945     1.000     0.972      52.0
         B-stoploc.city_name      0.952     1.000     0.976      20.0
                 B-city_name      0.868     0.579     0.695      57.0
                B-class_type      0.923     1.000     0.960      24.0
          B-arrive_time.time      0.892     0.971     0.930      34.0
 B-arrive_time.time_relative      0.882     0.968     0.923      31.0
                I-class_type      1.000     1.000     1.000      17.0
               B-flight_stop      0.955     1.000     0.977      21.0
          I-arrive_time.time      0.971     0.971     0.971      35.0
              B-airline_code      1.000     0.853     0.921      34.0
    I-depart_date.day_number      1.000     1.000     1.000      15.0
      I-fromloc.airport_name      0.429     1.000     0.600      15.0
      B-fromloc.airport_name      0.355     0.917     0.512      12.0
      B-arrive_date.day_name      0.647     1.000     0.786      11.0
          B-toloc.state_code      0.947     1.000     0.973      18.0
B-depart_date.today_relative      1.000     0.889     0.941       9.0
             B-flight_number      0.429     0.818     0.562      11.0
 B-depart_date.date_relative      0.875     0.824     0.848      17.0
          B-toloc.state_name      0.875     1.000     0.933      28.0
           B-fare_basis_code      0.889     0.941     0.914      17.0
               B-flight_time      1.000     1.000     1.000       1.0
                        B-or      1.000     1.000     1.000       3.0
 B-arrive_time.period_of_day      0.750     1.000     0.857       6.0
          B-meal_description      1.000     0.900     0.947      10.0
             I-cost_relative      1.000     0.667     0.800       3.0
              I-airport_name      0.846     0.379     0.524      29.0
               B-fare_amount      0.500     1.000     0.667       2.0
               I-fare_amount      1.000     1.000     1.000       2.0
                 I-city_name      1.000     0.533     0.696      30.0
        I-toloc.airport_name      0.750     1.000     0.857       3.0
            B-transport_type      1.000     0.800     0.889      10.0
    B-arrive_date.month_name      0.714     0.833     0.769       6.0
    B-arrive_date.day_number      0.714     0.833     0.769       6.0
         I-stoploc.city_name      0.909     1.000     0.952      10.0
                      B-meal      1.000     1.000     1.000      16.0
        B-fromloc.state_code      1.000     0.957     0.978      23.0
    B-depart_time.period_mod      0.667     0.800     0.727       5.0
                   B-connect      1.000     1.000     1.000       6.0
               B-flight_days      1.000     1.000     1.000      10.0
        B-toloc.airport_name      1.000     0.333     0.500       3.0
        B-fromloc.state_name      0.944     1.000     0.971      17.0
              B-airport_name      0.800     0.381     0.516      21.0
                   B-economy      1.000     1.000     1.000       6.0
               I-flight_time      1.000     1.000     1.000       1.0
             B-aircraft_code      0.962     0.758     0.847      33.0
                       B-mod      0.000     0.000     0.000       2.0
              B-airport_code      0.000     0.000     0.000       9.0
    B-depart_time.start_time      0.667     0.667     0.667       3.0
      B-depart_time.end_time      1.000     0.667     0.800       3.0
          B-depart_date.year      1.000     0.667     0.800       3.0
            I-transport_type      0.000     0.000     0.000       1.0
          B-restriction_code      0.300     0.750     0.429       4.0
    B-arrive_time.start_time      0.875     0.875     0.875       8.0
        B-toloc.airport_code      1.000     0.500     0.667       4.0
      B-arrive_time.end_time      0.875     0.875     0.875       8.0
      I-arrive_time.end_time      0.875     0.875     0.875       8.0
      I-depart_time.end_time      1.000     0.333     0.500       3.0
               I-flight_stop      0.000     0.000     0.000       0.0
      B-fromloc.airport_code      1.000     1.000     1.000       5.0
          I-restriction_code      0.600     1.000     0.750       3.0
    I-depart_time.start_time      0.500     1.000     0.667       1.0
          I-toloc.state_name      1.000     1.000     1.000       1.0
I-depart_date.today_relative      0.000     0.000     0.000       0.0
 B-arrive_date.date_relative      1.000     1.000     1.000       2.0
                I-flight_mod      1.000     0.167     0.286       6.0
                   I-economy      0.000     0.000     0.000       0.0
 B-return_date.date_relative      0.333     0.333     0.333       3.0
        I-fromloc.state_name      1.000     1.000     1.000       1.0
                B-state_code      1.000     1.000     1.000       1.0
    I-arrive_time.start_time      1.000     1.000     1.000       1.0
    I-arrive_date.day_number      0.000     0.000     0.000       0.0
                 B-meal_code      0.000     0.000     0.000       1.0
 I-depart_time.period_of_day      0.000     0.000     0.000       1.0
                  B-day_name      1.000     0.500     0.667       2.0
             B-period_of_day      1.000     0.500     0.667       4.0
        B-stoploc.state_code      0.000     0.000     0.000       0.0
    B-return_date.month_name      0.000     0.000     0.000       0.0
    B-return_date.day_number      0.000     0.000     0.000       0.0
    B-arrive_time.period_mod      0.000     0.000     0.000       0.0
                 I-meal_code      0.000     0.000     0.000       0.0
        B-toloc.country_name      1.000     1.000     1.000       1.0
                 B-days_code      1.000     1.000     1.000       1.0
 I-arrive_time.period_of_day      0.000     0.000     0.000       0.0
            I-today_relative      0.000     0.000     0.000       0.0
 B-return_time.period_of_day      0.000     0.000     0.000       0.0
                      B-time      0.000     0.000     0.000       0.0
           I-fare_basis_code      0.000     0.000     0.000       0.0
 I-arrive_time.time_relative      1.000     0.500     0.667       4.0
 I-depart_time.time_relative      0.000     0.000     0.000       1.0
            B-today_relative      0.000     0.000     0.000       0.0
                B-state_name      0.000     0.000     0.000       9.0
B-arrive_date.today_relative      0.000     0.000     0.000       0.0
    B-return_time.period_mod      0.000     0.000     0.000       0.0
                B-month_name      0.000     0.000     0.000       0.0
                B-day_number      0.000     0.000     0.000       0.0
 I-return_date.date_relative      0.600     1.000     0.750       3.0
I-return_date.today_relative      0.000     0.000     0.000       0.0
      B-stoploc.airport_name      0.000     0.000     0.000       0.0
             B-time_relative      0.000     0.000     0.000       0.0
                      I-time      0.000     0.000     0.000       0.0
    I-return_date.day_number      0.000     0.000     0.000       0.0
          I-meal_description      0.000     0.000     0.000       0.0
B-return_date.today_relative      0.000     0.000     0.000       0.0
      B-return_date.day_name      0.000     0.000     0.000       2.0

                   micro avg      0.945     0.947     0.946    3657.0
                   macro avg      0.633     0.618     0.608    3657.0
                weighted avg      0.952     0.947     0.944    3657.0

I0717 07:10:04.832902 140695199004544 interactiveshell.py:2882] Best Slot F1: 0.947
Reading ../data/atis.train.w-intent.iob
I0717 07:10:06.262478 140695199004544 interactiveshell.py:2882] Step 12800 | Loss: 0.0023 | Loss_intent: 0.0000 | Loss_slots: 0.0022 | Spent: 6.7 secs | LR: 0.000078
I0717 07:10:10.450864 140695199004544 interactiveshell.py:2882] Step 12850 | Loss: 0.0055 | Loss_intent: 0.0000 | Loss_slots: 0.0055 | Spent: 4.2 secs | LR: 0.000077
I0717 07:10:14.674801 140695199004544 interactiveshell.py:2882] Step 12900 | Loss: 0.0029 | Loss_intent: 0.0001 | Loss_slots: 0.0028 | Spent: 4.2 secs | LR: 0.000077
I0717 07:10:18.880299 140695199004544 interactiveshell.py:2882] Step 12950 | Loss: 0.0092 | Loss_intent: 0.0000 | Loss_slots: 0.0092 | Spent: 4.2 secs | LR: 0.000077
I0717 07:10:23.071536 140695199004544 interactiveshell.py:2882] Step 13000 | Loss: 0.0059 | Loss_intent: 0.0001 | Loss_slots: 0.0059 | Spent: 4.2 secs | LR: 0.000076
I0717 07:10:27.262769 140695199004544 interactiveshell.py:2882] Step 13050 | Loss: 0.0160 | Loss_intent: 0.0000 | Loss_slots: 0.0160 | Spent: 4.2 secs | LR: 0.000076
I0717 07:10:31.452487 140695199004544 interactiveshell.py:2882] Step 13100 | Loss: 0.0063 | Loss_intent: 0.0000 | Loss_slots: 0.0063 | Spent: 4.2 secs | LR: 0.000075
Reading ../data/atis.test.w-intent.iob
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.
  'precision', 'predicted', average, warn_for)
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1439: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples.
  'recall', 'true', average, warn_for)
I0717 07:10:33.428923 140695199004544 interactiveshell.py:2882] 
                                          precision    recall  f1-score   support

                             atis_flight      0.983     0.984     0.983       632
                            atis_airfare      0.923     1.000     0.960        48
                     atis_ground_service      1.000     1.000     1.000        36
                            atis_airline      0.974     1.000     0.987        38
                       atis_abbreviation      1.000     1.000     1.000        33
                           atis_aircraft      0.889     0.889     0.889         9
                        atis_flight_time      1.000     1.000     1.000         1
                           atis_quantity      0.375     1.000     0.545         3
                atis_flight#atis_airfare      0.800     0.333     0.471        12
                            atis_airport      0.947     1.000     0.973        18
                           atis_distance      1.000     1.000     1.000        10
                               atis_city      0.500     0.667     0.571         6
                        atis_ground_fare      1.000     0.714     0.833         7
                           atis_capacity      1.000     1.000     1.000        21
                          atis_flight_no      0.889     1.000     0.941         8
                               atis_meal      1.000     0.833     0.909         6
                        atis_restriction      0.000     0.000     0.000         0
             atis_airline#atis_flight_no      0.000     0.000     0.000         0
    atis_ground_service#atis_ground_fare      0.000     0.000     0.000         0
           atis_airfare#atis_flight_time      0.000     0.000     0.000         0
                           atis_cheapest      0.000     0.000     0.000         0
atis_aircraft#atis_flight#atis_flight_no      0.000     0.000     0.000         0

                               micro avg      0.968     0.973     0.970       888
                               macro avg      0.649     0.655     0.639       888
                            weighted avg      0.971     0.973     0.970       888

I0717 07:10:33.469022 140695199004544 interactiveshell.py:2882] 
                              precision    recall  f1-score   support

                           O      0.000     0.000     0.000       0.0
           B-toloc.city_name      0.973     0.994     0.983     716.0
         B-fromloc.city_name      0.985     0.994     0.989     704.0
           I-toloc.city_name      0.964     1.000     0.981     265.0
      B-depart_date.day_name      0.995     0.976     0.986     212.0
              B-airline_name      0.971     0.990     0.980     101.0
         I-fromloc.city_name      0.989     0.977     0.983     177.0
 B-depart_time.period_of_day      0.975     0.915     0.944     130.0
              I-airline_name      1.000     1.000     1.000      65.0
    B-depart_date.day_number      0.981     0.927     0.953      55.0
    B-depart_date.month_name      0.981     0.946     0.964      56.0
          B-depart_time.time      0.812     0.982     0.889      57.0
                B-round_trip      1.000     0.986     0.993      73.0
             B-cost_relative      1.000     0.973     0.986      37.0
                I-round_trip      1.000     1.000     1.000      71.0
                B-flight_mod      1.000     0.958     0.979      24.0
 B-depart_time.time_relative      0.984     0.954     0.969      65.0
          I-depart_time.time      0.944     0.981     0.962      52.0
         B-stoploc.city_name      0.952     1.000     0.976      20.0
                 B-city_name      0.868     0.579     0.695      57.0
                B-class_type      0.889     1.000     0.941      24.0
          B-arrive_time.time      0.971     0.971     0.971      34.0
 B-arrive_time.time_relative      0.882     0.968     0.923      31.0
                I-class_type      1.000     1.000     1.000      17.0
               B-flight_stop      0.955     1.000     0.977      21.0
          I-arrive_time.time      0.944     0.971     0.958      35.0
              B-airline_code      1.000     0.853     0.921      34.0
    I-depart_date.day_number      1.000     1.000     1.000      15.0
      I-fromloc.airport_name      0.429     1.000     0.600      15.0
      B-fromloc.airport_name      0.423     0.917     0.579      12.0
      B-arrive_date.day_name      0.611     1.000     0.759      11.0
          B-toloc.state_code      0.947     1.000     0.973      18.0
B-depart_date.today_relative      1.000     1.000     1.000       9.0
             B-flight_number      0.375     0.818     0.514      11.0
 B-depart_date.date_relative      0.933     0.824     0.875      17.0
          B-toloc.state_name      0.900     0.964     0.931      28.0
           B-fare_basis_code      0.917     0.647     0.759      17.0
               B-flight_time      1.000     1.000     1.000       1.0
                        B-or      1.000     1.000     1.000       3.0
 B-arrive_time.period_of_day      0.750     1.000     0.857       6.0
          B-meal_description      1.000     0.900     0.947      10.0
             I-cost_relative      1.000     0.667     0.800       3.0
              I-airport_name      0.846     0.379     0.524      29.0
               B-fare_amount      0.333     1.000     0.500       2.0
               I-fare_amount      1.000     1.000     1.000       2.0
                 I-city_name      1.000     0.467     0.636      30.0
        I-toloc.airport_name      0.750     1.000     0.857       3.0
            B-transport_type      1.000     0.700     0.824      10.0
    B-arrive_date.month_name      0.625     0.833     0.714       6.0
    B-arrive_date.day_number      0.625     0.833     0.714       6.0
         I-stoploc.city_name      0.769     1.000     0.870      10.0
                      B-meal      1.000     1.000     1.000      16.0
        B-fromloc.state_code      1.000     0.957     0.978      23.0
    B-depart_time.period_mod      1.000     0.600     0.750       5.0
                   B-connect      1.000     1.000     1.000       6.0
               B-flight_days      1.000     1.000     1.000      10.0
        B-toloc.airport_name      1.000     0.333     0.500       3.0
        B-fromloc.state_name      0.895     1.000     0.944      17.0
              B-airport_name      0.615     0.381     0.471      21.0
                   B-economy      1.000     1.000     1.000       6.0
               I-flight_time      1.000     1.000     1.000       1.0
             B-aircraft_code      0.929     0.788     0.852      33.0
                       B-mod      0.000     0.000     0.000       2.0
              B-airport_code      0.000     0.000     0.000       9.0
    B-depart_time.start_time      0.667     0.667     0.667       3.0
      B-depart_time.end_time      1.000     0.667     0.800       3.0
          B-depart_date.year      1.000     0.667     0.800       3.0
            I-transport_type      0.000     0.000     0.000       1.0
          B-restriction_code      0.333     0.750     0.462       4.0
    B-arrive_time.start_time      0.889     1.000     0.941       8.0
        B-toloc.airport_code      1.000     0.500     0.667       4.0
      B-arrive_time.end_time      0.889     1.000     0.941       8.0
      I-arrive_time.end_time      0.875     0.875     0.875       8.0
      I-depart_time.end_time      1.000     0.333     0.500       3.0
               I-flight_stop      0.000     0.000     0.000       0.0
      B-fromloc.airport_code      1.000     1.000     1.000       5.0
          I-restriction_code      0.750     1.000     0.857       3.0
    I-depart_time.start_time      0.500     1.000     0.667       1.0
          I-toloc.state_name      1.000     1.000     1.000       1.0
I-depart_date.today_relative      0.000     0.000     0.000       0.0
 B-arrive_date.date_relative      0.667     1.000     0.800       2.0
                I-flight_mod      1.000     0.167     0.286       6.0
                   I-economy      0.000     0.000     0.000       0.0
 B-return_date.date_relative      0.500     0.333     0.400       3.0
        I-fromloc.state_name      1.000     1.000     1.000       1.0
                B-state_code      1.000     1.000     1.000       1.0
    I-arrive_time.start_time      1.000     1.000     1.000       1.0
    I-arrive_date.day_number      0.000     0.000     0.000       0.0
                 B-meal_code      1.000     1.000     1.000       1.0
 I-depart_time.period_of_day      0.000     0.000     0.000       1.0
                  B-day_name      1.000     0.500     0.667       2.0
             B-period_of_day      1.000     0.500     0.667       4.0
        B-stoploc.state_code      0.000     0.000     0.000       0.0
    B-return_date.month_name      0.000     0.000     0.000       0.0
    B-return_date.day_number      0.000     0.000     0.000       0.0
    B-arrive_time.period_mod      0.000     0.000     0.000       0.0
                 I-meal_code      0.000     0.000     0.000       0.0
        B-toloc.country_name      1.000     1.000     1.000       1.0
                 B-days_code      1.000     1.000     1.000       1.0
 I-arrive_time.period_of_day      0.000     0.000     0.000       0.0
            I-today_relative      0.000     0.000     0.000       0.0
 B-return_time.period_of_day      0.000     0.000     0.000       0.0
                      B-time      0.000     0.000     0.000       0.0
           I-fare_basis_code      0.000     0.000     0.000       0.0
 I-arrive_time.time_relative      1.000     0.500     0.667       4.0
 I-depart_time.time_relative      0.000     0.000     0.000       1.0
            B-today_relative      0.000     0.000     0.000       0.0
                B-state_name      1.000     0.111     0.200       9.0
B-arrive_date.today_relative      0.000     0.000     0.000       0.0
    B-return_time.period_mod      0.000     0.000     0.000       0.0
                B-month_name      0.000     0.000     0.000       0.0
                B-day_number      0.000     0.000     0.000       0.0
 I-return_date.date_relative      0.600     1.000     0.750       3.0
I-return_date.today_relative      0.000     0.000     0.000       0.0
      B-stoploc.airport_name      0.000     0.000     0.000       0.0
             B-time_relative      0.000     0.000     0.000       0.0
                      I-time      0.000     0.000     0.000       0.0
    I-return_date.day_number      0.000     0.000     0.000       0.0
          I-meal_description      0.000     0.000     0.000       0.0
B-return_date.today_relative      0.000     0.000     0.000       0.0
      B-return_date.day_name      0.000     0.000     0.000       2.0

                   micro avg      0.944     0.945     0.945    3657.0
                   macro avg      0.648     0.624     0.614    3657.0
                weighted avg      0.953     0.945     0.943    3657.0

I0717 07:10:33.469820 140695199004544 interactiveshell.py:2882] Best Slot F1: 0.947
Reading ../data/atis.train.w-intent.iob
I0717 07:10:38.049268 140695199004544 interactiveshell.py:2882] Step 13150 | Loss: 0.0135 | Loss_intent: 0.0000 | Loss_slots: 0.0135 | Spent: 6.6 secs | LR: 0.000075
I0717 07:10:42.236283 140695199004544 interactiveshell.py:2882] Step 13200 | Loss: 0.0210 | Loss_intent: 0.0066 | Loss_slots: 0.0144 | Spent: 4.2 secs | LR: 0.000075
I0717 07:10:46.417019 140695199004544 interactiveshell.py:2882] Step 13250 | Loss: 0.0051 | Loss_intent: 0.0000 | Loss_slots: 0.0051 | Spent: 4.2 secs | LR: 0.000074
I0717 07:10:50.601023 140695199004544 interactiveshell.py:2882] Step 13300 | Loss: 0.0118 | Loss_intent: 0.0025 | Loss_slots: 0.0093 | Spent: 4.2 secs | LR: 0.000074
I0717 07:10:54.765142 140695199004544 interactiveshell.py:2882] Step 13350 | Loss: 0.0030 | Loss_intent: 0.0000 | Loss_slots: 0.0030 | Spent: 4.2 secs | LR: 0.000073
I0717 07:10:58.965339 140695199004544 interactiveshell.py:2882] Step 13400 | Loss: 0.0065 | Loss_intent: 0.0000 | Loss_slots: 0.0065 | Spent: 4.2 secs | LR: 0.000073
Reading ../data/atis.test.w-intent.iob
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.
  'precision', 'predicted', average, warn_for)
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1439: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples.
  'recall', 'true', average, warn_for)
I0717 07:11:01.929519 140695199004544 interactiveshell.py:2882] 
                                          precision    recall  f1-score   support

                             atis_flight      0.983     0.986     0.984       632
                            atis_airfare      0.923     1.000     0.960        48
                     atis_ground_service      1.000     1.000     1.000        36
                            atis_airline      0.974     1.000     0.987        38
                       atis_abbreviation      0.971     1.000     0.985        33
                           atis_aircraft      1.000     0.889     0.941         9
                        atis_flight_time      1.000     1.000     1.000         1
                           atis_quantity      0.286     0.667     0.400         3
                atis_flight#atis_airfare      0.800     0.333     0.471        12
                            atis_airport      1.000     1.000     1.000        18
                           atis_distance      1.000     1.000     1.000        10
                               atis_city      0.500     0.667     0.571         6
                        atis_ground_fare      1.000     0.714     0.833         7
                           atis_capacity      0.955     1.000     0.977        21
                          atis_flight_no      0.889     1.000     0.941         8
                               atis_meal      1.000     0.833     0.909         6
                        atis_restriction      0.000     0.000     0.000         0
             atis_airline#atis_flight_no      0.000     0.000     0.000         0
    atis_ground_service#atis_ground_fare      0.000     0.000     0.000         0
           atis_airfare#atis_flight_time      0.000     0.000     0.000         0
                           atis_cheapest      0.000     0.000     0.000         0
atis_aircraft#atis_flight#atis_flight_no      0.000     0.000     0.000         0

                               micro avg      0.968     0.973     0.970       888
                               macro avg      0.649     0.640     0.635       888
                            weighted avg      0.971     0.973     0.970       888

I0717 07:11:01.969538 140695199004544 interactiveshell.py:2882] 
                              precision    recall  f1-score   support

                           O      0.000     0.000     0.000       0.0
           B-toloc.city_name      0.973     0.994     0.983     716.0
         B-fromloc.city_name      0.987     0.991     0.989     704.0
           I-toloc.city_name      0.970     0.992     0.981     265.0
      B-depart_date.day_name      0.991     0.986     0.988     212.0
              B-airline_name      0.980     0.990     0.985     101.0
         I-fromloc.city_name      0.972     0.989     0.980     177.0
 B-depart_time.period_of_day      0.992     0.900     0.944     130.0
              I-airline_name      1.000     1.000     1.000      65.0
    B-depart_date.day_number      0.963     0.945     0.954      55.0
    B-depart_date.month_name      0.982     0.964     0.973      56.0
          B-depart_time.time      0.789     0.982     0.875      57.0
                B-round_trip      1.000     0.986     0.993      73.0
             B-cost_relative      1.000     0.973     0.986      37.0
                I-round_trip      1.000     1.000     1.000      71.0
                B-flight_mod      0.885     0.958     0.920      24.0
 B-depart_time.time_relative      0.984     0.969     0.977      65.0
          I-depart_time.time      0.929     1.000     0.963      52.0
         B-stoploc.city_name      1.000     1.000     1.000      20.0
                 B-city_name      0.846     0.579     0.688      57.0
                B-class_type      0.923     1.000     0.960      24.0
          B-arrive_time.time      0.917     0.971     0.943      34.0
 B-arrive_time.time_relative      0.909     0.968     0.937      31.0
                I-class_type      1.000     1.000     1.000      17.0
               B-flight_stop      0.955     1.000     0.977      21.0
          I-arrive_time.time      0.971     0.971     0.971      35.0
              B-airline_code      1.000     0.853     0.921      34.0
    I-depart_date.day_number      1.000     1.000     1.000      15.0
      I-fromloc.airport_name      0.429     1.000     0.600      15.0
      B-fromloc.airport_name      0.367     0.917     0.524      12.0
      B-arrive_date.day_name      0.733     1.000     0.846      11.0
          B-toloc.state_code      0.947     1.000     0.973      18.0
B-depart_date.today_relative      1.000     1.000     1.000       9.0
             B-flight_number      0.375     0.818     0.514      11.0
 B-depart_date.date_relative      0.941     0.941     0.941      17.0
          B-toloc.state_name      0.897     0.929     0.912      28.0
           B-fare_basis_code      0.917     0.647     0.759      17.0
               B-flight_time      1.000     1.000     1.000       1.0
                        B-or      1.000     1.000     1.000       3.0
 B-arrive_time.period_of_day      0.750     1.000     0.857       6.0
          B-meal_description      1.000     0.900     0.947      10.0
             I-cost_relative      1.000     0.667     0.800       3.0
              I-airport_name      0.846     0.379     0.524      29.0
               B-fare_amount      0.333     0.500     0.400       2.0
               I-fare_amount      1.000     1.000     1.000       2.0
                 I-city_name      1.000     0.533     0.696      30.0
        I-toloc.airport_name      0.750     1.000     0.857       3.0
            B-transport_type      1.000     0.800     0.889      10.0
    B-arrive_date.month_name      0.714     0.833     0.769       6.0
    B-arrive_date.day_number      0.714     0.833     0.769       6.0
         I-stoploc.city_name      0.909     1.000     0.952      10.0
                      B-meal      1.000     1.000     1.000      16.0
        B-fromloc.state_code      1.000     0.957     0.978      23.0
    B-depart_time.period_mod      0.833     1.000     0.909       5.0
                   B-connect      1.000     1.000     1.000       6.0
               B-flight_days      1.000     1.000     1.000      10.0
        B-toloc.airport_name      1.000     0.333     0.500       3.0
        B-fromloc.state_name      0.850     1.000     0.919      17.0
              B-airport_name      0.800     0.381     0.516      21.0
                   B-economy      1.000     1.000     1.000       6.0
               I-flight_time      1.000     1.000     1.000       1.0
             B-aircraft_code      0.931     0.818     0.871      33.0
                       B-mod      0.000     0.000     0.000       2.0
              B-airport_code      0.000     0.000     0.000       9.0
    B-depart_time.start_time      0.667     0.667     0.667       3.0
      B-depart_time.end_time      1.000     0.667     0.800       3.0
          B-depart_date.year      1.000     0.667     0.800       3.0
            I-transport_type      0.000     0.000     0.000       1.0
          B-restriction_code      0.333     0.750     0.462       4.0
    B-arrive_time.start_time      0.875     0.875     0.875       8.0
        B-toloc.airport_code      1.000     0.500     0.667       4.0
      B-arrive_time.end_time      0.889     1.000     0.941       8.0
      I-arrive_time.end_time      0.875     0.875     0.875       8.0
      I-depart_time.end_time      1.000     0.333     0.500       3.0
               I-flight_stop      0.000     0.000     0.000       0.0
      B-fromloc.airport_code      1.000     1.000     1.000       5.0
          I-restriction_code      0.750     1.000     0.857       3.0
    I-depart_time.start_time      0.500     1.000     0.667       1.0
          I-toloc.state_name      1.000     1.000     1.000       1.0
I-depart_date.today_relative      0.000     0.000     0.000       0.0
 B-arrive_date.date_relative      1.000     1.000     1.000       2.0
                I-flight_mod      1.000     0.167     0.286       6.0
                   I-economy      0.000     0.000     0.000       0.0
 B-return_date.date_relative      0.500     0.333     0.400       3.0
        I-fromloc.state_name      1.000     1.000     1.000       1.0
                B-state_code      1.000     1.000     1.000       1.0
    I-arrive_time.start_time      1.000     1.000     1.000       1.0
    I-arrive_date.day_number      0.000     0.000     0.000       0.0
                 B-meal_code      1.000     1.000     1.000       1.0
 I-depart_time.period_of_day      0.000     0.000     0.000       1.0
                  B-day_name      1.000     0.500     0.667       2.0
             B-period_of_day      1.000     0.500     0.667       4.0
        B-stoploc.state_code      0.000     0.000     0.000       0.0
    B-return_date.month_name      0.000     0.000     0.000       0.0
    B-return_date.day_number      0.000     0.000     0.000       0.0
    B-arrive_time.period_mod      0.000     0.000     0.000       0.0
                 I-meal_code      0.000     0.000     0.000       0.0
        B-toloc.country_name      1.000     1.000     1.000       1.0
                 B-days_code      1.000     1.000     1.000       1.0
 I-arrive_time.period_of_day      0.000     0.000     0.000       0.0
            I-today_relative      0.000     0.000     0.000       0.0
 B-return_time.period_of_day      0.000     0.000     0.000       0.0
                      B-time      0.000     0.000     0.000       0.0
           I-fare_basis_code      0.000     0.000     0.000       0.0
 I-arrive_time.time_relative      1.000     0.500     0.667       4.0
 I-depart_time.time_relative      0.000     0.000     0.000       1.0
            B-today_relative      0.000     0.000     0.000       0.0
                B-state_name      1.000     0.111     0.200       9.0
B-arrive_date.today_relative      0.000     0.000     0.000       0.0
    B-return_time.period_mod      0.000     0.000     0.000       0.0
                B-month_name      0.000     0.000     0.000       0.0
                B-day_number      0.000     0.000     0.000       0.0
 I-return_date.date_relative      0.667     0.667     0.667       3.0
I-return_date.today_relative      0.000     0.000     0.000       0.0
      B-stoploc.airport_name      0.000     0.000     0.000       0.0
             B-time_relative      0.000     0.000     0.000       0.0
                      I-time      0.000     0.000     0.000       0.0
    I-return_date.day_number      0.000     0.000     0.000       0.0
          I-meal_description      0.000     0.000     0.000       0.0
B-return_date.today_relative      0.000     0.000     0.000       0.0
      B-return_date.day_name      0.000     0.000     0.000       2.0

                   micro avg      0.945     0.947     0.946    3657.0
                   macro avg      0.653     0.622     0.618    3657.0
                weighted avg      0.954     0.947     0.944    3657.0

I0717 07:11:01.970335 140695199004544 interactiveshell.py:2882] Best Slot F1: 0.947
Reading ../data/atis.train.w-intent.iob
I0717 07:11:05.551928 140695199004544 interactiveshell.py:2882] Step 13450 | Loss: 0.0145 | Loss_intent: 0.0000 | Loss_slots: 0.0145 | Spent: 6.6 secs | LR: 0.000073
I0717 07:11:09.760756 140695199004544 interactiveshell.py:2882] Step 13500 | Loss: 0.0051 | Loss_intent: 0.0000 | Loss_slots: 0.0051 | Spent: 4.2 secs | LR: 0.000072
I0717 07:11:13.928624 140695199004544 interactiveshell.py:2882] Step 13550 | Loss: 0.0125 | Loss_intent: 0.0001 | Loss_slots: 0.0124 | Spent: 4.2 secs | LR: 0.000072
I0717 07:11:18.134912 140695199004544 interactiveshell.py:2882] Step 13600 | Loss: 0.0254 | Loss_intent: 0.0000 | Loss_slots: 0.0254 | Spent: 4.2 secs | LR: 0.000072
I0717 07:11:22.404248 140695199004544 interactiveshell.py:2882] Step 13650 | Loss: 0.0020 | Loss_intent: 0.0002 | Loss_slots: 0.0018 | Spent: 4.3 secs | LR: 0.000071
I0717 07:11:26.646952 140695199004544 interactiveshell.py:2882] Step 13700 | Loss: 0.0028 | Loss_intent: 0.0000 | Loss_slots: 0.0028 | Spent: 4.2 secs | LR: 0.000071
Reading ../data/atis.test.w-intent.iob
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.
  'precision', 'predicted', average, warn_for)
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1439: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples.
  'recall', 'true', average, warn_for)
I0717 07:11:30.624337 140695199004544 interactiveshell.py:2882] 
                                          precision    recall  f1-score   support

                             atis_flight      0.980     0.989     0.984       632
                            atis_airfare      0.906     1.000     0.950        48
                     atis_ground_service      1.000     1.000     1.000        36
                            atis_airline      1.000     0.974     0.987        38
                       atis_abbreviation      0.971     1.000     0.985        33
                           atis_aircraft      1.000     0.889     0.941         9
                        atis_flight_time      1.000     1.000     1.000         1
                           atis_quantity      0.375     1.000     0.545         3
                atis_flight#atis_airfare      0.800     0.333     0.471        12
                            atis_airport      1.000     1.000     1.000        18
                           atis_distance      1.000     1.000     1.000        10
                               atis_city      0.667     0.667     0.667         6
                        atis_ground_fare      1.000     0.714     0.833         7
                           atis_capacity      1.000     0.952     0.976        21
                          atis_flight_no      0.889     1.000     0.941         8
                               atis_meal      1.000     0.833     0.909         6
                        atis_restriction      0.000     0.000     0.000         0
             atis_airline#atis_flight_no      0.000     0.000     0.000         0
    atis_ground_service#atis_ground_fare      0.000     0.000     0.000         0
           atis_airfare#atis_flight_time      0.000     0.000     0.000         0
                           atis_cheapest      0.000     0.000     0.000         0
atis_aircraft#atis_flight#atis_flight_no      0.000     0.000     0.000         0

                               micro avg      0.969     0.974     0.971       888
                               macro avg      0.663     0.652     0.645       888
                            weighted avg      0.971     0.974     0.970       888

I0717 07:11:30.665703 140695199004544 interactiveshell.py:2882] 
                              precision    recall  f1-score   support

                           O      0.000     0.000     0.000       0.0
           B-toloc.city_name      0.971     0.994     0.983     716.0
         B-fromloc.city_name      0.989     0.993     0.991     704.0
           I-toloc.city_name      0.960     0.992     0.976     265.0
      B-depart_date.day_name      0.995     0.972     0.983     212.0
              B-airline_name      0.990     1.000     0.995     101.0
         I-fromloc.city_name      0.972     0.983     0.978     177.0
 B-depart_time.period_of_day      0.991     0.892     0.939     130.0
              I-airline_name      1.000     1.000     1.000      65.0
    B-depart_date.day_number      0.980     0.909     0.943      55.0
    B-depart_date.month_name      0.981     0.946     0.964      56.0
          B-depart_time.time      0.848     0.982     0.911      57.0
                B-round_trip      1.000     0.986     0.993      73.0
             B-cost_relative      1.000     0.973     0.986      37.0
                I-round_trip      1.000     1.000     1.000      71.0
                B-flight_mod      0.960     1.000     0.980      24.0
 B-depart_time.time_relative      0.984     0.954     0.969      65.0
          I-depart_time.time      0.945     1.000     0.972      52.0
         B-stoploc.city_name      0.952     1.000     0.976      20.0
                 B-city_name      0.941     0.561     0.703      57.0
                B-class_type      0.923     1.000     0.960      24.0
          B-arrive_time.time      0.892     0.971     0.930      34.0
 B-arrive_time.time_relative      0.882     0.968     0.923      31.0
                I-class_type      1.000     1.000     1.000      17.0
               B-flight_stop      1.000     1.000     1.000      21.0
          I-arrive_time.time      0.971     0.971     0.971      35.0
              B-airline_code      1.000     0.912     0.954      34.0
    I-depart_date.day_number      1.000     1.000     1.000      15.0
      I-fromloc.airport_name      0.429     1.000     0.600      15.0
      B-fromloc.airport_name      0.367     0.917     0.524      12.0
      B-arrive_date.day_name      0.579     1.000     0.733      11.0
          B-toloc.state_code      0.947     1.000     0.973      18.0
B-depart_date.today_relative      1.000     1.000     1.000       9.0
             B-flight_number      0.400     0.909     0.556      11.0
 B-depart_date.date_relative      0.929     0.765     0.839      17.0
          B-toloc.state_name      0.800     1.000     0.889      28.0
           B-fare_basis_code      0.909     0.588     0.714      17.0
               B-flight_time      1.000     1.000     1.000       1.0
                        B-or      1.000     1.000     1.000       3.0
 B-arrive_time.period_of_day      0.750     1.000     0.857       6.0
          B-meal_description      1.000     0.900     0.947      10.0
             I-cost_relative      1.000     0.667     0.800       3.0
              I-airport_name      0.846     0.379     0.524      29.0
               B-fare_amount      0.333     1.000     0.500       2.0
               I-fare_amount      1.000     1.000     1.000       2.0
                 I-city_name      1.000     0.500     0.667      30.0
        I-toloc.airport_name      0.750     1.000     0.857       3.0
            B-transport_type      1.000     0.700     0.824      10.0
    B-arrive_date.month_name      0.625     0.833     0.714       6.0
    B-arrive_date.day_number      0.625     0.833     0.714       6.0
         I-stoploc.city_name      1.000     1.000     1.000      10.0
                      B-meal      1.000     1.000     1.000      16.0
        B-fromloc.state_code      1.000     0.957     0.978      23.0
    B-depart_time.period_mod      0.625     1.000     0.769       5.0
                   B-connect      1.000     1.000     1.000       6.0
               B-flight_days      1.000     1.000     1.000      10.0
        B-toloc.airport_name      1.000     0.333     0.500       3.0
        B-fromloc.state_name      0.944     1.000     0.971      17.0
              B-airport_name      0.750     0.286     0.414      21.0
                   B-economy      1.000     1.000     1.000       6.0
               I-flight_time      1.000     1.000     1.000       1.0
             B-aircraft_code      0.929     0.788     0.852      33.0
                       B-mod      0.000     0.000     0.000       2.0
              B-airport_code      0.000     0.000     0.000       9.0
    B-depart_time.start_time      0.667     0.667     0.667       3.0
      B-depart_time.end_time      1.000     0.667     0.800       3.0
          B-depart_date.year      1.000     0.667     0.800       3.0
            I-transport_type      0.250     1.000     0.400       1.0
          B-restriction_code      0.300     0.750     0.429       4.0
    B-arrive_time.start_time      0.875     0.875     0.875       8.0
        B-toloc.airport_code      1.000     0.500     0.667       4.0
      B-arrive_time.end_time      0.875     0.875     0.875       8.0
      I-arrive_time.end_time      0.875     0.875     0.875       8.0
      I-depart_time.end_time      1.000     0.333     0.500       3.0
               I-flight_stop      0.000     0.000     0.000       0.0
      B-fromloc.airport_code      1.000     1.000     1.000       5.0
          I-restriction_code      0.500     1.000     0.667       3.0
    I-depart_time.start_time      0.500     1.000     0.667       1.0
          I-toloc.state_name      1.000     1.000     1.000       1.0
I-depart_date.today_relative      0.000     0.000     0.000       0.0
 B-arrive_date.date_relative      1.000     1.000     1.000       2.0
                I-flight_mod      1.000     0.167     0.286       6.0
                   I-economy      0.000     0.000     0.000       0.0
 B-return_date.date_relative      0.333     0.333     0.333       3.0
        I-fromloc.state_name      1.000     1.000     1.000       1.0
                B-state_code      1.000     1.000     1.000       1.0
    I-arrive_time.start_time      1.000     1.000     1.000       1.0
    I-arrive_date.day_number      0.000     0.000     0.000       0.0
                 B-meal_code      0.000     0.000     0.000       1.0
 I-depart_time.period_of_day      0.000     0.000     0.000       1.0
                  B-day_name      1.000     0.500     0.667       2.0
             B-period_of_day      1.000     0.500     0.667       4.0
        B-stoploc.state_code      0.000     0.000     0.000       0.0
    B-return_date.month_name      0.000     0.000     0.000       0.0
    B-return_date.day_number      0.000     0.000     0.000       0.0
    B-arrive_time.period_mod      0.000     0.000     0.000       0.0
                 I-meal_code      0.000     0.000     0.000       0.0
        B-toloc.country_name      1.000     1.000     1.000       1.0
                 B-days_code      1.000     1.000     1.000       1.0
 I-arrive_time.period_of_day      0.000     0.000     0.000       0.0
            I-today_relative      0.000     0.000     0.000       0.0
 B-return_time.period_of_day      0.000     0.000     0.000       0.0
                      B-time      0.000     0.000     0.000       0.0
           I-fare_basis_code      0.000     0.000     0.000       0.0
 I-arrive_time.time_relative      1.000     0.500     0.667       4.0
 I-depart_time.time_relative      0.000     0.000     0.000       1.0
            B-today_relative      0.000     0.000     0.000       0.0
                B-state_name      0.000     0.000     0.000       9.0
B-arrive_date.today_relative      0.000     0.000     0.000       0.0
    B-return_time.period_mod      0.000     0.000     0.000       0.0
                B-month_name      0.000     0.000     0.000       0.0
                B-day_number      0.000     0.000     0.000       0.0
 I-return_date.date_relative      0.600     1.000     0.750       3.0
I-return_date.today_relative      0.000     0.000     0.000       0.0
      B-stoploc.airport_name      0.000     0.000     0.000       0.0
             B-time_relative      0.000     0.000     0.000       0.0
                      I-time      0.000     0.000     0.000       0.0
    I-return_date.day_number      0.000     0.000     0.000       0.0
          I-meal_description      0.000     0.000     0.000       0.0
B-return_date.today_relative      0.000     0.000     0.000       0.0
      B-return_date.day_name      0.000     0.000     0.000       2.0

                   micro avg      0.942     0.944     0.943    3657.0
                   macro avg      0.632     0.624     0.606    3657.0
                weighted avg      0.952     0.944     0.942    3657.0

I0717 07:11:30.666527 140695199004544 interactiveshell.py:2882] Best Slot F1: 0.947
Reading ../data/atis.train.w-intent.iob
I0717 07:11:33.255311 140695199004544 interactiveshell.py:2882] Step 13750 | Loss: 0.0060 | Loss_intent: 0.0000 | Loss_slots: 0.0060 | Spent: 6.6 secs | LR: 0.000070
I0717 07:11:37.434088 140695199004544 interactiveshell.py:2882] Step 13800 | Loss: 0.0037 | Loss_intent: 0.0001 | Loss_slots: 0.0036 | Spent: 4.2 secs | LR: 0.000070
I0717 07:11:41.630036 140695199004544 interactiveshell.py:2882] Step 13850 | Loss: 0.0088 | Loss_intent: 0.0000 | Loss_slots: 0.0088 | Spent: 4.2 secs | LR: 0.000070
I0717 07:11:45.811309 140695199004544 interactiveshell.py:2882] Step 13900 | Loss: 0.0125 | Loss_intent: 0.0000 | Loss_slots: 0.0124 | Spent: 4.2 secs | LR: 0.000069
I0717 07:11:49.996835 140695199004544 interactiveshell.py:2882] Step 13950 | Loss: 0.0181 | Loss_intent: 0.0001 | Loss_slots: 0.0180 | Spent: 4.2 secs | LR: 0.000069
I0717 07:11:54.179274 140695199004544 interactiveshell.py:2882] Step 14000 | Loss: 0.0030 | Loss_intent: 0.0000 | Loss_slots: 0.0029 | Spent: 4.2 secs | LR: 0.000069
Reading ../data/atis.test.w-intent.iob
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.
  'precision', 'predicted', average, warn_for)
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1439: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples.
  'recall', 'true', average, warn_for)
I0717 07:11:59.231339 140695199004544 interactiveshell.py:2882] 
                                          precision    recall  f1-score   support

                             atis_flight      0.986     0.983     0.984       632
                            atis_airfare      0.906     1.000     0.950        48
                     atis_ground_service      1.000     1.000     1.000        36
                            atis_airline      0.974     1.000     0.987        38
                       atis_abbreviation      1.000     0.970     0.985        33
                           atis_aircraft      1.000     0.889     0.941         9
                        atis_flight_time      1.000     1.000     1.000         1
                           atis_quantity      0.375     1.000     0.545         3
                atis_flight#atis_airfare      0.833     0.417     0.556        12
                            atis_airport      0.947     1.000     0.973        18
                           atis_distance      1.000     1.000     1.000        10
                               atis_city      0.444     0.667     0.533         6
                        atis_ground_fare      1.000     0.714     0.833         7
                           atis_capacity      0.955     1.000     0.977        21
                          atis_flight_no      0.889     1.000     0.941         8
                               atis_meal      1.000     0.833     0.909         6
                        atis_restriction      0.000     0.000     0.000         0
             atis_airline#atis_flight_no      0.000     0.000     0.000         0
    atis_ground_service#atis_ground_fare      0.000     0.000     0.000         0
           atis_airfare#atis_flight_time      0.000     0.000     0.000         0
                           atis_cheapest      0.000     0.000     0.000         0
atis_aircraft#atis_flight#atis_flight_no      0.000     0.000     0.000         0

                               micro avg      0.966     0.972     0.969       888
                               macro avg      0.650     0.658     0.642       888
                            weighted avg      0.972     0.972     0.970       888

I0717 07:11:59.276688 140695199004544 interactiveshell.py:2882] 
                              precision    recall  f1-score   support

                           O      0.000     0.000     0.000       0.0
           B-toloc.city_name      0.975     0.994     0.985     716.0
         B-fromloc.city_name      0.989     0.990     0.989     704.0
           I-toloc.city_name      0.974     1.000     0.987     265.0
      B-depart_date.day_name      0.990     0.976     0.983     212.0
              B-airline_name      0.952     0.990     0.971     101.0
         I-fromloc.city_name      0.983     0.983     0.983     177.0
 B-depart_time.period_of_day      0.992     0.908     0.948     130.0
              I-airline_name      1.000     1.000     1.000      65.0
    B-depart_date.day_number      0.981     0.945     0.963      55.0
    B-depart_date.month_name      0.982     0.964     0.973      56.0
          B-depart_time.time      0.838     1.000     0.912      57.0
                B-round_trip      1.000     0.986     0.993      73.0
             B-cost_relative      1.000     0.973     0.986      37.0
                I-round_trip      1.000     1.000     1.000      71.0
                B-flight_mod      1.000     0.958     0.979      24.0
 B-depart_time.time_relative      0.984     0.954     0.969      65.0
          I-depart_time.time      0.945     1.000     0.972      52.0
         B-stoploc.city_name      0.952     1.000     0.976      20.0
                 B-city_name      0.791     0.596     0.680      57.0
                B-class_type      0.889     1.000     0.941      24.0
          B-arrive_time.time      0.971     0.971     0.971      34.0
 B-arrive_time.time_relative      0.882     0.968     0.923      31.0
                I-class_type      1.000     1.000     1.000      17.0
               B-flight_stop      0.955     1.000     0.977      21.0
          I-arrive_time.time      0.971     0.971     0.971      35.0
              B-airline_code      1.000     0.853     0.921      34.0
    I-depart_date.day_number      1.000     1.000     1.000      15.0
      I-fromloc.airport_name      0.429     1.000     0.600      15.0
      B-fromloc.airport_name      0.423     0.917     0.579      12.0
      B-arrive_date.day_name      0.647     1.000     0.786      11.0
          B-toloc.state_code      0.947     1.000     0.973      18.0
B-depart_date.today_relative      1.000     1.000     1.000       9.0
             B-flight_number      0.435     0.909     0.588      11.0
 B-depart_date.date_relative      0.933     0.824     0.875      17.0
          B-toloc.state_name      0.900     0.964     0.931      28.0
           B-fare_basis_code      0.909     0.588     0.714      17.0
               B-flight_time      1.000     1.000     1.000       1.0
                        B-or      1.000     1.000     1.000       3.0
 B-arrive_time.period_of_day      0.750     1.000     0.857       6.0
          B-meal_description      1.000     0.900     0.947      10.0
             I-cost_relative      1.000     0.667     0.800       3.0
              I-airport_name      0.846     0.379     0.524      29.0
               B-fare_amount      0.333     1.000     0.500       2.0
               I-fare_amount      1.000     1.000     1.000       2.0
                 I-city_name      1.000     0.600     0.750      30.0
        I-toloc.airport_name      0.750     1.000     0.857       3.0
            B-transport_type      1.000     0.700     0.824      10.0
    B-arrive_date.month_name      0.714     0.833     0.769       6.0
    B-arrive_date.day_number      0.714     0.833     0.769       6.0
         I-stoploc.city_name      0.909     1.000     0.952      10.0
                      B-meal      1.000     1.000     1.000      16.0
        B-fromloc.state_code      1.000     0.957     0.978      23.0
    B-depart_time.period_mod      0.833     1.000     0.909       5.0
                   B-connect      1.000     1.000     1.000       6.0
               B-flight_days      1.000     1.000     1.000      10.0
        B-toloc.airport_name      1.000     0.667     0.800       3.0
        B-fromloc.state_name      0.895     1.000     0.944      17.0
              B-airport_name      0.750     0.429     0.545      21.0
                   B-economy      1.000     1.000     1.000       6.0
               I-flight_time      1.000     1.000     1.000       1.0
             B-aircraft_code      0.926     0.758     0.833      33.0
                       B-mod      0.000     0.000     0.000       2.0
              B-airport_code      0.000     0.000     0.000       9.0
    B-depart_time.start_time      0.667     0.667     0.667       3.0
      B-depart_time.end_time      1.000     0.667     0.800       3.0
          B-depart_date.year      1.000     0.667     0.800       3.0
            I-transport_type      0.250     1.000     0.400       1.0
          B-restriction_code      0.333     1.000     0.500       4.0
    B-arrive_time.start_time      0.889     1.000     0.941       8.0
        B-toloc.airport_code      1.000     0.500     0.667       4.0
      B-arrive_time.end_time      0.889     1.000     0.941       8.0
      I-arrive_time.end_time      0.875     0.875     0.875       8.0
      I-depart_time.end_time      1.000     0.333     0.500       3.0
               I-flight_stop      0.000     0.000     0.000       0.0
      B-fromloc.airport_code      1.000     1.000     1.000       5.0
          I-restriction_code      0.750     1.000     0.857       3.0
    I-depart_time.start_time      0.500     1.000     0.667       1.0
          I-toloc.state_name      1.000     1.000     1.000       1.0
I-depart_date.today_relative      0.000     0.000     0.000       0.0
 B-arrive_date.date_relative      0.500     1.000     0.667       2.0
                I-flight_mod      1.000     0.167     0.286       6.0
                   I-economy      0.000     0.000     0.000       0.0
 B-return_date.date_relative      1.000     0.333     0.500       3.0
        I-fromloc.state_name      1.000     1.000     1.000       1.0
                B-state_code      1.000     1.000     1.000       1.0
    I-arrive_time.start_time      1.000     1.000     1.000       1.0
    I-arrive_date.day_number      0.000     0.000     0.000       0.0
                 B-meal_code      0.000     0.000     0.000       1.0
 I-depart_time.period_of_day      0.000     0.000     0.000       1.0
                  B-day_name      1.000     0.500     0.667       2.0
             B-period_of_day      1.000     0.500     0.667       4.0
        B-stoploc.state_code      0.000     0.000     0.000       0.0
    B-return_date.month_name      0.000     0.000     0.000       0.0
    B-return_date.day_number      0.000     0.000     0.000       0.0
    B-arrive_time.period_mod      0.000     0.000     0.000       0.0
                 I-meal_code      0.000     0.000     0.000       0.0
        B-toloc.country_name      1.000     1.000     1.000       1.0
                 B-days_code      1.000     1.000     1.000       1.0
 I-arrive_time.period_of_day      0.000     0.000     0.000       0.0
            I-today_relative      0.000     0.000     0.000       0.0
 B-return_time.period_of_day      0.000     0.000     0.000       0.0
                      B-time      0.000     0.000     0.000       0.0
           I-fare_basis_code      0.000     0.000     0.000       0.0
 I-arrive_time.time_relative      1.000     0.500     0.667       4.0
 I-depart_time.time_relative      0.000     0.000     0.000       1.0
            B-today_relative      0.000     0.000     0.000       0.0
                B-state_name      0.000     0.000     0.000       9.0
B-arrive_date.today_relative      0.000     0.000     0.000       0.0
    B-return_time.period_mod      0.000     0.000     0.000       0.0
                B-month_name      0.000     0.000     0.000       0.0
                B-day_number      0.000     0.000     0.000       0.0
 I-return_date.date_relative      0.600     1.000     0.750       3.0
I-return_date.today_relative      0.000     0.000     0.000       0.0
      B-stoploc.airport_name      0.000     0.000     0.000       0.0
             B-time_relative      0.000     0.000     0.000       0.0
                      I-time      0.000     0.000     0.000       0.0
    I-return_date.day_number      0.000     0.000     0.000       0.0
          I-meal_description      0.000     0.000     0.000       0.0
B-return_date.today_relative      0.000     0.000     0.000       0.0
      B-return_date.day_name      0.000     0.000     0.000       2.0

                   micro avg      0.946     0.948     0.947    3657.0
                   macro avg      0.639     0.633     0.615    3657.0
                weighted avg      0.953     0.948     0.945    3657.0

I0717 07:11:59.277656 140695199004544 interactiveshell.py:2882] Best Slot F1: 0.947
Reading ../data/atis.train.w-intent.iob
I0717 07:12:00.923331 140695199004544 interactiveshell.py:2882] Step 14050 | Loss: 0.0100 | Loss_intent: 0.0000 | Loss_slots: 0.0100 | Spent: 6.7 secs | LR: 0.000068
I0717 07:12:05.320405 140695199004544 interactiveshell.py:2882] Step 14100 | Loss: 0.0191 | Loss_intent: 0.0011 | Loss_slots: 0.0180 | Spent: 4.4 secs | LR: 0.000068
I0717 07:12:09.499826 140695199004544 interactiveshell.py:2882] Step 14150 | Loss: 0.0049 | Loss_intent: 0.0000 | Loss_slots: 0.0049 | Spent: 4.2 secs | LR: 0.000068
I0717 07:12:13.685975 140695199004544 interactiveshell.py:2882] Step 14200 | Loss: 0.0073 | Loss_intent: 0.0001 | Loss_slots: 0.0071 | Spent: 4.2 secs | LR: 0.000067
I0717 07:12:17.863866 140695199004544 interactiveshell.py:2882] Step 14250 | Loss: 0.0078 | Loss_intent: 0.0001 | Loss_slots: 0.0077 | Spent: 4.2 secs | LR: 0.000067
I0717 07:12:22.062161 140695199004544 interactiveshell.py:2882] Step 14300 | Loss: 0.0051 | Loss_intent: 0.0000 | Loss_slots: 0.0051 | Spent: 4.2 secs | LR: 0.000066
I0717 07:12:26.260833 140695199004544 interactiveshell.py:2882] Step 14350 | Loss: 0.0060 | Loss_intent: 0.0000 | Loss_slots: 0.0060 | Spent: 4.2 secs | LR: 0.000066
Reading ../data/atis.test.w-intent.iob
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.
  'precision', 'predicted', average, warn_for)
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1439: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples.
  'recall', 'true', average, warn_for)
I0717 07:12:28.067962 140695199004544 interactiveshell.py:2882] 
                                          precision    recall  f1-score   support

                             atis_flight      0.980     0.984     0.982       632
                            atis_airfare      0.922     0.979     0.949        48
                     atis_ground_service      1.000     1.000     1.000        36
                            atis_airline      0.974     1.000     0.987        38
                       atis_abbreviation      1.000     1.000     1.000        33
                           atis_aircraft      0.889     0.889     0.889         9
                        atis_flight_time      1.000     1.000     1.000         1
                           atis_quantity      0.375     1.000     0.545         3
                atis_flight#atis_airfare      0.800     0.333     0.471        12
                            atis_airport      0.947     1.000     0.973        18
                           atis_distance      1.000     1.000     1.000        10
                               atis_city      0.500     0.667     0.571         6
                        atis_ground_fare      1.000     0.714     0.833         7
                           atis_capacity      1.000     0.952     0.976        21
                          atis_flight_no      0.889     1.000     0.941         8
                               atis_meal      1.000     0.833     0.909         6
                        atis_restriction      0.000     0.000     0.000         0
             atis_airline#atis_flight_no      0.000     0.000     0.000         0
    atis_ground_service#atis_ground_fare      0.000     0.000     0.000         0
           atis_airfare#atis_flight_time      0.000     0.000     0.000         0
                           atis_cheapest      0.000     0.000     0.000         0
atis_aircraft#atis_flight#atis_flight_no      0.000     0.000     0.000         0

                               micro avg      0.965     0.971     0.968       888
                               macro avg      0.649     0.652     0.638       888
                            weighted avg      0.969     0.971     0.968       888

I0717 07:12:28.108288 140695199004544 interactiveshell.py:2882] 
                              precision    recall  f1-score   support

                           O      0.000     0.000     0.000       0.0
           B-toloc.city_name      0.974     0.994     0.984     716.0
         B-fromloc.city_name      0.989     0.994     0.992     704.0
           I-toloc.city_name      0.971     1.000     0.985     265.0
      B-depart_date.day_name      0.990     0.981     0.986     212.0
              B-airline_name      0.952     0.990     0.971     101.0
         I-fromloc.city_name      0.978     0.983     0.980     177.0
 B-depart_time.period_of_day      0.983     0.915     0.948     130.0
              I-airline_name      1.000     1.000     1.000      65.0
    B-depart_date.day_number      0.964     0.964     0.964      55.0
    B-depart_date.month_name      0.982     0.964     0.973      56.0
          B-depart_time.time      0.803     1.000     0.891      57.0
                B-round_trip      1.000     0.973     0.986      73.0
             B-cost_relative      1.000     0.973     0.986      37.0
                I-round_trip      1.000     1.000     1.000      71.0
                B-flight_mod      1.000     0.958     0.979      24.0
 B-depart_time.time_relative      0.984     0.969     0.977      65.0
          I-depart_time.time      0.945     1.000     0.972      52.0
         B-stoploc.city_name      0.952     1.000     0.976      20.0
                 B-city_name      0.825     0.579     0.680      57.0
                B-class_type      0.960     1.000     0.980      24.0
          B-arrive_time.time      0.943     0.971     0.957      34.0
 B-arrive_time.time_relative      0.909     0.968     0.937      31.0
                I-class_type      1.000     1.000     1.000      17.0
               B-flight_stop      0.913     1.000     0.955      21.0
          I-arrive_time.time      0.971     0.971     0.971      35.0
              B-airline_code      1.000     0.853     0.921      34.0
    I-depart_date.day_number      1.000     1.000     1.000      15.0
      I-fromloc.airport_name      0.429     1.000     0.600      15.0
      B-fromloc.airport_name      0.440     0.917     0.595      12.0
      B-arrive_date.day_name      0.688     1.000     0.815      11.0
          B-toloc.state_code      0.947     1.000     0.973      18.0
B-depart_date.today_relative      1.000     1.000     1.000       9.0
             B-flight_number      0.346     0.818     0.486      11.0
 B-depart_date.date_relative      0.933     0.824     0.875      17.0
          B-toloc.state_name      0.897     0.929     0.912      28.0
           B-fare_basis_code      0.917     0.647     0.759      17.0
               B-flight_time      1.000     1.000     1.000       1.0
                        B-or      1.000     1.000     1.000       3.0
 B-arrive_time.period_of_day      0.750     1.000     0.857       6.0
          B-meal_description      1.000     0.900     0.947      10.0
             I-cost_relative      1.000     0.667     0.800       3.0
              I-airport_name      0.917     0.379     0.537      29.0
               B-fare_amount      1.000     0.500     0.667       2.0
               I-fare_amount      1.000     1.000     1.000       2.0
                 I-city_name      1.000     0.567     0.723      30.0
        I-toloc.airport_name      0.750     1.000     0.857       3.0
            B-transport_type      1.000     0.800     0.889      10.0
    B-arrive_date.month_name      0.714     0.833     0.769       6.0
    B-arrive_date.day_number      0.714     0.833     0.769       6.0
         I-stoploc.city_name      0.900     0.900     0.900      10.0
                      B-meal      1.000     1.000     1.000      16.0
        B-fromloc.state_code      1.000     0.957     0.978      23.0
    B-depart_time.period_mod      1.000     0.800     0.889       5.0
                   B-connect      1.000     1.000     1.000       6.0
               B-flight_days      1.000     1.000     1.000      10.0
        B-toloc.airport_name      1.000     0.333     0.500       3.0
        B-fromloc.state_name      0.850     1.000     0.919      17.0
              B-airport_name      0.727     0.381     0.500      21.0
                   B-economy      1.000     1.000     1.000       6.0
               I-flight_time      1.000     1.000     1.000       1.0
             B-aircraft_code      0.926     0.758     0.833      33.0
                       B-mod      1.000     0.500     0.667       2.0
              B-airport_code      0.000     0.000     0.000       9.0
    B-depart_time.start_time      0.667     0.667     0.667       3.0
      B-depart_time.end_time      1.000     0.667     0.800       3.0
          B-depart_date.year      1.000     0.667     0.800       3.0
            I-transport_type      0.333     1.000     0.500       1.0
          B-restriction_code      0.400     1.000     0.571       4.0
    B-arrive_time.start_time      0.875     0.875     0.875       8.0
        B-toloc.airport_code      1.000     0.500     0.667       4.0
      B-arrive_time.end_time      0.889     1.000     0.941       8.0
      I-arrive_time.end_time      0.875     0.875     0.875       8.0
      I-depart_time.end_time      1.000     0.333     0.500       3.0
               I-flight_stop      0.000     0.000     0.000       0.0
      B-fromloc.airport_code      0.833     1.000     0.909       5.0
          I-restriction_code      1.000     1.000     1.000       3.0
    I-depart_time.start_time      0.500     1.000     0.667       1.0
          I-toloc.state_name      1.000     1.000     1.000       1.0
I-depart_date.today_relative      0.000     0.000     0.000       0.0
 B-arrive_date.date_relative      1.000     1.000     1.000       2.0
                I-flight_mod      1.000     0.167     0.286       6.0
                   I-economy      0.000     0.000     0.000       0.0
 B-return_date.date_relative      0.333     0.333     0.333       3.0
        I-fromloc.state_name      1.000     1.000     1.000       1.0
                B-state_code      1.000     1.000     1.000       1.0
    I-arrive_time.start_time      1.000     1.000     1.000       1.0
    I-arrive_date.day_number      0.000     0.000     0.000       0.0
                 B-meal_code      1.000     1.000     1.000       1.0
 I-depart_time.period_of_day      0.000     0.000     0.000       1.0
                  B-day_name      1.000     0.500     0.667       2.0
             B-period_of_day      1.000     0.500     0.667       4.0
        B-stoploc.state_code      0.000     0.000     0.000       0.0
    B-return_date.month_name      0.000     0.000     0.000       0.0
    B-return_date.day_number      0.000     0.000     0.000       0.0
    B-arrive_time.period_mod      0.000     0.000     0.000       0.0
                 I-meal_code      0.000     0.000     0.000       0.0
        B-toloc.country_name      1.000     1.000     1.000       1.0
                 B-days_code      1.000     1.000     1.000       1.0
 I-arrive_time.period_of_day      0.000     0.000     0.000       0.0
            I-today_relative      0.000     0.000     0.000       0.0
 B-return_time.period_of_day      0.000     0.000     0.000       0.0
                      B-time      0.000     0.000     0.000       0.0
           I-fare_basis_code      0.000     0.000     0.000       0.0
 I-arrive_time.time_relative      1.000     0.500     0.667       4.0
 I-depart_time.time_relative      0.000     0.000     0.000       1.0
            B-today_relative      0.000     0.000     0.000       0.0
                B-state_name      1.000     0.111     0.200       9.0
B-arrive_date.today_relative      0.000     0.000     0.000       0.0
    B-return_time.period_mod      0.000     0.000     0.000       0.0
                B-month_name      0.000     0.000     0.000       0.0
                B-day_number      0.000     0.000     0.000       0.0
 I-return_date.date_relative      0.600     1.000     0.750       3.0
I-return_date.today_relative      0.000     0.000     0.000       0.0
      B-stoploc.airport_name      0.000     0.000     0.000       0.0
             B-time_relative      0.000     0.000     0.000       0.0
                      I-time      0.000     0.000     0.000       0.0
    I-return_date.day_number      0.000     0.000     0.000       0.0
          I-meal_description      0.000     0.000     0.000       0.0
B-return_date.today_relative      0.000     0.000     0.000       0.0
      B-return_date.day_name      0.000     0.000     0.000       2.0

                   micro avg      0.946     0.948     0.947    3657.0
                   macro avg      0.671     0.636     0.631    3657.0
                weighted avg      0.956     0.948     0.946    3657.0

I0717 07:12:28.109200 140695199004544 interactiveshell.py:2882] Best Slot F1: 0.947
Reading ../data/atis.train.w-intent.iob
I0717 07:12:32.848890 140695199004544 interactiveshell.py:2882] Step 14400 | Loss: 0.0035 | Loss_intent: 0.0001 | Loss_slots: 0.0034 | Spent: 6.6 secs | LR: 0.000066
I0717 07:12:37.036815 140695199004544 interactiveshell.py:2882] Step 14450 | Loss: 0.0022 | Loss_intent: 0.0000 | Loss_slots: 0.0022 | Spent: 4.2 secs | LR: 0.000065
I0717 07:12:41.310458 140695199004544 interactiveshell.py:2882] Step 14500 | Loss: 0.0037 | Loss_intent: 0.0001 | Loss_slots: 0.0036 | Spent: 4.3 secs | LR: 0.000065
I0717 07:12:45.579168 140695199004544 interactiveshell.py:2882] Step 14550 | Loss: 0.0020 | Loss_intent: 0.0000 | Loss_slots: 0.0020 | Spent: 4.3 secs | LR: 0.000065
I0717 07:12:49.786779 140695199004544 interactiveshell.py:2882] Step 14600 | Loss: 0.0133 | Loss_intent: 0.0000 | Loss_slots: 0.0133 | Spent: 4.2 secs | LR: 0.000064
I0717 07:12:53.978777 140695199004544 interactiveshell.py:2882] Step 14650 | Loss: 0.0056 | Loss_intent: 0.0000 | Loss_slots: 0.0056 | Spent: 4.2 secs | LR: 0.000064
Reading ../data/atis.test.w-intent.iob
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.
  'precision', 'predicted', average, warn_for)
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1439: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples.
  'recall', 'true', average, warn_for)
I0717 07:12:56.778736 140695199004544 interactiveshell.py:2882] 
                                          precision    recall  f1-score   support

                             atis_flight      0.989     0.983     0.986       632
                            atis_airfare      0.889     1.000     0.941        48
                     atis_ground_service      1.000     1.000     1.000        36
                            atis_airline      0.974     1.000     0.987        38
                       atis_abbreviation      0.971     1.000     0.985        33
                           atis_aircraft      0.889     0.889     0.889         9
                        atis_flight_time      1.000     1.000     1.000         1
                           atis_quantity      0.429     1.000     0.600         3
                atis_flight#atis_airfare      0.875     0.583     0.700        12
                            atis_airport      1.000     1.000     1.000        18
                           atis_distance      1.000     1.000     1.000        10
                               atis_city      0.444     0.667     0.533         6
                        atis_ground_fare      1.000     0.714     0.833         7
                           atis_capacity      1.000     0.952     0.976        21
                          atis_flight_no      0.889     1.000     0.941         8
                               atis_meal      1.000     0.833     0.909         6
                        atis_restriction      0.000     0.000     0.000         0
             atis_airline#atis_flight_no      0.000     0.000     0.000         0
    atis_ground_service#atis_ground_fare      0.000     0.000     0.000         0
           atis_airfare#atis_flight_time      0.000     0.000     0.000         0
                           atis_cheapest      0.000     0.000     0.000         0
atis_aircraft#atis_flight#atis_flight_no      0.000     0.000     0.000         0

                               micro avg      0.969     0.974     0.971       888
                               macro avg      0.652     0.665     0.649       888
                            weighted avg      0.974     0.974     0.973       888

I0717 07:12:56.828884 140695199004544 interactiveshell.py:2882] 
                              precision    recall  f1-score   support

                           O      0.000     0.000     0.000       0.0
           B-toloc.city_name      0.973     0.993     0.983     716.0
         B-fromloc.city_name      0.986     0.993     0.989     704.0
           I-toloc.city_name      0.967     0.992     0.980     265.0
      B-depart_date.day_name      0.990     0.981     0.986     212.0
              B-airline_name      0.981     1.000     0.990     101.0
         I-fromloc.city_name      0.967     0.983     0.975     177.0
 B-depart_time.period_of_day      1.000     0.908     0.952     130.0
              I-airline_name      1.000     1.000     1.000      65.0
    B-depart_date.day_number      0.963     0.945     0.954      55.0
    B-depart_date.month_name      0.982     0.964     0.973      56.0
          B-depart_time.time      0.838     1.000     0.912      57.0
                B-round_trip      1.000     0.986     0.993      73.0
             B-cost_relative      1.000     0.973     0.986      37.0
                I-round_trip      1.000     1.000     1.000      71.0
                B-flight_mod      1.000     0.958     0.979      24.0
 B-depart_time.time_relative      0.984     0.954     0.969      65.0
          I-depart_time.time      0.945     1.000     0.972      52.0
         B-stoploc.city_name      0.950     0.950     0.950      20.0
                 B-city_name      0.868     0.579     0.695      57.0
                B-class_type      0.923     1.000     0.960      24.0
          B-arrive_time.time      0.971     0.971     0.971      34.0
 B-arrive_time.time_relative      0.882     0.968     0.923      31.0
                I-class_type      1.000     1.000     1.000      17.0
               B-flight_stop      1.000     1.000     1.000      21.0
          I-arrive_time.time      0.971     0.971     0.971      35.0
              B-airline_code      1.000     0.912     0.954      34.0
    I-depart_date.day_number      1.000     1.000     1.000      15.0
      I-fromloc.airport_name      0.417     1.000     0.588      15.0
      B-fromloc.airport_name      0.367     0.917     0.524      12.0
      B-arrive_date.day_name      0.688     1.000     0.815      11.0
          B-toloc.state_code      0.947     1.000     0.973      18.0
B-depart_date.today_relative      1.000     1.000     1.000       9.0
             B-flight_number      0.323     0.909     0.476      11.0
 B-depart_date.date_relative      0.933     0.824     0.875      17.0
          B-toloc.state_name      0.900     0.964     0.931      28.0
           B-fare_basis_code      0.917     0.647     0.759      17.0
               B-flight_time      1.000     1.000     1.000       1.0
                        B-or      1.000     1.000     1.000       3.0
 B-arrive_time.period_of_day      0.750     1.000     0.857       6.0
          B-meal_description      1.000     0.900     0.947      10.0
             I-cost_relative      1.000     0.667     0.800       3.0
              I-airport_name      0.846     0.379     0.524      29.0
               B-fare_amount      0.500     1.000     0.667       2.0
               I-fare_amount      1.000     1.000     1.000       2.0
                 I-city_name      1.000     0.533     0.696      30.0
        I-toloc.airport_name      0.750     1.000     0.857       3.0
            B-transport_type      1.000     0.800     0.889      10.0
    B-arrive_date.month_name      0.714     0.833     0.769       6.0
    B-arrive_date.day_number      0.714     0.833     0.769       6.0
         I-stoploc.city_name      0.900     0.900     0.900      10.0
                      B-meal      1.000     1.000     1.000      16.0
        B-fromloc.state_code      1.000     0.957     0.978      23.0
    B-depart_time.period_mod      0.833     1.000     0.909       5.0
                   B-connect      1.000     1.000     1.000       6.0
               B-flight_days      1.000     1.000     1.000      10.0
        B-toloc.airport_name      1.000     0.333     0.500       3.0
        B-fromloc.state_name      0.895     1.000     0.944      17.0
              B-airport_name      0.727     0.381     0.500      21.0
                   B-economy      1.000     1.000     1.000       6.0
               I-flight_time      1.000     1.000     1.000       1.0
             B-aircraft_code      0.926     0.758     0.833      33.0
                       B-mod      0.000     0.000     0.000       2.0
              B-airport_code      0.000     0.000     0.000       9.0
    B-depart_time.start_time      0.500     0.333     0.400       3.0
      B-depart_time.end_time      0.667     0.667     0.667       3.0
          B-depart_date.year      1.000     0.667     0.800       3.0
            I-transport_type      0.000     0.000     0.000       1.0
          B-restriction_code      0.600     0.750     0.667       4.0
    B-arrive_time.start_time      0.889     1.000     0.941       8.0
        B-toloc.airport_code      1.000     0.500     0.667       4.0
      B-arrive_time.end_time      0.889     1.000     0.941       8.0
      I-arrive_time.end_time      0.875     0.875     0.875       8.0
      I-depart_time.end_time      1.000     0.333     0.500       3.0
               I-flight_stop      0.000     0.000     0.000       0.0
      B-fromloc.airport_code      1.000     1.000     1.000       5.0
          I-restriction_code      0.750     1.000     0.857       3.0
    I-depart_time.start_time      0.500     1.000     0.667       1.0
          I-toloc.state_name      1.000     1.000     1.000       1.0
I-depart_date.today_relative      0.000     0.000     0.000       0.0
 B-arrive_date.date_relative      1.000     1.000     1.000       2.0
                I-flight_mod      1.000     0.167     0.286       6.0
                   I-economy      0.000     0.000     0.000       0.0
 B-return_date.date_relative      0.400     0.667     0.500       3.0
        I-fromloc.state_name      1.000     1.000     1.000       1.0
                B-state_code      1.000     1.000     1.000       1.0
    I-arrive_time.start_time      1.000     1.000     1.000       1.0
    I-arrive_date.day_number      0.000     0.000     0.000       0.0
                 B-meal_code      1.000     1.000     1.000       1.0
 I-depart_time.period_of_day      1.000     1.000     1.000       1.0
                  B-day_name      1.000     0.500     0.667       2.0
             B-period_of_day      1.000     0.500     0.667       4.0
        B-stoploc.state_code      0.000     0.000     0.000       0.0
    B-return_date.month_name      0.000     0.000     0.000       0.0
    B-return_date.day_number      0.000     0.000     0.000       0.0
    B-arrive_time.period_mod      0.000     0.000     0.000       0.0
                 I-meal_code      0.000     0.000     0.000       0.0
        B-toloc.country_name      1.000     1.000     1.000       1.0
                 B-days_code      1.000     1.000     1.000       1.0
 I-arrive_time.period_of_day      0.000     0.000     0.000       0.0
            I-today_relative      0.000     0.000     0.000       0.0
 B-return_time.period_of_day      0.000     0.000     0.000       0.0
                      B-time      0.000     0.000     0.000       0.0
           I-fare_basis_code      0.000     0.000     0.000       0.0
 I-arrive_time.time_relative      1.000     0.500     0.667       4.0
 I-depart_time.time_relative      0.000     0.000     0.000       1.0
            B-today_relative      0.000     0.000     0.000       0.0
                B-state_name      1.000     0.111     0.200       9.0
B-arrive_date.today_relative      0.000     0.000     0.000       0.0
    B-return_time.period_mod      0.000     0.000     0.000       0.0
                B-month_name      0.000     0.000     0.000       0.0
                B-day_number      0.000     0.000     0.000       0.0
 I-return_date.date_relative      1.000     1.000     1.000       3.0
I-return_date.today_relative      0.000     0.000     0.000       0.0
      B-stoploc.airport_name      0.000     0.000     0.000       0.0
             B-time_relative      0.000     0.000     0.000       0.0
                      I-time      0.000     0.000     0.000       0.0
    I-return_date.day_number      0.000     0.000     0.000       0.0
          I-meal_description      0.000     0.000     0.000       0.0
B-return_date.today_relative      0.000     0.000     0.000       0.0
      B-return_date.day_name      0.000     0.000     0.000       2.0

                   micro avg      0.946     0.947     0.947    3657.0
                   macro avg      0.663     0.637     0.630    3657.0
                weighted avg      0.956     0.947     0.945    3657.0

I0717 07:12:56.829870 140695199004544 interactiveshell.py:2882] Best Slot F1: 0.947
Reading ../data/atis.train.w-intent.iob
I0717 07:13:00.566971 140695199004544 interactiveshell.py:2882] Step 14700 | Loss: 0.0091 | Loss_intent: 0.0000 | Loss_slots: 0.0091 | Spent: 6.6 secs | LR: 0.000064
I0717 07:13:04.766425 140695199004544 interactiveshell.py:2882] Step 14750 | Loss: 0.0071 | Loss_intent: 0.0002 | Loss_slots: 0.0069 | Spent: 4.2 secs | LR: 0.000063
I0717 07:13:08.943408 140695199004544 interactiveshell.py:2882] Step 14800 | Loss: 0.0088 | Loss_intent: 0.0025 | Loss_slots: 0.0063 | Spent: 4.2 secs | LR: 0.000063
I0717 07:13:13.147660 140695199004544 interactiveshell.py:2882] Step 14850 | Loss: 0.0030 | Loss_intent: 0.0000 | Loss_slots: 0.0030 | Spent: 4.2 secs | LR: 0.000063
I0717 07:13:17.329801 140695199004544 interactiveshell.py:2882] Step 14900 | Loss: 0.0022 | Loss_intent: 0.0000 | Loss_slots: 0.0022 | Spent: 4.2 secs | LR: 0.000062
I0717 07:13:21.512699 140695199004544 interactiveshell.py:2882] Step 14950 | Loss: 0.0064 | Loss_intent: 0.0000 | Loss_slots: 0.0064 | Spent: 4.2 secs | LR: 0.000062
Reading ../data/atis.test.w-intent.iob
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.
  'precision', 'predicted', average, warn_for)
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1439: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples.
  'recall', 'true', average, warn_for)
I0717 07:13:25.320992 140695199004544 interactiveshell.py:2882] 
                                          precision    recall  f1-score   support

                             atis_flight      0.990     0.983     0.986       632
                            atis_airfare      0.889     1.000     0.941        48
                     atis_ground_service      1.000     1.000     1.000        36
                            atis_airline      0.974     1.000     0.987        38
                       atis_abbreviation      0.971     1.000     0.985        33
                           atis_aircraft      1.000     0.889     0.941         9
                        atis_flight_time      0.500     1.000     0.667         1
                           atis_quantity      0.375     1.000     0.545         3
                atis_flight#atis_airfare      0.857     0.500     0.632        12
                            atis_airport      0.947     1.000     0.973        18
                           atis_distance      1.000     1.000     1.000        10
                               atis_city      0.444     0.667     0.533         6
                        atis_ground_fare      1.000     0.714     0.833         7
                           atis_capacity      1.000     1.000     1.000        21
                          atis_flight_no      0.889     1.000     0.941         8
                               atis_meal      1.000     0.833     0.909         6
                        atis_restriction      0.000     0.000     0.000         0
             atis_airline#atis_flight_no      0.000     0.000     0.000         0
    atis_ground_service#atis_ground_fare      0.000     0.000     0.000         0
           atis_airfare#atis_flight_time      0.000     0.000     0.000         0
                           atis_cheapest      0.000     0.000     0.000         0
atis_aircraft#atis_flight#atis_flight_no      0.000     0.000     0.000         0

                               micro avg      0.969     0.974     0.971       888
                               macro avg      0.629     0.663     0.631       888
                            weighted avg      0.975     0.974     0.972       888

I0717 07:13:25.361564 140695199004544 interactiveshell.py:2882] 
                              precision    recall  f1-score   support

                           O      0.000     0.000     0.000       0.0
           B-toloc.city_name      0.975     0.994     0.985     716.0
         B-fromloc.city_name      0.987     0.990     0.989     704.0
           I-toloc.city_name      0.960     1.000     0.980     265.0
      B-depart_date.day_name      0.990     0.976     0.983     212.0
              B-airline_name      0.980     0.990     0.985     101.0
         I-fromloc.city_name      0.989     0.983     0.986     177.0
 B-depart_time.period_of_day      0.992     0.900     0.944     130.0
              I-airline_name      1.000     1.000     1.000      65.0
    B-depart_date.day_number      0.962     0.927     0.944      55.0
    B-depart_date.month_name      0.982     0.964     0.973      56.0
          B-depart_time.time      0.851     1.000     0.919      57.0
                B-round_trip      1.000     0.986     0.993      73.0
             B-cost_relative      1.000     0.973     0.986      37.0
                I-round_trip      1.000     1.000     1.000      71.0
                B-flight_mod      0.958     0.958     0.958      24.0
 B-depart_time.time_relative      0.984     0.954     0.969      65.0
          I-depart_time.time      0.943     0.962     0.952      52.0
         B-stoploc.city_name      0.952     1.000     0.976      20.0
                 B-city_name      0.825     0.579     0.680      57.0
                B-class_type      0.923     1.000     0.960      24.0
          B-arrive_time.time      0.917     0.971     0.943      34.0
 B-arrive_time.time_relative      0.882     0.968     0.923      31.0
                I-class_type      1.000     1.000     1.000      17.0
               B-flight_stop      1.000     1.000     1.000      21.0
          I-arrive_time.time      0.919     0.971     0.944      35.0
              B-airline_code      1.000     0.882     0.938      34.0
    I-depart_date.day_number      1.000     1.000     1.000      15.0
      I-fromloc.airport_name      0.417     1.000     0.588      15.0
      B-fromloc.airport_name      0.355     0.917     0.512      12.0
      B-arrive_date.day_name      0.647     1.000     0.786      11.0
          B-toloc.state_code      0.947     1.000     0.973      18.0
B-depart_date.today_relative      1.000     1.000     1.000       9.0
             B-flight_number      0.400     0.909     0.556      11.0
 B-depart_date.date_relative      0.938     0.882     0.909      17.0
          B-toloc.state_name      0.900     0.964     0.931      28.0
           B-fare_basis_code      0.917     0.647     0.759      17.0
               B-flight_time      1.000     1.000     1.000       1.0
                        B-or      1.000     1.000     1.000       3.0
 B-arrive_time.period_of_day      0.750     1.000     0.857       6.0
          B-meal_description      1.000     0.900     0.947      10.0
             I-cost_relative      1.000     0.667     0.800       3.0
              I-airport_name      0.846     0.379     0.524      29.0
               B-fare_amount      0.400     1.000     0.571       2.0
               I-fare_amount      1.000     1.000     1.000       2.0
                 I-city_name      1.000     0.500     0.667      30.0
        I-toloc.airport_name      0.750     1.000     0.857       3.0
            B-transport_type      1.000     0.700     0.824      10.0
    B-arrive_date.month_name      0.714     0.833     0.769       6.0
    B-arrive_date.day_number      0.625     0.833     0.714       6.0
         I-stoploc.city_name      0.909     1.000     0.952      10.0
                      B-meal      1.000     1.000     1.000      16.0
        B-fromloc.state_code      1.000     0.957     0.978      23.0
    B-depart_time.period_mod      0.714     1.000     0.833       5.0
                   B-connect      1.000     1.000     1.000       6.0
               B-flight_days      1.000     1.000     1.000      10.0
        B-toloc.airport_name      1.000     1.000     1.000       3.0
        B-fromloc.state_name      0.944     1.000     0.971      17.0
              B-airport_name      0.727     0.381     0.500      21.0
                   B-economy      1.000     1.000     1.000       6.0
               I-flight_time      1.000     1.000     1.000       1.0
             B-aircraft_code      0.929     0.788     0.852      33.0
                       B-mod      0.000     0.000     0.000       2.0
              B-airport_code      0.000     0.000     0.000       9.0
    B-depart_time.start_time      0.667     0.667     0.667       3.0
      B-depart_time.end_time      1.000     0.667     0.800       3.0
          B-depart_date.year      1.000     0.667     0.800       3.0
            I-transport_type      0.000     0.000     0.000       1.0
          B-restriction_code      0.364     1.000     0.533       4.0
    B-arrive_time.start_time      0.875     0.875     0.875       8.0
        B-toloc.airport_code      1.000     0.500     0.667       4.0
      B-arrive_time.end_time      0.889     1.000     0.941       8.0
      I-arrive_time.end_time      0.875     0.875     0.875       8.0
      I-depart_time.end_time      1.000     0.333     0.500       3.0
               I-flight_stop      0.000     0.000     0.000       0.0
      B-fromloc.airport_code      1.000     1.000     1.000       5.0
          I-restriction_code      1.000     1.000     1.000       3.0
    I-depart_time.start_time      0.500     1.000     0.667       1.0
          I-toloc.state_name      1.000     1.000     1.000       1.0
I-depart_date.today_relative      0.000     0.000     0.000       0.0
 B-arrive_date.date_relative      0.500     1.000     0.667       2.0
                I-flight_mod      1.000     0.167     0.286       6.0
                   I-economy      0.000     0.000     0.000       0.0
 B-return_date.date_relative      1.000     0.333     0.500       3.0
        I-fromloc.state_name      1.000     1.000     1.000       1.0
                B-state_code      1.000     1.000     1.000       1.0
    I-arrive_time.start_time      1.000     1.000     1.000       1.0
    I-arrive_date.day_number      0.000     0.000     0.000       0.0
                 B-meal_code      1.000     1.000     1.000       1.0
 I-depart_time.period_of_day      0.000     0.000     0.000       1.0
                  B-day_name      1.000     0.500     0.667       2.0
             B-period_of_day      1.000     0.500     0.667       4.0
        B-stoploc.state_code      0.000     0.000     0.000       0.0
    B-return_date.month_name      0.000     0.000     0.000       0.0
    B-return_date.day_number      0.000     0.000     0.000       0.0
    B-arrive_time.period_mod      0.000     0.000     0.000       0.0
                 I-meal_code      0.000     0.000     0.000       0.0
        B-toloc.country_name      1.000     1.000     1.000       1.0
                 B-days_code      1.000     1.000     1.000       1.0
 I-arrive_time.period_of_day      0.000     0.000     0.000       0.0
            I-today_relative      0.000     0.000     0.000       0.0
 B-return_time.period_of_day      0.000     0.000     0.000       0.0
                      B-time      0.000     0.000     0.000       0.0
           I-fare_basis_code      0.000     0.000     0.000       0.0
 I-arrive_time.time_relative      1.000     0.500     0.667       4.0
 I-depart_time.time_relative      0.000     0.000     0.000       1.0
            B-today_relative      0.000     0.000     0.000       0.0
                B-state_name      1.000     0.111     0.200       9.0
B-arrive_date.today_relative      0.000     0.000     0.000       0.0
    B-return_time.period_mod      0.000     0.000     0.000       0.0
                B-month_name      0.000     0.000     0.000       0.0
                B-day_number      0.000     0.000     0.000       0.0
 I-return_date.date_relative      0.750     1.000     0.857       3.0
I-return_date.today_relative      0.000     0.000     0.000       0.0
      B-stoploc.airport_name      0.000     0.000     0.000       0.0
             B-time_relative      0.000     0.000     0.000       0.0
                      I-time      0.000     0.000     0.000       0.0
    I-return_date.day_number      0.000     0.000     0.000       0.0
          I-meal_description      0.000     0.000     0.000       0.0
B-return_date.today_relative      0.000     0.000     0.000       0.0
      B-return_date.day_name      0.000     0.000     0.000       2.0

                   micro avg      0.945     0.947     0.946    3657.0
                   macro avg      0.655     0.635     0.623    3657.0
                weighted avg      0.955     0.947     0.944    3657.0

I0717 07:13:25.362364 140695199004544 interactiveshell.py:2882] Best Slot F1: 0.947
Reading ../data/atis.train.w-intent.iob
I0717 07:13:28.102679 140695199004544 interactiveshell.py:2882] Step 15000 | Loss: 0.0028 | Loss_intent: 0.0000 | Loss_slots: 0.0028 | Spent: 6.6 secs | LR: 0.000062
I0717 07:13:32.279748 140695199004544 interactiveshell.py:2882] Step 15050 | Loss: 0.0019 | Loss_intent: 0.0002 | Loss_slots: 0.0017 | Spent: 4.2 secs | LR: 0.000061
I0717 07:13:36.459770 140695199004544 interactiveshell.py:2882] Step 15100 | Loss: 0.0094 | Loss_intent: 0.0001 | Loss_slots: 0.0093 | Spent: 4.2 secs | LR: 0.000061
I0717 07:13:40.639065 140695199004544 interactiveshell.py:2882] Step 15150 | Loss: 0.0066 | Loss_intent: 0.0000 | Loss_slots: 0.0066 | Spent: 4.2 secs | LR: 0.000061
I0717 07:13:44.849027 140695199004544 interactiveshell.py:2882] Step 15200 | Loss: 0.0016 | Loss_intent: 0.0000 | Loss_slots: 0.0016 | Spent: 4.2 secs | LR: 0.000060
I0717 07:13:49.016605 140695199004544 interactiveshell.py:2882] Step 15250 | Loss: 0.0038 | Loss_intent: 0.0000 | Loss_slots: 0.0038 | Spent: 4.2 secs | LR: 0.000060
Reading ../data/atis.test.w-intent.iob
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.
  'precision', 'predicted', average, warn_for)
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1439: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples.
  'recall', 'true', average, warn_for)
I0717 07:13:53.840553 140695199004544 interactiveshell.py:2882] 
                                          precision    recall  f1-score   support

                             atis_flight      0.984     0.983     0.983       632
                            atis_airfare      0.960     1.000     0.980        48
                     atis_ground_service      1.000     1.000     1.000        36
                            atis_airline      0.974     1.000     0.987        38
                       atis_abbreviation      1.000     1.000     1.000        33
                           atis_aircraft      0.889     0.889     0.889         9
                        atis_flight_time      1.000     1.000     1.000         1
                           atis_quantity      0.375     1.000     0.545         3
                atis_flight#atis_airfare      0.857     0.500     0.632        12
                            atis_airport      0.944     0.944     0.944        18
                           atis_distance      1.000     1.000     1.000        10
                               atis_city      0.444     0.667     0.533         6
                        atis_ground_fare      1.000     1.000     1.000         7
                           atis_capacity      1.000     1.000     1.000        21
                          atis_flight_no      0.889     1.000     0.941         8
                               atis_meal      1.000     0.833     0.909         6
                        atis_restriction      0.000     0.000     0.000         0
             atis_airline#atis_flight_no      0.000     0.000     0.000         0
    atis_ground_service#atis_ground_fare      0.000     0.000     0.000         0
           atis_airfare#atis_flight_time      0.000     0.000     0.000         0
                           atis_cheapest      0.000     0.000     0.000         0
atis_aircraft#atis_flight#atis_flight_no      0.000     0.000     0.000         0

                               micro avg      0.970     0.975     0.972       888
                               macro avg      0.651     0.673     0.652       888
                            weighted avg      0.974     0.975     0.973       888

I0717 07:13:53.880754 140695199004544 interactiveshell.py:2882] 
                              precision    recall  f1-score   support

                           O      0.000     0.000     0.000       0.0
           B-toloc.city_name      0.977     0.994     0.985     716.0
         B-fromloc.city_name      0.989     0.993     0.991     704.0
           I-toloc.city_name      0.967     1.000     0.983     265.0
      B-depart_date.day_name      0.990     0.981     0.986     212.0
              B-airline_name      0.971     0.990     0.980     101.0
         I-fromloc.city_name      0.983     0.989     0.986     177.0
 B-depart_time.period_of_day      0.992     0.908     0.948     130.0
              I-airline_name      1.000     1.000     1.000      65.0
    B-depart_date.day_number      0.981     0.927     0.953      55.0
    B-depart_date.month_name      0.982     0.964     0.973      56.0
          B-depart_time.time      0.826     1.000     0.905      57.0
                B-round_trip      1.000     0.986     0.993      73.0
             B-cost_relative      1.000     0.973     0.986      37.0
                I-round_trip      1.000     1.000     1.000      71.0
                B-flight_mod      0.793     0.958     0.868      24.0
 B-depart_time.time_relative      0.984     0.969     0.977      65.0
          I-depart_time.time      0.963     1.000     0.981      52.0
         B-stoploc.city_name      0.952     1.000     0.976      20.0
                 B-city_name      0.829     0.596     0.694      57.0
                B-class_type      0.889     1.000     0.941      24.0
          B-arrive_time.time      0.943     0.971     0.957      34.0
 B-arrive_time.time_relative      0.909     0.968     0.937      31.0
                I-class_type      1.000     1.000     1.000      17.0
               B-flight_stop      0.955     1.000     0.977      21.0
          I-arrive_time.time      0.971     0.971     0.971      35.0
              B-airline_code      1.000     0.853     0.921      34.0
    I-depart_date.day_number      1.000     1.000     1.000      15.0
      I-fromloc.airport_name      0.429     1.000     0.600      15.0
      B-fromloc.airport_name      0.440     0.917     0.595      12.0
      B-arrive_date.day_name      0.688     1.000     0.815      11.0
          B-toloc.state_code      1.000     1.000     1.000      18.0
B-depart_date.today_relative      1.000     0.889     0.941       9.0
             B-flight_number      0.529     0.818     0.643      11.0
 B-depart_date.date_relative      0.895     1.000     0.944      17.0
          B-toloc.state_name      0.900     0.964     0.931      28.0
           B-fare_basis_code      0.944     1.000     0.971      17.0
               B-flight_time      1.000     1.000     1.000       1.0
                        B-or      1.000     1.000     1.000       3.0
 B-arrive_time.period_of_day      0.750     1.000     0.857       6.0
          B-meal_description      1.000     0.900     0.947      10.0
             I-cost_relative      1.000     0.667     0.800       3.0
              I-airport_name      0.846     0.379     0.524      29.0
               B-fare_amount      0.400     1.000     0.571       2.0
               I-fare_amount      1.000     1.000     1.000       2.0
                 I-city_name      1.000     0.600     0.750      30.0
        I-toloc.airport_name      0.750     1.000     0.857       3.0
            B-transport_type      1.000     0.800     0.889      10.0
    B-arrive_date.month_name      0.714     0.833     0.769       6.0
    B-arrive_date.day_number      0.625     0.833     0.714       6.0
         I-stoploc.city_name      1.000     1.000     1.000      10.0
                      B-meal      1.000     1.000     1.000      16.0
        B-fromloc.state_code      1.000     1.000     1.000      23.0
    B-depart_time.period_mod      1.000     1.000     1.000       5.0
                   B-connect      1.000     1.000     1.000       6.0
               B-flight_days      1.000     1.000     1.000      10.0
        B-toloc.airport_name      1.000     1.000     1.000       3.0
        B-fromloc.state_name      0.944     1.000     0.971      17.0
              B-airport_name      0.727     0.381     0.500      21.0
                   B-economy      1.000     1.000     1.000       6.0
               I-flight_time      1.000     1.000     1.000       1.0
             B-aircraft_code      0.900     0.818     0.857      33.0
                       B-mod      0.000     0.000     0.000       2.0
              B-airport_code      0.000     0.000     0.000       9.0
    B-depart_time.start_time      0.500     0.333     0.400       3.0
      B-depart_time.end_time      1.000     0.333     0.500       3.0
          B-depart_date.year      1.000     0.667     0.800       3.0
            I-transport_type      0.333     1.000     0.500       1.0
          B-restriction_code      0.364     1.000     0.533       4.0
    B-arrive_time.start_time      0.875     0.875     0.875       8.0
        B-toloc.airport_code      1.000     0.500     0.667       4.0
      B-arrive_time.end_time      0.800     1.000     0.889       8.0
      I-arrive_time.end_time      0.875     0.875     0.875       8.0
      I-depart_time.end_time      1.000     0.333     0.500       3.0
               I-flight_stop      0.000     0.000     0.000       0.0
      B-fromloc.airport_code      0.833     1.000     0.909       5.0
          I-restriction_code      1.000     1.000     1.000       3.0
    I-depart_time.start_time      0.500     1.000     0.667       1.0
          I-toloc.state_name      1.000     1.000     1.000       1.0
I-depart_date.today_relative      0.000     0.000     0.000       0.0
 B-arrive_date.date_relative      1.000     1.000     1.000       2.0
                I-flight_mod      1.000     0.167     0.286       6.0
                   I-economy      0.000     0.000     0.000       0.0
 B-return_date.date_relative      1.000     0.667     0.800       3.0
        I-fromloc.state_name      1.000     1.000     1.000       1.0
                B-state_code      1.000     1.000     1.000       1.0
    I-arrive_time.start_time      1.000     1.000     1.000       1.0
    I-arrive_date.day_number      0.000     0.000     0.000       0.0
                 B-meal_code      1.000     1.000     1.000       1.0
 I-depart_time.period_of_day      0.000     0.000     0.000       1.0
                  B-day_name      1.000     0.500     0.667       2.0
             B-period_of_day      1.000     0.500     0.667       4.0
        B-stoploc.state_code      0.000     0.000     0.000       0.0
    B-return_date.month_name      0.000     0.000     0.000       0.0
    B-return_date.day_number      0.000     0.000     0.000       0.0
    B-arrive_time.period_mod      0.000     0.000     0.000       0.0
                 I-meal_code      0.000     0.000     0.000       0.0
        B-toloc.country_name      1.000     1.000     1.000       1.0
                 B-days_code      1.000     1.000     1.000       1.0
 I-arrive_time.period_of_day      0.000     0.000     0.000       0.0
            I-today_relative      0.000     0.000     0.000       0.0
 B-return_time.period_of_day      0.000     0.000     0.000       0.0
                      B-time      0.000     0.000     0.000       0.0
           I-fare_basis_code      0.000     0.000     0.000       0.0
 I-arrive_time.time_relative      1.000     0.500     0.667       4.0
 I-depart_time.time_relative      0.000     0.000     0.000       1.0
            B-today_relative      0.000     0.000     0.000       0.0
                B-state_name      1.000     0.111     0.200       9.0
B-arrive_date.today_relative      0.000     0.000     0.000       0.0
    B-return_time.period_mod      0.000     0.000     0.000       0.0
                B-month_name      0.000     0.000     0.000       0.0
                B-day_number      0.000     0.000     0.000       0.0
 I-return_date.date_relative      1.000     1.000     1.000       3.0
I-return_date.today_relative      0.000     0.000     0.000       0.0
      B-stoploc.airport_name      0.000     0.000     0.000       0.0
             B-time_relative      0.000     0.000     0.000       0.0
                      I-time      0.000     0.000     0.000       0.0
    I-return_date.day_number      0.000     0.000     0.000       0.0
          I-meal_description      0.000     0.000     0.000       0.0
B-return_date.today_relative      0.000     0.000     0.000       0.0
      B-return_date.day_name      0.000     0.000     0.000       2.0

                   micro avg      0.951     0.952     0.951    3657.0
                   macro avg      0.665     0.646     0.634    3657.0
                weighted avg      0.957     0.952     0.949    3657.0

I0717 07:13:53.881693 140695199004544 interactiveshell.py:2882] Best Slot F1: 0.951
Reading ../data/atis.train.w-intent.iob
I0717 07:13:55.612569 140695199004544 interactiveshell.py:2882] Step 15300 | Loss: 0.0051 | Loss_intent: 0.0001 | Loss_slots: 0.0050 | Spent: 6.6 secs | LR: 0.000060
I0717 07:13:59.836955 140695199004544 interactiveshell.py:2882] Step 15350 | Loss: 0.0059 | Loss_intent: 0.0000 | Loss_slots: 0.0059 | Spent: 4.2 secs | LR: 0.000060
I0717 07:14:04.111049 140695199004544 interactiveshell.py:2882] Step 15400 | Loss: 0.0117 | Loss_intent: 0.0000 | Loss_slots: 0.0117 | Spent: 4.3 secs | LR: 0.000059
I0717 07:14:08.343145 140695199004544 interactiveshell.py:2882] Step 15450 | Loss: 0.0203 | Loss_intent: 0.0000 | Loss_slots: 0.0203 | Spent: 4.2 secs | LR: 0.000059
I0717 07:14:12.526320 140695199004544 interactiveshell.py:2882] Step 15500 | Loss: 0.0196 | Loss_intent: 0.0000 | Loss_slots: 0.0196 | Spent: 4.2 secs | LR: 0.000059
I0717 07:14:16.687836 140695199004544 interactiveshell.py:2882] Step 15550 | Loss: 0.0058 | Loss_intent: 0.0000 | Loss_slots: 0.0058 | Spent: 4.2 secs | LR: 0.000058
Reading ../data/atis.test.w-intent.iob
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.
  'precision', 'predicted', average, warn_for)
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1439: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples.
  'recall', 'true', average, warn_for)
I0717 07:14:22.506429 140695199004544 interactiveshell.py:2882] 
                                          precision    recall  f1-score   support

                             atis_flight      0.986     0.983     0.984       632
                            atis_airfare      0.959     0.979     0.969        48
                     atis_ground_service      1.000     1.000     1.000        36
                            atis_airline      0.974     0.974     0.974        38
                       atis_abbreviation      0.971     1.000     0.985        33
                           atis_aircraft      1.000     0.889     0.941         9
                        atis_flight_time      0.500     1.000     0.667         1
                           atis_quantity      0.333     1.000     0.500         3
                atis_flight#atis_airfare      0.857     0.500     0.632        12
                            atis_airport      0.944     0.944     0.944        18
                           atis_distance      1.000     1.000     1.000        10
                               atis_city      0.545     1.000     0.706         6
                        atis_ground_fare      1.000     1.000     1.000         7
                           atis_capacity      1.000     0.952     0.976        21
                          atis_flight_no      0.889     1.000     0.941         8
                               atis_meal      1.000     0.833     0.909         6
                        atis_restriction      0.000     0.000     0.000         0
             atis_airline#atis_flight_no      0.000     0.000     0.000         0
    atis_ground_service#atis_ground_fare      0.000     0.000     0.000         0
           atis_airfare#atis_flight_time      0.000     0.000     0.000         0
                           atis_cheapest      0.000     0.000     0.000         0
atis_aircraft#atis_flight#atis_flight_no      0.000     0.000     0.000         0

                               micro avg      0.969     0.974     0.971       888
                               macro avg      0.634     0.684     0.642       888
                            weighted avg      0.975     0.974     0.973       888

I0717 07:14:22.546967 140695199004544 interactiveshell.py:2882] 
                              precision    recall  f1-score   support

                           O      0.000     0.000     0.000       0.0
           B-toloc.city_name      0.979     0.993     0.986     716.0
         B-fromloc.city_name      0.991     0.990     0.991     704.0
           I-toloc.city_name      0.974     0.992     0.983     265.0
      B-depart_date.day_name      0.990     0.981     0.986     212.0
              B-airline_name      0.962     1.000     0.981     101.0
         I-fromloc.city_name      0.983     0.983     0.983     177.0
 B-depart_time.period_of_day      0.991     0.885     0.935     130.0
              I-airline_name      1.000     1.000     1.000      65.0
    B-depart_date.day_number      0.964     0.964     0.964      55.0
    B-depart_date.month_name      0.982     0.964     0.973      56.0
          B-depart_time.time      0.814     1.000     0.898      57.0
                B-round_trip      1.000     0.986     0.993      73.0
             B-cost_relative      1.000     0.973     0.986      37.0
                I-round_trip      1.000     1.000     1.000      71.0
                B-flight_mod      0.920     0.958     0.939      24.0
 B-depart_time.time_relative      0.984     0.969     0.977      65.0
          I-depart_time.time      0.929     1.000     0.963      52.0
         B-stoploc.city_name      0.909     1.000     0.952      20.0
                 B-city_name      0.822     0.649     0.725      57.0
                B-class_type      0.960     1.000     0.980      24.0
          B-arrive_time.time      0.943     0.971     0.957      34.0
 B-arrive_time.time_relative      0.909     0.968     0.937      31.0
                I-class_type      1.000     1.000     1.000      17.0
               B-flight_stop      0.955     1.000     0.977      21.0
          I-arrive_time.time      0.971     0.971     0.971      35.0
              B-airline_code      1.000     0.853     0.921      34.0
    I-depart_date.day_number      1.000     1.000     1.000      15.0
      I-fromloc.airport_name      0.441     1.000     0.612      15.0
      B-fromloc.airport_name      0.407     0.917     0.564      12.0
      B-arrive_date.day_name      0.688     1.000     0.815      11.0
          B-toloc.state_code      0.947     1.000     0.973      18.0
B-depart_date.today_relative      1.000     1.000     1.000       9.0
             B-flight_number      0.562     0.818     0.667      11.0
 B-depart_date.date_relative      0.895     1.000     0.944      17.0
          B-toloc.state_name      0.900     0.964     0.931      28.0
           B-fare_basis_code      0.941     0.941     0.941      17.0
               B-flight_time      1.000     1.000     1.000       1.0
                        B-or      1.000     1.000     1.000       3.0
 B-arrive_time.period_of_day      0.750     1.000     0.857       6.0
          B-meal_description      1.000     0.900     0.947      10.0
             I-cost_relative      1.000     0.667     0.800       3.0
              I-airport_name      0.917     0.379     0.537      29.0
               B-fare_amount      0.400     1.000     0.571       2.0
               I-fare_amount      1.000     1.000     1.000       2.0
                 I-city_name      1.000     0.600     0.750      30.0
        I-toloc.airport_name      1.000     1.000     1.000       3.0
            B-transport_type      1.000     0.900     0.947      10.0
    B-arrive_date.month_name      0.714     0.833     0.769       6.0
    B-arrive_date.day_number      0.714     0.833     0.769       6.0
         I-stoploc.city_name      0.714     1.000     0.833      10.0
                      B-meal      1.000     1.000     1.000      16.0
        B-fromloc.state_code      1.000     0.957     0.978      23.0
    B-depart_time.period_mod      0.625     1.000     0.769       5.0
                   B-connect      1.000     1.000     1.000       6.0
               B-flight_days      1.000     1.000     1.000      10.0
        B-toloc.airport_name      0.750     1.000     0.857       3.0
        B-fromloc.state_name      0.944     1.000     0.971      17.0
              B-airport_name      0.818     0.429     0.562      21.0
                   B-economy      1.000     1.000     1.000       6.0
               I-flight_time      1.000     1.000     1.000       1.0
             B-aircraft_code      0.963     0.788     0.867      33.0
                       B-mod      0.000     0.000     0.000       2.0
              B-airport_code      0.000     0.000     0.000       9.0
    B-depart_time.start_time      0.667     0.667     0.667       3.0
      B-depart_time.end_time      0.667     0.667     0.667       3.0
          B-depart_date.year      1.000     0.667     0.800       3.0
            I-transport_type      0.000     0.000     0.000       1.0
          B-restriction_code      0.333     1.000     0.500       4.0
    B-arrive_time.start_time      0.875     0.875     0.875       8.0
        B-toloc.airport_code      1.000     0.500     0.667       4.0
      B-arrive_time.end_time      0.889     1.000     0.941       8.0
      I-arrive_time.end_time      0.875     0.875     0.875       8.0
      I-depart_time.end_time      1.000     0.333     0.500       3.0
               I-flight_stop      0.000     0.000     0.000       0.0
      B-fromloc.airport_code      0.833     1.000     0.909       5.0
          I-restriction_code      1.000     1.000     1.000       3.0
    I-depart_time.start_time      0.500     1.000     0.667       1.0
          I-toloc.state_name      1.000     1.000     1.000       1.0
I-depart_date.today_relative      0.000     0.000     0.000       0.0
 B-arrive_date.date_relative      1.000     0.500     0.667       2.0
                I-flight_mod      1.000     0.167     0.286       6.0
                   I-economy      0.000     0.000     0.000       0.0
 B-return_date.date_relative      1.000     0.333     0.500       3.0
        I-fromloc.state_name      1.000     1.000     1.000       1.0
                B-state_code      1.000     1.000     1.000       1.0
    I-arrive_time.start_time      1.000     1.000     1.000       1.0
    I-arrive_date.day_number      0.000     0.000     0.000       0.0
                 B-meal_code      1.000     1.000     1.000       1.0
 I-depart_time.period_of_day      0.000     0.000     0.000       1.0
                  B-day_name      1.000     0.500     0.667       2.0
             B-period_of_day      1.000     0.500     0.667       4.0
        B-stoploc.state_code      0.000     0.000     0.000       0.0
    B-return_date.month_name      0.000     0.000     0.000       0.0
    B-return_date.day_number      0.000     0.000     0.000       0.0
    B-arrive_time.period_mod      0.000     0.000     0.000       0.0
                 I-meal_code      0.000     0.000     0.000       0.0
        B-toloc.country_name      1.000     1.000     1.000       1.0
                 B-days_code      1.000     1.000     1.000       1.0
 I-arrive_time.period_of_day      0.000     0.000     0.000       0.0
            I-today_relative      0.000     0.000     0.000       0.0
 B-return_time.period_of_day      0.000     0.000     0.000       0.0
                      B-time      0.000     0.000     0.000       0.0
           I-fare_basis_code      0.000     0.000     0.000       0.0
 I-arrive_time.time_relative      1.000     0.500     0.667       4.0
 I-depart_time.time_relative      0.000     0.000     0.000       1.0
            B-today_relative      0.000     0.000     0.000       0.0
                B-state_name      1.000     0.111     0.200       9.0
B-arrive_date.today_relative      0.000     0.000     0.000       0.0
    B-return_time.period_mod      0.000     0.000     0.000       0.0
                B-month_name      0.000     0.000     0.000       0.0
                B-day_number      0.000     0.000     0.000       0.0
 I-return_date.date_relative      0.750     1.000     0.857       3.0
I-return_date.today_relative      0.000     0.000     0.000       0.0
      B-stoploc.airport_name      0.000     0.000     0.000       0.0
             B-time_relative      0.000     0.000     0.000       0.0
                      I-time      0.000     0.000     0.000       0.0
    I-return_date.day_number      0.000     0.000     0.000       0.0
          I-meal_description      0.000     0.000     0.000       0.0
B-return_date.today_relative      0.000     0.000     0.000       0.0
      B-return_date.day_name      0.000     0.000     0.000       2.0

                   micro avg      0.949     0.951     0.950    3657.0
                   macro avg      0.656     0.638     0.626    3657.0
                weighted avg      0.958     0.951     0.949    3657.0

I0717 07:14:22.547859 140695199004544 interactiveshell.py:2882] Best Slot F1: 0.951
Reading ../data/atis.train.w-intent.iob
I0717 07:14:23.278026 140695199004544 interactiveshell.py:2882] Step 15600 | Loss: 0.0166 | Loss_intent: 0.0003 | Loss_slots: 0.0164 | Spent: 6.6 secs | LR: 0.000058
I0717 07:14:27.457775 140695199004544 interactiveshell.py:2882] Step 15650 | Loss: 0.0028 | Loss_intent: 0.0000 | Loss_slots: 0.0028 | Spent: 4.2 secs | LR: 0.000058
I0717 07:14:31.631729 140695199004544 interactiveshell.py:2882] Step 15700 | Loss: 0.0036 | Loss_intent: 0.0024 | Loss_slots: 0.0012 | Spent: 4.2 secs | LR: 0.000057
I0717 07:14:35.844974 140695199004544 interactiveshell.py:2882] Step 15750 | Loss: 0.0033 | Loss_intent: 0.0000 | Loss_slots: 0.0033 | Spent: 4.2 secs | LR: 0.000057
I0717 07:14:40.035785 140695199004544 interactiveshell.py:2882] Step 15800 | Loss: 0.0127 | Loss_intent: 0.0000 | Loss_slots: 0.0127 | Spent: 4.2 secs | LR: 0.000057
I0717 07:14:44.215340 140695199004544 interactiveshell.py:2882] Step 15850 | Loss: 0.0111 | Loss_intent: 0.0000 | Loss_slots: 0.0111 | Spent: 4.2 secs | LR: 0.000056
I0717 07:14:48.408071 140695199004544 interactiveshell.py:2882] Step 15900 | Loss: 0.0136 | Loss_intent: 0.0000 | Loss_slots: 0.0136 | Spent: 4.2 secs | LR: 0.000056
Reading ../data/atis.test.w-intent.iob
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.
  'precision', 'predicted', average, warn_for)
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1439: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples.
  'recall', 'true', average, warn_for)
I0717 07:14:51.062433 140695199004544 interactiveshell.py:2882] 
                                          precision    recall  f1-score   support

                             atis_flight      0.980     0.986     0.983       632
                            atis_airfare      0.959     0.979     0.969        48
                     atis_ground_service      1.000     1.000     1.000        36
                            atis_airline      0.974     0.974     0.974        38
                       atis_abbreviation      1.000     1.000     1.000        33
                           atis_aircraft      1.000     0.889     0.941         9
                        atis_flight_time      1.000     1.000     1.000         1
                           atis_quantity      0.333     1.000     0.500         3
                atis_flight#atis_airfare      0.875     0.583     0.700        12
                            atis_airport      1.000     0.944     0.971        18
                           atis_distance      1.000     1.000     1.000        10
                               atis_city      0.500     0.667     0.571         6
                        atis_ground_fare      1.000     0.857     0.923         7
                           atis_capacity      1.000     0.952     0.976        21
                          atis_flight_no      0.889     1.000     0.941         8
                               atis_meal      1.000     0.833     0.909         6
                        atis_restriction      0.000     0.000     0.000         0
             atis_airline#atis_flight_no      0.000     0.000     0.000         0
    atis_ground_service#atis_ground_fare      0.000     0.000     0.000         0
           atis_airfare#atis_flight_time      0.000     0.000     0.000         0
                           atis_cheapest      0.000     0.000     0.000         0
atis_aircraft#atis_flight#atis_flight_no      0.000     0.000     0.000         0

                               micro avg      0.969     0.974     0.971       888
                               macro avg      0.660     0.667     0.653       888
                            weighted avg      0.974     0.974     0.973       888

I0717 07:14:51.103051 140695199004544 interactiveshell.py:2882] 
                              precision    recall  f1-score   support

                           O      0.000     0.000     0.000       0.0
           B-toloc.city_name      0.975     0.994     0.985     716.0
         B-fromloc.city_name      0.989     0.993     0.991     704.0
           I-toloc.city_name      0.967     0.992     0.980     265.0
      B-depart_date.day_name      0.991     0.986     0.988     212.0
              B-airline_name      0.981     1.000     0.990     101.0
         I-fromloc.city_name      0.983     0.983     0.983     177.0
 B-depart_time.period_of_day      0.992     0.900     0.944     130.0
              I-airline_name      1.000     1.000     1.000      65.0
    B-depart_date.day_number      0.964     0.964     0.964      55.0
    B-depart_date.month_name      0.982     0.964     0.973      56.0
          B-depart_time.time      0.838     1.000     0.912      57.0
                B-round_trip      1.000     0.986     0.993      73.0
             B-cost_relative      1.000     0.973     0.986      37.0
                I-round_trip      1.000     1.000     1.000      71.0
                B-flight_mod      0.719     0.958     0.821      24.0
 B-depart_time.time_relative      0.984     0.969     0.977      65.0
          I-depart_time.time      0.963     1.000     0.981      52.0
         B-stoploc.city_name      0.952     1.000     0.976      20.0
                 B-city_name      0.868     0.579     0.695      57.0
                B-class_type      0.960     1.000     0.980      24.0
          B-arrive_time.time      0.943     0.971     0.957      34.0
 B-arrive_time.time_relative      0.909     0.968     0.937      31.0
                I-class_type      1.000     1.000     1.000      17.0
               B-flight_stop      1.000     1.000     1.000      21.0
          I-arrive_time.time      1.000     0.971     0.986      35.0
              B-airline_code      1.000     0.853     0.921      34.0
    I-depart_date.day_number      1.000     1.000     1.000      15.0
      I-fromloc.airport_name      0.417     1.000     0.588      15.0
      B-fromloc.airport_name      0.379     0.917     0.537      12.0
      B-arrive_date.day_name      0.733     1.000     0.846      11.0
          B-toloc.state_code      0.947     1.000     0.973      18.0
B-depart_date.today_relative      1.000     1.000     1.000       9.0
             B-flight_number      0.321     0.818     0.462      11.0
 B-depart_date.date_relative      0.882     0.882     0.882      17.0
          B-toloc.state_name      0.897     0.929     0.912      28.0
           B-fare_basis_code      0.917     0.647     0.759      17.0
               B-flight_time      1.000     1.000     1.000       1.0
                        B-or      1.000     1.000     1.000       3.0
 B-arrive_time.period_of_day      0.750     1.000     0.857       6.0
          B-meal_description      1.000     0.900     0.947      10.0
             I-cost_relative      1.000     0.667     0.800       3.0
              I-airport_name      0.917     0.379     0.537      29.0
               B-fare_amount      1.000     1.000     1.000       2.0
               I-fare_amount      1.000     1.000     1.000       2.0
                 I-city_name      1.000     0.567     0.723      30.0
        I-toloc.airport_name      1.000     1.000     1.000       3.0
            B-transport_type      1.000     0.700     0.824      10.0
    B-arrive_date.month_name      0.714     0.833     0.769       6.0
    B-arrive_date.day_number      0.714     0.833     0.769       6.0
         I-stoploc.city_name      0.769     1.000     0.870      10.0
                      B-meal      1.000     1.000     1.000      16.0
        B-fromloc.state_code      1.000     0.957     0.978      23.0
    B-depart_time.period_mod      0.714     1.000     0.833       5.0
                   B-connect      1.000     1.000     1.000       6.0
               B-flight_days      1.000     1.000     1.000      10.0
        B-toloc.airport_name      0.667     0.667     0.667       3.0
        B-fromloc.state_name      0.895     1.000     0.944      17.0
              B-airport_name      0.800     0.381     0.516      21.0
                   B-economy      1.000     1.000     1.000       6.0
               I-flight_time      1.000     1.000     1.000       1.0
             B-aircraft_code      0.903     0.848     0.875      33.0
                       B-mod      0.000     0.000     0.000       2.0
              B-airport_code      0.000     0.000     0.000       9.0
    B-depart_time.start_time      0.667     0.667     0.667       3.0
      B-depart_time.end_time      1.000     0.667     0.800       3.0
          B-depart_date.year      1.000     0.667     0.800       3.0
            I-transport_type      0.200     1.000     0.333       1.0
          B-restriction_code      0.400     1.000     0.571       4.0
    B-arrive_time.start_time      0.889     1.000     0.941       8.0
        B-toloc.airport_code      1.000     0.500     0.667       4.0
      B-arrive_time.end_time      0.889     1.000     0.941       8.0
      I-arrive_time.end_time      0.889     1.000     0.941       8.0
      I-depart_time.end_time      1.000     0.333     0.500       3.0
               I-flight_stop      0.000     0.000     0.000       0.0
      B-fromloc.airport_code      0.833     1.000     0.909       5.0
          I-restriction_code      1.000     1.000     1.000       3.0
    I-depart_time.start_time      0.500     1.000     0.667       1.0
          I-toloc.state_name      1.000     1.000     1.000       1.0
I-depart_date.today_relative      0.000     0.000     0.000       0.0
 B-arrive_date.date_relative      1.000     0.500     0.667       2.0
                I-flight_mod      1.000     0.167     0.286       6.0
                   I-economy      0.000     0.000     0.000       0.0
 B-return_date.date_relative      0.500     0.333     0.400       3.0
        I-fromloc.state_name      1.000     1.000     1.000       1.0
                B-state_code      1.000     1.000     1.000       1.0
    I-arrive_time.start_time      1.000     1.000     1.000       1.0
    I-arrive_date.day_number      0.000     0.000     0.000       0.0
                 B-meal_code      0.000     0.000     0.000       1.0
 I-depart_time.period_of_day      0.000     0.000     0.000       1.0
                  B-day_name      1.000     0.500     0.667       2.0
             B-period_of_day      1.000     0.500     0.667       4.0
        B-stoploc.state_code      0.000     0.000     0.000       0.0
    B-return_date.month_name      0.000     0.000     0.000       0.0
    B-return_date.day_number      0.000     0.000     0.000       0.0
    B-arrive_time.period_mod      0.000     0.000     0.000       0.0
                 I-meal_code      0.000     0.000     0.000       0.0
        B-toloc.country_name      1.000     1.000     1.000       1.0
                 B-days_code      1.000     1.000     1.000       1.0
 I-arrive_time.period_of_day      0.000     0.000     0.000       0.0
            I-today_relative      0.000     0.000     0.000       0.0
 B-return_time.period_of_day      0.000     0.000     0.000       0.0
                      B-time      0.000     0.000     0.000       0.0
           I-fare_basis_code      0.000     0.000     0.000       0.0
 I-arrive_time.time_relative      1.000     0.500     0.667       4.0
 I-depart_time.time_relative      0.000     0.000     0.000       1.0
            B-today_relative      0.000     0.000     0.000       0.0
                B-state_name      1.000     0.111     0.200       9.0
B-arrive_date.today_relative      0.000     0.000     0.000       0.0
    B-return_time.period_mod      0.000     0.000     0.000       0.0
                B-month_name      0.000     0.000     0.000       0.0
                B-day_number      0.000     0.000     0.000       0.0
 I-return_date.date_relative      0.750     1.000     0.857       3.0
I-return_date.today_relative      0.000     0.000     0.000       0.0
      B-stoploc.airport_name      0.000     0.000     0.000       0.0
             B-time_relative      0.000     0.000     0.000       0.0
                      I-time      0.000     0.000     0.000       0.0
    I-return_date.day_number      0.000     0.000     0.000       0.0
          I-meal_description      0.000     0.000     0.000       0.0
B-return_date.today_relative      0.000     0.000     0.000       0.0
      B-return_date.day_name      0.000     0.000     0.000       2.0

                   micro avg      0.948     0.949     0.948    3657.0
                   macro avg      0.651     0.631     0.619    3657.0
                weighted avg      0.956     0.949     0.946    3657.0

I0717 07:14:51.103854 140695199004544 interactiveshell.py:2882] Best Slot F1: 0.951
Reading ../data/atis.train.w-intent.iob
I0717 07:14:55.004951 140695199004544 interactiveshell.py:2882] Step 15950 | Loss: 0.0055 | Loss_intent: 0.0000 | Loss_slots: 0.0055 | Spent: 6.6 secs | LR: 0.000056
I0717 07:14:59.196345 140695199004544 interactiveshell.py:2882] Step 16000 | Loss: 0.0041 | Loss_intent: 0.0000 | Loss_slots: 0.0041 | Spent: 4.2 secs | LR: 0.000056
I0717 07:15:03.884916 140695199004544 interactiveshell.py:2882] Step 16050 | Loss: 0.0152 | Loss_intent: 0.0000 | Loss_slots: 0.0152 | Spent: 4.7 secs | LR: 0.000055
I0717 07:15:08.634106 140695199004544 interactiveshell.py:2882] Step 16100 | Loss: 0.0054 | Loss_intent: 0.0000 | Loss_slots: 0.0054 | Spent: 4.7 secs | LR: 0.000055
I0717 07:15:12.985124 140695199004544 interactiveshell.py:2882] Step 16150 | Loss: 0.0099 | Loss_intent: 0.0000 | Loss_slots: 0.0099 | Spent: 4.3 secs | LR: 0.000055
I0717 07:15:17.169754 140695199004544 interactiveshell.py:2882] Step 16200 | Loss: 0.0403 | Loss_intent: 0.0000 | Loss_slots: 0.0403 | Spent: 4.2 secs | LR: 0.000054
Reading ../data/atis.test.w-intent.iob
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.
  'precision', 'predicted', average, warn_for)
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1439: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples.
  'recall', 'true', average, warn_for)
I0717 07:15:20.900602 140695199004544 interactiveshell.py:2882] 
                                          precision    recall  f1-score   support

                             atis_flight      0.980     0.984     0.982       632
                            atis_airfare      0.940     0.979     0.959        48
                     atis_ground_service      1.000     1.000     1.000        36
                            atis_airline      0.974     1.000     0.987        38
                       atis_abbreviation      1.000     1.000     1.000        33
                           atis_aircraft      0.889     0.889     0.889         9
                        atis_flight_time      1.000     1.000     1.000         1
                           atis_quantity      0.375     1.000     0.545         3
                atis_flight#atis_airfare      0.857     0.500     0.632        12
                            atis_airport      1.000     0.944     0.971        18
                           atis_distance      1.000     1.000     1.000        10
                               atis_city      0.444     0.667     0.533         6
                        atis_ground_fare      1.000     0.714     0.833         7
                           atis_capacity      1.000     0.952     0.976        21
                          atis_flight_no      0.889     1.000     0.941         8
                               atis_meal      1.000     0.833     0.909         6
                        atis_restriction      0.000     0.000     0.000         0
             atis_airline#atis_flight_no      0.000     0.000     0.000         0
    atis_ground_service#atis_ground_fare      0.000     0.000     0.000         0
           atis_airfare#atis_flight_time      0.000     0.000     0.000         0
                           atis_cheapest      0.000     0.000     0.000         0
atis_aircraft#atis_flight#atis_flight_no      0.000     0.000     0.000         0

                               micro avg      0.966     0.972     0.969       888
                               macro avg      0.652     0.657     0.644       888
                            weighted avg      0.971     0.972     0.970       888

I0717 07:15:20.940192 140695199004544 interactiveshell.py:2882] 
                              precision    recall  f1-score   support

                           O      0.000     0.000     0.000       0.0
           B-toloc.city_name      0.973     0.994     0.983     716.0
         B-fromloc.city_name      0.989     0.993     0.991     704.0
           I-toloc.city_name      0.964     1.000     0.981     265.0
      B-depart_date.day_name      0.991     0.986     0.988     212.0
              B-airline_name      0.981     1.000     0.990     101.0
         I-fromloc.city_name      0.978     0.983     0.980     177.0
 B-depart_time.period_of_day      0.992     0.908     0.948     130.0
              I-airline_name      1.000     1.000     1.000      65.0
    B-depart_date.day_number      0.981     0.964     0.972      55.0
    B-depart_date.month_name      0.982     0.964     0.973      56.0
          B-depart_time.time      0.826     1.000     0.905      57.0
                B-round_trip      1.000     0.986     0.993      73.0
             B-cost_relative      1.000     0.973     0.986      37.0
                I-round_trip      1.000     1.000     1.000      71.0
                B-flight_mod      0.852     0.958     0.902      24.0
 B-depart_time.time_relative      0.984     0.969     0.977      65.0
          I-depart_time.time      0.963     1.000     0.981      52.0
         B-stoploc.city_name      0.952     1.000     0.976      20.0
                 B-city_name      0.875     0.614     0.722      57.0
                B-class_type      0.923     1.000     0.960      24.0
          B-arrive_time.time      0.943     0.971     0.957      34.0
 B-arrive_time.time_relative      0.909     0.968     0.937      31.0
                I-class_type      1.000     1.000     1.000      17.0
               B-flight_stop      1.000     1.000     1.000      21.0
          I-arrive_time.time      0.971     0.971     0.971      35.0
              B-airline_code      1.000     0.882     0.938      34.0
    I-depart_date.day_number      1.000     1.000     1.000      15.0
      I-fromloc.airport_name      0.441     1.000     0.612      15.0
      B-fromloc.airport_name      0.370     0.833     0.513      12.0
      B-arrive_date.day_name      0.733     1.000     0.846      11.0
          B-toloc.state_code      0.947     1.000     0.973      18.0
B-depart_date.today_relative      1.000     1.000     1.000       9.0
             B-flight_number      0.360     0.818     0.500      11.0
 B-depart_date.date_relative      0.941     0.941     0.941      17.0
          B-toloc.state_name      0.897     0.929     0.912      28.0
           B-fare_basis_code      0.909     0.588     0.714      17.0
               B-flight_time      1.000     1.000     1.000       1.0
                        B-or      1.000     1.000     1.000       3.0
 B-arrive_time.period_of_day      0.750     1.000     0.857       6.0
          B-meal_description      1.000     0.900     0.947      10.0
             I-cost_relative      1.000     0.667     0.800       3.0
              I-airport_name      0.917     0.379     0.537      29.0
               B-fare_amount      0.667     1.000     0.800       2.0
               I-fare_amount      1.000     1.000     1.000       2.0
                 I-city_name      1.000     0.567     0.723      30.0
        I-toloc.airport_name      1.000     1.000     1.000       3.0
            B-transport_type      1.000     0.900     0.947      10.0
    B-arrive_date.month_name      0.714     0.833     0.769       6.0
    B-arrive_date.day_number      0.714     0.833     0.769       6.0
         I-stoploc.city_name      1.000     1.000     1.000      10.0
                      B-meal      1.000     1.000     1.000      16.0
        B-fromloc.state_code      1.000     0.957     0.978      23.0
    B-depart_time.period_mod      0.714     1.000     0.833       5.0
                   B-connect      1.000     1.000     1.000       6.0
               B-flight_days      1.000     1.000     1.000      10.0
        B-toloc.airport_name      0.500     0.333     0.400       3.0
        B-fromloc.state_name      0.895     1.000     0.944      17.0
              B-airport_name      0.818     0.429     0.562      21.0
                   B-economy      1.000     1.000     1.000       6.0
               I-flight_time      1.000     1.000     1.000       1.0
             B-aircraft_code      0.931     0.818     0.871      33.0
                       B-mod      0.000     0.000     0.000       2.0
              B-airport_code      0.000     0.000     0.000       9.0
    B-depart_time.start_time      0.667     0.667     0.667       3.0
      B-depart_time.end_time      1.000     0.667     0.800       3.0
          B-depart_date.year      1.000     0.667     0.800       3.0
            I-transport_type      0.333     1.000     0.500       1.0
          B-restriction_code      0.333     1.000     0.500       4.0
    B-arrive_time.start_time      0.875     0.875     0.875       8.0
        B-toloc.airport_code      1.000     0.500     0.667       4.0
      B-arrive_time.end_time      0.889     1.000     0.941       8.0
      I-arrive_time.end_time      0.875     0.875     0.875       8.0
      I-depart_time.end_time      1.000     0.333     0.500       3.0
               I-flight_stop      0.000     0.000     0.000       0.0
      B-fromloc.airport_code      1.000     1.000     1.000       5.0
          I-restriction_code      1.000     1.000     1.000       3.0
    I-depart_time.start_time      0.500     1.000     0.667       1.0
          I-toloc.state_name      1.000     1.000     1.000       1.0
I-depart_date.today_relative      0.000     0.000     0.000       0.0
 B-arrive_date.date_relative      1.000     1.000     1.000       2.0
                I-flight_mod      1.000     0.167     0.286       6.0
                   I-economy      0.000     0.000     0.000       0.0
 B-return_date.date_relative      0.500     0.333     0.400       3.0
        I-fromloc.state_name      1.000     1.000     1.000       1.0
                B-state_code      1.000     1.000     1.000       1.0
    I-arrive_time.start_time      1.000     1.000     1.000       1.0
    I-arrive_date.day_number      0.000     0.000     0.000       0.0
                 B-meal_code      1.000     1.000     1.000       1.0
 I-depart_time.period_of_day      0.000     0.000     0.000       1.0
                  B-day_name      1.000     0.500     0.667       2.0
             B-period_of_day      1.000     0.500     0.667       4.0
        B-stoploc.state_code      0.000     0.000     0.000       0.0
    B-return_date.month_name      0.000     0.000     0.000       0.0
    B-return_date.day_number      0.000     0.000     0.000       0.0
    B-arrive_time.period_mod      0.000     0.000     0.000       0.0
                 I-meal_code      0.000     0.000     0.000       0.0
        B-toloc.country_name      1.000     1.000     1.000       1.0
                 B-days_code      1.000     1.000     1.000       1.0
 I-arrive_time.period_of_day      0.000     0.000     0.000       0.0
            I-today_relative      0.000     0.000     0.000       0.0
 B-return_time.period_of_day      0.000     0.000     0.000       0.0
                      B-time      0.000     0.000     0.000       0.0
           I-fare_basis_code      0.000     0.000     0.000       0.0
 I-arrive_time.time_relative      0.000     0.000     0.000       4.0
 I-depart_time.time_relative      0.000     0.000     0.000       1.0
            B-today_relative      0.000     0.000     0.000       0.0
                B-state_name      1.000     0.111     0.200       9.0
B-arrive_date.today_relative      0.000     0.000     0.000       0.0
    B-return_time.period_mod      0.000     0.000     0.000       0.0
                B-month_name      0.000     0.000     0.000       0.0
                B-day_number      0.000     0.000     0.000       0.0
 I-return_date.date_relative      0.750     1.000     0.857       3.0
I-return_date.today_relative      0.000     0.000     0.000       0.0
      B-stoploc.airport_name      0.000     0.000     0.000       0.0
             B-time_relative      0.000     0.000     0.000       0.0
                      I-time      0.000     0.000     0.000       0.0
    I-return_date.day_number      0.000     0.000     0.000       0.0
          I-meal_description      0.000     0.000     0.000       0.0
B-return_date.today_relative      0.000     0.000     0.000       0.0
      B-return_date.day_name      0.000     0.000     0.000       2.0

                   micro avg      0.949     0.950     0.949    3657.0
                   macro avg      0.653     0.636     0.625    3657.0
                weighted avg      0.956     0.950     0.947    3657.0

I0717 07:15:20.940957 140695199004544 interactiveshell.py:2882] Best Slot F1: 0.951
Reading ../data/atis.train.w-intent.iob
I0717 07:15:23.892106 140695199004544 interactiveshell.py:2882] Step 16250 | Loss: 0.0045 | Loss_intent: 0.0000 | Loss_slots: 0.0045 | Spent: 6.7 secs | LR: 0.000054
I0717 07:15:28.140586 140695199004544 interactiveshell.py:2882] Step 16300 | Loss: 0.0064 | Loss_intent: 0.0000 | Loss_slots: 0.0064 | Spent: 4.2 secs | LR: 0.000054
I0717 07:15:32.325695 140695199004544 interactiveshell.py:2882] Step 16350 | Loss: 0.0034 | Loss_intent: 0.0001 | Loss_slots: 0.0033 | Spent: 4.2 secs | LR: 0.000054
I0717 07:15:36.519267 140695199004544 interactiveshell.py:2882] Step 16400 | Loss: 0.0051 | Loss_intent: 0.0000 | Loss_slots: 0.0051 | Spent: 4.2 secs | LR: 0.000053
I0717 07:15:40.705425 140695199004544 interactiveshell.py:2882] Step 16450 | Loss: 0.0104 | Loss_intent: 0.0000 | Loss_slots: 0.0104 | Spent: 4.2 secs | LR: 0.000053
I0717 07:15:44.913280 140695199004544 interactiveshell.py:2882] Step 16500 | Loss: 0.0091 | Loss_intent: 0.0000 | Loss_slots: 0.0091 | Spent: 4.2 secs | LR: 0.000053
Reading ../data/atis.test.w-intent.iob
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.
  'precision', 'predicted', average, warn_for)
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1439: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples.
  'recall', 'true', average, warn_for)
I0717 07:15:49.588286 140695199004544 interactiveshell.py:2882] 
                                          precision    recall  f1-score   support

                             atis_flight      0.987     0.981     0.984       632
                            atis_airfare      0.923     1.000     0.960        48
                     atis_ground_service      1.000     1.000     1.000        36
                            atis_airline      0.974     1.000     0.987        38
                       atis_abbreviation      0.971     1.000     0.985        33
                           atis_aircraft      0.889     0.889     0.889         9
                        atis_flight_time      1.000     1.000     1.000         1
                           atis_quantity      0.375     1.000     0.545         3
                atis_flight#atis_airfare      0.857     0.500     0.632        12
                            atis_airport      0.947     1.000     0.973        18
                           atis_distance      1.000     1.000     1.000        10
                               atis_city      0.545     1.000     0.706         6
                        atis_ground_fare      1.000     0.714     0.833         7
                           atis_capacity      1.000     0.952     0.976        21
                          atis_flight_no      0.889     1.000     0.941         8
                               atis_meal      1.000     0.833     0.909         6
                        atis_restriction      0.000     0.000     0.000         0
             atis_airline#atis_flight_no      0.000     0.000     0.000         0
    atis_ground_service#atis_ground_fare      0.000     0.000     0.000         0
           atis_airfare#atis_flight_time      0.000     0.000     0.000         0
                           atis_cheapest      0.000     0.000     0.000         0
atis_aircraft#atis_flight#atis_flight_no      0.000     0.000     0.000         0

                               micro avg      0.969     0.974     0.971       888
                               macro avg      0.653     0.676     0.651       888
                            weighted avg      0.974     0.974     0.972       888

I0717 07:15:49.628928 140695199004544 interactiveshell.py:2882] 
                              precision    recall  f1-score   support

                           O      0.000     0.000     0.000       0.0
           B-toloc.city_name      0.971     0.994     0.983     716.0
         B-fromloc.city_name      0.991     0.990     0.991     704.0
           I-toloc.city_name      0.960     1.000     0.980     265.0
      B-depart_date.day_name      0.990     0.981     0.986     212.0
              B-airline_name      0.990     1.000     0.995     101.0
         I-fromloc.city_name      0.983     0.983     0.983     177.0
 B-depart_time.period_of_day      1.000     0.908     0.952     130.0
              I-airline_name      1.000     1.000     1.000      65.0
    B-depart_date.day_number      0.964     0.964     0.964      55.0
    B-depart_date.month_name      0.982     0.964     0.973      56.0
          B-depart_time.time      0.826     1.000     0.905      57.0
                B-round_trip      1.000     0.986     0.993      73.0
             B-cost_relative      1.000     0.973     0.986      37.0
                I-round_trip      1.000     1.000     1.000      71.0
                B-flight_mod      0.920     0.958     0.939      24.0
 B-depart_time.time_relative      0.984     0.969     0.977      65.0
          I-depart_time.time      0.963     1.000     0.981      52.0
         B-stoploc.city_name      0.952     1.000     0.976      20.0
                 B-city_name      0.829     0.596     0.694      57.0
                B-class_type      0.960     1.000     0.980      24.0
          B-arrive_time.time      0.971     0.971     0.971      34.0
 B-arrive_time.time_relative      0.909     0.968     0.937      31.0
                I-class_type      1.000     1.000     1.000      17.0
               B-flight_stop      0.955     1.000     0.977      21.0
          I-arrive_time.time      1.000     0.971     0.986      35.0
              B-airline_code      1.000     0.882     0.938      34.0
    I-depart_date.day_number      1.000     1.000     1.000      15.0
      I-fromloc.airport_name      0.429     1.000     0.600      15.0
      B-fromloc.airport_name      0.367     0.917     0.524      12.0
      B-arrive_date.day_name      0.688     1.000     0.815      11.0
          B-toloc.state_code      0.947     1.000     0.973      18.0
B-depart_date.today_relative      1.000     1.000     1.000       9.0
             B-flight_number      0.450     0.818     0.581      11.0
 B-depart_date.date_relative      0.941     0.941     0.941      17.0
          B-toloc.state_name      0.897     0.929     0.912      28.0
           B-fare_basis_code      0.917     0.647     0.759      17.0
               B-flight_time      1.000     1.000     1.000       1.0
                        B-or      1.000     1.000     1.000       3.0
 B-arrive_time.period_of_day      0.750     1.000     0.857       6.0
          B-meal_description      1.000     0.900     0.947      10.0
             I-cost_relative      1.000     0.667     0.800       3.0
              I-airport_name      0.917     0.379     0.537      29.0
               B-fare_amount      1.000     1.000     1.000       2.0
               I-fare_amount      1.000     1.000     1.000       2.0
                 I-city_name      1.000     0.500     0.667      30.0
        I-toloc.airport_name      1.000     1.000     1.000       3.0
            B-transport_type      1.000     0.700     0.824      10.0
    B-arrive_date.month_name      0.714     0.833     0.769       6.0
    B-arrive_date.day_number      0.714     0.833     0.769       6.0
         I-stoploc.city_name      0.909     1.000     0.952      10.0
                      B-meal      1.000     1.000     1.000      16.0
        B-fromloc.state_code      1.000     0.957     0.978      23.0
    B-depart_time.period_mod      0.833     1.000     0.909       5.0
                   B-connect      1.000     1.000     1.000       6.0
               B-flight_days      1.000     1.000     1.000      10.0
        B-toloc.airport_name      0.500     0.333     0.400       3.0
        B-fromloc.state_name      0.895     1.000     0.944      17.0
              B-airport_name      0.800     0.381     0.516      21.0
                   B-economy      1.000     1.000     1.000       6.0
               I-flight_time      1.000     1.000     1.000       1.0
             B-aircraft_code      0.966     0.848     0.903      33.0
                       B-mod      1.000     0.500     0.667       2.0
              B-airport_code      0.000     0.000     0.000       9.0
    B-depart_time.start_time      0.667     0.667     0.667       3.0
      B-depart_time.end_time      1.000     0.667     0.800       3.0
          B-depart_date.year      1.000     0.667     0.800       3.0
            I-transport_type      0.250     1.000     0.400       1.0
          B-restriction_code      0.364     1.000     0.533       4.0
    B-arrive_time.start_time      0.889     1.000     0.941       8.0
        B-toloc.airport_code      1.000     0.500     0.667       4.0
      B-arrive_time.end_time      0.889     1.000     0.941       8.0
      I-arrive_time.end_time      0.889     1.000     0.941       8.0
      I-depart_time.end_time      1.000     0.333     0.500       3.0
               I-flight_stop      0.000     0.000     0.000       0.0
      B-fromloc.airport_code      0.833     1.000     0.909       5.0
          I-restriction_code      0.300     1.000     0.462       3.0
    I-depart_time.start_time      0.500     1.000     0.667       1.0
          I-toloc.state_name      1.000     1.000     1.000       1.0
I-depart_date.today_relative      0.000     0.000     0.000       0.0
 B-arrive_date.date_relative      1.000     1.000     1.000       2.0
                I-flight_mod      1.000     0.167     0.286       6.0
                   I-economy      0.000     0.000     0.000       0.0
 B-return_date.date_relative      0.500     0.333     0.400       3.0
        I-fromloc.state_name      1.000     1.000     1.000       1.0
                B-state_code      1.000     1.000     1.000       1.0
    I-arrive_time.start_time      1.000     1.000     1.000       1.0
    I-arrive_date.day_number      0.000     0.000     0.000       0.0
                 B-meal_code      0.000     0.000     0.000       1.0
 I-depart_time.period_of_day      1.000     1.000     1.000       1.0
                  B-day_name      1.000     0.500     0.667       2.0
             B-period_of_day      1.000     0.500     0.667       4.0
        B-stoploc.state_code      0.000     0.000     0.000       0.0
    B-return_date.month_name      0.000     0.000     0.000       0.0
    B-return_date.day_number      0.000     0.000     0.000       0.0
    B-arrive_time.period_mod      0.000     0.000     0.000       0.0
                 I-meal_code      0.000     0.000     0.000       0.0
        B-toloc.country_name      1.000     1.000     1.000       1.0
                 B-days_code      1.000     1.000     1.000       1.0
 I-arrive_time.period_of_day      0.000     0.000     0.000       0.0
            I-today_relative      0.000     0.000     0.000       0.0
 B-return_time.period_of_day      0.000     0.000     0.000       0.0
                      B-time      0.000     0.000     0.000       0.0
           I-fare_basis_code      0.000     0.000     0.000       0.0
 I-arrive_time.time_relative      1.000     0.500     0.667       4.0
 I-depart_time.time_relative      0.000     0.000     0.000       1.0
            B-today_relative      0.000     0.000     0.000       0.0
                B-state_name      1.000     0.111     0.200       9.0
B-arrive_date.today_relative      0.000     0.000     0.000       0.0
    B-return_time.period_mod      0.000     0.000     0.000       0.0
                B-month_name      0.000     0.000     0.000       0.0
                B-day_number      0.000     0.000     0.000       0.0
 I-return_date.date_relative      0.750     1.000     0.857       3.0
I-return_date.today_relative      0.000     0.000     0.000       0.0
      B-stoploc.airport_name      0.000     0.000     0.000       0.0
             B-time_relative      0.000     0.000     0.000       0.0
                      I-time      0.000     0.000     0.000       0.0
    I-return_date.day_number      0.000     0.000     0.000       0.0
          I-meal_description      0.000     0.000     0.000       0.0
B-return_date.today_relative      0.000     0.000     0.000       0.0
      B-return_date.day_name      0.000     0.000     0.000       2.0

                   micro avg      0.948     0.950     0.949    3657.0
                   macro avg      0.666     0.645     0.633    3657.0
                weighted avg      0.958     0.950     0.948    3657.0

I0717 07:15:49.629884 140695199004544 interactiveshell.py:2882] Best Slot F1: 0.951
Reading ../data/atis.train.w-intent.iob
I0717 07:15:51.534317 140695199004544 interactiveshell.py:2882] Step 16550 | Loss: 0.0037 | Loss_intent: 0.0007 | Loss_slots: 0.0030 | Spent: 6.6 secs | LR: 0.000052
I0717 07:15:55.744883 140695199004544 interactiveshell.py:2882] Step 16600 | Loss: 0.0018 | Loss_intent: 0.0001 | Loss_slots: 0.0017 | Spent: 4.2 secs | LR: 0.000052
I0717 07:15:59.949020 140695199004544 interactiveshell.py:2882] Step 16650 | Loss: 0.0007 | Loss_intent: 0.0000 | Loss_slots: 0.0007 | Spent: 4.2 secs | LR: 0.000052
I0717 07:16:04.138455 140695199004544 interactiveshell.py:2882] Step 16700 | Loss: 0.0054 | Loss_intent: 0.0000 | Loss_slots: 0.0054 | Spent: 4.2 secs | LR: 0.000052
I0717 07:16:08.321025 140695199004544 interactiveshell.py:2882] Step 16750 | Loss: 0.0025 | Loss_intent: 0.0000 | Loss_slots: 0.0025 | Spent: 4.2 secs | LR: 0.000051
I0717 07:16:12.496573 140695199004544 interactiveshell.py:2882] Step 16800 | Loss: 0.0038 | Loss_intent: 0.0000 | Loss_slots: 0.0038 | Spent: 4.2 secs | LR: 0.000051
Reading ../data/atis.test.w-intent.iob
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.
  'precision', 'predicted', average, warn_for)
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1439: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples.
  'recall', 'true', average, warn_for)
I0717 07:16:18.150051 140695199004544 interactiveshell.py:2882] 
                                          precision    recall  f1-score   support

                             atis_flight      0.984     0.987     0.986       632
                            atis_airfare      0.922     0.979     0.949        48
                     atis_ground_service      1.000     1.000     1.000        36
                            atis_airline      0.974     1.000     0.987        38
                       atis_abbreviation      1.000     1.000     1.000        33
                           atis_aircraft      0.889     0.889     0.889         9
                        atis_flight_time      1.000     1.000     1.000         1
                           atis_quantity      0.429     1.000     0.600         3
                atis_flight#atis_airfare      0.833     0.417     0.556        12
                            atis_airport      1.000     1.000     1.000        18
                           atis_distance      1.000     1.000     1.000        10
                               atis_city      0.667     1.000     0.800         6
                        atis_ground_fare      1.000     0.714     0.833         7
                           atis_capacity      1.000     0.952     0.976        21
                          atis_flight_no      0.889     1.000     0.941         8
                               atis_meal      1.000     0.833     0.909         6
                        atis_restriction      0.000     0.000     0.000         0
             atis_airline#atis_flight_no      0.000     0.000     0.000         0
    atis_ground_service#atis_ground_fare      0.000     0.000     0.000         0
           atis_airfare#atis_flight_time      0.000     0.000     0.000         0
                           atis_cheapest      0.000     0.000     0.000         0
atis_aircraft#atis_flight#atis_flight_no      0.000     0.000     0.000         0

                               micro avg      0.971     0.976     0.974       888
                               macro avg      0.663     0.671     0.656       888
                            weighted avg      0.975     0.976     0.974       888

I0717 07:16:18.190067 140695199004544 interactiveshell.py:2882] 
                              precision    recall  f1-score   support

                           O      0.000     0.000     0.000       0.0
           B-toloc.city_name      0.973     0.994     0.983     716.0
         B-fromloc.city_name      0.989     0.994     0.992     704.0
           I-toloc.city_name      0.960     1.000     0.980     265.0
      B-depart_date.day_name      0.991     0.986     0.988     212.0
              B-airline_name      0.990     1.000     0.995     101.0
         I-fromloc.city_name      0.983     0.983     0.983     177.0
 B-depart_time.period_of_day      0.992     0.908     0.948     130.0
              I-airline_name      1.000     1.000     1.000      65.0
    B-depart_date.day_number      0.964     0.964     0.964      55.0
    B-depart_date.month_name      0.982     0.964     0.973      56.0
          B-depart_time.time      0.814     1.000     0.898      57.0
                B-round_trip      1.000     0.986     0.993      73.0
             B-cost_relative      1.000     0.973     0.986      37.0
                I-round_trip      1.000     1.000     1.000      71.0
                B-flight_mod      0.958     0.958     0.958      24.0
 B-depart_time.time_relative      0.984     0.969     0.977      65.0
          I-depart_time.time      0.963     1.000     0.981      52.0
         B-stoploc.city_name      0.952     1.000     0.976      20.0
                 B-city_name      0.865     0.561     0.681      57.0
                B-class_type      0.960     1.000     0.980      24.0
          B-arrive_time.time      0.971     0.971     0.971      34.0
 B-arrive_time.time_relative      0.909     0.968     0.937      31.0
                I-class_type      1.000     1.000     1.000      17.0
               B-flight_stop      1.000     1.000     1.000      21.0
          I-arrive_time.time      0.971     0.971     0.971      35.0
              B-airline_code      1.000     0.882     0.938      34.0
    I-depart_date.day_number      1.000     1.000     1.000      15.0
      I-fromloc.airport_name      0.429     1.000     0.600      15.0
      B-fromloc.airport_name      0.367     0.917     0.524      12.0
      B-arrive_date.day_name      0.733     1.000     0.846      11.0
          B-toloc.state_code      0.947     1.000     0.973      18.0
B-depart_date.today_relative      1.000     1.000     1.000       9.0
             B-flight_number      0.300     0.818     0.439      11.0
 B-depart_date.date_relative      0.941     0.941     0.941      17.0
          B-toloc.state_name      0.903     1.000     0.949      28.0
           B-fare_basis_code      0.917     0.647     0.759      17.0
               B-flight_time      1.000     1.000     1.000       1.0
                        B-or      1.000     1.000     1.000       3.0
 B-arrive_time.period_of_day      0.750     1.000     0.857       6.0
          B-meal_description      1.000     0.900     0.947      10.0
             I-cost_relative      1.000     0.667     0.800       3.0
              I-airport_name      0.917     0.379     0.537      29.0
               B-fare_amount      1.000     0.500     0.667       2.0
               I-fare_amount      1.000     1.000     1.000       2.0
                 I-city_name      1.000     0.500     0.667      30.0
        I-toloc.airport_name      1.000     1.000     1.000       3.0
            B-transport_type      1.000     0.700     0.824      10.0
    B-arrive_date.month_name      0.714     0.833     0.769       6.0
    B-arrive_date.day_number      0.714     0.833     0.769       6.0
         I-stoploc.city_name      1.000     1.000     1.000      10.0
                      B-meal      1.000     1.000     1.000      16.0
        B-fromloc.state_code      1.000     0.957     0.978      23.0
    B-depart_time.period_mod      0.714     1.000     0.833       5.0
                   B-connect      1.000     1.000     1.000       6.0
               B-flight_days      1.000     1.000     1.000      10.0
        B-toloc.airport_name      0.500     0.333     0.400       3.0
        B-fromloc.state_name      0.944     1.000     0.971      17.0
              B-airport_name      0.800     0.381     0.516      21.0
                   B-economy      1.000     1.000     1.000       6.0
               I-flight_time      1.000     1.000     1.000       1.0
             B-aircraft_code      0.933     0.848     0.889      33.0
                       B-mod      1.000     0.500     0.667       2.0
              B-airport_code      0.000     0.000     0.000       9.0
    B-depart_time.start_time      0.667     0.667     0.667       3.0
      B-depart_time.end_time      1.000     0.667     0.800       3.0
          B-depart_date.year      1.000     0.667     0.800       3.0
            I-transport_type      0.250     1.000     0.400       1.0
          B-restriction_code      0.600     0.750     0.667       4.0
    B-arrive_time.start_time      0.889     1.000     0.941       8.0
        B-toloc.airport_code      1.000     0.500     0.667       4.0
      B-arrive_time.end_time      0.889     1.000     0.941       8.0
      I-arrive_time.end_time      0.875     0.875     0.875       8.0
      I-depart_time.end_time      1.000     0.333     0.500       3.0
               I-flight_stop      0.000     0.000     0.000       0.0
      B-fromloc.airport_code      1.000     1.000     1.000       5.0
          I-restriction_code      0.500     1.000     0.667       3.0
    I-depart_time.start_time      0.500     1.000     0.667       1.0
          I-toloc.state_name      1.000     1.000     1.000       1.0
I-depart_date.today_relative      0.000     0.000     0.000       0.0
 B-arrive_date.date_relative      1.000     1.000     1.000       2.0
                I-flight_mod      1.000     0.167     0.286       6.0
                   I-economy      0.000     0.000     0.000       0.0
 B-return_date.date_relative      0.500     0.333     0.400       3.0
        I-fromloc.state_name      1.000     1.000     1.000       1.0
                B-state_code      1.000     1.000     1.000       1.0
    I-arrive_time.start_time      1.000     1.000     1.000       1.0
    I-arrive_date.day_number      0.000     0.000     0.000       0.0
                 B-meal_code      0.000     0.000     0.000       1.0
 I-depart_time.period_of_day      0.000     0.000     0.000       1.0
                  B-day_name      1.000     0.500     0.667       2.0
             B-period_of_day      1.000     0.500     0.667       4.0
        B-stoploc.state_code      0.000     0.000     0.000       0.0
    B-return_date.month_name      0.000     0.000     0.000       0.0
    B-return_date.day_number      0.000     0.000     0.000       0.0
    B-arrive_time.period_mod      0.000     0.000     0.000       0.0
                 I-meal_code      0.000     0.000     0.000       0.0
        B-toloc.country_name      1.000     1.000     1.000       1.0
                 B-days_code      1.000     1.000     1.000       1.0
 I-arrive_time.period_of_day      0.000     0.000     0.000       0.0
            I-today_relative      0.000     0.000     0.000       0.0
 B-return_time.period_of_day      0.000     0.000     0.000       0.0
                      B-time      0.000     0.000     0.000       0.0
           I-fare_basis_code      0.000     0.000     0.000       0.0
 I-arrive_time.time_relative      1.000     0.500     0.667       4.0
 I-depart_time.time_relative      0.000     0.000     0.000       1.0
            B-today_relative      0.000     0.000     0.000       0.0
                B-state_name      1.000     0.111     0.200       9.0
B-arrive_date.today_relative      0.000     0.000     0.000       0.0
    B-return_time.period_mod      0.000     0.000     0.000       0.0
                B-month_name      0.000     0.000     0.000       0.0
                B-day_number      0.000     0.000     0.000       0.0
 I-return_date.date_relative      0.750     1.000     0.857       3.0
I-return_date.today_relative      0.000     0.000     0.000       0.0
      B-stoploc.airport_name      0.000     0.000     0.000       0.0
             B-time_relative      0.000     0.000     0.000       0.0
                      I-time      0.000     0.000     0.000       0.0
    I-return_date.day_number      0.000     0.000     0.000       0.0
          I-meal_description      0.000     0.000     0.000       0.0
B-return_date.today_relative      0.000     0.000     0.000       0.0
      B-return_date.day_name      0.000     0.000     0.000       2.0

                   micro avg      0.948     0.950     0.949    3657.0
                   macro avg      0.662     0.630     0.624    3657.0
                weighted avg      0.958     0.950     0.948    3657.0

I0717 07:16:18.190871 140695199004544 interactiveshell.py:2882] Best Slot F1: 0.951
Reading ../data/atis.train.w-intent.iob
I0717 07:16:19.091081 140695199004544 interactiveshell.py:2882] Step 16850 | Loss: 0.0022 | Loss_intent: 0.0000 | Loss_slots: 0.0022 | Spent: 6.6 secs | LR: 0.000051
I0717 07:16:23.310116 140695199004544 interactiveshell.py:2882] Step 16900 | Loss: 0.0160 | Loss_intent: 0.0000 | Loss_slots: 0.0160 | Spent: 4.2 secs | LR: 0.000051
I0717 07:16:27.509814 140695199004544 interactiveshell.py:2882] Step 16950 | Loss: 0.0195 | Loss_intent: 0.0002 | Loss_slots: 0.0193 | Spent: 4.2 secs | LR: 0.000050
I0717 07:16:31.687762 140695199004544 interactiveshell.py:2882] Step 17000 | Loss: 0.0049 | Loss_intent: 0.0000 | Loss_slots: 0.0048 | Spent: 4.2 secs | LR: 0.000050
I0717 07:16:35.867108 140695199004544 interactiveshell.py:2882] Step 17050 | Loss: 0.0016 | Loss_intent: 0.0002 | Loss_slots: 0.0014 | Spent: 4.2 secs | LR: 0.000050
I0717 07:16:40.087414 140695199004544 interactiveshell.py:2882] Step 17100 | Loss: 0.0015 | Loss_intent: 0.0000 | Loss_slots: 0.0015 | Spent: 4.2 secs | LR: 0.000050
I0717 07:16:44.354962 140695199004544 interactiveshell.py:2882] Step 17150 | Loss: 0.0106 | Loss_intent: 0.0000 | Loss_slots: 0.0106 | Spent: 4.3 secs | LR: 0.000049
Reading ../data/atis.test.w-intent.iob
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.
  'precision', 'predicted', average, warn_for)
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1439: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples.
  'recall', 'true', average, warn_for)
I0717 07:16:46.862094 140695199004544 interactiveshell.py:2882] 
                                          precision    recall  f1-score   support

                             atis_flight      0.977     0.989     0.983       632
                            atis_airfare      0.922     0.979     0.949        48
                     atis_ground_service      1.000     1.000     1.000        36
                            atis_airline      0.974     1.000     0.987        38
                       atis_abbreviation      1.000     0.970     0.985        33
                           atis_aircraft      0.889     0.889     0.889         9
                        atis_flight_time      1.000     1.000     1.000         1
                           atis_quantity      0.375     1.000     0.545         3
                atis_flight#atis_airfare      0.800     0.333     0.471        12
                            atis_airport      1.000     0.944     0.971        18
                           atis_distance      1.000     1.000     1.000        10
                               atis_city      0.667     0.667     0.667         6
                        atis_ground_fare      1.000     0.714     0.833         7
                           atis_capacity      1.000     0.952     0.976        21
                          atis_flight_no      0.889     1.000     0.941         8
                               atis_meal      1.000     0.667     0.800         6
                        atis_restriction      0.000     0.000     0.000         0
             atis_airline#atis_flight_no      0.000     0.000     0.000         0
    atis_ground_service#atis_ground_fare      0.000     0.000     0.000         0
           atis_airfare#atis_flight_time      0.000     0.000     0.000         0
                           atis_cheapest      0.000     0.000     0.000         0
atis_aircraft#atis_flight#atis_flight_no      0.000     0.000     0.000         0

                               micro avg      0.965     0.971     0.968       888
                               macro avg      0.659     0.641     0.636       888
                            weighted avg      0.969     0.971     0.967       888

I0717 07:16:46.901127 140695199004544 interactiveshell.py:2882] 
                              precision    recall  f1-score   support

                           O      0.000     0.000     0.000       0.0
           B-toloc.city_name      0.974     0.994     0.984     716.0
         B-fromloc.city_name      0.987     0.994     0.991     704.0
           I-toloc.city_name      0.967     1.000     0.983     265.0
      B-depart_date.day_name      0.990     0.981     0.986     212.0
              B-airline_name      0.971     1.000     0.985     101.0
         I-fromloc.city_name      0.983     0.983     0.983     177.0
 B-depart_time.period_of_day      1.000     0.892     0.943     130.0
              I-airline_name      1.000     1.000     1.000      65.0
    B-depart_date.day_number      0.964     0.964     0.964      55.0
    B-depart_date.month_name      0.982     0.964     0.973      56.0
          B-depart_time.time      0.803     1.000     0.891      57.0
                B-round_trip      1.000     0.986     0.993      73.0
             B-cost_relative      1.000     0.973     0.986      37.0
                I-round_trip      1.000     1.000     1.000      71.0
                B-flight_mod      0.719     0.958     0.821      24.0
 B-depart_time.time_relative      0.984     0.969     0.977      65.0
          I-depart_time.time      0.945     1.000     0.972      52.0
         B-stoploc.city_name      1.000     1.000     1.000      20.0
                 B-city_name      0.868     0.579     0.695      57.0
                B-class_type      0.960     1.000     0.980      24.0
          B-arrive_time.time      0.943     0.971     0.957      34.0
 B-arrive_time.time_relative      0.909     0.968     0.937      31.0
                I-class_type      1.000     1.000     1.000      17.0
               B-flight_stop      1.000     1.000     1.000      21.0
          I-arrive_time.time      1.000     0.971     0.986      35.0
              B-airline_code      1.000     0.853     0.921      34.0
    I-depart_date.day_number      1.000     1.000     1.000      15.0
      I-fromloc.airport_name      0.441     1.000     0.612      15.0
      B-fromloc.airport_name      0.367     0.917     0.524      12.0
      B-arrive_date.day_name      0.688     1.000     0.815      11.0
          B-toloc.state_code      0.947     1.000     0.973      18.0
B-depart_date.today_relative      1.000     1.000     1.000       9.0
             B-flight_number      0.375     0.818     0.514      11.0
 B-depart_date.date_relative      0.944     1.000     0.971      17.0
          B-toloc.state_name      0.903     1.000     0.949      28.0
           B-fare_basis_code      0.917     0.647     0.759      17.0
               B-flight_time      1.000     1.000     1.000       1.0
                        B-or      1.000     1.000     1.000       3.0
 B-arrive_time.period_of_day      0.750     1.000     0.857       6.0
          B-meal_description      1.000     0.900     0.947      10.0
             I-cost_relative      1.000     0.667     0.800       3.0
              I-airport_name      0.923     0.414     0.571      29.0
               B-fare_amount      0.667     1.000     0.800       2.0
               I-fare_amount      1.000     1.000     1.000       2.0
                 I-city_name      1.000     0.567     0.723      30.0
        I-toloc.airport_name      1.000     1.000     1.000       3.0
            B-transport_type      1.000     0.900     0.947      10.0
    B-arrive_date.month_name      0.714     0.833     0.769       6.0
    B-arrive_date.day_number      0.714     0.833     0.769       6.0
         I-stoploc.city_name      1.000     1.000     1.000      10.0
                      B-meal      1.000     1.000     1.000      16.0
        B-fromloc.state_code      1.000     0.957     0.978      23.0
    B-depart_time.period_mod      1.000     1.000     1.000       5.0
                   B-connect      1.000     1.000     1.000       6.0
               B-flight_days      1.000     1.000     1.000      10.0
        B-toloc.airport_name      0.500     0.333     0.400       3.0
        B-fromloc.state_name      0.944     1.000     0.971      17.0
              B-airport_name      0.800     0.381     0.516      21.0
                   B-economy      1.000     1.000     1.000       6.0
               I-flight_time      1.000     1.000     1.000       1.0
             B-aircraft_code      0.964     0.818     0.885      33.0
                       B-mod      0.000     0.000     0.000       2.0
              B-airport_code      0.000     0.000     0.000       9.0
    B-depart_time.start_time      0.667     0.667     0.667       3.0
      B-depart_time.end_time      1.000     0.667     0.800       3.0
          B-depart_date.year      1.000     0.667     0.800       3.0
            I-transport_type      0.500     1.000     0.667       1.0
          B-restriction_code      0.333     0.750     0.462       4.0
    B-arrive_time.start_time      0.875     0.875     0.875       8.0
        B-toloc.airport_code      1.000     0.500     0.667       4.0
      B-arrive_time.end_time      0.889     1.000     0.941       8.0
      I-arrive_time.end_time      0.889     1.000     0.941       8.0
      I-depart_time.end_time      1.000     0.333     0.500       3.0
               I-flight_stop      0.000     0.000     0.000       0.0
      B-fromloc.airport_code      1.000     1.000     1.000       5.0
          I-restriction_code      0.500     1.000     0.667       3.0
    I-depart_time.start_time      0.500     1.000     0.667       1.0
          I-toloc.state_name      1.000     1.000     1.000       1.0
I-depart_date.today_relative      0.000     0.000     0.000       0.0
 B-arrive_date.date_relative      1.000     1.000     1.000       2.0
                I-flight_mod      1.000     0.167     0.286       6.0
                   I-economy      0.000     0.000     0.000       0.0
 B-return_date.date_relative      1.000     0.333     0.500       3.0
        I-fromloc.state_name      1.000     1.000     1.000       1.0
                B-state_code      1.000     1.000     1.000       1.0
    I-arrive_time.start_time      1.000     1.000     1.000       1.0
    I-arrive_date.day_number      0.000     0.000     0.000       0.0
                 B-meal_code      0.000     0.000     0.000       1.0
 I-depart_time.period_of_day      1.000     1.000     1.000       1.0
                  B-day_name      1.000     0.500     0.667       2.0
             B-period_of_day      1.000     0.500     0.667       4.0
        B-stoploc.state_code      0.000     0.000     0.000       0.0
    B-return_date.month_name      0.000     0.000     0.000       0.0
    B-return_date.day_number      0.000     0.000     0.000       0.0
    B-arrive_time.period_mod      0.000     0.000     0.000       0.0
                 I-meal_code      0.000     0.000     0.000       0.0
        B-toloc.country_name      1.000     1.000     1.000       1.0
                 B-days_code      1.000     1.000     1.000       1.0
 I-arrive_time.period_of_day      0.000     0.000     0.000       0.0
            I-today_relative      0.000     0.000     0.000       0.0
 B-return_time.period_of_day      0.000     0.000     0.000       0.0
                      B-time      0.000     0.000     0.000       0.0
           I-fare_basis_code      0.000     0.000     0.000       0.0
 I-arrive_time.time_relative      1.000     0.500     0.667       4.0
 I-depart_time.time_relative      0.000     0.000     0.000       1.0
            B-today_relative      0.000     0.000     0.000       0.0
                B-state_name      1.000     0.111     0.200       9.0
B-arrive_date.today_relative      0.000     0.000     0.000       0.0
    B-return_time.period_mod      0.000     0.000     0.000       0.0
                B-month_name      0.000     0.000     0.000       0.0
                B-day_number      0.000     0.000     0.000       0.0
 I-return_date.date_relative      0.667     0.667     0.667       3.0
I-return_date.today_relative      0.000     0.000     0.000       0.0
      B-stoploc.airport_name      0.000     0.000     0.000       0.0
             B-time_relative      0.000     0.000     0.000       0.0
                      I-time      0.000     0.000     0.000       0.0
    I-return_date.day_number      0.000     0.000     0.000       0.0
          I-meal_description      0.000     0.000     0.000       0.0
B-return_date.today_relative      0.000     0.000     0.000       0.0
      B-return_date.day_name      0.000     0.000     0.000       2.0

                   micro avg      0.949     0.951     0.950    3657.0
                   macro avg      0.664     0.638     0.630    3657.0
                weighted avg      0.958     0.951     0.948    3657.0

I0717 07:16:46.902327 140695199004544 interactiveshell.py:2882] Best Slot F1: 0.951
Reading ../data/atis.train.w-intent.iob
I0717 07:16:50.999181 140695199004544 interactiveshell.py:2882] Step 17200 | Loss: 0.0077 | Loss_intent: 0.0000 | Loss_slots: 0.0077 | Spent: 6.6 secs | LR: 0.000049
I0717 07:16:55.185637 140695199004544 interactiveshell.py:2882] Step 17250 | Loss: 0.0013 | Loss_intent: 0.0001 | Loss_slots: 0.0013 | Spent: 4.2 secs | LR: 0.000049
I0717 07:16:59.355044 140695199004544 interactiveshell.py:2882] Step 17300 | Loss: 0.0013 | Loss_intent: 0.0000 | Loss_slots: 0.0013 | Spent: 4.2 secs | LR: 0.000048
I0717 07:17:03.536587 140695199004544 interactiveshell.py:2882] Step 17350 | Loss: 0.0087 | Loss_intent: 0.0000 | Loss_slots: 0.0087 | Spent: 4.2 secs | LR: 0.000048
I0717 07:17:08.043799 140695199004544 interactiveshell.py:2882] Step 17400 | Loss: 0.0090 | Loss_intent: 0.0000 | Loss_slots: 0.0090 | Spent: 4.5 secs | LR: 0.000048
I0717 07:17:12.298329 140695199004544 interactiveshell.py:2882] Step 17450 | Loss: 0.0058 | Loss_intent: 0.0009 | Loss_slots: 0.0049 | Spent: 4.3 secs | LR: 0.000048
Reading ../data/atis.test.w-intent.iob
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.
  'precision', 'predicted', average, warn_for)
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1439: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples.
  'recall', 'true', average, warn_for)
I0717 07:17:15.775067 140695199004544 interactiveshell.py:2882] 
                                          precision    recall  f1-score   support

                             atis_flight      0.979     0.983     0.981       632
                            atis_airfare      0.940     0.979     0.959        48
                     atis_ground_service      1.000     1.000     1.000        36
                            atis_airline      0.974     1.000     0.987        38
                       atis_abbreviation      1.000     1.000     1.000        33
                           atis_aircraft      0.889     0.889     0.889         9
                        atis_flight_time      1.000     1.000     1.000         1
                           atis_quantity      0.429     1.000     0.600         3
                atis_flight#atis_airfare      0.800     0.333     0.471        12
                            atis_airport      0.947     1.000     0.973        18
                           atis_distance      1.000     1.000     1.000        10
                               atis_city      0.545     1.000     0.706         6
                        atis_ground_fare      1.000     0.714     0.833         7
                           atis_capacity      1.000     0.952     0.976        21
                          atis_flight_no      0.889     1.000     0.941         8
                               atis_meal      1.000     0.667     0.800         6
                        atis_restriction      0.000     0.000     0.000         0
             atis_airline#atis_flight_no      0.000     0.000     0.000         0
    atis_ground_service#atis_ground_fare      0.000     0.000     0.000         0
           atis_airfare#atis_flight_time      0.000     0.000     0.000         0
                           atis_cheapest      0.000     0.000     0.000         0
atis_aircraft#atis_flight#atis_flight_no      0.000     0.000     0.000         0

                               micro avg      0.965     0.971     0.968       888
                               macro avg      0.654     0.660     0.642       888
                            weighted avg      0.970     0.971     0.968       888

I0717 07:17:15.815323 140695199004544 interactiveshell.py:2882] 
                              precision    recall  f1-score   support

                           O      0.000     0.000     0.000       0.0
           B-toloc.city_name      0.974     0.993     0.983     716.0
         B-fromloc.city_name      0.987     0.991     0.989     704.0
           I-toloc.city_name      0.974     0.992     0.983     265.0
      B-depart_date.day_name      0.990     0.981     0.986     212.0
              B-airline_name      0.971     1.000     0.985     101.0
         I-fromloc.city_name      0.978     0.989     0.983     177.0
 B-depart_time.period_of_day      0.992     0.915     0.952     130.0
              I-airline_name      1.000     1.000     1.000      65.0
    B-depart_date.day_number      0.964     0.964     0.964      55.0
    B-depart_date.month_name      0.982     0.964     0.973      56.0
          B-depart_time.time      0.792     1.000     0.884      57.0
                B-round_trip      1.000     0.986     0.993      73.0
             B-cost_relative      1.000     0.973     0.986      37.0
                I-round_trip      1.000     1.000     1.000      71.0
                B-flight_mod      0.821     0.958     0.885      24.0
 B-depart_time.time_relative      0.984     0.969     0.977      65.0
          I-depart_time.time      0.963     1.000     0.981      52.0
         B-stoploc.city_name      0.952     1.000     0.976      20.0
                 B-city_name      0.821     0.561     0.667      57.0
                B-class_type      0.960     1.000     0.980      24.0
          B-arrive_time.time      0.971     0.971     0.971      34.0
 B-arrive_time.time_relative      0.909     0.968     0.937      31.0
                I-class_type      1.000     1.000     1.000      17.0
               B-flight_stop      1.000     1.000     1.000      21.0
          I-arrive_time.time      0.971     0.971     0.971      35.0
              B-airline_code      1.000     0.853     0.921      34.0
    I-depart_date.day_number      1.000     1.000     1.000      15.0
      I-fromloc.airport_name      0.429     1.000     0.600      15.0
      B-fromloc.airport_name      0.379     0.917     0.537      12.0
      B-arrive_date.day_name      0.688     1.000     0.815      11.0
          B-toloc.state_code      1.000     1.000     1.000      18.0
B-depart_date.today_relative      1.000     1.000     1.000       9.0
             B-flight_number      0.346     0.818     0.486      11.0
 B-depart_date.date_relative      0.941     0.941     0.941      17.0
          B-toloc.state_name      0.903     1.000     0.949      28.0
           B-fare_basis_code      0.917     0.647     0.759      17.0
               B-flight_time      1.000     1.000     1.000       1.0
                        B-or      1.000     1.000     1.000       3.0
 B-arrive_time.period_of_day      0.750     1.000     0.857       6.0
          B-meal_description      1.000     0.900     0.947      10.0
             I-cost_relative      1.000     0.667     0.800       3.0
              I-airport_name      0.917     0.379     0.537      29.0
               B-fare_amount      1.000     0.500     0.667       2.0
               I-fare_amount      1.000     1.000     1.000       2.0
                 I-city_name      1.000     0.600     0.750      30.0
        I-toloc.airport_name      1.000     1.000     1.000       3.0
            B-transport_type      1.000     0.700     0.824      10.0
    B-arrive_date.month_name      0.714     0.833     0.769       6.0
    B-arrive_date.day_number      0.714     0.833     0.769       6.0
         I-stoploc.city_name      0.833     1.000     0.909      10.0
                      B-meal      1.000     1.000     1.000      16.0
        B-fromloc.state_code      1.000     1.000     1.000      23.0
    B-depart_time.period_mod      1.000     1.000     1.000       5.0
                   B-connect      1.000     1.000     1.000       6.0
               B-flight_days      1.000     1.000     1.000      10.0
        B-toloc.airport_name      0.667     0.667     0.667       3.0
        B-fromloc.state_name      0.944     1.000     0.971      17.0
              B-airport_name      0.800     0.381     0.516      21.0
                   B-economy      1.000     1.000     1.000       6.0
               I-flight_time      1.000     1.000     1.000       1.0
             B-aircraft_code      0.963     0.788     0.867      33.0
                       B-mod      1.000     0.500     0.667       2.0
              B-airport_code      0.000     0.000     0.000       9.0
    B-depart_time.start_time      0.667     0.667     0.667       3.0
      B-depart_time.end_time      1.000     0.667     0.800       3.0
          B-depart_date.year      1.000     0.667     0.800       3.0
            I-transport_type      0.250     1.000     0.400       1.0
          B-restriction_code      0.333     0.750     0.462       4.0
    B-arrive_time.start_time      0.889     1.000     0.941       8.0
        B-toloc.airport_code      1.000     0.500     0.667       4.0
      B-arrive_time.end_time      0.889     1.000     0.941       8.0
      I-arrive_time.end_time      0.875     0.875     0.875       8.0
      I-depart_time.end_time      1.000     0.333     0.500       3.0
               I-flight_stop      0.000     0.000     0.000       0.0
      B-fromloc.airport_code      0.833     1.000     0.909       5.0
          I-restriction_code      0.600     1.000     0.750       3.0
    I-depart_time.start_time      0.500     1.000     0.667       1.0
          I-toloc.state_name      1.000     1.000     1.000       1.0
I-depart_date.today_relative      0.000     0.000     0.000       0.0
 B-arrive_date.date_relative      1.000     1.000     1.000       2.0
                I-flight_mod      1.000     0.167     0.286       6.0
                   I-economy      0.000     0.000     0.000       0.0
 B-return_date.date_relative      0.500     0.333     0.400       3.0
        I-fromloc.state_name      1.000     1.000     1.000       1.0
                B-state_code      1.000     1.000     1.000       1.0
    I-arrive_time.start_time      1.000     1.000     1.000       1.0
    I-arrive_date.day_number      0.000     0.000     0.000       0.0
                 B-meal_code      1.000     1.000     1.000       1.0
 I-depart_time.period_of_day      0.000     0.000     0.000       1.0
                  B-day_name      1.000     0.500     0.667       2.0
             B-period_of_day      1.000     0.500     0.667       4.0
        B-stoploc.state_code      0.000     0.000     0.000       0.0
    B-return_date.month_name      0.000     0.000     0.000       0.0
    B-return_date.day_number      0.000     0.000     0.000       0.0
    B-arrive_time.period_mod      0.000     0.000     0.000       0.0
                 I-meal_code      0.000     0.000     0.000       0.0
        B-toloc.country_name      1.000     1.000     1.000       1.0
                 B-days_code      1.000     1.000     1.000       1.0
 I-arrive_time.period_of_day      0.000     0.000     0.000       0.0
            I-today_relative      0.000     0.000     0.000       0.0
 B-return_time.period_of_day      0.000     0.000     0.000       0.0
                      B-time      0.000     0.000     0.000       0.0
           I-fare_basis_code      0.000     0.000     0.000       0.0
 I-arrive_time.time_relative      1.000     0.500     0.667       4.0
 I-depart_time.time_relative      0.000     0.000     0.000       1.0
            B-today_relative      0.000     0.000     0.000       0.0
                B-state_name      1.000     0.111     0.200       9.0
B-arrive_date.today_relative      0.000     0.000     0.000       0.0
    B-return_time.period_mod      0.000     0.000     0.000       0.0
                B-month_name      0.000     0.000     0.000       0.0
                B-day_number      0.000     0.000     0.000       0.0
 I-return_date.date_relative      0.750     1.000     0.857       3.0
I-return_date.today_relative      0.000     0.000     0.000       0.0
      B-stoploc.airport_name      0.000     0.000     0.000       0.0
             B-time_relative      0.000     0.000     0.000       0.0
                      I-time      0.000     0.000     0.000       0.0
    I-return_date.day_number      0.000     0.000     0.000       0.0
          I-meal_description      0.000     0.000     0.000       0.0
B-return_date.today_relative      0.000     0.000     0.000       0.0
      B-return_date.day_name      0.000     0.000     0.000       2.0

                   micro avg      0.948     0.950     0.949    3657.0
                   macro avg      0.669     0.642     0.633    3657.0
                weighted avg      0.957     0.950     0.947    3657.0

I0717 07:17:15.816229 140695199004544 interactiveshell.py:2882] Best Slot F1: 0.951
Reading ../data/atis.train.w-intent.iob
I0717 07:17:18.891889 140695199004544 interactiveshell.py:2882] Step 17500 | Loss: 0.0026 | Loss_intent: 0.0010 | Loss_slots: 0.0016 | Spent: 6.6 secs | LR: 0.000047
I0717 07:17:23.097444 140695199004544 interactiveshell.py:2882] Step 17550 | Loss: 0.0085 | Loss_intent: 0.0000 | Loss_slots: 0.0085 | Spent: 4.2 secs | LR: 0.000047
I0717 07:17:27.277808 140695199004544 interactiveshell.py:2882] Step 17600 | Loss: 0.0016 | Loss_intent: 0.0000 | Loss_slots: 0.0015 | Spent: 4.2 secs | LR: 0.000047
I0717 07:17:31.452935 140695199004544 interactiveshell.py:2882] Step 17650 | Loss: 0.0144 | Loss_intent: 0.0024 | Loss_slots: 0.0120 | Spent: 4.2 secs | LR: 0.000047
I0717 07:17:35.628699 140695199004544 interactiveshell.py:2882] Step 17700 | Loss: 0.0013 | Loss_intent: 0.0001 | Loss_slots: 0.0012 | Spent: 4.2 secs | LR: 0.000046
I0717 07:17:39.823620 140695199004544 interactiveshell.py:2882] Step 17750 | Loss: 0.0071 | Loss_intent: 0.0001 | Loss_slots: 0.0070 | Spent: 4.2 secs | LR: 0.000046
Reading ../data/atis.test.w-intent.iob
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.
  'precision', 'predicted', average, warn_for)
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1439: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples.
  'recall', 'true', average, warn_for)
I0717 07:17:44.295534 140695199004544 interactiveshell.py:2882] 
                                          precision    recall  f1-score   support

                             atis_flight      0.987     0.981     0.984       632
                            atis_airfare      0.940     0.979     0.959        48
                     atis_ground_service      1.000     1.000     1.000        36
                            atis_airline      0.974     0.974     0.974        38
                       atis_abbreviation      1.000     0.970     0.985        33
                           atis_aircraft      0.800     0.889     0.842         9
                        atis_flight_time      1.000     1.000     1.000         1
                           atis_quantity      0.375     1.000     0.545         3
                atis_flight#atis_airfare      0.875     0.583     0.700        12
                            atis_airport      0.947     1.000     0.973        18
                           atis_distance      1.000     1.000     1.000        10
                               atis_city      0.545     1.000     0.706         6
                        atis_ground_fare      1.000     0.714     0.833         7
                           atis_capacity      1.000     1.000     1.000        21
                          atis_flight_no      0.889     1.000     0.941         8
                               atis_meal      1.000     0.833     0.909         6
                        atis_restriction      0.000     0.000     0.000         0
             atis_airline#atis_flight_no      0.000     0.000     0.000         0
    atis_ground_service#atis_ground_fare      0.000     0.000     0.000         0
           atis_airfare#atis_flight_time      0.000     0.000     0.000         0
                           atis_cheapest      0.000     0.000     0.000         0
atis_aircraft#atis_flight#atis_flight_no      0.000     0.000     0.000         0

                               micro avg      0.968     0.973     0.970       888
                               macro avg      0.651     0.678     0.652       888
                            weighted avg      0.976     0.973     0.973       888

I0717 07:17:44.343163 140695199004544 interactiveshell.py:2882] 
                              precision    recall  f1-score   support

                           O      0.000     0.000     0.000       0.0
           B-toloc.city_name      0.975     0.994     0.985     716.0
         B-fromloc.city_name      0.991     0.993     0.992     704.0
           I-toloc.city_name      0.964     1.000     0.981     265.0
      B-depart_date.day_name      0.990     0.976     0.983     212.0
              B-airline_name      0.981     1.000     0.990     101.0
         I-fromloc.city_name      0.989     0.983     0.986     177.0
 B-depart_time.period_of_day      1.000     0.892     0.943     130.0
              I-airline_name      1.000     1.000     1.000      65.0
    B-depart_date.day_number      0.964     0.964     0.964      55.0
    B-depart_date.month_name      0.982     0.964     0.973      56.0
          B-depart_time.time      0.814     1.000     0.898      57.0
                B-round_trip      1.000     0.986     0.993      73.0
             B-cost_relative      1.000     0.973     0.986      37.0
                I-round_trip      1.000     1.000     1.000      71.0
                B-flight_mod      1.000     0.958     0.979      24.0
 B-depart_time.time_relative      0.984     0.969     0.977      65.0
          I-depart_time.time      0.929     1.000     0.963      52.0
         B-stoploc.city_name      0.952     1.000     0.976      20.0
                 B-city_name      0.833     0.614     0.707      57.0
                B-class_type      0.960     1.000     0.980      24.0
          B-arrive_time.time      0.971     0.971     0.971      34.0
 B-arrive_time.time_relative      0.909     0.968     0.937      31.0
                I-class_type      1.000     1.000     1.000      17.0
               B-flight_stop      1.000     1.000     1.000      21.0
          I-arrive_time.time      0.971     0.971     0.971      35.0
              B-airline_code      1.000     0.853     0.921      34.0
    I-depart_date.day_number      1.000     1.000     1.000      15.0
      I-fromloc.airport_name      0.417     1.000     0.588      15.0
      B-fromloc.airport_name      0.423     0.917     0.579      12.0
      B-arrive_date.day_name      0.647     1.000     0.786      11.0
          B-toloc.state_code      1.000     1.000     1.000      18.0
B-depart_date.today_relative      1.000     1.000     1.000       9.0
             B-flight_number      0.375     0.818     0.514      11.0
 B-depart_date.date_relative      0.944     1.000     0.971      17.0
          B-toloc.state_name      0.929     0.929     0.929      28.0
           B-fare_basis_code      0.917     0.647     0.759      17.0
               B-flight_time      1.000     1.000     1.000       1.0
                        B-or      1.000     1.000     1.000       3.0
 B-arrive_time.period_of_day      0.750     1.000     0.857       6.0
          B-meal_description      1.000     0.900     0.947      10.0
             I-cost_relative      1.000     0.667     0.800       3.0
              I-airport_name      0.917     0.379     0.537      29.0
               B-fare_amount      0.500     0.500     0.500       2.0
               I-fare_amount      1.000     1.000     1.000       2.0
                 I-city_name      1.000     0.567     0.723      30.0
        I-toloc.airport_name      1.000     1.000     1.000       3.0
            B-transport_type      1.000     0.700     0.824      10.0
    B-arrive_date.month_name      0.714     0.833     0.769       6.0
    B-arrive_date.day_number      0.714     0.833     0.769       6.0
         I-stoploc.city_name      1.000     1.000     1.000      10.0
                      B-meal      1.000     1.000     1.000      16.0
        B-fromloc.state_code      1.000     1.000     1.000      23.0
    B-depart_time.period_mod      0.714     1.000     0.833       5.0
                   B-connect      1.000     1.000     1.000       6.0
               B-flight_days      1.000     1.000     1.000      10.0
        B-toloc.airport_name      0.667     0.667     0.667       3.0
        B-fromloc.state_name      0.895     1.000     0.944      17.0
              B-airport_name      0.800     0.381     0.516      21.0
                   B-economy      1.000     1.000     1.000       6.0
               I-flight_time      1.000     1.000     1.000       1.0
             B-aircraft_code      0.906     0.879     0.892      33.0
                       B-mod      1.000     0.500     0.667       2.0
              B-airport_code      0.000     0.000     0.000       9.0
    B-depart_time.start_time      0.667     0.667     0.667       3.0
      B-depart_time.end_time      1.000     0.667     0.800       3.0
          B-depart_date.year      1.000     0.667     0.800       3.0
            I-transport_type      0.200     1.000     0.333       1.0
          B-restriction_code      0.364     1.000     0.533       4.0
    B-arrive_time.start_time      0.889     1.000     0.941       8.0
        B-toloc.airport_code      1.000     0.500     0.667       4.0
      B-arrive_time.end_time      0.889     1.000     0.941       8.0
      I-arrive_time.end_time      0.875     0.875     0.875       8.0
      I-depart_time.end_time      1.000     0.333     0.500       3.0
               I-flight_stop      0.000     0.000     0.000       0.0
      B-fromloc.airport_code      0.833     1.000     0.909       5.0
          I-restriction_code      1.000     1.000     1.000       3.0
    I-depart_time.start_time      0.500     1.000     0.667       1.0
          I-toloc.state_name      1.000     1.000     1.000       1.0
I-depart_date.today_relative      0.000     0.000     0.000       0.0
 B-arrive_date.date_relative      1.000     1.000     1.000       2.0
                I-flight_mod      1.000     0.167     0.286       6.0
                   I-economy      0.000     0.000     0.000       0.0
 B-return_date.date_relative      1.000     0.333     0.500       3.0
        I-fromloc.state_name      1.000     1.000     1.000       1.0
                B-state_code      1.000     1.000     1.000       1.0
    I-arrive_time.start_time      1.000     1.000     1.000       1.0
    I-arrive_date.day_number      0.000     0.000     0.000       0.0
                 B-meal_code      1.000     1.000     1.000       1.0
 I-depart_time.period_of_day      1.000     1.000     1.000       1.0
                  B-day_name      1.000     0.500     0.667       2.0
             B-period_of_day      1.000     0.500     0.667       4.0
        B-stoploc.state_code      0.000     0.000     0.000       0.0
    B-return_date.month_name      0.000     0.000     0.000       0.0
    B-return_date.day_number      0.000     0.000     0.000       0.0
    B-arrive_time.period_mod      0.000     0.000     0.000       0.0
                 I-meal_code      0.000     0.000     0.000       0.0
        B-toloc.country_name      1.000     1.000     1.000       1.0
                 B-days_code      1.000     1.000     1.000       1.0
 I-arrive_time.period_of_day      0.000     0.000     0.000       0.0
            I-today_relative      0.000     0.000     0.000       0.0
 B-return_time.period_of_day      0.000     0.000     0.000       0.0
                      B-time      0.000     0.000     0.000       0.0
           I-fare_basis_code      0.000     0.000     0.000       0.0
 I-arrive_time.time_relative      1.000     0.500     0.667       4.0
 I-depart_time.time_relative      0.000     0.000     0.000       1.0
            B-today_relative      0.000     0.000     0.000       0.0
                B-state_name      1.000     0.222     0.364       9.0
B-arrive_date.today_relative      0.000     0.000     0.000       0.0
    B-return_time.period_mod      0.000     0.000     0.000       0.0
                B-month_name      0.000     0.000     0.000       0.0
                B-day_number      0.000     0.000     0.000       0.0
 I-return_date.date_relative      0.750     1.000     0.857       3.0
I-return_date.today_relative      0.000     0.000     0.000       0.0
      B-stoploc.airport_name      0.000     0.000     0.000       0.0
             B-time_relative      0.000     0.000     0.000       0.0
                      I-time      0.000     0.000     0.000       0.0
    I-return_date.day_number      0.000     0.000     0.000       0.0
          I-meal_description      0.000     0.000     0.000       0.0
B-return_date.today_relative      0.000     0.000     0.000       0.0
      B-return_date.day_name      0.000     0.000     0.000       2.0

                   micro avg      0.950     0.951     0.951    3657.0
                   macro avg      0.681     0.654     0.645    3657.0
                weighted avg      0.960     0.951     0.950    3657.0

I0717 07:17:44.343908 140695199004544 interactiveshell.py:2882] Best Slot F1: 0.951
Reading ../data/atis.train.w-intent.iob
I0717 07:17:46.428725 140695199004544 interactiveshell.py:2882] Step 17800 | Loss: 0.0045 | Loss_intent: 0.0001 | Loss_slots: 0.0044 | Spent: 6.6 secs | LR: 0.000046
I0717 07:17:50.606248 140695199004544 interactiveshell.py:2882] Step 17850 | Loss: 0.0033 | Loss_intent: 0.0001 | Loss_slots: 0.0032 | Spent: 4.2 secs | LR: 0.000046
I0717 07:17:54.835326 140695199004544 interactiveshell.py:2882] Step 17900 | Loss: 0.0027 | Loss_intent: 0.0000 | Loss_slots: 0.0027 | Spent: 4.2 secs | LR: 0.000046
I0717 07:17:59.020902 140695199004544 interactiveshell.py:2882] Step 17950 | Loss: 0.0164 | Loss_intent: 0.0074 | Loss_slots: 0.0090 | Spent: 4.2 secs | LR: 0.000045
I0717 07:18:03.271838 140695199004544 interactiveshell.py:2882] Step 18000 | Loss: 0.0088 | Loss_intent: 0.0000 | Loss_slots: 0.0088 | Spent: 4.2 secs | LR: 0.000045
I0717 07:18:07.534608 140695199004544 interactiveshell.py:2882] Step 18050 | Loss: 0.0020 | Loss_intent: 0.0000 | Loss_slots: 0.0020 | Spent: 4.3 secs | LR: 0.000045
Reading ../data/atis.test.w-intent.iob
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.
  'precision', 'predicted', average, warn_for)
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1439: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples.
  'recall', 'true', average, warn_for)
I0717 07:18:13.017204 140695199004544 interactiveshell.py:2882] 
                                          precision    recall  f1-score   support

                             atis_flight      0.984     0.981     0.983       632
                            atis_airfare      0.923     1.000     0.960        48
                     atis_ground_service      1.000     1.000     1.000        36
                            atis_airline      0.974     1.000     0.987        38
                       atis_abbreviation      1.000     1.000     1.000        33
                           atis_aircraft      0.818     1.000     0.900         9
                        atis_flight_time      1.000     1.000     1.000         1
                           atis_quantity      0.429     1.000     0.600         3
                atis_flight#atis_airfare      0.800     0.333     0.471        12
                            atis_airport      0.947     1.000     0.973        18
                           atis_distance      1.000     1.000     1.000        10
                               atis_city      0.545     1.000     0.706         6
                        atis_ground_fare      1.000     0.714     0.833         7
                           atis_capacity      1.000     0.952     0.976        21
                          atis_flight_no      0.889     1.000     0.941         8
                               atis_meal      1.000     0.833     0.909         6
                        atis_restriction      0.000     0.000     0.000         0
             atis_airline#atis_flight_no      0.000     0.000     0.000         0
    atis_ground_service#atis_ground_fare      0.000     0.000     0.000         0
           atis_airfare#atis_flight_time      0.000     0.000     0.000         0
                           atis_cheapest      0.000     0.000     0.000         0
atis_aircraft#atis_flight#atis_flight_no      0.000     0.000     0.000         0

                               micro avg      0.968     0.973     0.970       888
                               macro avg      0.650     0.673     0.647       888
                            weighted avg      0.972     0.973     0.970       888

I0717 07:18:13.058234 140695199004544 interactiveshell.py:2882] 
                              precision    recall  f1-score   support

                           O      0.000     0.000     0.000       0.0
           B-toloc.city_name      0.977     0.993     0.985     716.0
         B-fromloc.city_name      0.990     0.991     0.991     704.0
           I-toloc.city_name      0.971     0.996     0.983     265.0
      B-depart_date.day_name      0.990     0.981     0.986     212.0
              B-airline_name      0.971     1.000     0.985     101.0
         I-fromloc.city_name      0.983     0.983     0.983     177.0
 B-depart_time.period_of_day      1.000     0.908     0.952     130.0
              I-airline_name      1.000     1.000     1.000      65.0
    B-depart_date.day_number      0.981     0.964     0.972      55.0
    B-depart_date.month_name      0.982     0.964     0.973      56.0
          B-depart_time.time      0.838     1.000     0.912      57.0
                B-round_trip      1.000     0.986     0.993      73.0
             B-cost_relative      1.000     0.973     0.986      37.0
                I-round_trip      1.000     1.000     1.000      71.0
                B-flight_mod      1.000     0.958     0.979      24.0
 B-depart_time.time_relative      0.984     0.969     0.977      65.0
          I-depart_time.time      0.963     1.000     0.981      52.0
         B-stoploc.city_name      0.952     1.000     0.976      20.0
                 B-city_name      0.833     0.614     0.707      57.0
                B-class_type      0.923     1.000     0.960      24.0
          B-arrive_time.time      0.971     0.971     0.971      34.0
 B-arrive_time.time_relative      0.909     0.968     0.937      31.0
                I-class_type      1.000     1.000     1.000      17.0
               B-flight_stop      0.955     1.000     0.977      21.0
          I-arrive_time.time      1.000     0.971     0.986      35.0
              B-airline_code      1.000     0.853     0.921      34.0
    I-depart_date.day_number      1.000     1.000     1.000      15.0
      I-fromloc.airport_name      0.429     1.000     0.600      15.0
      B-fromloc.airport_name      0.393     0.917     0.550      12.0
      B-arrive_date.day_name      0.688     1.000     0.815      11.0
          B-toloc.state_code      1.000     1.000     1.000      18.0
B-depart_date.today_relative      1.000     1.000     1.000       9.0
             B-flight_number      0.333     0.818     0.474      11.0
 B-depart_date.date_relative      0.944     1.000     0.971      17.0
          B-toloc.state_name      0.929     0.929     0.929      28.0
           B-fare_basis_code      0.917     0.647     0.759      17.0
               B-flight_time      1.000     1.000     1.000       1.0
                        B-or      1.000     1.000     1.000       3.0
 B-arrive_time.period_of_day      0.750     1.000     0.857       6.0
          B-meal_description      1.000     0.900     0.947      10.0
             I-cost_relative      1.000     0.667     0.800       3.0
              I-airport_name      0.917     0.379     0.537      29.0
               B-fare_amount      0.667     1.000     0.800       2.0
               I-fare_amount      1.000     1.000     1.000       2.0
                 I-city_name      1.000     0.600     0.750      30.0
        I-toloc.airport_name      1.000     1.000     1.000       3.0
            B-transport_type      1.000     1.000     1.000      10.0
    B-arrive_date.month_name      0.714     0.833     0.769       6.0
    B-arrive_date.day_number      0.714     0.833     0.769       6.0
         I-stoploc.city_name      0.833     1.000     0.909      10.0
                      B-meal      1.000     1.000     1.000      16.0
        B-fromloc.state_code      1.000     1.000     1.000      23.0
    B-depart_time.period_mod      0.714     1.000     0.833       5.0
                   B-connect      1.000     1.000     1.000       6.0
               B-flight_days      1.000     1.000     1.000      10.0
        B-toloc.airport_name      0.750     1.000     0.857       3.0
        B-fromloc.state_name      0.850     1.000     0.919      17.0
              B-airport_name      0.800     0.381     0.516      21.0
                   B-economy      1.000     1.000     1.000       6.0
               I-flight_time      1.000     1.000     1.000       1.0
             B-aircraft_code      0.962     0.758     0.847      33.0
                       B-mod      1.000     0.500     0.667       2.0
              B-airport_code      0.000     0.000     0.000       9.0
    B-depart_time.start_time      0.667     0.667     0.667       3.0
      B-depart_time.end_time      1.000     0.667     0.800       3.0
          B-depart_date.year      1.000     0.667     0.800       3.0
            I-transport_type      1.000     1.000     1.000       1.0
          B-restriction_code      0.364     1.000     0.533       4.0
    B-arrive_time.start_time      0.889     1.000     0.941       8.0
        B-toloc.airport_code      1.000     0.500     0.667       4.0
      B-arrive_time.end_time      0.889     1.000     0.941       8.0
      I-arrive_time.end_time      0.889     1.000     0.941       8.0
      I-depart_time.end_time      1.000     0.333     0.500       3.0
               I-flight_stop      0.000     0.000     0.000       0.0
      B-fromloc.airport_code      0.833     1.000     0.909       5.0
          I-restriction_code      1.000     1.000     1.000       3.0
    I-depart_time.start_time      0.500     1.000     0.667       1.0
          I-toloc.state_name      1.000     1.000     1.000       1.0
I-depart_date.today_relative      0.000     0.000     0.000       0.0
 B-arrive_date.date_relative      1.000     1.000     1.000       2.0
                I-flight_mod      1.000     0.167     0.286       6.0
                   I-economy      0.000     0.000     0.000       0.0
 B-return_date.date_relative      1.000     0.333     0.500       3.0
        I-fromloc.state_name      1.000     1.000     1.000       1.0
                B-state_code      1.000     1.000     1.000       1.0
    I-arrive_time.start_time      1.000     1.000     1.000       1.0
    I-arrive_date.day_number      0.000     0.000     0.000       0.0
                 B-meal_code      1.000     1.000     1.000       1.0
 I-depart_time.period_of_day      1.000     1.000     1.000       1.0
                  B-day_name      1.000     0.500     0.667       2.0
             B-period_of_day      1.000     0.500     0.667       4.0
        B-stoploc.state_code      0.000     0.000     0.000       0.0
    B-return_date.month_name      0.000     0.000     0.000       0.0
    B-return_date.day_number      0.000     0.000     0.000       0.0
    B-arrive_time.period_mod      0.000     0.000     0.000       0.0
                 I-meal_code      0.000     0.000     0.000       0.0
        B-toloc.country_name      1.000     1.000     1.000       1.0
                 B-days_code      1.000     1.000     1.000       1.0
 I-arrive_time.period_of_day      0.000     0.000     0.000       0.0
            I-today_relative      0.000     0.000     0.000       0.0
 B-return_time.period_of_day      0.000     0.000     0.000       0.0
                      B-time      0.000     0.000     0.000       0.0
           I-fare_basis_code      0.000     0.000     0.000       0.0
 I-arrive_time.time_relative      1.000     0.500     0.667       4.0
 I-depart_time.time_relative      0.000     0.000     0.000       1.0
            B-today_relative      0.000     0.000     0.000       0.0
                B-state_name      1.000     0.222     0.364       9.0
B-arrive_date.today_relative      0.000     0.000     0.000       0.0
    B-return_time.period_mod      0.000     0.000     0.000       0.0
                B-month_name      0.000     0.000     0.000       0.0
                B-day_number      0.000     0.000     0.000       0.0
 I-return_date.date_relative      0.750     1.000     0.857       3.0
I-return_date.today_relative      0.000     0.000     0.000       0.0
      B-stoploc.airport_name      0.000     0.000     0.000       0.0
             B-time_relative      0.000     0.000     0.000       0.0
                      I-time      0.000     0.000     0.000       0.0
    I-return_date.day_number      0.000     0.000     0.000       0.0
          I-meal_description      0.000     0.000     0.000       0.0
B-return_date.today_relative      0.000     0.000     0.000       0.0
      B-return_date.day_name      0.000     0.000     0.000       2.0

                   micro avg      0.951     0.952     0.951    3657.0
                   macro avg      0.688     0.663     0.656    3657.0
                weighted avg      0.961     0.952     0.951    3657.0

I0717 07:18:13.059054 140695199004544 interactiveshell.py:2882] Best Slot F1: 0.951
Reading ../data/atis.train.w-intent.iob
I0717 07:18:14.140707 140695199004544 interactiveshell.py:2882] Step 18100 | Loss: 0.0073 | Loss_intent: 0.0000 | Loss_slots: 0.0073 | Spent: 6.6 secs | LR: 0.000045
I0717 07:18:18.317097 140695199004544 interactiveshell.py:2882] Step 18150 | Loss: 0.0025 | Loss_intent: 0.0000 | Loss_slots: 0.0025 | Spent: 4.2 secs | LR: 0.000044
I0717 07:18:22.509021 140695199004544 interactiveshell.py:2882] Step 18200 | Loss: 0.0067 | Loss_intent: 0.0000 | Loss_slots: 0.0067 | Spent: 4.2 secs | LR: 0.000044
I0717 07:18:26.689043 140695199004544 interactiveshell.py:2882] Step 18250 | Loss: 0.0043 | Loss_intent: 0.0006 | Loss_slots: 0.0038 | Spent: 4.2 secs | LR: 0.000044
I0717 07:18:30.873675 140695199004544 interactiveshell.py:2882] Step 18300 | Loss: 0.0044 | Loss_intent: 0.0000 | Loss_slots: 0.0044 | Spent: 4.2 secs | LR: 0.000044
I0717 07:18:35.069429 140695199004544 interactiveshell.py:2882] Step 18350 | Loss: 0.0044 | Loss_intent: 0.0000 | Loss_slots: 0.0044 | Spent: 4.2 secs | LR: 0.000043
I0717 07:18:39.253667 140695199004544 interactiveshell.py:2882] Step 18400 | Loss: 0.0009 | Loss_intent: 0.0000 | Loss_slots: 0.0009 | Spent: 4.2 secs | LR: 0.000043
Reading ../data/atis.test.w-intent.iob
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.
  'precision', 'predicted', average, warn_for)
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1439: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples.
  'recall', 'true', average, warn_for)
I0717 07:18:41.690083 140695199004544 interactiveshell.py:2882] 
                                          precision    recall  f1-score   support

                             atis_flight      0.983     0.983     0.983       632
                            atis_airfare      0.923     1.000     0.960        48
                     atis_ground_service      1.000     1.000     1.000        36
                            atis_airline      0.974     0.974     0.974        38
                       atis_abbreviation      1.000     1.000     1.000        33
                           atis_aircraft      0.900     1.000     0.947         9
                        atis_flight_time      1.000     1.000     1.000         1
                           atis_quantity      0.429     1.000     0.600         3
                atis_flight#atis_airfare      0.800     0.333     0.471        12
                            atis_airport      0.947     1.000     0.973        18
                           atis_distance      1.000     1.000     1.000        10
                               atis_city      0.545     1.000     0.706         6
                        atis_ground_fare      1.000     0.714     0.833         7
                           atis_capacity      1.000     0.952     0.976        21
                          atis_flight_no      0.889     1.000     0.941         8
                               atis_meal      1.000     0.833     0.909         6
                        atis_restriction      0.000     0.000     0.000         0
             atis_airline#atis_flight_no      0.000     0.000     0.000         0
    atis_ground_service#atis_ground_fare      0.000     0.000     0.000         0
           atis_airfare#atis_flight_time      0.000     0.000     0.000         0
                           atis_cheapest      0.000     0.000     0.000         0
atis_aircraft#atis_flight#atis_flight_no      0.000     0.000     0.000         0

                               micro avg      0.968     0.973     0.970       888
                               macro avg      0.654     0.672     0.649       888
                            weighted avg      0.972     0.973     0.970       888

I0717 07:18:41.730012 140695199004544 interactiveshell.py:2882] 
                              precision    recall  f1-score   support

                           O      0.000     0.000     0.000       0.0
           B-toloc.city_name      0.973     0.993     0.983     716.0
         B-fromloc.city_name      0.986     0.994     0.990     704.0
           I-toloc.city_name      0.967     0.996     0.981     265.0
      B-depart_date.day_name      0.990     0.981     0.986     212.0
              B-airline_name      0.981     1.000     0.990     101.0
         I-fromloc.city_name      0.989     0.983     0.986     177.0
 B-depart_time.period_of_day      1.000     0.900     0.947     130.0
              I-airline_name      1.000     1.000     1.000      65.0
    B-depart_date.day_number      0.964     0.964     0.964      55.0
    B-depart_date.month_name      0.982     0.964     0.973      56.0
          B-depart_time.time      0.838     1.000     0.912      57.0
                B-round_trip      1.000     0.986     0.993      73.0
             B-cost_relative      1.000     0.973     0.986      37.0
                I-round_trip      1.000     1.000     1.000      71.0
                B-flight_mod      0.920     0.958     0.939      24.0
 B-depart_time.time_relative      0.984     0.969     0.977      65.0
          I-depart_time.time      0.945     1.000     0.972      52.0
         B-stoploc.city_name      0.952     1.000     0.976      20.0
                 B-city_name      0.868     0.579     0.695      57.0
                B-class_type      0.960     1.000     0.980      24.0
          B-arrive_time.time      0.971     0.971     0.971      34.0
 B-arrive_time.time_relative      0.909     0.968     0.937      31.0
                I-class_type      1.000     1.000     1.000      17.0
               B-flight_stop      0.955     1.000     0.977      21.0
          I-arrive_time.time      0.971     0.971     0.971      35.0
              B-airline_code      1.000     0.882     0.938      34.0
    I-depart_date.day_number      1.000     1.000     1.000      15.0
      I-fromloc.airport_name      0.429     1.000     0.600      15.0
      B-fromloc.airport_name      0.423     0.917     0.579      12.0
      B-arrive_date.day_name      0.688     1.000     0.815      11.0
          B-toloc.state_code      0.947     1.000     0.973      18.0
B-depart_date.today_relative      1.000     1.000     1.000       9.0
             B-flight_number      0.333     0.818     0.474      11.0
 B-depart_date.date_relative      0.944     1.000     0.971      17.0
          B-toloc.state_name      0.897     0.929     0.912      28.0
           B-fare_basis_code      0.917     0.647     0.759      17.0
               B-flight_time      1.000     1.000     1.000       1.0
                        B-or      1.000     1.000     1.000       3.0
 B-arrive_time.period_of_day      0.750     1.000     0.857       6.0
          B-meal_description      1.000     0.900     0.947      10.0
             I-cost_relative      1.000     0.667     0.800       3.0
              I-airport_name      0.917     0.379     0.537      29.0
               B-fare_amount      1.000     1.000     1.000       2.0
               I-fare_amount      1.000     1.000     1.000       2.0
                 I-city_name      1.000     0.567     0.723      30.0
        I-toloc.airport_name      0.750     1.000     0.857       3.0
            B-transport_type      1.000     0.800     0.889      10.0
    B-arrive_date.month_name      0.714     0.833     0.769       6.0
    B-arrive_date.day_number      0.714     0.833     0.769       6.0
         I-stoploc.city_name      0.769     1.000     0.870      10.0
                      B-meal      1.000     1.000     1.000      16.0
        B-fromloc.state_code      1.000     0.957     0.978      23.0
    B-depart_time.period_mod      1.000     1.000     1.000       5.0
                   B-connect      1.000     1.000     1.000       6.0
               B-flight_days      1.000     1.000     1.000      10.0
        B-toloc.airport_name      1.000     0.333     0.500       3.0
        B-fromloc.state_name      0.895     1.000     0.944      17.0
              B-airport_name      0.800     0.381     0.516      21.0
                   B-economy      1.000     1.000     1.000       6.0
               I-flight_time      1.000     1.000     1.000       1.0
             B-aircraft_code      0.929     0.788     0.852      33.0
                       B-mod      0.000     0.000     0.000       2.0
              B-airport_code      0.000     0.000     0.000       9.0
    B-depart_time.start_time      0.667     0.667     0.667       3.0
      B-depart_time.end_time      1.000     0.667     0.800       3.0
          B-depart_date.year      1.000     0.667     0.800       3.0
            I-transport_type      0.250     1.000     0.400       1.0
          B-restriction_code      0.364     1.000     0.533       4.0
    B-arrive_time.start_time      0.889     1.000     0.941       8.0
        B-toloc.airport_code      1.000     0.500     0.667       4.0
      B-arrive_time.end_time      0.889     1.000     0.941       8.0
      I-arrive_time.end_time      0.875     0.875     0.875       8.0
      I-depart_time.end_time      1.000     0.333     0.500       3.0
               I-flight_stop      0.000     0.000     0.000       0.0
      B-fromloc.airport_code      0.833     1.000     0.909       5.0
          I-restriction_code      1.000     1.000     1.000       3.0
    I-depart_time.start_time      0.500     1.000     0.667       1.0
          I-toloc.state_name      1.000     1.000     1.000       1.0
I-depart_date.today_relative      0.000     0.000     0.000       0.0
 B-arrive_date.date_relative      1.000     1.000     1.000       2.0
                I-flight_mod      1.000     0.167     0.286       6.0
                   I-economy      0.000     0.000     0.000       0.0
 B-return_date.date_relative      1.000     0.333     0.500       3.0
        I-fromloc.state_name      1.000     1.000     1.000       1.0
                B-state_code      1.000     1.000     1.000       1.0
    I-arrive_time.start_time      1.000     1.000     1.000       1.0
    I-arrive_date.day_number      0.000     0.000     0.000       0.0
                 B-meal_code      1.000     1.000     1.000       1.0
 I-depart_time.period_of_day      1.000     1.000     1.000       1.0
                  B-day_name      1.000     0.500     0.667       2.0
             B-period_of_day      1.000     0.500     0.667       4.0
        B-stoploc.state_code      0.000     0.000     0.000       0.0
    B-return_date.month_name      0.000     0.000     0.000       0.0
    B-return_date.day_number      0.000     0.000     0.000       0.0
    B-arrive_time.period_mod      0.000     0.000     0.000       0.0
                 I-meal_code      0.000     0.000     0.000       0.0
        B-toloc.country_name      1.000     1.000     1.000       1.0
                 B-days_code      1.000     1.000     1.000       1.0
 I-arrive_time.period_of_day      0.000     0.000     0.000       0.0
            I-today_relative      0.000     0.000     0.000       0.0
 B-return_time.period_of_day      0.000     0.000     0.000       0.0
                      B-time      0.000     0.000     0.000       0.0
           I-fare_basis_code      0.000     0.000     0.000       0.0
 I-arrive_time.time_relative      1.000     0.500     0.667       4.0
 I-depart_time.time_relative      0.000     0.000     0.000       1.0
            B-today_relative      0.000     0.000     0.000       0.0
                B-state_name      1.000     0.111     0.200       9.0
B-arrive_date.today_relative      0.000     0.000     0.000       0.0
    B-return_time.period_mod      0.000     0.000     0.000       0.0
                B-month_name      0.000     0.000     0.000       0.0
                B-day_number      0.000     0.000     0.000       0.0
 I-return_date.date_relative      0.750     1.000     0.857       3.0
I-return_date.today_relative      0.000     0.000     0.000       0.0
      B-stoploc.airport_name      0.000     0.000     0.000       0.0
             B-time_relative      0.000     0.000     0.000       0.0
                      I-time      0.000     0.000     0.000       0.0
    I-return_date.day_number      0.000     0.000     0.000       0.0
          I-meal_description      0.000     0.000     0.000       0.0
B-return_date.today_relative      0.000     0.000     0.000       0.0
      B-return_date.day_name      0.000     0.000     0.000       2.0

                   micro avg      0.948     0.950     0.949    3657.0
                   macro avg      0.677     0.650     0.640    3657.0
                weighted avg      0.958     0.950     0.948    3657.0

I0717 07:18:41.730792 140695199004544 interactiveshell.py:2882] Best Slot F1: 0.951
Reading ../data/atis.train.w-intent.iob
I0717 07:18:45.992314 140695199004544 interactiveshell.py:2882] Step 18450 | Loss: 0.0083 | Loss_intent: 0.0000 | Loss_slots: 0.0083 | Spent: 6.7 secs | LR: 0.000043
I0717 07:18:50.178429 140695199004544 interactiveshell.py:2882] Step 18500 | Loss: 0.0020 | Loss_intent: 0.0000 | Loss_slots: 0.0020 | Spent: 4.2 secs | LR: 0.000043
I0717 07:18:54.371531 140695199004544 interactiveshell.py:2882] Step 18550 | Loss: 0.0082 | Loss_intent: 0.0000 | Loss_slots: 0.0082 | Spent: 4.2 secs | LR: 0.000042
I0717 07:18:58.573544 140695199004544 interactiveshell.py:2882] Step 18600 | Loss: 0.0225 | Loss_intent: 0.0000 | Loss_slots: 0.0225 | Spent: 4.2 secs | LR: 0.000042
I0717 07:19:02.754323 140695199004544 interactiveshell.py:2882] Step 18650 | Loss: 0.0045 | Loss_intent: 0.0000 | Loss_slots: 0.0045 | Spent: 4.2 secs | LR: 0.000042
I0717 07:19:06.945436 140695199004544 interactiveshell.py:2882] Step 18700 | Loss: 0.0034 | Loss_intent: 0.0000 | Loss_slots: 0.0034 | Spent: 4.2 secs | LR: 0.000042
Reading ../data/atis.test.w-intent.iob
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.
  'precision', 'predicted', average, warn_for)
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1439: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples.
  'recall', 'true', average, warn_for)
I0717 07:19:10.267065 140695199004544 interactiveshell.py:2882] 
                                          precision    recall  f1-score   support

                             atis_flight      0.981     0.983     0.982       632
                            atis_airfare      0.940     0.979     0.959        48
                     atis_ground_service      1.000     1.000     1.000        36
                            atis_airline      0.974     1.000     0.987        38
                       atis_abbreviation      1.000     1.000     1.000        33
                           atis_aircraft      0.900     1.000     0.947         9
                        atis_flight_time      1.000     1.000     1.000         1
                           atis_quantity      0.429     1.000     0.600         3
                atis_flight#atis_airfare      0.800     0.333     0.471        12
                            atis_airport      0.947     1.000     0.973        18
                           atis_distance      1.000     1.000     1.000        10
                               atis_city      0.545     1.000     0.706         6
                        atis_ground_fare      1.000     0.714     0.833         7
                           atis_capacity      1.000     0.952     0.976        21
                          atis_flight_no      0.889     1.000     0.941         8
                               atis_meal      1.000     0.833     0.909         6
                        atis_restriction      0.000     0.000     0.000         0
             atis_airline#atis_flight_no      0.000     0.000     0.000         0
    atis_ground_service#atis_ground_fare      0.000     0.000     0.000         0
           atis_airfare#atis_flight_time      0.000     0.000     0.000         0
                           atis_cheapest      0.000     0.000     0.000         0
atis_aircraft#atis_flight#atis_flight_no      0.000     0.000     0.000         0

                               micro avg      0.968     0.973     0.970       888
                               macro avg      0.655     0.673     0.649       888
                            weighted avg      0.971     0.973     0.970       888

I0717 07:19:10.307251 140695199004544 interactiveshell.py:2882] 
                              precision    recall  f1-score   support

                           O      0.000     0.000     0.000       0.0
           B-toloc.city_name      0.975     0.993     0.984     716.0
         B-fromloc.city_name      0.986     0.993     0.989     704.0
           I-toloc.city_name      0.964     0.996     0.980     265.0
      B-depart_date.day_name      0.990     0.981     0.986     212.0
              B-airline_name      0.990     1.000     0.995     101.0
         I-fromloc.city_name      0.978     0.983     0.980     177.0
 B-depart_time.period_of_day      0.992     0.915     0.952     130.0
              I-airline_name      1.000     1.000     1.000      65.0
    B-depart_date.day_number      0.964     0.964     0.964      55.0
    B-depart_date.month_name      0.982     0.964     0.973      56.0
          B-depart_time.time      0.838     1.000     0.912      57.0
                B-round_trip      1.000     0.986     0.993      73.0
             B-cost_relative      1.000     0.973     0.986      37.0
                I-round_trip      1.000     1.000     1.000      71.0
                B-flight_mod      1.000     0.958     0.979      24.0
 B-depart_time.time_relative      0.984     0.969     0.977      65.0
          I-depart_time.time      0.945     1.000     0.972      52.0
         B-stoploc.city_name      1.000     1.000     1.000      20.0
                 B-city_name      0.868     0.579     0.695      57.0
                B-class_type      0.960     1.000     0.980      24.0
          B-arrive_time.time      0.971     0.971     0.971      34.0
 B-arrive_time.time_relative      0.909     0.968     0.937      31.0
                I-class_type      1.000     1.000     1.000      17.0
               B-flight_stop      1.000     1.000     1.000      21.0
          I-arrive_time.time      0.971     0.971     0.971      35.0
              B-airline_code      1.000     0.882     0.938      34.0
    I-depart_date.day_number      1.000     1.000     1.000      15.0
      I-fromloc.airport_name      0.429     1.000     0.600      15.0
      B-fromloc.airport_name      0.440     0.917     0.595      12.0
      B-arrive_date.day_name      0.688     1.000     0.815      11.0
          B-toloc.state_code      1.000     1.000     1.000      18.0
B-depart_date.today_relative      1.000     1.000     1.000       9.0
             B-flight_number      0.345     0.909     0.500      11.0
 B-depart_date.date_relative      0.941     0.941     0.941      17.0
          B-toloc.state_name      0.897     0.929     0.912      28.0
           B-fare_basis_code      0.909     0.588     0.714      17.0
               B-flight_time      1.000     1.000     1.000       1.0
                        B-or      1.000     1.000     1.000       3.0
 B-arrive_time.period_of_day      0.750     1.000     0.857       6.0
          B-meal_description      1.000     0.900     0.947      10.0
             I-cost_relative      1.000     0.667     0.800       3.0
              I-airport_name      0.917     0.379     0.537      29.0
               B-fare_amount      1.000     1.000     1.000       2.0
               I-fare_amount      1.000     1.000     1.000       2.0
                 I-city_name      1.000     0.600     0.750      30.0
        I-toloc.airport_name      1.000     1.000     1.000       3.0
            B-transport_type      1.000     0.800     0.889      10.0
    B-arrive_date.month_name      0.714     0.833     0.769       6.0
    B-arrive_date.day_number      0.714     0.833     0.769       6.0
         I-stoploc.city_name      1.000     1.000     1.000      10.0
                      B-meal      1.000     1.000     1.000      16.0
        B-fromloc.state_code      1.000     1.000     1.000      23.0
    B-depart_time.period_mod      1.000     1.000     1.000       5.0
                   B-connect      1.000     1.000     1.000       6.0
               B-flight_days      1.000     1.000     1.000      10.0
        B-toloc.airport_name      0.750     1.000     0.857       3.0
        B-fromloc.state_name      0.895     1.000     0.944      17.0
              B-airport_name      0.750     0.429     0.545      21.0
                   B-economy      1.000     1.000     1.000       6.0
               I-flight_time      1.000     1.000     1.000       1.0
             B-aircraft_code      0.931     0.818     0.871      33.0
                       B-mod      1.000     0.500     0.667       2.0
              B-airport_code      0.000     0.000     0.000       9.0
    B-depart_time.start_time      0.667     0.667     0.667       3.0
      B-depart_time.end_time      1.000     0.667     0.800       3.0
          B-depart_date.year      1.000     0.667     0.800       3.0
            I-transport_type      0.000     0.000     0.000       1.0
          B-restriction_code      0.333     1.000     0.500       4.0
    B-arrive_time.start_time      0.889     1.000     0.941       8.0
        B-toloc.airport_code      1.000     0.500     0.667       4.0
      B-arrive_time.end_time      0.889     1.000     0.941       8.0
      I-arrive_time.end_time      0.875     0.875     0.875       8.0
      I-depart_time.end_time      1.000     0.333     0.500       3.0
               I-flight_stop      0.000     0.000     0.000       0.0
      B-fromloc.airport_code      0.833     1.000     0.909       5.0
          I-restriction_code      1.000     1.000     1.000       3.0
    I-depart_time.start_time      0.500     1.000     0.667       1.0
          I-toloc.state_name      1.000     1.000     1.000       1.0
I-depart_date.today_relative      0.000     0.000     0.000       0.0
 B-arrive_date.date_relative      0.667     1.000     0.800       2.0
                I-flight_mod      1.000     0.167     0.286       6.0
                   I-economy      0.000     0.000     0.000       0.0
 B-return_date.date_relative      1.000     0.333     0.500       3.0
        I-fromloc.state_name      1.000     1.000     1.000       1.0
                B-state_code      1.000     1.000     1.000       1.0
    I-arrive_time.start_time      1.000     1.000     1.000       1.0
    I-arrive_date.day_number      0.000     0.000     0.000       0.0
                 B-meal_code      1.000     1.000     1.000       1.0
 I-depart_time.period_of_day      0.000     0.000     0.000       1.0
                  B-day_name      1.000     0.500     0.667       2.0
             B-period_of_day      1.000     0.500     0.667       4.0
        B-stoploc.state_code      0.000     0.000     0.000       0.0
    B-return_date.month_name      0.000     0.000     0.000       0.0
    B-return_date.day_number      0.000     0.000     0.000       0.0
    B-arrive_time.period_mod      0.000     0.000     0.000       0.0
                 I-meal_code      0.000     0.000     0.000       0.0
        B-toloc.country_name      1.000     1.000     1.000       1.0
                 B-days_code      1.000     1.000     1.000       1.0
 I-arrive_time.period_of_day      0.000     0.000     0.000       0.0
            I-today_relative      0.000     0.000     0.000       0.0
 B-return_time.period_of_day      0.000     0.000     0.000       0.0
                      B-time      0.000     0.000     0.000       0.0
           I-fare_basis_code      0.000     0.000     0.000       0.0
 I-arrive_time.time_relative      1.000     0.500     0.667       4.0
 I-depart_time.time_relative      0.000     0.000     0.000       1.0
            B-today_relative      0.000     0.000     0.000       0.0
                B-state_name      1.000     0.111     0.200       9.0
B-arrive_date.today_relative      0.000     0.000     0.000       0.0
    B-return_time.period_mod      0.000     0.000     0.000       0.0
                B-month_name      0.000     0.000     0.000       0.0
                B-day_number      0.000     0.000     0.000       0.0
 I-return_date.date_relative      0.750     1.000     0.857       3.0
I-return_date.today_relative      0.000     0.000     0.000       0.0
      B-stoploc.airport_name      0.000     0.000     0.000       0.0
             B-time_relative      0.000     0.000     0.000       0.0
                      I-time      0.000     0.000     0.000       0.0
    I-return_date.day_number      0.000     0.000     0.000       0.0
          I-meal_description      0.000     0.000     0.000       0.0
B-return_date.today_relative      0.000     0.000     0.000       0.0
      B-return_date.day_name      0.000     0.000     0.000       2.0

                   micro avg      0.950     0.951     0.951    3657.0
                   macro avg      0.676     0.644     0.639    3657.0
                weighted avg      0.960     0.951     0.949    3657.0

I0717 07:19:10.308166 140695199004544 interactiveshell.py:2882] Best Slot F1: 0.951
Reading ../data/atis.train.w-intent.iob
I0717 07:19:13.537889 140695199004544 interactiveshell.py:2882] Step 18750 | Loss: 0.0033 | Loss_intent: 0.0000 | Loss_slots: 0.0032 | Spent: 6.6 secs | LR: 0.000042
I0717 07:19:17.717000 140695199004544 interactiveshell.py:2882] Step 18800 | Loss: 0.0085 | Loss_intent: 0.0000 | Loss_slots: 0.0085 | Spent: 4.2 secs | LR: 0.000041
I0717 07:19:21.966057 140695199004544 interactiveshell.py:2882] Step 18850 | Loss: 0.0033 | Loss_intent: 0.0000 | Loss_slots: 0.0033 | Spent: 4.2 secs | LR: 0.000041
I0717 07:19:26.228581 140695199004544 interactiveshell.py:2882] Step 18900 | Loss: 0.0052 | Loss_intent: 0.0000 | Loss_slots: 0.0052 | Spent: 4.3 secs | LR: 0.000041
I0717 07:19:30.456960 140695199004544 interactiveshell.py:2882] Step 18950 | Loss: 0.0141 | Loss_intent: 0.0000 | Loss_slots: 0.0141 | Spent: 4.2 secs | LR: 0.000041
I0717 07:19:34.645341 140695199004544 interactiveshell.py:2882] Step 19000 | Loss: 0.0015 | Loss_intent: 0.0001 | Loss_slots: 0.0014 | Spent: 4.2 secs | LR: 0.000041
Reading ../data/atis.test.w-intent.iob
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.
  'precision', 'predicted', average, warn_for)
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1439: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples.
  'recall', 'true', average, warn_for)
I0717 07:19:38.962058 140695199004544 interactiveshell.py:2882] 
                                          precision    recall  f1-score   support

                             atis_flight      0.984     0.983     0.983       632
                            atis_airfare      0.923     1.000     0.960        48
                     atis_ground_service      1.000     1.000     1.000        36
                            atis_airline      0.974     1.000     0.987        38
                       atis_abbreviation      1.000     1.000     1.000        33
                           atis_aircraft      0.889     0.889     0.889         9
                        atis_flight_time      1.000     1.000     1.000         1
                           atis_quantity      0.429     1.000     0.600         3
                atis_flight#atis_airfare      0.800     0.333     0.471        12
                            atis_airport      0.900     1.000     0.947        18
                           atis_distance      1.000     1.000     1.000        10
                               atis_city      0.545     1.000     0.706         6
                        atis_ground_fare      1.000     0.714     0.833         7
                           atis_capacity      1.000     0.952     0.976        21
                          atis_flight_no      0.889     1.000     0.941         8
                               atis_meal      1.000     0.833     0.909         6
                        atis_restriction      0.000     0.000     0.000         0
             atis_airline#atis_flight_no      0.000     0.000     0.000         0
    atis_ground_service#atis_ground_fare      0.000     0.000     0.000         0
           atis_airfare#atis_flight_time      0.000     0.000     0.000         0
                           atis_cheapest      0.000     0.000     0.000         0
atis_aircraft#atis_flight#atis_flight_no      0.000     0.000     0.000         0

                               micro avg      0.968     0.973     0.970       888
                               macro avg      0.652     0.668     0.646       888
                            weighted avg      0.972     0.973     0.970       888

I0717 07:19:39.002943 140695199004544 interactiveshell.py:2882] 
                              precision    recall  f1-score   support

                           O      0.000     0.000     0.000       0.0
           B-toloc.city_name      0.977     0.993     0.985     716.0
         B-fromloc.city_name      0.987     0.994     0.991     704.0
           I-toloc.city_name      0.967     0.992     0.980     265.0
      B-depart_date.day_name      0.990     0.981     0.986     212.0
              B-airline_name      0.990     1.000     0.995     101.0
         I-fromloc.city_name      0.967     0.983     0.975     177.0
 B-depart_time.period_of_day      1.000     0.900     0.947     130.0
              I-airline_name      1.000     1.000     1.000      65.0
    B-depart_date.day_number      0.981     0.964     0.972      55.0
    B-depart_date.month_name      0.982     0.964     0.973      56.0
          B-depart_time.time      0.851     1.000     0.919      57.0
                B-round_trip      1.000     0.986     0.993      73.0
             B-cost_relative      1.000     0.973     0.986      37.0
                I-round_trip      1.000     1.000     1.000      71.0
                B-flight_mod      0.857     1.000     0.923      24.0
 B-depart_time.time_relative      0.984     0.969     0.977      65.0
          I-depart_time.time      0.945     1.000     0.972      52.0
         B-stoploc.city_name      0.952     1.000     0.976      20.0
                 B-city_name      0.868     0.579     0.695      57.0
                B-class_type      0.960     1.000     0.980      24.0
          B-arrive_time.time      0.971     0.971     0.971      34.0
 B-arrive_time.time_relative      0.909     0.968     0.937      31.0
                I-class_type      1.000     1.000     1.000      17.0
               B-flight_stop      1.000     1.000     1.000      21.0
          I-arrive_time.time      0.971     0.971     0.971      35.0
              B-airline_code      1.000     0.882     0.938      34.0
    I-depart_date.day_number      1.000     1.000     1.000      15.0
      I-fromloc.airport_name      0.441     1.000     0.612      15.0
      B-fromloc.airport_name      0.440     0.917     0.595      12.0
      B-arrive_date.day_name      0.688     1.000     0.815      11.0
          B-toloc.state_code      0.947     1.000     0.973      18.0
B-depart_date.today_relative      1.000     1.000     1.000       9.0
             B-flight_number      0.333     1.000     0.500      11.0
 B-depart_date.date_relative      0.944     1.000     0.971      17.0
          B-toloc.state_name      0.897     0.929     0.912      28.0
           B-fare_basis_code      0.917     0.647     0.759      17.0
               B-flight_time      1.000     1.000     1.000       1.0
                        B-or      1.000     1.000     1.000       3.0
 B-arrive_time.period_of_day      0.750     1.000     0.857       6.0
          B-meal_description      1.000     0.900     0.947      10.0
             I-cost_relative      1.000     0.667     0.800       3.0
              I-airport_name      0.917     0.379     0.537      29.0
               B-fare_amount      1.000     1.000     1.000       2.0
               I-fare_amount      1.000     1.000     1.000       2.0
                 I-city_name      1.000     0.600     0.750      30.0
        I-toloc.airport_name      1.000     1.000     1.000       3.0
            B-transport_type      1.000     0.800     0.889      10.0
    B-arrive_date.month_name      0.714     0.833     0.769       6.0
    B-arrive_date.day_number      0.714     0.833     0.769       6.0
         I-stoploc.city_name      0.909     1.000     0.952      10.0
                      B-meal      1.000     1.000     1.000      16.0
        B-fromloc.state_code      1.000     0.957     0.978      23.0
    B-depart_time.period_mod      0.833     1.000     0.909       5.0
                   B-connect      1.000     1.000     1.000       6.0
               B-flight_days      1.000     1.000     1.000      10.0
        B-toloc.airport_name      0.750     1.000     0.857       3.0
        B-fromloc.state_name      0.895     1.000     0.944      17.0
              B-airport_name      0.750     0.429     0.545      21.0
                   B-economy      1.000     1.000     1.000       6.0
               I-flight_time      1.000     1.000     1.000       1.0
             B-aircraft_code      0.931     0.818     0.871      33.0
                       B-mod      1.000     0.500     0.667       2.0
              B-airport_code      0.000     0.000     0.000       9.0
    B-depart_time.start_time      0.500     0.333     0.400       3.0
      B-depart_time.end_time      1.000     0.333     0.500       3.0
          B-depart_date.year      1.000     0.667     0.800       3.0
            I-transport_type      0.333     1.000     0.500       1.0
          B-restriction_code      0.400     1.000     0.571       4.0
    B-arrive_time.start_time      0.889     1.000     0.941       8.0
        B-toloc.airport_code      1.000     0.500     0.667       4.0
      B-arrive_time.end_time      0.800     1.000     0.889       8.0
      I-arrive_time.end_time      0.875     0.875     0.875       8.0
      I-depart_time.end_time      1.000     0.667     0.800       3.0
               I-flight_stop      0.000     0.000     0.000       0.0
      B-fromloc.airport_code      0.833     1.000     0.909       5.0
          I-restriction_code      1.000     1.000     1.000       3.0
    I-depart_time.start_time      1.000     1.000     1.000       1.0
          I-toloc.state_name      1.000     1.000     1.000       1.0
I-depart_date.today_relative      0.000     0.000     0.000       0.0
 B-arrive_date.date_relative      1.000     1.000     1.000       2.0
                I-flight_mod      1.000     0.167     0.286       6.0
                   I-economy      0.000     0.000     0.000       0.0
 B-return_date.date_relative      1.000     0.333     0.500       3.0
        I-fromloc.state_name      1.000     1.000     1.000       1.0
                B-state_code      1.000     1.000     1.000       1.0
    I-arrive_time.start_time      1.000     1.000     1.000       1.0
    I-arrive_date.day_number      0.000     0.000     0.000       0.0
                 B-meal_code      0.000     0.000     0.000       1.0
 I-depart_time.period_of_day      1.000     1.000     1.000       1.0
                  B-day_name      1.000     0.500     0.667       2.0
             B-period_of_day      1.000     0.500     0.667       4.0
        B-stoploc.state_code      0.000     0.000     0.000       0.0
    B-return_date.month_name      0.000     0.000     0.000       0.0
    B-return_date.day_number      0.000     0.000     0.000       0.0
    B-arrive_time.period_mod      0.000     0.000     0.000       0.0
                 I-meal_code      0.000     0.000     0.000       0.0
        B-toloc.country_name      1.000     1.000     1.000       1.0
                 B-days_code      1.000     1.000     1.000       1.0
 I-arrive_time.period_of_day      0.000     0.000     0.000       0.0
            I-today_relative      0.000     0.000     0.000       0.0
 B-return_time.period_of_day      0.000     0.000     0.000       0.0
                      B-time      0.000     0.000     0.000       0.0
           I-fare_basis_code      0.000     0.000     0.000       0.0
 I-arrive_time.time_relative      1.000     0.500     0.667       4.0
 I-depart_time.time_relative      0.000     0.000     0.000       1.0
            B-today_relative      0.000     0.000     0.000       0.0
                B-state_name      1.000     0.111     0.200       9.0
B-arrive_date.today_relative      0.000     0.000     0.000       0.0
    B-return_time.period_mod      0.000     0.000     0.000       0.0
                B-month_name      0.000     0.000     0.000       0.0
                B-day_number      0.000     0.000     0.000       0.0
 I-return_date.date_relative      0.667     0.667     0.667       3.0
I-return_date.today_relative      0.000     0.000     0.000       0.0
      B-stoploc.airport_name      0.000     0.000     0.000       0.0
             B-time_relative      0.000     0.000     0.000       0.0
                      I-time      0.000     0.000     0.000       0.0
    I-return_date.day_number      0.000     0.000     0.000       0.0
          I-meal_description      0.000     0.000     0.000       0.0
B-return_date.today_relative      0.000     0.000     0.000       0.0
      B-return_date.day_name      0.000     0.000     0.000       2.0

                   micro avg      0.950     0.951     0.951    3657.0
                   macro avg      0.679     0.648     0.642    3657.0
                weighted avg      0.959     0.951     0.949    3657.0

I0717 07:19:39.003856 140695199004544 interactiveshell.py:2882] Best Slot F1: 0.951
Reading ../data/atis.train.w-intent.iob
I0717 07:19:41.243783 140695199004544 interactiveshell.py:2882] Step 19050 | Loss: 0.0025 | Loss_intent: 0.0000 | Loss_slots: 0.0025 | Spent: 6.6 secs | LR: 0.000040
I0717 07:19:45.416293 140695199004544 interactiveshell.py:2882] Step 19100 | Loss: 0.0024 | Loss_intent: 0.0000 | Loss_slots: 0.0024 | Spent: 4.2 secs | LR: 0.000040
I0717 07:19:49.606454 140695199004544 interactiveshell.py:2882] Step 19150 | Loss: 0.0009 | Loss_intent: 0.0000 | Loss_slots: 0.0009 | Spent: 4.2 secs | LR: 0.000040
I0717 07:19:53.807183 140695199004544 interactiveshell.py:2882] Step 19200 | Loss: 0.0024 | Loss_intent: 0.0000 | Loss_slots: 0.0024 | Spent: 4.2 secs | LR: 0.000040
I0717 07:19:57.986964 140695199004544 interactiveshell.py:2882] Step 19250 | Loss: 0.0064 | Loss_intent: 0.0000 | Loss_slots: 0.0064 | Spent: 4.2 secs | LR: 0.000039
I0717 07:20:02.173227 140695199004544 interactiveshell.py:2882] Step 19300 | Loss: 0.0021 | Loss_intent: 0.0000 | Loss_slots: 0.0021 | Spent: 4.2 secs | LR: 0.000039
Reading ../data/atis.test.w-intent.iob
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.
  'precision', 'predicted', average, warn_for)
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1439: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples.
  'recall', 'true', average, warn_for)
I0717 07:20:07.490386 140695199004544 interactiveshell.py:2882] 
                                          precision    recall  f1-score   support

                             atis_flight      0.989     0.979     0.984       632
                            atis_airfare      0.906     1.000     0.950        48
                     atis_ground_service      1.000     1.000     1.000        36
                            atis_airline      0.974     0.974     0.974        38
                       atis_abbreviation      1.000     1.000     1.000        33
                           atis_aircraft      0.800     0.889     0.842         9
                        atis_flight_time      1.000     1.000     1.000         1
                           atis_quantity      0.375     1.000     0.545         3
                atis_flight#atis_airfare      0.857     0.500     0.632        12
                            atis_airport      0.947     1.000     0.973        18
                           atis_distance      1.000     1.000     1.000        10
                               atis_city      0.500     1.000     0.667         6
                        atis_ground_fare      1.000     0.714     0.833         7
                           atis_capacity      1.000     0.952     0.976        21
                          atis_flight_no      0.889     1.000     0.941         8
                               atis_meal      1.000     0.833     0.909         6
                        atis_restriction      0.000     0.000     0.000         0
             atis_airline#atis_flight_no      0.000     0.000     0.000         0
    atis_ground_service#atis_ground_fare      0.000     0.000     0.000         0
           atis_airfare#atis_flight_time      0.000     0.000     0.000         0
                           atis_cheapest      0.000     0.000     0.000         0
atis_aircraft#atis_flight#atis_flight_no      0.000     0.000     0.000         0

                               micro avg      0.966     0.972     0.969       888
                               macro avg      0.647     0.675     0.647       888
                            weighted avg      0.974     0.972     0.971       888

I0717 07:20:07.530584 140695199004544 interactiveshell.py:2882] 
                              precision    recall  f1-score   support

                           O      0.000     0.000     0.000       0.0
           B-toloc.city_name      0.974     0.993     0.983     716.0
         B-fromloc.city_name      0.990     0.990     0.990     704.0
           I-toloc.city_name      0.967     0.992     0.980     265.0
      B-depart_date.day_name      0.990     0.981     0.986     212.0
              B-airline_name      0.981     1.000     0.990     101.0
         I-fromloc.city_name      0.978     0.983     0.980     177.0
 B-depart_time.period_of_day      0.992     0.915     0.952     130.0
              I-airline_name      1.000     1.000     1.000      65.0
    B-depart_date.day_number      0.981     0.964     0.972      55.0
    B-depart_date.month_name      0.982     0.964     0.973      56.0
          B-depart_time.time      0.838     1.000     0.912      57.0
                B-round_trip      1.000     0.973     0.986      73.0
             B-cost_relative      1.000     0.973     0.986      37.0
                I-round_trip      1.000     1.000     1.000      71.0
                B-flight_mod      1.000     1.000     1.000      24.0
 B-depart_time.time_relative      0.984     0.954     0.969      65.0
          I-depart_time.time      0.945     1.000     0.972      52.0
         B-stoploc.city_name      0.952     1.000     0.976      20.0
                 B-city_name      0.833     0.614     0.707      57.0
                B-class_type      0.960     1.000     0.980      24.0
          B-arrive_time.time      0.943     0.971     0.957      34.0
 B-arrive_time.time_relative      0.882     0.968     0.923      31.0
                I-class_type      1.000     1.000     1.000      17.0
               B-flight_stop      0.913     1.000     0.955      21.0
          I-arrive_time.time      0.971     0.971     0.971      35.0
              B-airline_code      1.000     0.853     0.921      34.0
    I-depart_date.day_number      1.000     1.000     1.000      15.0
      I-fromloc.airport_name      0.429     1.000     0.600      15.0
      B-fromloc.airport_name      0.423     0.917     0.579      12.0
      B-arrive_date.day_name      0.688     1.000     0.815      11.0
          B-toloc.state_code      0.947     1.000     0.973      18.0
B-depart_date.today_relative      1.000     1.000     1.000       9.0
             B-flight_number      0.345     0.909     0.500      11.0
 B-depart_date.date_relative      0.941     0.941     0.941      17.0
          B-toloc.state_name      0.897     0.929     0.912      28.0
           B-fare_basis_code      0.917     0.647     0.759      17.0
               B-flight_time      1.000     1.000     1.000       1.0
                        B-or      1.000     1.000     1.000       3.0
 B-arrive_time.period_of_day      0.750     1.000     0.857       6.0
          B-meal_description      1.000     0.900     0.947      10.0
             I-cost_relative      1.000     0.667     0.800       3.0
              I-airport_name      0.917     0.379     0.537      29.0
               B-fare_amount      1.000     1.000     1.000       2.0
               I-fare_amount      1.000     1.000     1.000       2.0
                 I-city_name      1.000     0.567     0.723      30.0
        I-toloc.airport_name      1.000     1.000     1.000       3.0
            B-transport_type      1.000     1.000     1.000      10.0
    B-arrive_date.month_name      0.714     0.833     0.769       6.0
    B-arrive_date.day_number      0.714     0.833     0.769       6.0
         I-stoploc.city_name      0.833     1.000     0.909      10.0
                      B-meal      1.000     1.000     1.000      16.0
        B-fromloc.state_code      1.000     0.957     0.978      23.0
    B-depart_time.period_mod      1.000     1.000     1.000       5.0
                   B-connect      1.000     1.000     1.000       6.0
               B-flight_days      1.000     1.000     1.000      10.0
        B-toloc.airport_name      0.750     1.000     0.857       3.0
        B-fromloc.state_name      0.944     1.000     0.971      17.0
              B-airport_name      0.818     0.429     0.562      21.0
                   B-economy      1.000     1.000     1.000       6.0
               I-flight_time      1.000     1.000     1.000       1.0
             B-aircraft_code      0.931     0.818     0.871      33.0
                       B-mod      1.000     0.500     0.667       2.0
              B-airport_code      0.000     0.000     0.000       9.0
    B-depart_time.start_time      0.667     0.667     0.667       3.0
      B-depart_time.end_time      1.000     0.333     0.500       3.0
          B-depart_date.year      1.000     0.667     0.800       3.0
            I-transport_type      1.000     1.000     1.000       1.0
          B-restriction_code      0.400     1.000     0.571       4.0
    B-arrive_time.start_time      0.875     0.875     0.875       8.0
        B-toloc.airport_code      1.000     0.500     0.667       4.0
      B-arrive_time.end_time      0.800     1.000     0.889       8.0
      I-arrive_time.end_time      0.875     0.875     0.875       8.0
      I-depart_time.end_time      1.000     0.667     0.800       3.0
               I-flight_stop      0.000     0.000     0.000       0.0
      B-fromloc.airport_code      0.833     1.000     0.909       5.0
          I-restriction_code      0.750     1.000     0.857       3.0
    I-depart_time.start_time      1.000     1.000     1.000       1.0
          I-toloc.state_name      1.000     1.000     1.000       1.0
I-depart_date.today_relative      0.000     0.000     0.000       0.0
 B-arrive_date.date_relative      0.667     1.000     0.800       2.0
                I-flight_mod      1.000     0.167     0.286       6.0
                   I-economy      0.000     0.000     0.000       0.0
 B-return_date.date_relative      1.000     0.333     0.500       3.0
        I-fromloc.state_name      1.000     1.000     1.000       1.0
                B-state_code      1.000     1.000     1.000       1.0
    I-arrive_time.start_time      1.000     1.000     1.000       1.0
    I-arrive_date.day_number      0.000     0.000     0.000       0.0
                 B-meal_code      0.000     0.000     0.000       1.0
 I-depart_time.period_of_day      0.000     0.000     0.000       1.0
                  B-day_name      1.000     0.500     0.667       2.0
             B-period_of_day      1.000     0.500     0.667       4.0
        B-stoploc.state_code      0.000     0.000     0.000       0.0
    B-return_date.month_name      0.000     0.000     0.000       0.0
    B-return_date.day_number      0.000     0.000     0.000       0.0
    B-arrive_time.period_mod      0.000     0.000     0.000       0.0
                 I-meal_code      0.000     0.000     0.000       0.0
        B-toloc.country_name      1.000     1.000     1.000       1.0
                 B-days_code      1.000     1.000     1.000       1.0
 I-arrive_time.period_of_day      0.000     0.000     0.000       0.0
            I-today_relative      0.000     0.000     0.000       0.0
 B-return_time.period_of_day      0.000     0.000     0.000       0.0
                      B-time      0.000     0.000     0.000       0.0
           I-fare_basis_code      0.000     0.000     0.000       0.0
 I-arrive_time.time_relative      1.000     0.500     0.667       4.0
 I-depart_time.time_relative      0.000     0.000     0.000       1.0
            B-today_relative      0.000     0.000     0.000       0.0
                B-state_name      1.000     0.111     0.200       9.0
B-arrive_date.today_relative      0.000     0.000     0.000       0.0
    B-return_time.period_mod      0.000     0.000     0.000       0.0
                B-month_name      0.000     0.000     0.000       0.0
                B-day_number      0.000     0.000     0.000       0.0
 I-return_date.date_relative      0.750     1.000     0.857       3.0
I-return_date.today_relative      0.000     0.000     0.000       0.0
      B-stoploc.airport_name      0.000     0.000     0.000       0.0
             B-time_relative      0.000     0.000     0.000       0.0
                      I-time      0.000     0.000     0.000       0.0
    I-return_date.day_number      0.000     0.000     0.000       0.0
          I-meal_description      0.000     0.000     0.000       0.0
B-return_date.today_relative      0.000     0.000     0.000       0.0
      B-return_date.day_name      0.000     0.000     0.000       2.0

                   micro avg      0.949     0.951     0.950    3657.0
                   macro avg      0.675     0.644     0.639    3657.0
                weighted avg      0.959     0.951     0.948    3657.0

I0717 07:20:07.531388 140695199004544 interactiveshell.py:2882] Best Slot F1: 0.951
Reading ../data/atis.train.w-intent.iob
I0717 07:20:08.774271 140695199004544 interactiveshell.py:2882] Step 19350 | Loss: 0.0028 | Loss_intent: 0.0001 | Loss_slots: 0.0027 | Spent: 6.6 secs | LR: 0.000039
I0717 07:20:13.349466 140695199004544 interactiveshell.py:2882] Step 19400 | Loss: 0.0022 | Loss_intent: 0.0001 | Loss_slots: 0.0021 | Spent: 4.6 secs | LR: 0.000039
I0717 07:20:18.076716 140695199004544 interactiveshell.py:2882] Step 19450 | Loss: 0.0275 | Loss_intent: 0.0014 | Loss_slots: 0.0261 | Spent: 4.7 secs | LR: 0.000039
I0717 07:20:22.544690 140695199004544 interactiveshell.py:2882] Step 19500 | Loss: 0.0061 | Loss_intent: 0.0019 | Loss_slots: 0.0042 | Spent: 4.5 secs | LR: 0.000038
I0717 07:20:26.743532 140695199004544 interactiveshell.py:2882] Step 19550 | Loss: 0.0021 | Loss_intent: 0.0000 | Loss_slots: 0.0021 | Spent: 4.2 secs | LR: 0.000038
I0717 07:20:30.918980 140695199004544 interactiveshell.py:2882] Step 19600 | Loss: 0.0003 | Loss_intent: 0.0000 | Loss_slots: 0.0003 | Spent: 4.2 secs | LR: 0.000038
I0717 07:20:35.088401 140695199004544 interactiveshell.py:2882] Step 19650 | Loss: 0.0091 | Loss_intent: 0.0000 | Loss_slots: 0.0091 | Spent: 4.2 secs | LR: 0.000038
Reading ../data/atis.test.w-intent.iob
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.
  'precision', 'predicted', average, warn_for)
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1439: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples.
  'recall', 'true', average, warn_for)
I0717 07:20:37.233413 140695199004544 interactiveshell.py:2882] 
                                          precision    recall  f1-score   support

                             atis_flight      0.990     0.983     0.986       632
                            atis_airfare      0.906     1.000     0.950        48
                     atis_ground_service      1.000     1.000     1.000        36
                            atis_airline      0.974     0.974     0.974        38
                       atis_abbreviation      0.971     1.000     0.985        33
                           atis_aircraft      1.000     0.889     0.941         9
                        atis_flight_time      1.000     1.000     1.000         1
                           atis_quantity      0.333     1.000     0.500         3
                atis_flight#atis_airfare      0.875     0.583     0.700        12
                            atis_airport      0.947     1.000     0.973        18
                           atis_distance      1.000     1.000     1.000        10
                               atis_city      0.545     1.000     0.706         6
                        atis_ground_fare      1.000     0.714     0.833         7
                           atis_capacity      1.000     0.952     0.976        21
                          atis_flight_no      0.889     1.000     0.941         8
                               atis_meal      1.000     0.833     0.909         6
                        atis_restriction      0.000     0.000     0.000         0
             atis_airline#atis_flight_no      0.000     0.000     0.000         0
    atis_ground_service#atis_ground_fare      0.000     0.000     0.000         0
           atis_airfare#atis_flight_time      0.000     0.000     0.000         0
                           atis_cheapest      0.000     0.000     0.000         0
atis_aircraft#atis_flight#atis_flight_no      0.000     0.000     0.000         0

                               micro avg      0.970     0.975     0.972       888
                               macro avg      0.656     0.679     0.653       888
                            weighted avg      0.977     0.975     0.974       888

I0717 07:20:37.273307 140695199004544 interactiveshell.py:2882] 
                              precision    recall  f1-score   support

                           O      0.000     0.000     0.000       0.0
           B-toloc.city_name      0.975     0.993     0.984     716.0
         B-fromloc.city_name      0.987     0.994     0.991     704.0
           I-toloc.city_name      0.971     1.000     0.985     265.0
      B-depart_date.day_name      0.990     0.981     0.986     212.0
              B-airline_name      0.990     1.000     0.995     101.0
         I-fromloc.city_name      0.983     0.983     0.983     177.0
 B-depart_time.period_of_day      1.000     0.923     0.960     130.0
              I-airline_name      1.000     1.000     1.000      65.0
    B-depart_date.day_number      0.981     0.964     0.972      55.0
    B-depart_date.month_name      0.982     0.964     0.973      56.0
          B-depart_time.time      0.838     1.000     0.912      57.0
                B-round_trip      1.000     0.986     0.993      73.0
             B-cost_relative      1.000     0.973     0.986      37.0
                I-round_trip      1.000     1.000     1.000      71.0
                B-flight_mod      0.960     1.000     0.980      24.0
 B-depart_time.time_relative      0.984     0.969     0.977      65.0
          I-depart_time.time      0.963     1.000     0.981      52.0
         B-stoploc.city_name      0.952     1.000     0.976      20.0
                 B-city_name      0.872     0.596     0.708      57.0
                B-class_type      0.960     1.000     0.980      24.0
          B-arrive_time.time      0.971     0.971     0.971      34.0
 B-arrive_time.time_relative      0.909     0.968     0.937      31.0
                I-class_type      1.000     1.000     1.000      17.0
               B-flight_stop      0.955     1.000     0.977      21.0
          I-arrive_time.time      0.971     0.971     0.971      35.0
              B-airline_code      1.000     0.882     0.938      34.0
    I-depart_date.day_number      1.000     1.000     1.000      15.0
      I-fromloc.airport_name      0.429     1.000     0.600      15.0
      B-fromloc.airport_name      0.440     0.917     0.595      12.0
      B-arrive_date.day_name      0.688     1.000     0.815      11.0
          B-toloc.state_code      0.947     1.000     0.973      18.0
B-depart_date.today_relative      1.000     1.000     1.000       9.0
             B-flight_number      0.529     0.818     0.643      11.0
 B-depart_date.date_relative      0.944     1.000     0.971      17.0
          B-toloc.state_name      0.903     1.000     0.949      28.0
           B-fare_basis_code      0.944     1.000     0.971      17.0
               B-flight_time      1.000     1.000     1.000       1.0
                        B-or      1.000     1.000     1.000       3.0
 B-arrive_time.period_of_day      0.750     1.000     0.857       6.0
          B-meal_description      1.000     0.900     0.947      10.0
             I-cost_relative      1.000     0.667     0.800       3.0
              I-airport_name      0.917     0.379     0.537      29.0
               B-fare_amount      0.400     1.000     0.571       2.0
               I-fare_amount      1.000     1.000     1.000       2.0
                 I-city_name      1.000     0.600     0.750      30.0
        I-toloc.airport_name      1.000     1.000     1.000       3.0
            B-transport_type      1.000     1.000     1.000      10.0
    B-arrive_date.month_name      0.714     0.833     0.769       6.0
    B-arrive_date.day_number      0.714     0.833     0.769       6.0
         I-stoploc.city_name      0.909     1.000     0.952      10.0
                      B-meal      1.000     1.000     1.000      16.0
        B-fromloc.state_code      1.000     0.957     0.978      23.0
    B-depart_time.period_mod      1.000     1.000     1.000       5.0
                   B-connect      1.000     1.000     1.000       6.0
               B-flight_days      1.000     1.000     1.000      10.0
        B-toloc.airport_name      0.500     0.333     0.400       3.0
        B-fromloc.state_name      0.944     1.000     0.971      17.0
              B-airport_name      0.750     0.429     0.545      21.0
                   B-economy      1.000     1.000     1.000       6.0
               I-flight_time      1.000     1.000     1.000       1.0
             B-aircraft_code      0.933     0.848     0.889      33.0
                       B-mod      1.000     0.500     0.667       2.0
              B-airport_code      0.000     0.000     0.000       9.0
    B-depart_time.start_time      0.667     0.667     0.667       3.0
      B-depart_time.end_time      1.000     0.667     0.800       3.0
          B-depart_date.year      1.000     0.667     0.800       3.0
            I-transport_type      1.000     1.000     1.000       1.0
          B-restriction_code      0.364     1.000     0.533       4.0
    B-arrive_time.start_time      0.889     1.000     0.941       8.0
        B-toloc.airport_code      1.000     0.500     0.667       4.0
      B-arrive_time.end_time      0.889     1.000     0.941       8.0
      I-arrive_time.end_time      0.875     0.875     0.875       8.0
      I-depart_time.end_time      1.000     0.667     0.800       3.0
               I-flight_stop      0.000     0.000     0.000       0.0
      B-fromloc.airport_code      0.833     1.000     0.909       5.0
          I-restriction_code      1.000     1.000     1.000       3.0
    I-depart_time.start_time      1.000     1.000     1.000       1.0
          I-toloc.state_name      1.000     1.000     1.000       1.0
I-depart_date.today_relative      0.000     0.000     0.000       0.0
 B-arrive_date.date_relative      1.000     1.000     1.000       2.0
                I-flight_mod      1.000     0.167     0.286       6.0
                   I-economy      0.000     0.000     0.000       0.0
 B-return_date.date_relative      1.000     0.333     0.500       3.0
        I-fromloc.state_name      1.000     1.000     1.000       1.0
                B-state_code      1.000     1.000     1.000       1.0
    I-arrive_time.start_time      1.000     1.000     1.000       1.0
    I-arrive_date.day_number      0.000     0.000     0.000       0.0
                 B-meal_code      1.000     1.000     1.000       1.0
 I-depart_time.period_of_day      1.000     1.000     1.000       1.0
                  B-day_name      1.000     0.500     0.667       2.0
             B-period_of_day      1.000     0.500     0.667       4.0
        B-stoploc.state_code      0.000     0.000     0.000       0.0
    B-return_date.month_name      0.000     0.000     0.000       0.0
    B-return_date.day_number      0.000     0.000     0.000       0.0
    B-arrive_time.period_mod      0.000     0.000     0.000       0.0
                 I-meal_code      0.000     0.000     0.000       0.0
        B-toloc.country_name      1.000     1.000     1.000       1.0
                 B-days_code      1.000     1.000     1.000       1.0
 I-arrive_time.period_of_day      0.000     0.000     0.000       0.0
            I-today_relative      0.000     0.000     0.000       0.0
 B-return_time.period_of_day      0.000     0.000     0.000       0.0
                      B-time      0.000     0.000     0.000       0.0
           I-fare_basis_code      0.000     0.000     0.000       0.0
 I-arrive_time.time_relative      1.000     0.500     0.667       4.0
 I-depart_time.time_relative      0.000     0.000     0.000       1.0
            B-today_relative      0.000     0.000     0.000       0.0
                B-state_name      1.000     0.111     0.200       9.0
B-arrive_date.today_relative      0.000     0.000     0.000       0.0
    B-return_time.period_mod      0.000     0.000     0.000       0.0
                B-month_name      0.000     0.000     0.000       0.0
                B-day_number      0.000     0.000     0.000       0.0
 I-return_date.date_relative      0.667     0.667     0.667       3.0
I-return_date.today_relative      0.000     0.000     0.000       0.0
      B-stoploc.airport_name      0.000     0.000     0.000       0.0
             B-time_relative      0.000     0.000     0.000       0.0
                      I-time      0.000     0.000     0.000       0.0
    I-return_date.day_number      0.000     0.000     0.000       0.0
          I-meal_description      0.000     0.000     0.000       0.0
B-return_date.today_relative      0.000     0.000     0.000       0.0
      B-return_date.day_name      0.000     0.000     0.000       2.0

                   micro avg      0.954     0.956     0.955    3657.0
                   macro avg      0.692     0.661     0.658    3657.0
                weighted avg      0.962     0.956     0.953    3657.0

I0717 07:20:37.274080 140695199004544 interactiveshell.py:2882] Best Slot F1: 0.955
Reading ../data/atis.train.w-intent.iob
I0717 07:20:41.719130 140695199004544 interactiveshell.py:2882] Step 19700 | Loss: 0.0024 | Loss_intent: 0.0004 | Loss_slots: 0.0020 | Spent: 6.6 secs | LR: 0.000038
I0717 07:20:45.970895 140695199004544 interactiveshell.py:2882] Step 19750 | Loss: 0.0068 | Loss_intent: 0.0000 | Loss_slots: 0.0068 | Spent: 4.3 secs | LR: 0.000037
I0717 07:20:50.219272 140695199004544 interactiveshell.py:2882] Step 19800 | Loss: 0.0011 | Loss_intent: 0.0001 | Loss_slots: 0.0010 | Spent: 4.2 secs | LR: 0.000037
I0717 07:20:54.431279 140695199004544 interactiveshell.py:2882] Step 19850 | Loss: 0.0011 | Loss_intent: 0.0000 | Loss_slots: 0.0011 | Spent: 4.2 secs | LR: 0.000037
I0717 07:20:58.614575 140695199004544 interactiveshell.py:2882] Step 19900 | Loss: 0.0114 | Loss_intent: 0.0000 | Loss_slots: 0.0114 | Spent: 4.2 secs | LR: 0.000037
I0717 07:21:02.793770 140695199004544 interactiveshell.py:2882] Step 19950 | Loss: 0.0075 | Loss_intent: 0.0000 | Loss_slots: 0.0075 | Spent: 4.2 secs | LR: 0.000037
Reading ../data/atis.test.w-intent.iob
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.
  'precision', 'predicted', average, warn_for)
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1439: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples.
  'recall', 'true', average, warn_for)
I0717 07:21:05.940376 140695199004544 interactiveshell.py:2882] 
                                          precision    recall  f1-score   support

                             atis_flight      0.992     0.981     0.986       632
                            atis_airfare      0.906     1.000     0.950        48
                     atis_ground_service      1.000     1.000     1.000        36
                            atis_airline      0.974     1.000     0.987        38
                       atis_abbreviation      1.000     0.970     0.985        33
                           atis_aircraft      0.889     0.889     0.889         9
                        atis_flight_time      1.000     1.000     1.000         1
                           atis_quantity      0.375     1.000     0.545         3
                atis_flight#atis_airfare      0.875     0.583     0.700        12
                            atis_airport      0.947     1.000     0.973        18
                           atis_distance      1.000     1.000     1.000        10
                               atis_city      0.500     1.000     0.667         6
                        atis_ground_fare      1.000     0.714     0.833         7
                           atis_capacity      1.000     1.000     1.000        21
                          atis_flight_no      0.889     1.000     0.941         8
                               atis_meal      1.000     0.833     0.909         6
                        atis_restriction      0.000     0.000     0.000         0
             atis_airline#atis_flight_no      0.000     0.000     0.000         0
    atis_ground_service#atis_ground_fare      0.000     0.000     0.000         0
           atis_airfare#atis_flight_time      0.000     0.000     0.000         0
                           atis_cheapest      0.000     0.000     0.000         0
atis_aircraft#atis_flight#atis_flight_no      0.000     0.000     0.000         0

                               micro avg      0.970     0.975     0.972       888
                               macro avg      0.652     0.680     0.653       888
                            weighted avg      0.978     0.975     0.975       888

I0717 07:21:05.980569 140695199004544 interactiveshell.py:2882] 
                              precision    recall  f1-score   support

                           O      0.000     0.000     0.000       0.0
           B-toloc.city_name      0.974     0.993     0.983     716.0
         B-fromloc.city_name      0.987     0.994     0.991     704.0
           I-toloc.city_name      0.978     0.992     0.985     265.0
      B-depart_date.day_name      0.991     0.986     0.988     212.0
              B-airline_name      0.990     1.000     0.995     101.0
         I-fromloc.city_name      0.978     0.989     0.983     177.0
 B-depart_time.period_of_day      0.992     0.915     0.952     130.0
              I-airline_name      1.000     1.000     1.000      65.0
    B-depart_date.day_number      0.964     0.964     0.964      55.0
    B-depart_date.month_name      0.982     0.964     0.973      56.0
          B-depart_time.time      0.838     1.000     0.912      57.0
                B-round_trip      1.000     0.986     0.993      73.0
             B-cost_relative      1.000     0.973     0.986      37.0
                I-round_trip      1.000     1.000     1.000      71.0
                B-flight_mod      0.885     0.958     0.920      24.0
 B-depart_time.time_relative      0.984     0.969     0.977      65.0
          I-depart_time.time      0.945     1.000     0.972      52.0
         B-stoploc.city_name      0.952     1.000     0.976      20.0
                 B-city_name      0.868     0.579     0.695      57.0
                B-class_type      0.960     1.000     0.980      24.0
          B-arrive_time.time      0.971     0.971     0.971      34.0
 B-arrive_time.time_relative      0.909     0.968     0.937      31.0
                I-class_type      1.000     1.000     1.000      17.0
               B-flight_stop      1.000     1.000     1.000      21.0
          I-arrive_time.time      0.971     0.971     0.971      35.0
              B-airline_code      1.000     0.882     0.938      34.0
    I-depart_date.day_number      1.000     1.000     1.000      15.0
      I-fromloc.airport_name      0.429     1.000     0.600      15.0
      B-fromloc.airport_name      0.407     0.917     0.564      12.0
      B-arrive_date.day_name      0.733     1.000     0.846      11.0
          B-toloc.state_code      1.000     1.000     1.000      18.0
B-depart_date.today_relative      1.000     1.000     1.000       9.0
             B-flight_number      0.400     0.909     0.556      11.0
 B-depart_date.date_relative      0.944     1.000     0.971      17.0
          B-toloc.state_name      0.931     0.964     0.947      28.0
           B-fare_basis_code      0.917     0.647     0.759      17.0
               B-flight_time      1.000     1.000     1.000       1.0
                        B-or      1.000     1.000     1.000       3.0
 B-arrive_time.period_of_day      0.750     1.000     0.857       6.0
          B-meal_description      1.000     0.900     0.947      10.0
             I-cost_relative      1.000     0.667     0.800       3.0
              I-airport_name      0.917     0.379     0.537      29.0
               B-fare_amount      0.400     1.000     0.571       2.0
               I-fare_amount      1.000     1.000     1.000       2.0
                 I-city_name      1.000     0.600     0.750      30.0
        I-toloc.airport_name      1.000     1.000     1.000       3.0
            B-transport_type      1.000     1.000     1.000      10.0
    B-arrive_date.month_name      0.714     0.833     0.769       6.0
    B-arrive_date.day_number      0.714     0.833     0.769       6.0
         I-stoploc.city_name      0.769     1.000     0.870      10.0
                      B-meal      1.000     1.000     1.000      16.0
        B-fromloc.state_code      1.000     1.000     1.000      23.0
    B-depart_time.period_mod      1.000     0.800     0.889       5.0
                   B-connect      1.000     1.000     1.000       6.0
               B-flight_days      1.000     1.000     1.000      10.0
        B-toloc.airport_name      0.500     0.333     0.400       3.0
        B-fromloc.state_name      0.944     1.000     0.971      17.0
              B-airport_name      0.818     0.429     0.562      21.0
                   B-economy      1.000     1.000     1.000       6.0
               I-flight_time      1.000     1.000     1.000       1.0
             B-aircraft_code      0.906     0.879     0.892      33.0
                       B-mod      1.000     0.500     0.667       2.0
              B-airport_code      0.000     0.000     0.000       9.0
    B-depart_time.start_time      0.667     0.667     0.667       3.0
      B-depart_time.end_time      1.000     0.333     0.500       3.0
          B-depart_date.year      1.000     0.667     0.800       3.0
            I-transport_type      1.000     1.000     1.000       1.0
          B-restriction_code      0.400     1.000     0.571       4.0
    B-arrive_time.start_time      0.889     1.000     0.941       8.0
        B-toloc.airport_code      1.000     0.500     0.667       4.0
      B-arrive_time.end_time      0.800     1.000     0.889       8.0
      I-arrive_time.end_time      0.875     0.875     0.875       8.0
      I-depart_time.end_time      1.000     0.333     0.500       3.0
               I-flight_stop      0.000     0.000     0.000       0.0
      B-fromloc.airport_code      0.833     1.000     0.909       5.0
          I-restriction_code      1.000     1.000     1.000       3.0
    I-depart_time.start_time      0.500     1.000     0.667       1.0
          I-toloc.state_name      1.000     1.000     1.000       1.0
I-depart_date.today_relative      0.000     0.000     0.000       0.0
 B-arrive_date.date_relative      1.000     1.000     1.000       2.0
                I-flight_mod      1.000     0.167     0.286       6.0
                   I-economy      0.000     0.000     0.000       0.0
 B-return_date.date_relative      1.000     0.333     0.500       3.0
        I-fromloc.state_name      1.000     1.000     1.000       1.0
                B-state_code      1.000     1.000     1.000       1.0
    I-arrive_time.start_time      1.000     1.000     1.000       1.0
    I-arrive_date.day_number      0.000     0.000     0.000       0.0
                 B-meal_code      0.000     0.000     0.000       1.0
 I-depart_time.period_of_day      1.000     1.000     1.000       1.0
                  B-day_name      1.000     0.500     0.667       2.0
             B-period_of_day      1.000     0.500     0.667       4.0
        B-stoploc.state_code      0.000     0.000     0.000       0.0
    B-return_date.month_name      0.000     0.000     0.000       0.0
    B-return_date.day_number      0.000     0.000     0.000       0.0
    B-arrive_time.period_mod      0.000     0.000     0.000       0.0
                 I-meal_code      0.000     0.000     0.000       0.0
        B-toloc.country_name      1.000     1.000     1.000       1.0
                 B-days_code      1.000     1.000     1.000       1.0
 I-arrive_time.period_of_day      0.000     0.000     0.000       0.0
            I-today_relative      0.000     0.000     0.000       0.0
 B-return_time.period_of_day      0.000     0.000     0.000       0.0
                      B-time      0.000     0.000     0.000       0.0
           I-fare_basis_code      0.000     0.000     0.000       0.0
 I-arrive_time.time_relative      1.000     0.500     0.667       4.0
 I-depart_time.time_relative      0.000     0.000     0.000       1.0
            B-today_relative      0.000     0.000     0.000       0.0
                B-state_name      1.000     0.222     0.364       9.0
B-arrive_date.today_relative      0.000     0.000     0.000       0.0
    B-return_time.period_mod      0.000     0.000     0.000       0.0
                B-month_name      0.000     0.000     0.000       0.0
                B-day_number      0.000     0.000     0.000       0.0
 I-return_date.date_relative      0.667     0.667     0.667       3.0
I-return_date.today_relative      0.000     0.000     0.000       0.0
      B-stoploc.airport_name      0.000     0.000     0.000       0.0
             B-time_relative      0.000     0.000     0.000       0.0
                      I-time      0.000     0.000     0.000       0.0
    I-return_date.day_number      0.000     0.000     0.000       0.0
          I-meal_description      0.000     0.000     0.000       0.0
B-return_date.today_relative      0.000     0.000     0.000       0.0
      B-return_date.day_name      0.000     0.000     0.000       2.0

                   micro avg      0.951     0.953     0.952    3657.0
                   macro avg      0.677     0.644     0.639    3657.0
                weighted avg      0.960     0.953     0.950    3657.0

I0717 07:21:05.981540 140695199004544 interactiveshell.py:2882] Best Slot F1: 0.955
Reading ../data/atis.train.w-intent.iob
I0717 07:21:09.416496 140695199004544 interactiveshell.py:2882] Step 20000 | Loss: 0.0034 | Loss_intent: 0.0010 | Loss_slots: 0.0024 | Spent: 6.6 secs | LR: 0.000036
I0717 07:21:13.591029 140695199004544 interactiveshell.py:2882] Step 20050 | Loss: 0.0096 | Loss_intent: 0.0002 | Loss_slots: 0.0094 | Spent: 4.2 secs | LR: 0.000036
I0717 07:21:17.766064 140695199004544 interactiveshell.py:2882] Step 20100 | Loss: 0.0050 | Loss_intent: 0.0000 | Loss_slots: 0.0049 | Spent: 4.2 secs | LR: 0.000036
I0717 07:21:21.952064 140695199004544 interactiveshell.py:2882] Step 20150 | Loss: 0.0382 | Loss_intent: 0.0022 | Loss_slots: 0.0360 | Spent: 4.2 secs | LR: 0.000036
I0717 07:21:26.125392 140695199004544 interactiveshell.py:2882] Step 20200 | Loss: 0.0060 | Loss_intent: 0.0000 | Loss_slots: 0.0060 | Spent: 4.2 secs | LR: 0.000036
I0717 07:21:30.309333 140695199004544 interactiveshell.py:2882] Step 20250 | Loss: 0.0078 | Loss_intent: 0.0000 | Loss_slots: 0.0078 | Spent: 4.2 secs | LR: 0.000036
Reading ../data/atis.test.w-intent.iob
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.
  'precision', 'predicted', average, warn_for)
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1439: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples.
  'recall', 'true', average, warn_for)
I0717 07:21:34.465004 140695199004544 interactiveshell.py:2882] 
                                          precision    recall  f1-score   support

                             atis_flight      0.984     0.984     0.984       632
                            atis_airfare      0.940     0.979     0.959        48
                     atis_ground_service      1.000     1.000     1.000        36
                            atis_airline      0.974     1.000     0.987        38
                       atis_abbreviation      1.000     0.970     0.985        33
                           atis_aircraft      0.889     0.889     0.889         9
                        atis_flight_time      1.000     1.000     1.000         1
                           atis_quantity      0.429     1.000     0.600         3
                atis_flight#atis_airfare      0.857     0.500     0.632        12
                            atis_airport      1.000     1.000     1.000        18
                           atis_distance      1.000     1.000     1.000        10
                               atis_city      0.545     1.000     0.706         6
                        atis_ground_fare      1.000     0.714     0.833         7
                           atis_capacity      1.000     0.952     0.976        21
                          atis_flight_no      0.889     1.000     0.941         8
                               atis_meal      1.000     0.833     0.909         6
                        atis_restriction      0.000     0.000     0.000         0
             atis_airline#atis_flight_no      0.000     0.000     0.000         0
    atis_ground_service#atis_ground_fare      0.000     0.000     0.000         0
           atis_airfare#atis_flight_time      0.000     0.000     0.000         0
                           atis_cheapest      0.000     0.000     0.000         0
atis_aircraft#atis_flight#atis_flight_no      0.000     0.000     0.000         0

                               micro avg      0.969     0.974     0.971       888
                               macro avg      0.659     0.674     0.655       888
                            weighted avg      0.975     0.974     0.973       888

I0717 07:21:34.504161 140695199004544 interactiveshell.py:2882] 
                              precision    recall  f1-score   support

                           O      0.000     0.000     0.000       0.0
           B-toloc.city_name      0.974     0.993     0.983     716.0
         B-fromloc.city_name      0.986     0.994     0.990     704.0
           I-toloc.city_name      0.974     0.996     0.985     265.0
      B-depart_date.day_name      0.991     0.986     0.988     212.0
              B-airline_name      1.000     1.000     1.000     101.0
         I-fromloc.city_name      0.978     0.989     0.983     177.0
 B-depart_time.period_of_day      0.992     0.915     0.952     130.0
              I-airline_name      1.000     1.000     1.000      65.0
    B-depart_date.day_number      0.964     0.964     0.964      55.0
    B-depart_date.month_name      0.982     0.964     0.973      56.0
          B-depart_time.time      0.826     1.000     0.905      57.0
                B-round_trip      1.000     0.986     0.993      73.0
             B-cost_relative      1.000     0.973     0.986      37.0
                I-round_trip      1.000     1.000     1.000      71.0
                B-flight_mod      0.821     0.958     0.885      24.0
 B-depart_time.time_relative      0.984     0.954     0.969      65.0
          I-depart_time.time      0.945     1.000     0.972      52.0
         B-stoploc.city_name      0.952     1.000     0.976      20.0
                 B-city_name      0.846     0.579     0.688      57.0
                B-class_type      0.960     1.000     0.980      24.0
          B-arrive_time.time      0.943     0.971     0.957      34.0
 B-arrive_time.time_relative      0.882     0.968     0.923      31.0
                I-class_type      1.000     1.000     1.000      17.0
               B-flight_stop      1.000     1.000     1.000      21.0
          I-arrive_time.time      0.971     0.971     0.971      35.0
              B-airline_code      1.000     0.912     0.954      34.0
    I-depart_date.day_number      1.000     1.000     1.000      15.0
      I-fromloc.airport_name      0.441     1.000     0.612      15.0
      B-fromloc.airport_name      0.440     0.917     0.595      12.0
      B-arrive_date.day_name      0.733     1.000     0.846      11.0
          B-toloc.state_code      1.000     1.000     1.000      18.0
B-depart_date.today_relative      1.000     1.000     1.000       9.0
             B-flight_number      0.346     0.818     0.486      11.0
 B-depart_date.date_relative      0.941     0.941     0.941      17.0
          B-toloc.state_name      0.900     0.964     0.931      28.0
           B-fare_basis_code      0.917     0.647     0.759      17.0
               B-flight_time      1.000     1.000     1.000       1.0
                        B-or      1.000     1.000     1.000       3.0
 B-arrive_time.period_of_day      0.750     1.000     0.857       6.0
          B-meal_description      1.000     0.900     0.947      10.0
             I-cost_relative      1.000     0.667     0.800       3.0
              I-airport_name      0.923     0.414     0.571      29.0
               B-fare_amount      0.500     1.000     0.667       2.0
               I-fare_amount      1.000     1.000     1.000       2.0
                 I-city_name      1.000     0.633     0.776      30.0
        I-toloc.airport_name      1.000     1.000     1.000       3.0
            B-transport_type      1.000     0.900     0.947      10.0
    B-arrive_date.month_name      0.714     0.833     0.769       6.0
    B-arrive_date.day_number      0.714     0.833     0.769       6.0
         I-stoploc.city_name      0.909     1.000     0.952      10.0
                      B-meal      1.000     1.000     1.000      16.0
        B-fromloc.state_code      1.000     1.000     1.000      23.0
    B-depart_time.period_mod      1.000     1.000     1.000       5.0
                   B-connect      1.000     1.000     1.000       6.0
               B-flight_days      1.000     1.000     1.000      10.0
        B-toloc.airport_name      0.500     0.333     0.400       3.0
        B-fromloc.state_name      0.944     1.000     0.971      17.0
              B-airport_name      0.818     0.429     0.562      21.0
                   B-economy      1.000     1.000     1.000       6.0
               I-flight_time      1.000     1.000     1.000       1.0
             B-aircraft_code      0.900     0.818     0.857      33.0
                       B-mod      1.000     0.500     0.667       2.0
              B-airport_code      0.000     0.000     0.000       9.0
    B-depart_time.start_time      0.667     0.667     0.667       3.0
      B-depart_time.end_time      1.000     0.333     0.500       3.0
          B-depart_date.year      1.000     0.667     0.800       3.0
            I-transport_type      0.500     1.000     0.667       1.0
          B-restriction_code      0.364     1.000     0.533       4.0
    B-arrive_time.start_time      0.875     0.875     0.875       8.0
        B-toloc.airport_code      1.000     0.500     0.667       4.0
      B-arrive_time.end_time      0.800     1.000     0.889       8.0
      I-arrive_time.end_time      0.875     0.875     0.875       8.0
      I-depart_time.end_time      1.000     0.333     0.500       3.0
               I-flight_stop      0.000     0.000     0.000       0.0
      B-fromloc.airport_code      0.833     1.000     0.909       5.0
          I-restriction_code      1.000     1.000     1.000       3.0
    I-depart_time.start_time      0.500     1.000     0.667       1.0
          I-toloc.state_name      1.000     1.000     1.000       1.0
I-depart_date.today_relative      0.000     0.000     0.000       0.0
 B-arrive_date.date_relative      1.000     1.000     1.000       2.0
                I-flight_mod      1.000     0.167     0.286       6.0
                   I-economy      0.000     0.000     0.000       0.0
 B-return_date.date_relative      0.500     0.333     0.400       3.0
        I-fromloc.state_name      1.000     1.000     1.000       1.0
                B-state_code      1.000     1.000     1.000       1.0
    I-arrive_time.start_time      1.000     1.000     1.000       1.0
    I-arrive_date.day_number      0.000     0.000     0.000       0.0
                 B-meal_code      0.000     0.000     0.000       1.0
 I-depart_time.period_of_day      0.000     0.000     0.000       1.0
                  B-day_name      1.000     0.500     0.667       2.0
             B-period_of_day      1.000     0.500     0.667       4.0
        B-stoploc.state_code      0.000     0.000     0.000       0.0
    B-return_date.month_name      0.000     0.000     0.000       0.0
    B-return_date.day_number      0.000     0.000     0.000       0.0
    B-arrive_time.period_mod      0.000     0.000     0.000       0.0
                 I-meal_code      0.000     0.000     0.000       0.0
        B-toloc.country_name      1.000     1.000     1.000       1.0
                 B-days_code      1.000     1.000     1.000       1.0
 I-arrive_time.period_of_day      0.000     0.000     0.000       0.0
            I-today_relative      0.000     0.000     0.000       0.0
 B-return_time.period_of_day      0.000     0.000     0.000       0.0
                      B-time      0.000     0.000     0.000       0.0
           I-fare_basis_code      0.000     0.000     0.000       0.0
 I-arrive_time.time_relative      1.000     0.500     0.667       4.0
 I-depart_time.time_relative      0.000     0.000     0.000       1.0
            B-today_relative      0.000     0.000     0.000       0.0
                B-state_name      0.000     0.000     0.000       9.0
B-arrive_date.today_relative      0.000     0.000     0.000       0.0
    B-return_time.period_mod      0.000     0.000     0.000       0.0
                B-month_name      0.000     0.000     0.000       0.0
                B-day_number      0.000     0.000     0.000       0.0
 I-return_date.date_relative      0.750     1.000     0.857       3.0
I-return_date.today_relative      0.000     0.000     0.000       0.0
      B-stoploc.airport_name      0.000     0.000     0.000       0.0
             B-time_relative      0.000     0.000     0.000       0.0
                      I-time      0.000     0.000     0.000       0.0
    I-return_date.day_number      0.000     0.000     0.000       0.0
          I-meal_description      0.000     0.000     0.000       0.0
B-return_date.today_relative      0.000     0.000     0.000       0.0
      B-return_date.day_name      0.000     0.000     0.000       2.0

                   micro avg      0.950     0.952     0.951    3657.0
                   macro avg      0.653     0.635     0.626    3657.0
                weighted avg      0.955     0.952     0.949    3657.0

I0717 07:21:34.504858 140695199004544 interactiveshell.py:2882] Best Slot F1: 0.955
Reading ../data/atis.train.w-intent.iob
I0717 07:21:36.903867 140695199004544 interactiveshell.py:2882] Step 20300 | Loss: 0.0028 | Loss_intent: 0.0013 | Loss_slots: 0.0016 | Spent: 6.6 secs | LR: 0.000035
I0717 07:21:41.094336 140695199004544 interactiveshell.py:2882] Step 20350 | Loss: 0.0041 | Loss_intent: 0.0002 | Loss_slots: 0.0039 | Spent: 4.2 secs | LR: 0.000035
I0717 07:21:45.281578 140695199004544 interactiveshell.py:2882] Step 20400 | Loss: 0.0026 | Loss_intent: 0.0001 | Loss_slots: 0.0025 | Spent: 4.2 secs | LR: 0.000035
I0717 07:21:49.483051 140695199004544 interactiveshell.py:2882] Step 20450 | Loss: 0.0050 | Loss_intent: 0.0000 | Loss_slots: 0.0050 | Spent: 4.2 secs | LR: 0.000035
I0717 07:21:53.667385 140695199004544 interactiveshell.py:2882] Step 20500 | Loss: 0.0051 | Loss_intent: 0.0000 | Loss_slots: 0.0051 | Spent: 4.2 secs | LR: 0.000035
I0717 07:21:57.840869 140695199004544 interactiveshell.py:2882] Step 20550 | Loss: 0.0033 | Loss_intent: 0.0000 | Loss_slots: 0.0033 | Spent: 4.2 secs | LR: 0.000034
Reading ../data/atis.test.w-intent.iob
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.
  'precision', 'predicted', average, warn_for)
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1439: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples.
  'recall', 'true', average, warn_for)
I0717 07:22:03.031012 140695199004544 interactiveshell.py:2882] 
                                          precision    recall  f1-score   support

                             atis_flight      0.990     0.981     0.986       632
                            atis_airfare      0.906     1.000     0.950        48
                     atis_ground_service      1.000     1.000     1.000        36
                            atis_airline      0.974     1.000     0.987        38
                       atis_abbreviation      1.000     1.000     1.000        33
                           atis_aircraft      0.889     0.889     0.889         9
                        atis_flight_time      1.000     1.000     1.000         1
                           atis_quantity      0.429     1.000     0.600         3
                atis_flight#atis_airfare      0.857     0.500     0.632        12
                            atis_airport      0.947     1.000     0.973        18
                           atis_distance      1.000     1.000     1.000        10
                               atis_city      0.500     1.000     0.667         6
                        atis_ground_fare      1.000     0.714     0.833         7
                           atis_capacity      1.000     1.000     1.000        21
                          atis_flight_no      0.889     1.000     0.941         8
                               atis_meal      1.000     0.833     0.909         6
                        atis_restriction      0.000     0.000     0.000         0
             atis_airline#atis_flight_no      0.000     0.000     0.000         0
    atis_ground_service#atis_ground_fare      0.000     0.000     0.000         0
           atis_airfare#atis_flight_time      0.000     0.000     0.000         0
                           atis_cheapest      0.000     0.000     0.000         0
atis_aircraft#atis_flight#atis_flight_no      0.000     0.000     0.000         0

                               micro avg      0.970     0.975     0.972       888
                               macro avg      0.654     0.678     0.653       888
                            weighted avg      0.977     0.975     0.974       888

I0717 07:22:03.071322 140695199004544 interactiveshell.py:2882] 
                              precision    recall  f1-score   support

                           O      0.000     0.000     0.000       0.0
           B-toloc.city_name      0.974     0.992     0.983     716.0
         B-fromloc.city_name      0.985     0.994     0.989     704.0
           I-toloc.city_name      0.970     0.992     0.981     265.0
      B-depart_date.day_name      0.991     0.986     0.988     212.0
              B-airline_name      0.990     1.000     0.995     101.0
         I-fromloc.city_name      0.962     0.989     0.975     177.0
 B-depart_time.period_of_day      0.992     0.923     0.956     130.0
              I-airline_name      1.000     1.000     1.000      65.0
    B-depart_date.day_number      0.964     0.964     0.964      55.0
    B-depart_date.month_name      0.982     0.964     0.973      56.0
          B-depart_time.time      0.826     1.000     0.905      57.0
                B-round_trip      1.000     0.986     0.993      73.0
             B-cost_relative      1.000     0.973     0.986      37.0
                I-round_trip      1.000     1.000     1.000      71.0
                B-flight_mod      0.852     0.958     0.902      24.0
 B-depart_time.time_relative      0.984     0.954     0.969      65.0
          I-depart_time.time      0.963     1.000     0.981      52.0
         B-stoploc.city_name      0.952     1.000     0.976      20.0
                 B-city_name      0.846     0.579     0.688      57.0
                B-class_type      0.960     1.000     0.980      24.0
          B-arrive_time.time      0.971     0.971     0.971      34.0
 B-arrive_time.time_relative      0.857     0.968     0.909      31.0
                I-class_type      1.000     1.000     1.000      17.0
               B-flight_stop      1.000     1.000     1.000      21.0
          I-arrive_time.time      0.971     0.971     0.971      35.0
              B-airline_code      1.000     0.882     0.938      34.0
    I-depart_date.day_number      1.000     1.000     1.000      15.0
      I-fromloc.airport_name      0.455     1.000     0.625      15.0
      B-fromloc.airport_name      0.440     0.917     0.595      12.0
      B-arrive_date.day_name      0.733     1.000     0.846      11.0
          B-toloc.state_code      0.947     1.000     0.973      18.0
B-depart_date.today_relative      1.000     1.000     1.000       9.0
             B-flight_number      0.474     0.818     0.600      11.0
 B-depart_date.date_relative      0.941     0.941     0.941      17.0
          B-toloc.state_name      0.900     0.964     0.931      28.0
           B-fare_basis_code      0.944     1.000     0.971      17.0
               B-flight_time      1.000     1.000     1.000       1.0
                        B-or      1.000     1.000     1.000       3.0
 B-arrive_time.period_of_day      0.750     1.000     0.857       6.0
          B-meal_description      1.000     0.900     0.947      10.0
             I-cost_relative      1.000     0.667     0.800       3.0
              I-airport_name      0.917     0.379     0.537      29.0
               B-fare_amount      0.400     1.000     0.571       2.0
               I-fare_amount      1.000     1.000     1.000       2.0
                 I-city_name      1.000     0.567     0.723      30.0
        I-toloc.airport_name      1.000     1.000     1.000       3.0
            B-transport_type      1.000     1.000     1.000      10.0
    B-arrive_date.month_name      0.714     0.833     0.769       6.0
    B-arrive_date.day_number      0.714     0.833     0.769       6.0
         I-stoploc.city_name      0.909     1.000     0.952      10.0
                      B-meal      1.000     1.000     1.000      16.0
        B-fromloc.state_code      1.000     0.957     0.978      23.0
    B-depart_time.period_mod      1.000     1.000     1.000       5.0
                   B-connect      1.000     1.000     1.000       6.0
               B-flight_days      1.000     1.000     1.000      10.0
        B-toloc.airport_name      0.500     0.333     0.400       3.0
        B-fromloc.state_name      0.944     1.000     0.971      17.0
              B-airport_name      0.818     0.429     0.562      21.0
                   B-economy      1.000     1.000     1.000       6.0
               I-flight_time      1.000     1.000     1.000       1.0
             B-aircraft_code      0.900     0.818     0.857      33.0
                       B-mod      1.000     0.500     0.667       2.0
              B-airport_code      0.000     0.000     0.000       9.0
    B-depart_time.start_time      0.667     0.667     0.667       3.0
      B-depart_time.end_time      1.000     0.333     0.500       3.0
          B-depart_date.year      1.000     0.667     0.800       3.0
            I-transport_type      1.000     1.000     1.000       1.0
          B-restriction_code      0.400     1.000     0.571       4.0
    B-arrive_time.start_time      0.889     1.000     0.941       8.0
        B-toloc.airport_code      1.000     0.500     0.667       4.0
      B-arrive_time.end_time      0.800     1.000     0.889       8.0
      I-arrive_time.end_time      0.875     0.875     0.875       8.0
      I-depart_time.end_time      1.000     0.333     0.500       3.0
               I-flight_stop      0.000     0.000     0.000       0.0
      B-fromloc.airport_code      0.833     1.000     0.909       5.0
          I-restriction_code      0.750     1.000     0.857       3.0
    I-depart_time.start_time      0.500     1.000     0.667       1.0
          I-toloc.state_name      1.000     1.000     1.000       1.0
I-depart_date.today_relative      0.000     0.000     0.000       0.0
 B-arrive_date.date_relative      0.667     1.000     0.800       2.0
                I-flight_mod      1.000     0.167     0.286       6.0
                   I-economy      0.000     0.000     0.000       0.0
 B-return_date.date_relative      1.000     0.333     0.500       3.0
        I-fromloc.state_name      1.000     1.000     1.000       1.0
                B-state_code      1.000     1.000     1.000       1.0
    I-arrive_time.start_time      1.000     1.000     1.000       1.0
    I-arrive_date.day_number      0.000     0.000     0.000       0.0
                 B-meal_code      0.000     0.000     0.000       1.0
 I-depart_time.period_of_day      0.000     0.000     0.000       1.0
                  B-day_name      1.000     0.500     0.667       2.0
             B-period_of_day      1.000     0.500     0.667       4.0
        B-stoploc.state_code      0.000     0.000     0.000       0.0
    B-return_date.month_name      0.000     0.000     0.000       0.0
    B-return_date.day_number      0.000     0.000     0.000       0.0
    B-arrive_time.period_mod      0.000     0.000     0.000       0.0
                 I-meal_code      0.000     0.000     0.000       0.0
        B-toloc.country_name      1.000     1.000     1.000       1.0
                 B-days_code      1.000     1.000     1.000       1.0
 I-arrive_time.period_of_day      0.000     0.000     0.000       0.0
            I-today_relative      0.000     0.000     0.000       0.0
 B-return_time.period_of_day      0.000     0.000     0.000       0.0
                      B-time      0.000     0.000     0.000       0.0
           I-fare_basis_code      0.000     0.000     0.000       0.0
 I-arrive_time.time_relative      1.000     0.250     0.400       4.0
 I-depart_time.time_relative      0.000     0.000     0.000       1.0
            B-today_relative      0.000     0.000     0.000       0.0
                B-state_name      0.000     0.000     0.000       9.0
B-arrive_date.today_relative      0.000     0.000     0.000       0.0
    B-return_time.period_mod      0.000     0.000     0.000       0.0
                B-month_name      0.000     0.000     0.000       0.0
                B-day_number      0.000     0.000     0.000       0.0
 I-return_date.date_relative      0.750     1.000     0.857       3.0
I-return_date.today_relative      0.000     0.000     0.000       0.0
      B-stoploc.airport_name      0.000     0.000     0.000       0.0
             B-time_relative      0.000     0.000     0.000       0.0
                      I-time      0.000     0.000     0.000       0.0
    I-return_date.day_number      0.000     0.000     0.000       0.0
          I-meal_description      0.000     0.000     0.000       0.0
B-return_date.today_relative      0.000     0.000     0.000       0.0
      B-return_date.day_name      0.000     0.000     0.000       2.0

                   micro avg      0.951     0.952     0.951    3657.0
                   macro avg      0.657     0.637     0.627    3657.0
                weighted avg      0.954     0.952     0.948    3657.0

I0717 07:22:03.072200 140695199004544 interactiveshell.py:2882] Best Slot F1: 0.955
Reading ../data/atis.train.w-intent.iob
I0717 07:22:04.493694 140695199004544 interactiveshell.py:2882] Step 20600 | Loss: 0.0010 | Loss_intent: 0.0000 | Loss_slots: 0.0010 | Spent: 6.7 secs | LR: 0.000034
I0717 07:22:08.740529 140695199004544 interactiveshell.py:2882] Step 20650 | Loss: 0.0031 | Loss_intent: 0.0001 | Loss_slots: 0.0030 | Spent: 4.2 secs | LR: 0.000034
I0717 07:22:13.220536 140695199004544 interactiveshell.py:2882] Step 20700 | Loss: 0.0066 | Loss_intent: 0.0000 | Loss_slots: 0.0066 | Spent: 4.5 secs | LR: 0.000034
I0717 07:22:17.505726 140695199004544 interactiveshell.py:2882] Step 20750 | Loss: 0.0020 | Loss_intent: 0.0000 | Loss_slots: 0.0020 | Spent: 4.3 secs | LR: 0.000034
I0717 07:22:21.680547 140695199004544 interactiveshell.py:2882] Step 20800 | Loss: 0.0068 | Loss_intent: 0.0000 | Loss_slots: 0.0068 | Spent: 4.2 secs | LR: 0.000034
I0717 07:22:25.871946 140695199004544 interactiveshell.py:2882] Step 20850 | Loss: 0.0057 | Loss_intent: 0.0000 | Loss_slots: 0.0057 | Spent: 4.2 secs | LR: 0.000033
I0717 07:22:30.061193 140695199004544 interactiveshell.py:2882] Step 20900 | Loss: 0.0067 | Loss_intent: 0.0000 | Loss_slots: 0.0067 | Spent: 4.2 secs | LR: 0.000033
Reading ../data/atis.test.w-intent.iob
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.
  'precision', 'predicted', average, warn_for)
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1439: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples.
  'recall', 'true', average, warn_for)
I0717 07:22:32.031732 140695199004544 interactiveshell.py:2882] 
                                          precision    recall  f1-score   support

                             atis_flight      0.987     0.983     0.985       632
                            atis_airfare      0.904     0.979     0.940        48
                     atis_ground_service      1.000     1.000     1.000        36
                            atis_airline      0.974     1.000     0.987        38
                       atis_abbreviation      1.000     1.000     1.000        33
                           atis_aircraft      0.889     0.889     0.889         9
                        atis_flight_time      1.000     1.000     1.000         1
                           atis_quantity      0.429     1.000     0.600         3
                atis_flight#atis_airfare      0.857     0.500     0.632        12
                            atis_airport      0.947     1.000     0.973        18
                           atis_distance      1.000     1.000     1.000        10
                               atis_city      0.545     1.000     0.706         6
                        atis_ground_fare      1.000     0.714     0.833         7
                           atis_capacity      1.000     0.952     0.976        21
                          atis_flight_no      0.889     1.000     0.941         8
                               atis_meal      1.000     0.833     0.909         6
                        atis_restriction      0.000     0.000     0.000         0
             atis_airline#atis_flight_no      0.000     0.000     0.000         0
    atis_ground_service#atis_ground_fare      0.000     0.000     0.000         0
           atis_airfare#atis_flight_time      0.000     0.000     0.000         0
                           atis_cheapest      0.000     0.000     0.000         0
atis_aircraft#atis_flight#atis_flight_no      0.000     0.000     0.000         0

                               micro avg      0.969     0.974     0.971       888
                               macro avg      0.656     0.675     0.653       888
                            weighted avg      0.975     0.974     0.972       888

I0717 07:22:32.072451 140695199004544 interactiveshell.py:2882] 
                              precision    recall  f1-score   support

                           O      0.000     0.000     0.000       0.0
           B-toloc.city_name      0.975     0.993     0.984     716.0
         B-fromloc.city_name      0.987     0.994     0.991     704.0
           I-toloc.city_name      0.974     0.996     0.985     265.0
      B-depart_date.day_name      0.991     0.986     0.988     212.0
              B-airline_name      0.981     1.000     0.990     101.0
         I-fromloc.city_name      0.978     0.989     0.983     177.0
 B-depart_time.period_of_day      1.000     0.923     0.960     130.0
              I-airline_name      1.000     1.000     1.000      65.0
    B-depart_date.day_number      0.964     0.964     0.964      55.0
    B-depart_date.month_name      0.982     0.964     0.973      56.0
          B-depart_time.time      0.826     1.000     0.905      57.0
                B-round_trip      1.000     0.986     0.993      73.0
             B-cost_relative      1.000     0.973     0.986      37.0
                I-round_trip      1.000     1.000     1.000      71.0
                B-flight_mod      0.885     0.958     0.920      24.0
 B-depart_time.time_relative      0.984     0.954     0.969      65.0
          I-depart_time.time      0.963     1.000     0.981      52.0
         B-stoploc.city_name      0.952     1.000     0.976      20.0
                 B-city_name      0.842     0.561     0.674      57.0
                B-class_type      0.960     1.000     0.980      24.0
          B-arrive_time.time      0.971     0.971     0.971      34.0
 B-arrive_time.time_relative      0.857     0.968     0.909      31.0
                I-class_type      1.000     1.000     1.000      17.0
               B-flight_stop      0.955     1.000     0.977      21.0
          I-arrive_time.time      0.971     0.971     0.971      35.0
              B-airline_code      1.000     0.853     0.921      34.0
    I-depart_date.day_number      1.000     1.000     1.000      15.0
      I-fromloc.airport_name      0.429     1.000     0.600      15.0
      B-fromloc.airport_name      0.440     0.917     0.595      12.0
      B-arrive_date.day_name      0.733     1.000     0.846      11.0
          B-toloc.state_code      1.000     1.000     1.000      18.0
B-depart_date.today_relative      1.000     1.000     1.000       9.0
             B-flight_number      0.360     0.818     0.500      11.0
 B-depart_date.date_relative      0.944     1.000     0.971      17.0
          B-toloc.state_name      0.903     1.000     0.949      28.0
           B-fare_basis_code      0.917     0.647     0.759      17.0
               B-flight_time      1.000     1.000     1.000       1.0
                        B-or      1.000     1.000     1.000       3.0
 B-arrive_time.period_of_day      0.750     1.000     0.857       6.0
          B-meal_description      1.000     0.900     0.947      10.0
             I-cost_relative      1.000     0.667     0.800       3.0
              I-airport_name      0.917     0.379     0.537      29.0
               B-fare_amount      0.400     1.000     0.571       2.0
               I-fare_amount      1.000     1.000     1.000       2.0
                 I-city_name      1.000     0.567     0.723      30.0
        I-toloc.airport_name      1.000     1.000     1.000       3.0
            B-transport_type      1.000     1.000     1.000      10.0
    B-arrive_date.month_name      0.714     0.833     0.769       6.0
    B-arrive_date.day_number      0.714     0.833     0.769       6.0
         I-stoploc.city_name      0.909     1.000     0.952      10.0
                      B-meal      1.000     1.000     1.000      16.0
        B-fromloc.state_code      1.000     1.000     1.000      23.0
    B-depart_time.period_mod      1.000     1.000     1.000       5.0
                   B-connect      1.000     1.000     1.000       6.0
               B-flight_days      1.000     1.000     1.000      10.0
        B-toloc.airport_name      0.750     1.000     0.857       3.0
        B-fromloc.state_name      0.944     1.000     0.971      17.0
              B-airport_name      0.750     0.429     0.545      21.0
                   B-economy      1.000     1.000     1.000       6.0
               I-flight_time      1.000     1.000     1.000       1.0
             B-aircraft_code      0.931     0.818     0.871      33.0
                       B-mod      1.000     0.500     0.667       2.0
              B-airport_code      0.000     0.000     0.000       9.0
    B-depart_time.start_time      0.667     0.667     0.667       3.0
      B-depart_time.end_time      1.000     0.333     0.500       3.0
          B-depart_date.year      1.000     0.667     0.800       3.0
            I-transport_type      0.500     1.000     0.667       1.0
          B-restriction_code      0.400     1.000     0.571       4.0
    B-arrive_time.start_time      0.889     1.000     0.941       8.0
        B-toloc.airport_code      1.000     0.500     0.667       4.0
      B-arrive_time.end_time      0.800     1.000     0.889       8.0
      I-arrive_time.end_time      0.875     0.875     0.875       8.0
      I-depart_time.end_time      1.000     0.333     0.500       3.0
               I-flight_stop      0.000     0.000     0.000       0.0
      B-fromloc.airport_code      0.833     1.000     0.909       5.0
          I-restriction_code      0.750     1.000     0.857       3.0
    I-depart_time.start_time      0.500     1.000     0.667       1.0
          I-toloc.state_name      1.000     1.000     1.000       1.0
I-depart_date.today_relative      0.000     0.000     0.000       0.0
 B-arrive_date.date_relative      1.000     1.000     1.000       2.0
                I-flight_mod      1.000     0.167     0.286       6.0
                   I-economy      0.000     0.000     0.000       0.0
 B-return_date.date_relative      1.000     0.333     0.500       3.0
        I-fromloc.state_name      1.000     1.000     1.000       1.0
                B-state_code      1.000     1.000     1.000       1.0
    I-arrive_time.start_time      1.000     1.000     1.000       1.0
    I-arrive_date.day_number      0.000     0.000     0.000       0.0
                 B-meal_code      0.000     0.000     0.000       1.0
 I-depart_time.period_of_day      1.000     1.000     1.000       1.0
                  B-day_name      1.000     0.500     0.667       2.0
             B-period_of_day      1.000     0.500     0.667       4.0
        B-stoploc.state_code      0.000     0.000     0.000       0.0
    B-return_date.month_name      0.000     0.000     0.000       0.0
    B-return_date.day_number      0.000     0.000     0.000       0.0
    B-arrive_time.period_mod      0.000     0.000     0.000       0.0
                 I-meal_code      0.000     0.000     0.000       0.0
        B-toloc.country_name      1.000     1.000     1.000       1.0
                 B-days_code      1.000     1.000     1.000       1.0
 I-arrive_time.period_of_day      0.000     0.000     0.000       0.0
            I-today_relative      0.000     0.000     0.000       0.0
 B-return_time.period_of_day      0.000     0.000     0.000       0.0
                      B-time      0.000     0.000     0.000       0.0
           I-fare_basis_code      0.000     0.000     0.000       0.0
 I-arrive_time.time_relative      1.000     0.250     0.400       4.0
 I-depart_time.time_relative      0.000     0.000     0.000       1.0
            B-today_relative      0.000     0.000     0.000       0.0
                B-state_name      0.000     0.000     0.000       9.0
B-arrive_date.today_relative      0.000     0.000     0.000       0.0
    B-return_time.period_mod      0.000     0.000     0.000       0.0
                B-month_name      0.000     0.000     0.000       0.0
                B-day_number      0.000     0.000     0.000       0.0
 I-return_date.date_relative      0.750     1.000     0.857       3.0
I-return_date.today_relative      0.000     0.000     0.000       0.0
      B-stoploc.airport_name      0.000     0.000     0.000       0.0
             B-time_relative      0.000     0.000     0.000       0.0
                      I-time      0.000     0.000     0.000       0.0
    I-return_date.day_number      0.000     0.000     0.000       0.0
          I-meal_description      0.000     0.000     0.000       0.0
B-return_date.today_relative      0.000     0.000     0.000       0.0
      B-return_date.day_name      0.000     0.000     0.000       2.0

                   micro avg      0.951     0.952     0.951    3657.0
                   macro avg      0.665     0.648     0.635    3657.0
                weighted avg      0.956     0.952     0.949    3657.0

I0717 07:22:32.073280 140695199004544 interactiveshell.py:2882] Best Slot F1: 0.955
Reading ../data/atis.train.w-intent.iob
I0717 07:22:36.628161 140695199004544 interactiveshell.py:2882] Step 20950 | Loss: 0.0004 | Loss_intent: 0.0000 | Loss_slots: 0.0004 | Spent: 6.6 secs | LR: 0.000033
I0717 07:22:40.822519 140695199004544 interactiveshell.py:2882] Step 21000 | Loss: 0.0044 | Loss_intent: 0.0000 | Loss_slots: 0.0044 | Spent: 4.2 secs | LR: 0.000033
I0717 07:22:44.990102 140695199004544 interactiveshell.py:2882] Step 21050 | Loss: 0.0040 | Loss_intent: 0.0000 | Loss_slots: 0.0040 | Spent: 4.2 secs | LR: 0.000033
I0717 07:22:49.160007 140695199004544 interactiveshell.py:2882] Step 21100 | Loss: 0.0024 | Loss_intent: 0.0000 | Loss_slots: 0.0023 | Spent: 4.2 secs | LR: 0.000032
I0717 07:22:53.368381 140695199004544 interactiveshell.py:2882] Step 21150 | Loss: 0.0143 | Loss_intent: 0.0076 | Loss_slots: 0.0067 | Spent: 4.2 secs | LR: 0.000032
I0717 07:22:57.546345 140695199004544 interactiveshell.py:2882] Step 21200 | Loss: 0.0013 | Loss_intent: 0.0000 | Loss_slots: 0.0013 | Spent: 4.2 secs | LR: 0.000032
Reading ../data/atis.test.w-intent.iob
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.
  'precision', 'predicted', average, warn_for)
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1439: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples.
  'recall', 'true', average, warn_for)
I0717 07:23:00.520035 140695199004544 interactiveshell.py:2882] 
                                          precision    recall  f1-score   support

                             atis_flight      0.990     0.983     0.986       632
                            atis_airfare      0.906     1.000     0.950        48
                     atis_ground_service      1.000     1.000     1.000        36
                            atis_airline      0.974     1.000     0.987        38
                       atis_abbreviation      1.000     1.000     1.000        33
                           atis_aircraft      0.889     0.889     0.889         9
                        atis_flight_time      1.000     1.000     1.000         1
                           atis_quantity      0.429     1.000     0.600         3
                atis_flight#atis_airfare      0.857     0.500     0.632        12
                            atis_airport      0.947     1.000     0.973        18
                           atis_distance      1.000     1.000     1.000        10
                               atis_city      0.545     1.000     0.706         6
                        atis_ground_fare      1.000     0.714     0.833         7
                           atis_capacity      1.000     1.000     1.000        21
                          atis_flight_no      0.889     1.000     0.941         8
                               atis_meal      1.000     0.833     0.909         6
                        atis_restriction      0.000     0.000     0.000         0
             atis_airline#atis_flight_no      0.000     0.000     0.000         0
    atis_ground_service#atis_ground_fare      0.000     0.000     0.000         0
           atis_airfare#atis_flight_time      0.000     0.000     0.000         0
                           atis_cheapest      0.000     0.000     0.000         0
atis_aircraft#atis_flight#atis_flight_no      0.000     0.000     0.000         0

                               micro avg      0.971     0.976     0.974       888
                               macro avg      0.656     0.678     0.655       888
                            weighted avg      0.977     0.976     0.975       888

I0717 07:23:00.561228 140695199004544 interactiveshell.py:2882] 
                              precision    recall  f1-score   support

                           O      0.000     0.000     0.000       0.0
           B-toloc.city_name      0.974     0.993     0.983     716.0
         B-fromloc.city_name      0.986     0.993     0.989     704.0
           I-toloc.city_name      0.978     1.000     0.989     265.0
      B-depart_date.day_name      0.991     0.986     0.988     212.0
              B-airline_name      0.981     1.000     0.990     101.0
         I-fromloc.city_name      0.983     0.989     0.986     177.0
 B-depart_time.period_of_day      0.992     0.915     0.952     130.0
              I-airline_name      1.000     1.000     1.000      65.0
    B-depart_date.day_number      0.964     0.964     0.964      55.0
    B-depart_date.month_name      0.982     0.964     0.973      56.0
          B-depart_time.time      0.826     1.000     0.905      57.0
                B-round_trip      1.000     0.986     0.993      73.0
             B-cost_relative      1.000     0.973     0.986      37.0
                I-round_trip      1.000     1.000     1.000      71.0
                B-flight_mod      1.000     0.958     0.979      24.0
 B-depart_time.time_relative      0.984     0.954     0.969      65.0
          I-depart_time.time      0.963     1.000     0.981      52.0
         B-stoploc.city_name      0.952     1.000     0.976      20.0
                 B-city_name      0.842     0.561     0.674      57.0
                B-class_type      0.960     1.000     0.980      24.0
          B-arrive_time.time      0.971     0.971     0.971      34.0
 B-arrive_time.time_relative      0.833     0.968     0.896      31.0
                I-class_type      1.000     1.000     1.000      17.0
               B-flight_stop      1.000     1.000     1.000      21.0
          I-arrive_time.time      0.971     0.971     0.971      35.0
              B-airline_code      1.000     0.853     0.921      34.0
    I-depart_date.day_number      1.000     1.000     1.000      15.0
      I-fromloc.airport_name      0.429     1.000     0.600      15.0
      B-fromloc.airport_name      0.393     0.917     0.550      12.0
      B-arrive_date.day_name      0.733     1.000     0.846      11.0
          B-toloc.state_code      1.000     1.000     1.000      18.0
B-depart_date.today_relative      1.000     1.000     1.000       9.0
             B-flight_number      0.474     0.818     0.600      11.0
 B-depart_date.date_relative      0.944     1.000     0.971      17.0
          B-toloc.state_name      0.903     1.000     0.949      28.0
           B-fare_basis_code      0.944     1.000     0.971      17.0
               B-flight_time      1.000     1.000     1.000       1.0
                        B-or      1.000     1.000     1.000       3.0
 B-arrive_time.period_of_day      0.750     1.000     0.857       6.0
          B-meal_description      1.000     0.900     0.947      10.0
             I-cost_relative      1.000     0.667     0.800       3.0
              I-airport_name      0.917     0.379     0.537      29.0
               B-fare_amount      0.500     1.000     0.667       2.0
               I-fare_amount      1.000     1.000     1.000       2.0
                 I-city_name      1.000     0.600     0.750      30.0
        I-toloc.airport_name      1.000     1.000     1.000       3.0
            B-transport_type      1.000     1.000     1.000      10.0
    B-arrive_date.month_name      0.714     0.833     0.769       6.0
    B-arrive_date.day_number      0.714     0.833     0.769       6.0
         I-stoploc.city_name      0.833     1.000     0.909      10.0
                      B-meal      1.000     1.000     1.000      16.0
        B-fromloc.state_code      1.000     1.000     1.000      23.0
    B-depart_time.period_mod      0.833     1.000     0.909       5.0
                   B-connect      1.000     1.000     1.000       6.0
               B-flight_days      1.000     1.000     1.000      10.0
        B-toloc.airport_name      0.750     1.000     0.857       3.0
        B-fromloc.state_name      0.944     1.000     0.971      17.0
              B-airport_name      0.800     0.381     0.516      21.0
                   B-economy      1.000     1.000     1.000       6.0
               I-flight_time      1.000     1.000     1.000       1.0
             B-aircraft_code      0.964     0.818     0.885      33.0
                       B-mod      1.000     0.500     0.667       2.0
              B-airport_code      0.000     0.000     0.000       9.0
    B-depart_time.start_time      0.667     0.667     0.667       3.0
      B-depart_time.end_time      1.000     0.667     0.800       3.0
          B-depart_date.year      1.000     0.667     0.800       3.0
            I-transport_type      1.000     1.000     1.000       1.0
          B-restriction_code      0.364     1.000     0.533       4.0
    B-arrive_time.start_time      0.889     1.000     0.941       8.0
        B-toloc.airport_code      1.000     0.500     0.667       4.0
      B-arrive_time.end_time      0.889     1.000     0.941       8.0
      I-arrive_time.end_time      0.875     0.875     0.875       8.0
      I-depart_time.end_time      1.000     0.333     0.500       3.0
               I-flight_stop      0.000     0.000     0.000       0.0
      B-fromloc.airport_code      0.833     1.000     0.909       5.0
          I-restriction_code      1.000     1.000     1.000       3.0
    I-depart_time.start_time      0.500     1.000     0.667       1.0
          I-toloc.state_name      1.000     1.000     1.000       1.0
I-depart_date.today_relative      0.000     0.000     0.000       0.0
 B-arrive_date.date_relative      1.000     1.000     1.000       2.0
                I-flight_mod      1.000     0.167     0.286       6.0
                   I-economy      0.000     0.000     0.000       0.0
 B-return_date.date_relative      1.000     0.333     0.500       3.0
        I-fromloc.state_name      1.000     1.000     1.000       1.0
                B-state_code      1.000     1.000     1.000       1.0
    I-arrive_time.start_time      1.000     1.000     1.000       1.0
    I-arrive_date.day_number      0.000     0.000     0.000       0.0
                 B-meal_code      1.000     1.000     1.000       1.0
 I-depart_time.period_of_day      0.000     0.000     0.000       1.0
                  B-day_name      1.000     0.500     0.667       2.0
             B-period_of_day      1.000     0.500     0.667       4.0
        B-stoploc.state_code      0.000     0.000     0.000       0.0
    B-return_date.month_name      0.000     0.000     0.000       0.0
    B-return_date.day_number      0.000     0.000     0.000       0.0
    B-arrive_time.period_mod      0.000     0.000     0.000       0.0
                 I-meal_code      0.000     0.000     0.000       0.0
        B-toloc.country_name      1.000     1.000     1.000       1.0
                 B-days_code      1.000     1.000     1.000       1.0
 I-arrive_time.period_of_day      0.000     0.000     0.000       0.0
            I-today_relative      0.000     0.000     0.000       0.0
 B-return_time.period_of_day      0.000     0.000     0.000       0.0
                      B-time      0.000     0.000     0.000       0.0
           I-fare_basis_code      0.000     0.000     0.000       0.0
 I-arrive_time.time_relative      0.000     0.000     0.000       4.0
 I-depart_time.time_relative      0.000     0.000     0.000       1.0
            B-today_relative      0.000     0.000     0.000       0.0
                B-state_name      0.000     0.000     0.000       9.0
B-arrive_date.today_relative      0.000     0.000     0.000       0.0
    B-return_time.period_mod      0.000     0.000     0.000       0.0
                B-month_name      0.000     0.000     0.000       0.0
                B-day_number      0.000     0.000     0.000       0.0
 I-return_date.date_relative      0.750     1.000     0.857       3.0
I-return_date.today_relative      0.000     0.000     0.000       0.0
      B-stoploc.airport_name      0.000     0.000     0.000       0.0
             B-time_relative      0.000     0.000     0.000       0.0
                      I-time      0.000     0.000     0.000       0.0
    I-return_date.day_number      0.000     0.000     0.000       0.0
          I-meal_description      0.000     0.000     0.000       0.0
B-return_date.today_relative      0.000     0.000     0.000       0.0
      B-return_date.day_name      0.000     0.000     0.000       2.0

                   micro avg      0.952     0.954     0.953    3657.0
                   macro avg      0.665     0.652     0.641    3657.0
                weighted avg      0.957     0.954     0.950    3657.0

I0717 07:23:00.562172 140695199004544 interactiveshell.py:2882] Best Slot F1: 0.955
Reading ../data/atis.train.w-intent.iob
I0717 07:23:04.146968 140695199004544 interactiveshell.py:2882] Step 21250 | Loss: 0.0020 | Loss_intent: 0.0000 | Loss_slots: 0.0019 | Spent: 6.6 secs | LR: 0.000032
I0717 07:23:08.326789 140695199004544 interactiveshell.py:2882] Step 21300 | Loss: 0.0062 | Loss_intent: 0.0000 | Loss_slots: 0.0062 | Spent: 4.2 secs | LR: 0.000032
I0717 07:23:12.502645 140695199004544 interactiveshell.py:2882] Step 21350 | Loss: 0.0046 | Loss_intent: 0.0001 | Loss_slots: 0.0045 | Spent: 4.2 secs | LR: 0.000032
I0717 07:23:16.669955 140695199004544 interactiveshell.py:2882] Step 21400 | Loss: 0.0019 | Loss_intent: 0.0000 | Loss_slots: 0.0019 | Spent: 4.2 secs | LR: 0.000031
I0717 07:23:20.868788 140695199004544 interactiveshell.py:2882] Step 21450 | Loss: 0.0207 | Loss_intent: 0.0105 | Loss_slots: 0.0101 | Spent: 4.2 secs | LR: 0.000031
I0717 07:23:25.140593 140695199004544 interactiveshell.py:2882] Step 21500 | Loss: 0.0159 | Loss_intent: 0.0001 | Loss_slots: 0.0158 | Spent: 4.3 secs | LR: 0.000031
Reading ../data/atis.test.w-intent.iob
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.
  'precision', 'predicted', average, warn_for)
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1439: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples.
  'recall', 'true', average, warn_for)
I0717 07:23:29.170674 140695199004544 interactiveshell.py:2882] 
                                          precision    recall  f1-score   support

                             atis_flight      0.989     0.983     0.986       632
                            atis_airfare      0.906     1.000     0.950        48
                     atis_ground_service      1.000     1.000     1.000        36
                            atis_airline      0.974     1.000     0.987        38
                       atis_abbreviation      0.971     1.000     0.985        33
                           atis_aircraft      1.000     0.889     0.941         9
                        atis_flight_time      1.000     1.000     1.000         1
                           atis_quantity      0.375     1.000     0.545         3
                atis_flight#atis_airfare      0.833     0.417     0.556        12
                            atis_airport      0.947     1.000     0.973        18
                           atis_distance      1.000     1.000     1.000        10
                               atis_city      0.545     1.000     0.706         6
                        atis_ground_fare      1.000     0.857     0.923         7
                           atis_capacity      1.000     0.952     0.976        21
                          atis_flight_no      0.889     1.000     0.941         8
                               atis_meal      1.000     0.833     0.909         6
                        atis_restriction      0.000     0.000     0.000         0
             atis_airline#atis_flight_no      0.000     0.000     0.000         0
    atis_ground_service#atis_ground_fare      0.000     0.000     0.000         0
           atis_airfare#atis_flight_time      0.000     0.000     0.000         0
                           atis_cheapest      0.000     0.000     0.000         0
atis_aircraft#atis_flight#atis_flight_no      0.000     0.000     0.000         0

                               micro avg      0.970     0.975     0.972       888
                               macro avg      0.656     0.679     0.654       888
                            weighted avg      0.975     0.975     0.973       888

I0717 07:23:29.211250 140695199004544 interactiveshell.py:2882] 
                              precision    recall  f1-score   support

                           O      0.000     0.000     0.000       0.0
           B-toloc.city_name      0.975     0.993     0.984     716.0
         B-fromloc.city_name      0.990     0.994     0.992     704.0
           I-toloc.city_name      0.981     0.996     0.989     265.0
      B-depart_date.day_name      0.990     0.981     0.986     212.0
              B-airline_name      0.990     1.000     0.995     101.0
         I-fromloc.city_name      0.972     0.989     0.980     177.0
 B-depart_time.period_of_day      1.000     0.915     0.956     130.0
              I-airline_name      1.000     1.000     1.000      65.0
    B-depart_date.day_number      0.963     0.945     0.954      55.0
    B-depart_date.month_name      0.982     0.964     0.973      56.0
          B-depart_time.time      0.826     1.000     0.905      57.0
                B-round_trip      1.000     0.986     0.993      73.0
             B-cost_relative      1.000     0.973     0.986      37.0
                I-round_trip      1.000     1.000     1.000      71.0
                B-flight_mod      0.958     0.958     0.958      24.0
 B-depart_time.time_relative      0.984     0.954     0.969      65.0
          I-depart_time.time      0.963     1.000     0.981      52.0
         B-stoploc.city_name      0.952     1.000     0.976      20.0
                 B-city_name      0.875     0.614     0.722      57.0
                B-class_type      0.960     1.000     0.980      24.0
          B-arrive_time.time      0.971     0.971     0.971      34.0
 B-arrive_time.time_relative      0.833     0.968     0.896      31.0
                I-class_type      1.000     1.000     1.000      17.0
               B-flight_stop      0.955     1.000     0.977      21.0
          I-arrive_time.time      0.971     0.971     0.971      35.0
              B-airline_code      1.000     0.882     0.938      34.0
    I-depart_date.day_number      1.000     1.000     1.000      15.0
      I-fromloc.airport_name      0.455     1.000     0.625      15.0
      B-fromloc.airport_name      0.423     0.917     0.579      12.0
      B-arrive_date.day_name      0.688     1.000     0.815      11.0
          B-toloc.state_code      0.947     1.000     0.973      18.0
B-depart_date.today_relative      1.000     1.000     1.000       9.0
             B-flight_number      0.529     0.818     0.643      11.0
 B-depart_date.date_relative      0.944     1.000     0.971      17.0
          B-toloc.state_name      0.900     0.964     0.931      28.0
           B-fare_basis_code      0.941     0.941     0.941      17.0
               B-flight_time      1.000     1.000     1.000       1.0
                        B-or      1.000     1.000     1.000       3.0
 B-arrive_time.period_of_day      0.750     1.000     0.857       6.0
          B-meal_description      1.000     0.900     0.947      10.0
             I-cost_relative      1.000     0.667     0.800       3.0
              I-airport_name      0.923     0.414     0.571      29.0
               B-fare_amount      0.400     1.000     0.571       2.0
               I-fare_amount      1.000     1.000     1.000       2.0
                 I-city_name      1.000     0.633     0.776      30.0
        I-toloc.airport_name      1.000     1.000     1.000       3.0
            B-transport_type      1.000     1.000     1.000      10.0
    B-arrive_date.month_name      0.714     0.833     0.769       6.0
    B-arrive_date.day_number      0.714     0.833     0.769       6.0
         I-stoploc.city_name      0.909     1.000     0.952      10.0
                      B-meal      1.000     1.000     1.000      16.0
        B-fromloc.state_code      1.000     0.957     0.978      23.0
    B-depart_time.period_mod      0.833     1.000     0.909       5.0
                   B-connect      1.000     1.000     1.000       6.0
               B-flight_days      1.000     1.000     1.000      10.0
        B-toloc.airport_name      0.667     0.667     0.667       3.0
        B-fromloc.state_name      0.895     1.000     0.944      17.0
              B-airport_name      0.727     0.381     0.500      21.0
                   B-economy      1.000     1.000     1.000       6.0
               I-flight_time      1.000     1.000     1.000       1.0
             B-aircraft_code      0.933     0.848     0.889      33.0
                       B-mod      1.000     0.500     0.667       2.0
              B-airport_code      0.000     0.000     0.000       9.0
    B-depart_time.start_time      0.667     0.667     0.667       3.0
      B-depart_time.end_time      1.000     0.333     0.500       3.0
          B-depart_date.year      1.000     0.667     0.800       3.0
            I-transport_type      1.000     1.000     1.000       1.0
          B-restriction_code      0.333     1.000     0.500       4.0
    B-arrive_time.start_time      0.889     1.000     0.941       8.0
        B-toloc.airport_code      1.000     0.500     0.667       4.0
      B-arrive_time.end_time      0.800     1.000     0.889       8.0
      I-arrive_time.end_time      0.875     0.875     0.875       8.0
      I-depart_time.end_time      1.000     0.667     0.800       3.0
               I-flight_stop      0.000     0.000     0.000       0.0
      B-fromloc.airport_code      0.833     1.000     0.909       5.0
          I-restriction_code      0.750     1.000     0.857       3.0
    I-depart_time.start_time      1.000     1.000     1.000       1.0
          I-toloc.state_name      1.000     1.000     1.000       1.0
I-depart_date.today_relative      0.000     0.000     0.000       0.0
 B-arrive_date.date_relative      1.000     1.000     1.000       2.0
                I-flight_mod      1.000     0.167     0.286       6.0
                   I-economy      0.000     0.000     0.000       0.0
 B-return_date.date_relative      1.000     0.333     0.500       3.0
        I-fromloc.state_name      1.000     1.000     1.000       1.0
                B-state_code      1.000     1.000     1.000       1.0
    I-arrive_time.start_time      1.000     1.000     1.000       1.0
    I-arrive_date.day_number      0.000     0.000     0.000       0.0
                 B-meal_code      0.000     0.000     0.000       1.0
 I-depart_time.period_of_day      1.000     1.000     1.000       1.0
                  B-day_name      1.000     0.500     0.667       2.0
             B-period_of_day      1.000     0.500     0.667       4.0
        B-stoploc.state_code      0.000     0.000     0.000       0.0
    B-return_date.month_name      0.000     0.000     0.000       0.0
    B-return_date.day_number      0.000     0.000     0.000       0.0
    B-arrive_time.period_mod      0.000     0.000     0.000       0.0
                 I-meal_code      0.000     0.000     0.000       0.0
        B-toloc.country_name      1.000     1.000     1.000       1.0
                 B-days_code      1.000     1.000     1.000       1.0
 I-arrive_time.period_of_day      0.000     0.000     0.000       0.0
            I-today_relative      0.000     0.000     0.000       0.0
 B-return_time.period_of_day      0.000     0.000     0.000       0.0
                      B-time      0.000     0.000     0.000       0.0
           I-fare_basis_code      0.000     0.000     0.000       0.0
 I-arrive_time.time_relative      0.000     0.000     0.000       4.0
 I-depart_time.time_relative      0.000     0.000     0.000       1.0
            B-today_relative      0.000     0.000     0.000       0.0
                B-state_name      1.000     0.111     0.200       9.0
B-arrive_date.today_relative      0.000     0.000     0.000       0.0
    B-return_time.period_mod      0.000     0.000     0.000       0.0
                B-month_name      0.000     0.000     0.000       0.0
                B-day_number      0.000     0.000     0.000       0.0
 I-return_date.date_relative      0.667     0.667     0.667       3.0
I-return_date.today_relative      0.000     0.000     0.000       0.0
      B-stoploc.airport_name      0.000     0.000     0.000       0.0
             B-time_relative      0.000     0.000     0.000       0.0
                      I-time      0.000     0.000     0.000       0.0
    I-return_date.day_number      0.000     0.000     0.000       0.0
          I-meal_description      0.000     0.000     0.000       0.0
B-return_date.today_relative      0.000     0.000     0.000       0.0
      B-return_date.day_name      0.000     0.000     0.000       2.0

                   micro avg      0.952     0.954     0.953    3657.0
                   macro avg      0.671     0.647     0.640    3657.0
                weighted avg      0.959     0.954     0.951    3657.0

I0717 07:23:29.212164 140695199004544 interactiveshell.py:2882] Best Slot F1: 0.955
Reading ../data/atis.train.w-intent.iob
I0717 07:23:31.792960 140695199004544 interactiveshell.py:2882] Step 21550 | Loss: 0.0119 | Loss_intent: 0.0000 | Loss_slots: 0.0119 | Spent: 6.7 secs | LR: 0.000031
I0717 07:23:35.953640 140695199004544 interactiveshell.py:2882] Step 21600 | Loss: 0.0058 | Loss_intent: 0.0003 | Loss_slots: 0.0054 | Spent: 4.2 secs | LR: 0.000031
I0717 07:23:40.121634 140695199004544 interactiveshell.py:2882] Step 21650 | Loss: 0.0024 | Loss_intent: 0.0001 | Loss_slots: 0.0023 | Spent: 4.2 secs | LR: 0.000031
I0717 07:23:44.323268 140695199004544 interactiveshell.py:2882] Step 21700 | Loss: 0.0009 | Loss_intent: 0.0000 | Loss_slots: 0.0009 | Spent: 4.2 secs | LR: 0.000030
I0717 07:23:48.505392 140695199004544 interactiveshell.py:2882] Step 21750 | Loss: 0.0046 | Loss_intent: 0.0000 | Loss_slots: 0.0046 | Spent: 4.2 secs | LR: 0.000030
I0717 07:23:52.719291 140695199004544 interactiveshell.py:2882] Step 21800 | Loss: 0.0018 | Loss_intent: 0.0000 | Loss_slots: 0.0018 | Spent: 4.2 secs | LR: 0.000030
Reading ../data/atis.test.w-intent.iob
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.
  'precision', 'predicted', average, warn_for)
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1439: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples.
  'recall', 'true', average, warn_for)
I0717 07:23:57.677037 140695199004544 interactiveshell.py:2882] 
                                          precision    recall  f1-score   support

                             atis_flight      0.984     0.983     0.983       632
                            atis_airfare      0.923     1.000     0.960        48
                     atis_ground_service      1.000     1.000     1.000        36
                            atis_airline      0.974     1.000     0.987        38
                       atis_abbreviation      1.000     1.000     1.000        33
                           atis_aircraft      1.000     0.889     0.941         9
                        atis_flight_time      1.000     1.000     1.000         1
                           atis_quantity      0.429     1.000     0.600         3
                atis_flight#atis_airfare      0.833     0.417     0.556        12
                            atis_airport      0.947     1.000     0.973        18
                           atis_distance      1.000     1.000     1.000        10
                               atis_city      0.545     1.000     0.706         6
                        atis_ground_fare      1.000     0.714     0.833         7
                           atis_capacity      1.000     0.952     0.976        21
                          atis_flight_no      0.889     1.000     0.941         8
                               atis_meal      1.000     0.833     0.909         6
                        atis_restriction      0.000     0.000     0.000         0
             atis_airline#atis_flight_no      0.000     0.000     0.000         0
    atis_ground_service#atis_ground_fare      0.000     0.000     0.000         0
           atis_airfare#atis_flight_time      0.000     0.000     0.000         0
                           atis_cheapest      0.000     0.000     0.000         0
atis_aircraft#atis_flight#atis_flight_no      0.000     0.000     0.000         0

                               micro avg      0.969     0.974     0.971       888
                               macro avg      0.660     0.672     0.653       888
                            weighted avg      0.974     0.974     0.972       888

I0717 07:23:57.718102 140695199004544 interactiveshell.py:2882] 
                              precision    recall  f1-score   support

                           O      0.000     0.000     0.000       0.0
           B-toloc.city_name      0.974     0.993     0.983     716.0
         B-fromloc.city_name      0.987     0.993     0.990     704.0
           I-toloc.city_name      0.978     0.996     0.987     265.0
      B-depart_date.day_name      0.991     0.986     0.988     212.0
              B-airline_name      0.990     1.000     0.995     101.0
         I-fromloc.city_name      0.972     0.989     0.980     177.0
 B-depart_time.period_of_day      0.992     0.915     0.952     130.0
              I-airline_name      1.000     1.000     1.000      65.0
    B-depart_date.day_number      0.981     0.964     0.972      55.0
    B-depart_date.month_name      0.982     0.964     0.973      56.0
          B-depart_time.time      0.838     1.000     0.912      57.0
                B-round_trip      1.000     0.986     0.993      73.0
             B-cost_relative      1.000     0.973     0.986      37.0
                I-round_trip      1.000     1.000     1.000      71.0
                B-flight_mod      0.885     0.958     0.920      24.0
 B-depart_time.time_relative      0.984     0.969     0.977      65.0
          I-depart_time.time      0.963     1.000     0.981      52.0
         B-stoploc.city_name      0.952     1.000     0.976      20.0
                 B-city_name      0.865     0.561     0.681      57.0
                B-class_type      0.923     1.000     0.960      24.0
          B-arrive_time.time      0.971     0.971     0.971      34.0
 B-arrive_time.time_relative      0.857     0.968     0.909      31.0
                I-class_type      1.000     1.000     1.000      17.0
               B-flight_stop      0.955     1.000     0.977      21.0
          I-arrive_time.time      0.971     0.971     0.971      35.0
              B-airline_code      1.000     0.882     0.938      34.0
    I-depart_date.day_number      1.000     1.000     1.000      15.0
      I-fromloc.airport_name      0.441     1.000     0.612      15.0
      B-fromloc.airport_name      0.407     0.917     0.564      12.0
      B-arrive_date.day_name      0.733     1.000     0.846      11.0
          B-toloc.state_code      0.947     1.000     0.973      18.0
B-depart_date.today_relative      1.000     1.000     1.000       9.0
             B-flight_number      0.385     0.909     0.541      11.0
 B-depart_date.date_relative      0.944     1.000     0.971      17.0
          B-toloc.state_name      0.903     1.000     0.949      28.0
           B-fare_basis_code      0.917     0.647     0.759      17.0
               B-flight_time      1.000     1.000     1.000       1.0
                        B-or      1.000     1.000     1.000       3.0
 B-arrive_time.period_of_day      0.750     1.000     0.857       6.0
          B-meal_description      1.000     0.900     0.947      10.0
             I-cost_relative      1.000     0.667     0.800       3.0
              I-airport_name      0.917     0.379     0.537      29.0
               B-fare_amount      0.500     1.000     0.667       2.0
               I-fare_amount      1.000     1.000     1.000       2.0
                 I-city_name      1.000     0.600     0.750      30.0
        I-toloc.airport_name      1.000     1.000     1.000       3.0
            B-transport_type      1.000     1.000     1.000      10.0
    B-arrive_date.month_name      0.714     0.833     0.769       6.0
    B-arrive_date.day_number      0.714     0.833     0.769       6.0
         I-stoploc.city_name      0.909     1.000     0.952      10.0
                      B-meal      1.000     1.000     1.000      16.0
        B-fromloc.state_code      1.000     0.957     0.978      23.0
    B-depart_time.period_mod      0.833     1.000     0.909       5.0
                   B-connect      1.000     1.000     1.000       6.0
               B-flight_days      1.000     1.000     1.000      10.0
        B-toloc.airport_name      0.750     1.000     0.857       3.0
        B-fromloc.state_name      0.944     1.000     0.971      17.0
              B-airport_name      0.727     0.381     0.500      21.0
                   B-economy      1.000     1.000     1.000       6.0
               I-flight_time      1.000     1.000     1.000       1.0
             B-aircraft_code      0.933     0.848     0.889      33.0
                       B-mod      1.000     0.500     0.667       2.0
              B-airport_code      0.000     0.000     0.000       9.0
    B-depart_time.start_time      0.667     0.667     0.667       3.0
      B-depart_time.end_time      1.000     0.333     0.500       3.0
          B-depart_date.year      1.000     0.667     0.800       3.0
            I-transport_type      1.000     1.000     1.000       1.0
          B-restriction_code      0.364     1.000     0.533       4.0
    B-arrive_time.start_time      0.889     1.000     0.941       8.0
        B-toloc.airport_code      1.000     0.500     0.667       4.0
      B-arrive_time.end_time      0.800     1.000     0.889       8.0
      I-arrive_time.end_time      0.875     0.875     0.875       8.0
      I-depart_time.end_time      1.000     0.667     0.800       3.0
               I-flight_stop      0.000     0.000     0.000       0.0
      B-fromloc.airport_code      0.833     1.000     0.909       5.0
          I-restriction_code      1.000     1.000     1.000       3.0
    I-depart_time.start_time      1.000     1.000     1.000       1.0
          I-toloc.state_name      1.000     1.000     1.000       1.0
I-depart_date.today_relative      0.000     0.000     0.000       0.0
 B-arrive_date.date_relative      1.000     1.000     1.000       2.0
                I-flight_mod      1.000     0.167     0.286       6.0
                   I-economy      0.000     0.000     0.000       0.0
 B-return_date.date_relative      1.000     0.333     0.500       3.0
        I-fromloc.state_name      1.000     1.000     1.000       1.0
                B-state_code      1.000     1.000     1.000       1.0
    I-arrive_time.start_time      1.000     1.000     1.000       1.0
    I-arrive_date.day_number      0.000     0.000     0.000       0.0
                 B-meal_code      0.000     0.000     0.000       1.0
 I-depart_time.period_of_day      0.000     0.000     0.000       1.0
                  B-day_name      1.000     0.500     0.667       2.0
             B-period_of_day      1.000     0.500     0.667       4.0
        B-stoploc.state_code      0.000     0.000     0.000       0.0
    B-return_date.month_name      0.000     0.000     0.000       0.0
    B-return_date.day_number      0.000     0.000     0.000       0.0
    B-arrive_time.period_mod      0.000     0.000     0.000       0.0
                 I-meal_code      0.000     0.000     0.000       0.0
        B-toloc.country_name      1.000     1.000     1.000       1.0
                 B-days_code      1.000     1.000     1.000       1.0
 I-arrive_time.period_of_day      0.000     0.000     0.000       0.0
            I-today_relative      0.000     0.000     0.000       0.0
 B-return_time.period_of_day      0.000     0.000     0.000       0.0
                      B-time      0.000     0.000     0.000       0.0
           I-fare_basis_code      0.000     0.000     0.000       0.0
 I-arrive_time.time_relative      0.000     0.000     0.000       4.0
 I-depart_time.time_relative      0.000     0.000     0.000       1.0
            B-today_relative      0.000     0.000     0.000       0.0
                B-state_name      1.000     0.111     0.200       9.0
B-arrive_date.today_relative      0.000     0.000     0.000       0.0
    B-return_time.period_mod      0.000     0.000     0.000       0.0
                B-month_name      0.000     0.000     0.000       0.0
                B-day_number      0.000     0.000     0.000       0.0
 I-return_date.date_relative      0.750     1.000     0.857       3.0
I-return_date.today_relative      0.000     0.000     0.000       0.0
      B-stoploc.airport_name      0.000     0.000     0.000       0.0
             B-time_relative      0.000     0.000     0.000       0.0
                      I-time      0.000     0.000     0.000       0.0
    I-return_date.day_number      0.000     0.000     0.000       0.0
          I-meal_description      0.000     0.000     0.000       0.0
B-return_date.today_relative      0.000     0.000     0.000       0.0
      B-return_date.day_name      0.000     0.000     0.000       2.0

                   micro avg      0.951     0.952     0.952    3657.0
                   macro avg      0.666     0.642     0.634    3657.0
                weighted avg      0.958     0.952     0.949    3657.0

I0717 07:23:57.718972 140695199004544 interactiveshell.py:2882] Best Slot F1: 0.955
Reading ../data/atis.train.w-intent.iob
I0717 07:23:59.293716 140695199004544 interactiveshell.py:2882] Step 21850 | Loss: 0.0042 | Loss_intent: 0.0000 | Loss_slots: 0.0042 | Spent: 6.6 secs | LR: 0.000030
I0717 07:24:03.495720 140695199004544 interactiveshell.py:2882] Step 21900 | Loss: 0.0021 | Loss_intent: 0.0000 | Loss_slots: 0.0021 | Spent: 4.2 secs | LR: 0.000030
I0717 07:24:07.662338 140695199004544 interactiveshell.py:2882] Step 21950 | Loss: 0.0145 | Loss_intent: 0.0000 | Loss_slots: 0.0144 | Spent: 4.2 secs | LR: 0.000030
I0717 07:24:11.845880 140695199004544 interactiveshell.py:2882] Step 22000 | Loss: 0.0036 | Loss_intent: 0.0000 | Loss_slots: 0.0036 | Spent: 4.2 secs | LR: 0.000030
I0717 07:24:16.006955 140695199004544 interactiveshell.py:2882] Step 22050 | Loss: 0.0195 | Loss_intent: 0.0000 | Loss_slots: 0.0194 | Spent: 4.2 secs | LR: 0.000029
I0717 07:24:20.167808 140695199004544 interactiveshell.py:2882] Step 22100 | Loss: 0.0018 | Loss_intent: 0.0000 | Loss_slots: 0.0018 | Spent: 4.2 secs | LR: 0.000029
I0717 07:24:24.350809 140695199004544 interactiveshell.py:2882] Step 22150 | Loss: 0.0074 | Loss_intent: 0.0000 | Loss_slots: 0.0074 | Spent: 4.2 secs | LR: 0.000029
Reading ../data/atis.test.w-intent.iob
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.
  'precision', 'predicted', average, warn_for)
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1439: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples.
  'recall', 'true', average, warn_for)
I0717 07:24:26.157835 140695199004544 interactiveshell.py:2882] 
                                          precision    recall  f1-score   support

                             atis_flight      0.986     0.983     0.984       632
                            atis_airfare      0.941     1.000     0.970        48
                     atis_ground_service      1.000     1.000     1.000        36
                            atis_airline      0.974     1.000     0.987        38
                       atis_abbreviation      1.000     1.000     1.000        33
                           atis_aircraft      0.889     0.889     0.889         9
                        atis_flight_time      1.000     1.000     1.000         1
                           atis_quantity      0.429     1.000     0.600         3
                atis_flight#atis_airfare      0.857     0.500     0.632        12
                            atis_airport      0.947     1.000     0.973        18
                           atis_distance      1.000     1.000     1.000        10
                               atis_city      0.545     1.000     0.706         6
                        atis_ground_fare      1.000     0.714     0.833         7
                           atis_capacity      1.000     0.952     0.976        21
                          atis_flight_no      0.889     1.000     0.941         8
                               atis_meal      1.000     0.833     0.909         6
                        atis_restriction      0.000     0.000     0.000         0
             atis_airline#atis_flight_no      0.000     0.000     0.000         0
    atis_ground_service#atis_ground_fare      0.000     0.000     0.000         0
           atis_airfare#atis_flight_time      0.000     0.000     0.000         0
                           atis_cheapest      0.000     0.000     0.000         0
atis_aircraft#atis_flight#atis_flight_no      0.000     0.000     0.000         0

                               micro avg      0.970     0.975     0.972       888
                               macro avg      0.657     0.676     0.655       888
                            weighted avg      0.975     0.975     0.974       888

I0717 07:24:26.197561 140695199004544 interactiveshell.py:2882] 
                              precision    recall  f1-score   support

                           O      0.000     0.000     0.000       0.0
           B-toloc.city_name      0.975     0.992     0.983     716.0
         B-fromloc.city_name      0.986     0.994     0.990     704.0
           I-toloc.city_name      0.978     0.989     0.983     265.0
      B-depart_date.day_name      0.991     0.986     0.988     212.0
              B-airline_name      1.000     1.000     1.000     101.0
         I-fromloc.city_name      0.972     0.989     0.980     177.0
 B-depart_time.period_of_day      0.992     0.915     0.952     130.0
              I-airline_name      1.000     1.000     1.000      65.0
    B-depart_date.day_number      0.981     0.964     0.972      55.0
    B-depart_date.month_name      0.982     0.964     0.973      56.0
          B-depart_time.time      0.826     1.000     0.905      57.0
                B-round_trip      1.000     0.986     0.993      73.0
             B-cost_relative      1.000     0.973     0.986      37.0
                I-round_trip      1.000     1.000     1.000      71.0
                B-flight_mod      0.852     0.958     0.902      24.0
 B-depart_time.time_relative      0.984     0.969     0.977      65.0
          I-depart_time.time      0.963     1.000     0.981      52.0
         B-stoploc.city_name      0.952     1.000     0.976      20.0
                 B-city_name      0.865     0.561     0.681      57.0
                B-class_type      0.923     1.000     0.960      24.0
          B-arrive_time.time      0.971     0.971     0.971      34.0
 B-arrive_time.time_relative      0.882     0.968     0.923      31.0
                I-class_type      1.000     1.000     1.000      17.0
               B-flight_stop      0.955     1.000     0.977      21.0
          I-arrive_time.time      0.971     0.971     0.971      35.0
              B-airline_code      1.000     0.912     0.954      34.0
    I-depart_date.day_number      1.000     1.000     1.000      15.0
      I-fromloc.airport_name      0.429     1.000     0.600      15.0
      B-fromloc.airport_name      0.367     0.917     0.524      12.0
      B-arrive_date.day_name      0.733     1.000     0.846      11.0
          B-toloc.state_code      0.947     1.000     0.973      18.0
B-depart_date.today_relative      1.000     1.000     1.000       9.0
             B-flight_number      0.346     0.818     0.486      11.0
 B-depart_date.date_relative      0.944     1.000     0.971      17.0
          B-toloc.state_name      0.903     1.000     0.949      28.0
           B-fare_basis_code      0.917     0.647     0.759      17.0
               B-flight_time      1.000     1.000     1.000       1.0
                        B-or      1.000     1.000     1.000       3.0
 B-arrive_time.period_of_day      0.750     1.000     0.857       6.0
          B-meal_description      1.000     0.900     0.947      10.0
             I-cost_relative      1.000     0.667     0.800       3.0
              I-airport_name      0.917     0.379     0.537      29.0
               B-fare_amount      0.500     1.000     0.667       2.0
               I-fare_amount      1.000     1.000     1.000       2.0
                 I-city_name      1.000     0.567     0.723      30.0
        I-toloc.airport_name      1.000     1.000     1.000       3.0
            B-transport_type      1.000     1.000     1.000      10.0
    B-arrive_date.month_name      0.714     0.833     0.769       6.0
    B-arrive_date.day_number      0.714     0.833     0.769       6.0
         I-stoploc.city_name      0.769     1.000     0.870      10.0
                      B-meal      1.000     1.000     1.000      16.0
        B-fromloc.state_code      1.000     0.957     0.978      23.0
    B-depart_time.period_mod      0.833     1.000     0.909       5.0
                   B-connect      1.000     1.000     1.000       6.0
               B-flight_days      1.000     1.000     1.000      10.0
        B-toloc.airport_name      0.750     1.000     0.857       3.0
        B-fromloc.state_name      0.944     1.000     0.971      17.0
              B-airport_name      0.727     0.381     0.500      21.0
                   B-economy      1.000     1.000     1.000       6.0
               I-flight_time      1.000     1.000     1.000       1.0
             B-aircraft_code      0.964     0.818     0.885      33.0
                       B-mod      1.000     0.500     0.667       2.0
              B-airport_code      0.000     0.000     0.000       9.0
    B-depart_time.start_time      0.667     0.667     0.667       3.0
      B-depart_time.end_time      1.000     0.333     0.500       3.0
          B-depart_date.year      1.000     0.667     0.800       3.0
            I-transport_type      0.500     1.000     0.667       1.0
          B-restriction_code      0.400     1.000     0.571       4.0
    B-arrive_time.start_time      0.889     1.000     0.941       8.0
        B-toloc.airport_code      1.000     0.500     0.667       4.0
      B-arrive_time.end_time      0.800     1.000     0.889       8.0
      I-arrive_time.end_time      0.875     0.875     0.875       8.0
      I-depart_time.end_time      1.000     0.667     0.800       3.0
               I-flight_stop      0.000     0.000     0.000       0.0
      B-fromloc.airport_code      1.000     1.000     1.000       5.0
          I-restriction_code      0.750     1.000     0.857       3.0
    I-depart_time.start_time      1.000     1.000     1.000       1.0
          I-toloc.state_name      1.000     1.000     1.000       1.0
I-depart_date.today_relative      0.000     0.000     0.000       0.0
 B-arrive_date.date_relative      1.000     1.000     1.000       2.0
                I-flight_mod      1.000     0.167     0.286       6.0
                   I-economy      0.000     0.000     0.000       0.0
 B-return_date.date_relative      1.000     0.333     0.500       3.0
        I-fromloc.state_name      1.000     1.000     1.000       1.0
                B-state_code      1.000     1.000     1.000       1.0
    I-arrive_time.start_time      1.000     1.000     1.000       1.0
    I-arrive_date.day_number      0.000     0.000     0.000       0.0
                 B-meal_code      0.000     0.000     0.000       1.0
 I-depart_time.period_of_day      0.000     0.000     0.000       1.0
                  B-day_name      1.000     0.500     0.667       2.0
             B-period_of_day      1.000     0.500     0.667       4.0
        B-stoploc.state_code      0.000     0.000     0.000       0.0
    B-return_date.month_name      0.000     0.000     0.000       0.0
    B-return_date.day_number      0.000     0.000     0.000       0.0
    B-arrive_time.period_mod      0.000     0.000     0.000       0.0
                 I-meal_code      0.000     0.000     0.000       0.0
        B-toloc.country_name      1.000     1.000     1.000       1.0
                 B-days_code      1.000     1.000     1.000       1.0
 I-arrive_time.period_of_day      0.000     0.000     0.000       0.0
            I-today_relative      0.000     0.000     0.000       0.0
 B-return_time.period_of_day      0.000     0.000     0.000       0.0
                      B-time      0.000     0.000     0.000       0.0
           I-fare_basis_code      0.000     0.000     0.000       0.0
 I-arrive_time.time_relative      1.000     0.250     0.400       4.0
 I-depart_time.time_relative      0.000     0.000     0.000       1.0
            B-today_relative      0.000     0.000     0.000       0.0
                B-state_name      1.000     0.111     0.200       9.0
B-arrive_date.today_relative      0.000     0.000     0.000       0.0
    B-return_time.period_mod      0.000     0.000     0.000       0.0
                B-month_name      0.000     0.000     0.000       0.0
                B-day_number      0.000     0.000     0.000       0.0
 I-return_date.date_relative      0.667     0.667     0.667       3.0
I-return_date.today_relative      0.000     0.000     0.000       0.0
      B-stoploc.airport_name      0.000     0.000     0.000       0.0
             B-time_relative      0.000     0.000     0.000       0.0
                      I-time      0.000     0.000     0.000       0.0
    I-return_date.day_number      0.000     0.000     0.000       0.0
          I-meal_description      0.000     0.000     0.000       0.0
B-return_date.today_relative      0.000     0.000     0.000       0.0
      B-return_date.day_name      0.000     0.000     0.000       2.0

                   micro avg      0.950     0.951     0.951    3657.0
                   macro avg      0.667     0.641     0.632    3657.0
                weighted avg      0.958     0.951     0.948    3657.0

I0717 07:24:26.198366 140695199004544 interactiveshell.py:2882] Best Slot F1: 0.955
Reading ../data/atis.train.w-intent.iob
I0717 07:24:30.925088 140695199004544 interactiveshell.py:2882] Step 22200 | Loss: 0.0005 | Loss_intent: 0.0000 | Loss_slots: 0.0005 | Spent: 6.6 secs | LR: 0.000029
I0717 07:24:35.094518 140695199004544 interactiveshell.py:2882] Step 22250 | Loss: 0.0017 | Loss_intent: 0.0000 | Loss_slots: 0.0017 | Spent: 4.2 secs | LR: 0.000029
I0717 07:24:39.279579 140695199004544 interactiveshell.py:2882] Step 22300 | Loss: 0.2400 | Loss_intent: 0.2362 | Loss_slots: 0.0038 | Spent: 4.2 secs | LR: 0.000029
I0717 07:24:43.528877 140695199004544 interactiveshell.py:2882] Step 22350 | Loss: 0.0025 | Loss_intent: 0.0000 | Loss_slots: 0.0025 | Spent: 4.2 secs | LR: 0.000028
I0717 07:24:47.779606 140695199004544 interactiveshell.py:2882] Step 22400 | Loss: 0.0018 | Loss_intent: 0.0000 | Loss_slots: 0.0017 | Spent: 4.2 secs | LR: 0.000028
I0717 07:24:52.019331 140695199004544 interactiveshell.py:2882] Step 22450 | Loss: 0.0156 | Loss_intent: 0.0000 | Loss_slots: 0.0156 | Spent: 4.2 secs | LR: 0.000028
Reading ../data/atis.test.w-intent.iob
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.
  'precision', 'predicted', average, warn_for)
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1439: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples.
  'recall', 'true', average, warn_for)
I0717 07:24:54.840681 140695199004544 interactiveshell.py:2882] 
                                          precision    recall  f1-score   support

                             atis_flight      0.984     0.983     0.983       632
                            atis_airfare      0.941     1.000     0.970        48
                     atis_ground_service      1.000     1.000     1.000        36
                            atis_airline      0.974     0.974     0.974        38
                       atis_abbreviation      1.000     1.000     1.000        33
                           atis_aircraft      1.000     0.889     0.941         9
                        atis_flight_time      1.000     1.000     1.000         1
                           atis_quantity      0.333     1.000     0.500         3
                atis_flight#atis_airfare      0.857     0.500     0.632        12
                            atis_airport      0.947     1.000     0.973        18
                           atis_distance      1.000     1.000     1.000        10
                               atis_city      0.545     1.000     0.706         6
                        atis_ground_fare      1.000     0.714     0.833         7
                           atis_capacity      1.000     0.952     0.976        21
                          atis_flight_no      0.889     1.000     0.941         8
                               atis_meal      1.000     0.833     0.909         6
                        atis_restriction      0.000     0.000     0.000         0
             atis_airline#atis_flight_no      0.000     0.000     0.000         0
    atis_ground_service#atis_ground_fare      0.000     0.000     0.000         0
           atis_airfare#atis_flight_time      0.000     0.000     0.000         0
                           atis_cheapest      0.000     0.000     0.000         0
atis_aircraft#atis_flight#ati