In [1]:
"""
We use following lines because we are running on Google Colab
If you are running notebook on a local computer, you don't need this cell
"""
from google.colab import drive
drive.mount('/content/gdrive')
import os
os.chdir('/content/gdrive/My Drive/finch/tensorflow2/spoken_language_understanding/atis/main')
Drive already mounted at /content/gdrive; to attempt to forcibly remount, call drive.mount("/content/gdrive", force_remount=True).
In [2]:
!pip install tensorflow-gpu==2.0.0-beta1
Requirement already satisfied: tensorflow-gpu==2.0.0-beta1 in /usr/local/lib/python3.6/dist-packages (2.0.0b1)
Requirement already satisfied: numpy<2.0,>=1.14.5 in /usr/local/lib/python3.6/dist-packages (from tensorflow-gpu==2.0.0-beta1) (1.16.4)
Requirement already satisfied: keras-applications>=1.0.6 in /usr/local/lib/python3.6/dist-packages (from tensorflow-gpu==2.0.0-beta1) (1.0.8)
Requirement already satisfied: grpcio>=1.8.6 in /usr/local/lib/python3.6/dist-packages (from tensorflow-gpu==2.0.0-beta1) (1.15.0)
Requirement already satisfied: six>=1.10.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow-gpu==2.0.0-beta1) (1.12.0)
Requirement already satisfied: keras-preprocessing>=1.0.5 in /usr/local/lib/python3.6/dist-packages (from tensorflow-gpu==2.0.0-beta1) (1.1.0)
Requirement already satisfied: google-pasta>=0.1.6 in /usr/local/lib/python3.6/dist-packages (from tensorflow-gpu==2.0.0-beta1) (0.1.7)
Requirement already satisfied: wheel>=0.26 in /usr/local/lib/python3.6/dist-packages (from tensorflow-gpu==2.0.0-beta1) (0.33.4)
Requirement already satisfied: absl-py>=0.7.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow-gpu==2.0.0-beta1) (0.7.1)
Requirement already satisfied: termcolor>=1.1.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow-gpu==2.0.0-beta1) (1.1.0)
Requirement already satisfied: wrapt>=1.11.1 in /usr/local/lib/python3.6/dist-packages (from tensorflow-gpu==2.0.0-beta1) (1.11.2)
Requirement already satisfied: gast>=0.2.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow-gpu==2.0.0-beta1) (0.2.2)
Requirement already satisfied: tb-nightly<1.14.0a20190604,>=1.14.0a20190603 in /usr/local/lib/python3.6/dist-packages (from tensorflow-gpu==2.0.0-beta1) (1.14.0a20190603)
Requirement already satisfied: tf-estimator-nightly<1.14.0.dev2019060502,>=1.14.0.dev2019060501 in /usr/local/lib/python3.6/dist-packages (from tensorflow-gpu==2.0.0-beta1) (1.14.0.dev2019060501)
Requirement already satisfied: protobuf>=3.6.1 in /usr/local/lib/python3.6/dist-packages (from tensorflow-gpu==2.0.0-beta1) (3.7.1)
Requirement already satisfied: astor>=0.6.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow-gpu==2.0.0-beta1) (0.8.0)
Requirement already satisfied: h5py in /usr/local/lib/python3.6/dist-packages (from keras-applications>=1.0.6->tensorflow-gpu==2.0.0-beta1) (2.8.0)
Requirement already satisfied: werkzeug>=0.11.15 in /usr/local/lib/python3.6/dist-packages (from tb-nightly<1.14.0a20190604,>=1.14.0a20190603->tensorflow-gpu==2.0.0-beta1) (0.15.4)
Requirement already satisfied: markdown>=2.6.8 in /usr/local/lib/python3.6/dist-packages (from tb-nightly<1.14.0a20190604,>=1.14.0a20190603->tensorflow-gpu==2.0.0-beta1) (3.1.1)
Requirement already satisfied: setuptools>=41.0.0 in /usr/local/lib/python3.6/dist-packages (from tb-nightly<1.14.0a20190604,>=1.14.0a20190603->tensorflow-gpu==2.0.0-beta1) (41.0.1)
In [3]:
import tensorflow as tf
import pprint
import logging
import time
import numpy as np

from sklearn.metrics import classification_report, f1_score

print("TensorFlow Version", tf.__version__)
print('GPU Enabled:', tf.test.is_gpu_available())
TensorFlow Version 2.0.0-beta1
GPU Enabled: True
In [0]:
def get_vocab(vocab_path):
  word2idx = {}
  with open(vocab_path) as f:
    for i, line in enumerate(f):
      line = line.rstrip()
      word2idx[line] = i
  return word2idx
In [0]:
def data_generator(f_path, params):
  print('Reading', f_path)
  with open(f_path) as f:
    for line in f:
      line = line.rstrip()
      text, slot_intent = line.split('\t')
      words = text.split()[1:-1]
      slot_intent = slot_intent.split()
      slots, intent = slot_intent[1:-1], slot_intent[-1]
      assert len(words) == len(slots)
      
      words = [params['word2idx'].get(w, len(params['word2idx'])) for w in words]
      intent = params['intent2idx'].get(intent, len(params['intent2idx']))
      slots = [params['slot2idx'].get(s, len(params['slot2idx'])) for s in slots]
      
      yield (words, (intent, slots))
In [0]:
def dataset(is_training, params):
  _shapes = ([None], ((), [None]))
  _types = (tf.int32, (tf.int32, tf.int32))
  _pads = (0, (-1, 0))
  
  if is_training:
    ds = tf.data.Dataset.from_generator(
      lambda: data_generator(params['train_path'], params),
      output_shapes = _shapes,
      output_types = _types,)
    ds = ds.shuffle(params['num_samples'])
    ds = ds.padded_batch(params['batch_size'], _shapes, _pads)
    ds = ds.prefetch(tf.data.experimental.AUTOTUNE)
  else:
    ds = tf.data.Dataset.from_generator(
      lambda: data_generator(params['test_path'], params),
      output_shapes = _shapes,
      output_types = _types,)
    ds = ds.padded_batch(params['batch_size'], _shapes, _pads)
    ds = ds.prefetch(tf.data.experimental.AUTOTUNE)
  
  return ds
In [0]:
params = {
  'train_path': '../data/atis.train.w-intent.iob',
  'test_path': '../data/atis.test.w-intent.iob',
  'word_path': '../vocab/word.txt',
  'vocab_path': '../vocab/word.npy',
  'intent_path': '../vocab/intent.txt',
  'slot_path': '../vocab/slot.txt',
  'batch_size': 16,
  'num_samples': 4978,
  'rnn_units': 300,
  'dropout_rate': 0.2,
  'clip_norm': 5.0,
  'lr': 3e-4,
  'num_patience': 3,
}
In [0]:
class Model(tf.keras.Model):
  def __init__(self, params: dict):
    super().__init__()
    self.embedding = tf.Variable(np.load(params['vocab_path']),
                                 dtype=tf.float32,
                                 name='pretrained_embedding')
    self.input_dropout = tf.keras.layers.Dropout(params['dropout_rate'])
    self.bidir_gru = tf.keras.layers.Bidirectional(tf.keras.layers.GRU(
        params['rnn_units'], return_sequences=True))
    self.out_linear_intent = tf.keras.layers.Dense(params['intent_size'], name='output_intent')
    self.out_linear_slot = tf.keras.layers.Dense(params['slot_size'], name='output_slot')
  
  
  def call(self, inputs, training=False):
    if inputs.dtype != tf.int32:
      inputs = tf.cast(inputs, tf.int32)
    mask = tf.sign(inputs)
    
    x = tf.nn.embedding_lookup(self.embedding, inputs)
    x = self.input_dropout(x, training=training)
    x = self.bidir_gru(x, mask=None)
    
    x_intent = tf.reduce_max(x, 1)
    x_intent = self.out_linear_intent(x_intent)
    x_slot = self.out_linear_slot(x)
    return (x_intent, x_slot)
In [0]:
params['word2idx'] = get_vocab(params['word_path'])
params['intent2idx'] = get_vocab(params['intent_path'])
params['slot2idx'] = get_vocab(params['slot_path'])

params['word_size'] = len(params['word2idx']) + 1
params['intent_size'] = len(params['intent2idx']) + 1
params['slot_size'] = len(params['slot2idx']) + 1
In [0]:
def is_descending(history: list):
  history = history[-(params['num_patience']+1):]
  for i in range(1, len(history)):
    if history[i-1] <= history[i]:
      return False
  return True    
In [11]:
model = Model(params)
model.build(input_shape=(None, None))
pprint.pprint([(v.name, v.shape) for v in model.trainable_variables])

decay_lr = tf.optimizers.schedules.ExponentialDecay(params['lr'], 1000, 0.9)
optim = tf.optimizers.Adam(params['lr'])
global_step = 0

history_f1 = []
best_f1 = .0

t0 = time.time()
logger = logging.getLogger('tensorflow')
logger.setLevel(logging.INFO)

while True:
  # TRAINING
  for (words, (intent, slots)) in dataset(is_training=True, params=params):
    with tf.GradientTape() as tape:
      y_intent, y_slots = model(words, training=True)
      loss_intent = tf.nn.sparse_softmax_cross_entropy_with_logits(labels=intent, logits=y_intent)
      loss_intent = tf.reduce_mean(loss_intent)
      # weighted cross entropy (weight of 'O' is set to be small)
      loss_slots = tf.nn.sparse_softmax_cross_entropy_with_logits(labels=slots, logits=y_slots)
      weights = tf.cast(tf.sign(slots), tf.float32)
      padding = tf.constant(1e-2, tf.float32, weights.shape)
      weights = tf.where(tf.equal(weights, 0.), padding, weights)
      loss_slots = tf.reduce_mean(loss_slots * weights)
      # joint loss
      loss = loss_intent + loss_slots
    
    optim.lr.assign(decay_lr(global_step))
    grads = tape.gradient(loss, model.trainable_variables)
    grads, _ = tf.clip_by_global_norm(grads, params['clip_norm'])
    optim.apply_gradients(zip(grads, model.trainable_variables))

    if global_step % 50 == 0:
      logger.info("Step {} | Loss: {:.4f} | Loss_intent: {:.4f} | Loss_slots: {:.4f} | Spent: {:.1f} secs | LR: {:.6f}".format(
          global_step, loss.numpy().item(), loss_intent.numpy().item(), loss_slots.numpy().item(), time.time()-t0, optim.lr.numpy().item()))
      t0 = time.time()
    global_step += 1
    
  # EVALUATION
  intent_true = []
  intent_pred = []
  slot_true = []
  slot_pred = []

  for (words, (intent, slots)) in dataset(is_training=False, params=params):
    y_intent, y_slots = model(words, training=False)
    y_intent = tf.argmax(y_intent, -1)
    y_slots = tf.argmax(y_slots, -1)
    
    intent_true += intent.numpy().flatten().tolist()
    intent_pred += y_intent.numpy().flatten().tolist()
    slot_true += slots.numpy().flatten().tolist()
    slot_pred += y_slots.numpy().flatten().tolist()
    
  f1_slots = f1_score(y_true = slot_true,
                      y_pred = slot_pred,
                      labels = list(params['slot2idx'].values()),
                      sample_weight = np.sign(slot_true),
                      average='micro',)
  history_f1.append(f1_slots)
  
  logger.info('\n'+classification_report(y_true = intent_true,
                                         y_pred = intent_pred,
                                         labels = list(params['intent2idx'].values()),
                                         target_names = list(params['intent2idx'].keys()),
                                         digits=3))
  logger.info('\n'+classification_report(y_true = slot_true,
                                         y_pred = slot_pred,
                                         labels = list(params['slot2idx'].values()),
                                         target_names = list(params['slot2idx'].keys()),
                                         sample_weight = np.sign(slot_true),
                                         digits=3))
  
  if f1_slots > best_f1:
    best_f1 = f1_slots
    # you can save model here
  logger.info("Best Slot F1: {:.3f}".format(best_f1))
  
  if len(history_f1) > params['num_patience'] and is_descending(history_f1):
    logger.info("Testing Slot Accuracy not improved over {} epochs, Early Stop".format(params['num_patience']))
    break
WARNING: Logging before flag parsing goes to stderr.
W0715 05:50:11.816013 139833336047488 deprecation.py:323] From /usr/local/lib/python3.6/dist-packages/tensorflow/python/data/ops/dataset_ops.py:505: py_func (from tensorflow.python.ops.script_ops) is deprecated and will be removed in a future version.
Instructions for updating:
tf.py_func is deprecated in TF V2. Instead, there are two
    options available in V2.
    - tf.py_function takes a python function which manipulates tf eager
    tensors instead of numpy arrays. It's easy to convert a tf eager tensor to
    an ndarray (just call tensor.numpy()) but having access to eager tensors
    means `tf.py_function`s can use accelerators such as GPUs as well as
    being differentiable using a gradient tape.
    - tf.numpy_function maintains the semantics of the deprecated tf.py_func
    (it is not differentiable, and manipulates numpy arrays). It drops the
    stateful argument making all functions stateful.
    
[('bidirectional/forward_gru/kernel:0', TensorShape([300, 900])),
 ('bidirectional/forward_gru/recurrent_kernel:0', TensorShape([300, 900])),
 ('bidirectional/forward_gru/bias:0', TensorShape([2, 900])),
 ('bidirectional/backward_gru/kernel:0', TensorShape([300, 900])),
 ('bidirectional/backward_gru/recurrent_kernel:0', TensorShape([300, 900])),
 ('bidirectional/backward_gru/bias:0', TensorShape([2, 900])),
 ('output_intent/kernel:0', TensorShape([600, 23])),
 ('output_intent/bias:0', TensorShape([23])),
 ('output_slot/kernel:0', TensorShape([600, 122])),
 ('output_slot/bias:0', TensorShape([122])),
 ('pretrained_embedding:0', TensorShape([750, 300]))]
Reading ../data/atis.train.w-intent.iob
W0715 05:50:14.357132 139833336047488 deprecation.py:323] From /usr/local/lib/python3.6/dist-packages/tensorflow/python/ops/clip_ops.py:286: add_dispatch_support.<locals>.wrapper (from tensorflow.python.ops.array_ops) is deprecated and will be removed in a future version.
Instructions for updating:
Use tf.where in 2.0, which has the same broadcast rule as np.where
I0715 05:50:14.428290 139833336047488 interactiveshell.py:2882] Step 0 | Loss: 4.0889 | Loss_intent: 3.2019 | Loss_slots: 0.8870 | Spent: 2.6 secs | LR: 0.000300
I0715 05:50:16.281319 139833336047488 interactiveshell.py:2882] Step 50 | Loss: 1.3226 | Loss_intent: 0.7091 | Loss_slots: 0.6135 | Spent: 1.9 secs | LR: 0.000298
I0715 05:50:18.133465 139833336047488 interactiveshell.py:2882] Step 100 | Loss: 0.9466 | Loss_intent: 0.6550 | Loss_slots: 0.2916 | Spent: 1.9 secs | LR: 0.000297
I0715 05:50:19.999065 139833336047488 interactiveshell.py:2882] Step 150 | Loss: 0.5824 | Loss_intent: 0.4027 | Loss_slots: 0.1797 | Spent: 1.9 secs | LR: 0.000295
I0715 05:50:21.869239 139833336047488 interactiveshell.py:2882] Step 200 | Loss: 0.7166 | Loss_intent: 0.4970 | Loss_slots: 0.2197 | Spent: 1.9 secs | LR: 0.000294
I0715 05:50:23.732006 139833336047488 interactiveshell.py:2882] Step 250 | Loss: 0.2119 | Loss_intent: 0.0831 | Loss_slots: 0.1289 | Spent: 1.9 secs | LR: 0.000292
I0715 05:50:25.586753 139833336047488 interactiveshell.py:2882] Step 300 | Loss: 0.4679 | Loss_intent: 0.3680 | Loss_slots: 0.0999 | Spent: 1.9 secs | LR: 0.000291
Reading ../data/atis.test.w-intent.iob
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.
  'precision', 'predicted', average, warn_for)
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1439: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples.
  'recall', 'true', average, warn_for)
I0715 05:50:26.712162 139833336047488 interactiveshell.py:2882] 
                                          precision    recall  f1-score   support

                             atis_flight      0.896     0.998     0.945       632
                            atis_airfare      0.957     0.938     0.947        48
                     atis_ground_service      0.857     1.000     0.923        36
                            atis_airline      0.919     0.895     0.907        38
                       atis_abbreviation      0.750     1.000     0.857        33
                           atis_aircraft      0.474     1.000     0.643         9
                        atis_flight_time      0.000     0.000     0.000         1
                           atis_quantity      0.000     0.000     0.000         3
                atis_flight#atis_airfare      0.000     0.000     0.000        12
                            atis_airport      0.000     0.000     0.000        18
                           atis_distance      0.000     0.000     0.000        10
                               atis_city      0.000     0.000     0.000         6
                        atis_ground_fare      0.000     0.000     0.000         7
                           atis_capacity      0.000     0.000     0.000        21
                          atis_flight_no      0.000     0.000     0.000         8
                               atis_meal      0.000     0.000     0.000         6
                        atis_restriction      0.000     0.000     0.000         0
             atis_airline#atis_flight_no      0.000     0.000     0.000         0
    atis_ground_service#atis_ground_fare      0.000     0.000     0.000         0
           atis_airfare#atis_flight_time      0.000     0.000     0.000         0
                           atis_cheapest      0.000     0.000     0.000         0
atis_aircraft#atis_flight#atis_flight_no      0.000     0.000     0.000         0

                               micro avg      0.882     0.887     0.885       888
                               macro avg      0.221     0.265     0.237       888
                            weighted avg      0.796     0.887     0.838       888

I0715 05:50:26.752126 139833336047488 interactiveshell.py:2882] 
                              precision    recall  f1-score   support

                           O      0.000     0.000     0.000       0.0
           B-toloc.city_name      0.847     0.985     0.911     716.0
         B-fromloc.city_name      0.896     0.982     0.937     704.0
           I-toloc.city_name      0.786     0.917     0.847     265.0
      B-depart_date.day_name      0.918     1.000     0.957     212.0
              B-airline_name      0.939     0.911     0.925     101.0
         I-fromloc.city_name      0.821     0.904     0.860     177.0
 B-depart_time.period_of_day      0.809     0.877     0.841     130.0
              I-airline_name      0.891     0.877     0.884      65.0
    B-depart_date.day_number      0.697     0.964     0.809      55.0
    B-depart_date.month_name      0.900     0.964     0.931      56.0
          B-depart_time.time      0.419     0.912     0.575      57.0
                B-round_trip      0.969     0.849     0.905      73.0
             B-cost_relative      0.946     0.946     0.946      37.0
                I-round_trip      0.741     0.845     0.789      71.0
                B-flight_mod      0.513     0.833     0.635      24.0
 B-depart_time.time_relative      0.590     0.954     0.729      65.0
          I-depart_time.time      0.491     1.000     0.658      52.0
         B-stoploc.city_name      0.000     0.000     0.000      20.0
                 B-city_name      0.388     0.333     0.358      57.0
                B-class_type      0.720     0.750     0.735      24.0
          B-arrive_time.time      0.000     0.000     0.000      34.0
 B-arrive_time.time_relative      0.500     0.032     0.061      31.0
                I-class_type      1.000     1.000     1.000      17.0
               B-flight_stop      1.000     0.714     0.833      21.0
          I-arrive_time.time      0.000     0.000     0.000      35.0
              B-airline_code      0.789     0.441     0.566      34.0
    I-depart_date.day_number      0.000     0.000     0.000      15.0
      I-fromloc.airport_name      0.000     0.000     0.000      15.0
      B-fromloc.airport_name      0.000     0.000     0.000      12.0
      B-arrive_date.day_name      0.000     0.000     0.000      11.0
          B-toloc.state_code      0.000     0.000     0.000      18.0
B-depart_date.today_relative      0.000     0.000     0.000       9.0
             B-flight_number      0.000     0.000     0.000      11.0
 B-depart_date.date_relative      1.000     0.118     0.211      17.0
          B-toloc.state_name      0.000     0.000     0.000      28.0
           B-fare_basis_code      0.421     0.941     0.582      17.0
               B-flight_time      0.000     0.000     0.000       1.0
                        B-or      0.000     0.000     0.000       3.0
 B-arrive_time.period_of_day      0.000     0.000     0.000       6.0
          B-meal_description      0.000     0.000     0.000      10.0
             I-cost_relative      0.000     0.000     0.000       3.0
              I-airport_name      0.000     0.000     0.000      29.0
               B-fare_amount      0.000     0.000     0.000       2.0
               I-fare_amount      1.000     1.000     1.000       2.0
                 I-city_name      0.000     0.000     0.000      30.0
        I-toloc.airport_name      0.000     0.000     0.000       3.0
            B-transport_type      0.889     0.800     0.842      10.0
    B-arrive_date.month_name      0.000     0.000     0.000       6.0
    B-arrive_date.day_number      0.000     0.000     0.000       6.0
         I-stoploc.city_name      0.000     0.000     0.000      10.0
                      B-meal      1.000     0.062     0.118      16.0
        B-fromloc.state_code      0.000     0.000     0.000      23.0
    B-depart_time.period_mod      0.000     0.000     0.000       5.0
                   B-connect      0.000     0.000     0.000       6.0
               B-flight_days      0.000     0.000     0.000      10.0
        B-toloc.airport_name      0.000     0.000     0.000       3.0
        B-fromloc.state_name      0.000     0.000     0.000      17.0
              B-airport_name      0.000     0.000     0.000      21.0
                   B-economy      0.000     0.000     0.000       6.0
               I-flight_time      0.000     0.000     0.000       1.0
             B-aircraft_code      0.000     0.000     0.000      33.0
                       B-mod      0.000     0.000     0.000       2.0
              B-airport_code      0.000     0.000     0.000       9.0
    B-depart_time.start_time      0.000     0.000     0.000       3.0
      B-depart_time.end_time      0.000     0.000     0.000       3.0
          B-depart_date.year      0.000     0.000     0.000       3.0
            I-transport_type      0.000     0.000     0.000       1.0
          B-restriction_code      0.000     0.000     0.000       4.0
    B-arrive_time.start_time      0.000     0.000     0.000       8.0
        B-toloc.airport_code      0.000     0.000     0.000       4.0
      B-arrive_time.end_time      0.000     0.000     0.000       8.0
      I-arrive_time.end_time      0.000     0.000     0.000       8.0
      I-depart_time.end_time      0.000     0.000     0.000       3.0
               I-flight_stop      0.000     0.000     0.000       0.0
      B-fromloc.airport_code      0.000     0.000     0.000       5.0
          I-restriction_code      0.000     0.000     0.000       3.0
    I-depart_time.start_time      0.000     0.000     0.000       1.0
          I-toloc.state_name      0.000     0.000     0.000       1.0
I-depart_date.today_relative      0.000     0.000     0.000       0.0
 B-arrive_date.date_relative      0.000     0.000     0.000       2.0
                I-flight_mod      0.000     0.000     0.000       6.0
                   I-economy      0.000     0.000     0.000       0.0
 B-return_date.date_relative      0.000     0.000     0.000       3.0
        I-fromloc.state_name      0.000     0.000     0.000       1.0
                B-state_code      0.000     0.000     0.000       1.0
    I-arrive_time.start_time      0.000     0.000     0.000       1.0
    I-arrive_date.day_number      0.000     0.000     0.000       0.0
                 B-meal_code      0.000     0.000     0.000       1.0
 I-depart_time.period_of_day      0.000     0.000     0.000       1.0
                  B-day_name      0.000     0.000     0.000       2.0
             B-period_of_day      0.000     0.000     0.000       4.0
        B-stoploc.state_code      0.000     0.000     0.000       0.0
    B-return_date.month_name      0.000     0.000     0.000       0.0
    B-return_date.day_number      0.000     0.000     0.000       0.0
    B-arrive_time.period_mod      0.000     0.000     0.000       0.0
                 I-meal_code      0.000     0.000     0.000       0.0
        B-toloc.country_name      0.000     0.000     0.000       1.0
                 B-days_code      0.000     0.000     0.000       1.0
 I-arrive_time.period_of_day      0.000     0.000     0.000       0.0
            I-today_relative      0.000     0.000     0.000       0.0
 B-return_time.period_of_day      0.000     0.000     0.000       0.0
                      B-time      0.000     0.000     0.000       0.0
           I-fare_basis_code      0.000     0.000     0.000       0.0
 I-arrive_time.time_relative      0.000     0.000     0.000       4.0
 I-depart_time.time_relative      0.000     0.000     0.000       1.0
            B-today_relative      0.000     0.000     0.000       0.0
                B-state_name      0.000     0.000     0.000       9.0
B-arrive_date.today_relative      0.000     0.000     0.000       0.0
    B-return_time.period_mod      0.000     0.000     0.000       0.0
                B-month_name      0.000     0.000     0.000       0.0
                B-day_number      0.000     0.000     0.000       0.0
 I-return_date.date_relative      0.000     0.000     0.000       3.0
I-return_date.today_relative      0.000     0.000     0.000       0.0
      B-stoploc.airport_name      0.000     0.000     0.000       0.0
             B-time_relative      0.000     0.000     0.000       0.0
                      I-time      0.000     0.000     0.000       0.0
    I-return_date.day_number      0.000     0.000     0.000       0.0
          I-meal_description      0.000     0.000     0.000       0.0
B-return_date.today_relative      0.000     0.000     0.000       0.0
      B-return_date.day_name      0.000     0.000     0.000       2.0

                   micro avg      0.775     0.776     0.775    3657.0
                   macro avg      0.181     0.181     0.169    3657.0
                weighted avg      0.700     0.776     0.726    3657.0

I0715 05:50:26.753019 139833336047488 interactiveshell.py:2882] Best Slot F1: 0.775
Reading ../data/atis.train.w-intent.iob
I0715 05:50:28.892717 139833336047488 interactiveshell.py:2882] Step 350 | Loss: 0.3441 | Loss_intent: 0.1191 | Loss_slots: 0.2250 | Spent: 3.3 secs | LR: 0.000289
I0715 05:50:30.759025 139833336047488 interactiveshell.py:2882] Step 400 | Loss: 0.1952 | Loss_intent: 0.0782 | Loss_slots: 0.1170 | Spent: 1.9 secs | LR: 0.000288
I0715 05:50:32.613597 139833336047488 interactiveshell.py:2882] Step 450 | Loss: 0.3267 | Loss_intent: 0.1589 | Loss_slots: 0.1678 | Spent: 1.9 secs | LR: 0.000286
I0715 05:50:34.464898 139833336047488 interactiveshell.py:2882] Step 500 | Loss: 0.2522 | Loss_intent: 0.1510 | Loss_slots: 0.1013 | Spent: 1.8 secs | LR: 0.000285
I0715 05:50:36.332761 139833336047488 interactiveshell.py:2882] Step 550 | Loss: 0.1549 | Loss_intent: 0.0519 | Loss_slots: 0.1030 | Spent: 1.9 secs | LR: 0.000283
I0715 05:50:38.178257 139833336047488 interactiveshell.py:2882] Step 600 | Loss: 0.1607 | Loss_intent: 0.0195 | Loss_slots: 0.1412 | Spent: 1.8 secs | LR: 0.000282
Reading ../data/atis.test.w-intent.iob
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.
  'precision', 'predicted', average, warn_for)
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1439: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples.
  'recall', 'true', average, warn_for)
I0715 05:50:39.756924 139833336047488 interactiveshell.py:2882] 
                                          precision    recall  f1-score   support

                             atis_flight      0.937     0.992     0.964       632
                            atis_airfare      0.906     1.000     0.950        48
                     atis_ground_service      0.818     1.000     0.900        36
                            atis_airline      1.000     1.000     1.000        38
                       atis_abbreviation      0.892     1.000     0.943        33
                           atis_aircraft      0.529     1.000     0.692         9
                        atis_flight_time      1.000     1.000     1.000         1
                           atis_quantity      0.375     1.000     0.545         3
                atis_flight#atis_airfare      1.000     0.167     0.286        12
                            atis_airport      1.000     0.500     0.667        18
                           atis_distance      1.000     0.300     0.462        10
                               atis_city      0.000     0.000     0.000         6
                        atis_ground_fare      1.000     0.571     0.727         7
                           atis_capacity      1.000     0.381     0.552        21
                          atis_flight_no      0.000     0.000     0.000         8
                               atis_meal      0.000     0.000     0.000         6
                        atis_restriction      0.000     0.000     0.000         0
             atis_airline#atis_flight_no      0.000     0.000     0.000         0
    atis_ground_service#atis_ground_fare      0.000     0.000     0.000         0
           atis_airfare#atis_flight_time      0.000     0.000     0.000         0
                           atis_cheapest      0.000     0.000     0.000         0
atis_aircraft#atis_flight#atis_flight_no      0.000     0.000     0.000         0

                               micro avg      0.919     0.925     0.922       888
                               macro avg      0.521     0.451     0.440       888
                            weighted avg      0.909     0.925     0.903       888

I0715 05:50:39.796904 139833336047488 interactiveshell.py:2882] 
                              precision    recall  f1-score   support

                           O      0.000     0.000     0.000       0.0
           B-toloc.city_name      0.940     0.987     0.963     716.0
         B-fromloc.city_name      0.959     0.987     0.973     704.0
           I-toloc.city_name      0.852     0.996     0.918     265.0
      B-depart_date.day_name      0.934     1.000     0.966     212.0
              B-airline_name      0.962     0.990     0.976     101.0
         I-fromloc.city_name      0.910     0.966     0.937     177.0
 B-depart_time.period_of_day      0.870     0.877     0.874     130.0
              I-airline_name      1.000     0.969     0.984      65.0
    B-depart_date.day_number      0.902     1.000     0.948      55.0
    B-depart_date.month_name      0.902     0.982     0.940      56.0
          B-depart_time.time      0.726     0.789     0.756      57.0
                B-round_trip      0.986     0.959     0.972      73.0
             B-cost_relative      1.000     0.973     0.986      37.0
                I-round_trip      1.000     0.901     0.948      71.0
                B-flight_mod      0.778     0.875     0.824      24.0
 B-depart_time.time_relative      0.936     0.677     0.786      65.0
          I-depart_time.time      0.792     0.365     0.500      52.0
         B-stoploc.city_name      0.722     0.650     0.684      20.0
                 B-city_name      0.576     0.596     0.586      57.0
                B-class_type      0.923     1.000     0.960      24.0
          B-arrive_time.time      0.476     0.882     0.619      34.0
 B-arrive_time.time_relative      0.558     0.935     0.699      31.0
                I-class_type      1.000     1.000     1.000      17.0
               B-flight_stop      1.000     1.000     1.000      21.0
          I-arrive_time.time      0.415     0.971     0.581      35.0
              B-airline_code      0.857     0.882     0.870      34.0
    I-depart_date.day_number      1.000     1.000     1.000      15.0
      I-fromloc.airport_name      0.306     0.733     0.431      15.0
      B-fromloc.airport_name      0.000     0.000     0.000      12.0
      B-arrive_date.day_name      0.000     0.000     0.000      11.0
          B-toloc.state_code      0.947     1.000     0.973      18.0
B-depart_date.today_relative      1.000     0.778     0.875       9.0
             B-flight_number      0.714     0.455     0.556      11.0
 B-depart_date.date_relative      0.850     1.000     0.919      17.0
          B-toloc.state_name      0.636     0.500     0.560      28.0
           B-fare_basis_code      0.364     0.941     0.525      17.0
               B-flight_time      1.000     1.000     1.000       1.0
                        B-or      1.000     0.667     0.800       3.0
 B-arrive_time.period_of_day      0.000     0.000     0.000       6.0
          B-meal_description      1.000     0.900     0.947      10.0
             I-cost_relative      1.000     0.667     0.800       3.0
              I-airport_name      1.000     0.103     0.188      29.0
               B-fare_amount      0.667     1.000     0.800       2.0
               I-fare_amount      1.000     1.000     1.000       2.0
                 I-city_name      0.400     0.067     0.114      30.0
        I-toloc.airport_name      1.000     0.333     0.500       3.0
            B-transport_type      0.909     1.000     0.952      10.0
    B-arrive_date.month_name      0.000     0.000     0.000       6.0
    B-arrive_date.day_number      0.000     0.000     0.000       6.0
         I-stoploc.city_name      0.000     0.000     0.000      10.0
                      B-meal      1.000     0.938     0.968      16.0
        B-fromloc.state_code      1.000     0.913     0.955      23.0
    B-depart_time.period_mod      0.667     0.400     0.500       5.0
                   B-connect      1.000     0.500     0.667       6.0
               B-flight_days      1.000     0.700     0.824      10.0
        B-toloc.airport_name      1.000     0.333     0.500       3.0
        B-fromloc.state_name      0.500     0.059     0.105      17.0
              B-airport_name      0.000     0.000     0.000      21.0
                   B-economy      1.000     0.333     0.500       6.0
               I-flight_time      1.000     1.000     1.000       1.0
             B-aircraft_code      0.000     0.000     0.000      33.0
                       B-mod      0.000     0.000     0.000       2.0
              B-airport_code      1.000     0.111     0.200       9.0
    B-depart_time.start_time      0.000     0.000     0.000       3.0
      B-depart_time.end_time      0.000     0.000     0.000       3.0
          B-depart_date.year      0.000     0.000     0.000       3.0
            I-transport_type      0.000     0.000     0.000       1.0
          B-restriction_code      1.000     0.250     0.400       4.0
    B-arrive_time.start_time      0.000     0.000     0.000       8.0
        B-toloc.airport_code      0.000     0.000     0.000       4.0
      B-arrive_time.end_time      0.000     0.000     0.000       8.0
      I-arrive_time.end_time      0.000     0.000     0.000       8.0
      I-depart_time.end_time      0.000     0.000     0.000       3.0
               I-flight_stop      0.000     0.000     0.000       0.0
      B-fromloc.airport_code      0.000     0.000     0.000       5.0
          I-restriction_code      0.000     0.000     0.000       3.0
    I-depart_time.start_time      0.000     0.000     0.000       1.0
          I-toloc.state_name      0.000     0.000     0.000       1.0
I-depart_date.today_relative      0.000     0.000     0.000       0.0
 B-arrive_date.date_relative      0.000     0.000     0.000       2.0
                I-flight_mod      0.000     0.000     0.000       6.0
                   I-economy      0.000     0.000     0.000       0.0
 B-return_date.date_relative      0.000     0.000     0.000       3.0
        I-fromloc.state_name      0.000     0.000     0.000       1.0
                B-state_code      0.000     0.000     0.000       1.0
    I-arrive_time.start_time      0.000     0.000     0.000       1.0
    I-arrive_date.day_number      0.000     0.000     0.000       0.0
                 B-meal_code      0.000     0.000     0.000       1.0
 I-depart_time.period_of_day      0.000     0.000     0.000       1.0
                  B-day_name      0.000     0.000     0.000       2.0
             B-period_of_day      0.000     0.000     0.000       4.0
        B-stoploc.state_code      0.000     0.000     0.000       0.0
    B-return_date.month_name      0.000     0.000     0.000       0.0
    B-return_date.day_number      0.000     0.000     0.000       0.0
    B-arrive_time.period_mod      0.000     0.000     0.000       0.0
                 I-meal_code      0.000     0.000     0.000       0.0
        B-toloc.country_name      0.000     0.000     0.000       1.0
                 B-days_code      0.000     0.000     0.000       1.0
 I-arrive_time.period_of_day      0.000     0.000     0.000       0.0
            I-today_relative      0.000     0.000     0.000       0.0
 B-return_time.period_of_day      0.000     0.000     0.000       0.0
                      B-time      0.000     0.000     0.000       0.0
           I-fare_basis_code      0.000     0.000     0.000       0.0
 I-arrive_time.time_relative      0.000     0.000     0.000       4.0
 I-depart_time.time_relative      0.000     0.000     0.000       1.0
            B-today_relative      0.000     0.000     0.000       0.0
                B-state_name      0.000     0.000     0.000       9.0
B-arrive_date.today_relative      0.000     0.000     0.000       0.0
    B-return_time.period_mod      0.000     0.000     0.000       0.0
                B-month_name      0.000     0.000     0.000       0.0
                B-day_number      0.000     0.000     0.000       0.0
 I-return_date.date_relative      0.000     0.000     0.000       3.0
I-return_date.today_relative      0.000     0.000     0.000       0.0
      B-stoploc.airport_name      0.000     0.000     0.000       0.0
             B-time_relative      0.000     0.000     0.000       0.0
                      I-time      0.000     0.000     0.000       0.0
    I-return_date.day_number      0.000     0.000     0.000       0.0
          I-meal_description      0.000     0.000     0.000       0.0
B-return_date.today_relative      0.000     0.000     0.000       0.0
      B-return_date.day_name      0.000     0.000     0.000       2.0

                   micro avg      0.862     0.864     0.863    3657.0
                   macro avg      0.388     0.346     0.346    3657.0
                weighted avg      0.845     0.864     0.842    3657.0

I0715 05:50:39.797752 139833336047488 interactiveshell.py:2882] Best Slot F1: 0.863
Reading ../data/atis.train.w-intent.iob
I0715 05:50:41.486537 139833336047488 interactiveshell.py:2882] Step 650 | Loss: 0.1667 | Loss_intent: 0.0124 | Loss_slots: 0.1543 | Spent: 3.3 secs | LR: 0.000280
I0715 05:50:43.334608 139833336047488 interactiveshell.py:2882] Step 700 | Loss: 0.1782 | Loss_intent: 0.0912 | Loss_slots: 0.0870 | Spent: 1.8 secs | LR: 0.000279
I0715 05:50:45.196822 139833336047488 interactiveshell.py:2882] Step 750 | Loss: 0.1778 | Loss_intent: 0.0780 | Loss_slots: 0.0998 | Spent: 1.9 secs | LR: 0.000277
I0715 05:50:47.048586 139833336047488 interactiveshell.py:2882] Step 800 | Loss: 0.0962 | Loss_intent: 0.0348 | Loss_slots: 0.0614 | Spent: 1.9 secs | LR: 0.000276
I0715 05:50:48.947755 139833336047488 interactiveshell.py:2882] Step 850 | Loss: 0.2765 | Loss_intent: 0.2151 | Loss_slots: 0.0614 | Spent: 1.9 secs | LR: 0.000274
I0715 05:50:50.810707 139833336047488 interactiveshell.py:2882] Step 900 | Loss: 0.0537 | Loss_intent: 0.0054 | Loss_slots: 0.0484 | Spent: 1.9 secs | LR: 0.000273
Reading ../data/atis.test.w-intent.iob
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.
  'precision', 'predicted', average, warn_for)
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1439: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples.
  'recall', 'true', average, warn_for)
I0715 05:50:52.835446 139833336047488 interactiveshell.py:2882] 
                                          precision    recall  f1-score   support

                             atis_flight      0.944     0.994     0.968       632
                            atis_airfare      0.940     0.979     0.959        48
                     atis_ground_service      0.923     1.000     0.960        36
                            atis_airline      1.000     1.000     1.000        38
                       atis_abbreviation      1.000     1.000     1.000        33
                           atis_aircraft      0.750     1.000     0.857         9
                        atis_flight_time      1.000     1.000     1.000         1
                           atis_quantity      0.429     1.000     0.600         3
                atis_flight#atis_airfare      0.000     0.000     0.000        12
                            atis_airport      0.944     0.944     0.944        18
                           atis_distance      1.000     0.700     0.824        10
                               atis_city      1.000     0.333     0.500         6
                        atis_ground_fare      1.000     0.571     0.727         7
                           atis_capacity      1.000     0.810     0.895        21
                          atis_flight_no      0.000     0.000     0.000         8
                               atis_meal      0.000     0.000     0.000         6
                        atis_restriction      0.000     0.000     0.000         0
             atis_airline#atis_flight_no      0.000     0.000     0.000         0
    atis_ground_service#atis_ground_fare      0.000     0.000     0.000         0
           atis_airfare#atis_flight_time      0.000     0.000     0.000         0
                           atis_cheapest      0.000     0.000     0.000         0
atis_aircraft#atis_flight#atis_flight_no      0.000     0.000     0.000         0

                               micro avg      0.943     0.948     0.946       888
                               macro avg      0.542     0.515     0.511       888
                            weighted avg      0.919     0.948     0.930       888

I0715 05:50:52.875339 139833336047488 interactiveshell.py:2882] 
                              precision    recall  f1-score   support

                           O      0.000     0.000     0.000       0.0
           B-toloc.city_name      0.965     0.989     0.977     716.0
         B-fromloc.city_name      0.966     0.996     0.980     704.0
           I-toloc.city_name      0.930     0.996     0.962     265.0
      B-depart_date.day_name      0.934     1.000     0.966     212.0
              B-airline_name      0.980     0.990     0.985     101.0
         I-fromloc.city_name      0.901     0.977     0.938     177.0
 B-depart_time.period_of_day      0.905     0.877     0.891     130.0
              I-airline_name      1.000     0.969     0.984      65.0
    B-depart_date.day_number      0.902     1.000     0.948      55.0
    B-depart_date.month_name      0.902     0.982     0.940      56.0
          B-depart_time.time      0.772     0.772     0.772      57.0
                B-round_trip      1.000     0.973     0.986      73.0
             B-cost_relative      1.000     0.973     0.986      37.0
                I-round_trip      1.000     0.986     0.993      71.0
                B-flight_mod      0.767     0.958     0.852      24.0
 B-depart_time.time_relative      0.957     0.677     0.793      65.0
          I-depart_time.time      0.815     0.423     0.557      52.0
         B-stoploc.city_name      0.818     0.900     0.857      20.0
                 B-city_name      0.673     0.579     0.623      57.0
                B-class_type      0.960     1.000     0.980      24.0
          B-arrive_time.time      0.500     0.971     0.660      34.0
 B-arrive_time.time_relative      0.556     0.968     0.706      31.0
                I-class_type      1.000     1.000     1.000      17.0
               B-flight_stop      1.000     1.000     1.000      21.0
          I-arrive_time.time      0.443     1.000     0.614      35.0
              B-airline_code      0.969     0.912     0.939      34.0
    I-depart_date.day_number      1.000     1.000     1.000      15.0
      I-fromloc.airport_name      0.379     0.733     0.500      15.0
      B-fromloc.airport_name      0.308     0.667     0.421      12.0
      B-arrive_date.day_name      0.000     0.000     0.000      11.0
          B-toloc.state_code      0.947     1.000     0.973      18.0
B-depart_date.today_relative      1.000     0.889     0.941       9.0
             B-flight_number      0.727     0.727     0.727      11.0
 B-depart_date.date_relative      0.850     1.000     0.919      17.0
          B-toloc.state_name      0.719     0.821     0.767      28.0
           B-fare_basis_code      0.810     1.000     0.895      17.0
               B-flight_time      1.000     1.000     1.000       1.0
                        B-or      1.000     1.000     1.000       3.0
 B-arrive_time.period_of_day      0.000     0.000     0.000       6.0
          B-meal_description      1.000     0.900     0.947      10.0
             I-cost_relative      1.000     0.667     0.800       3.0
              I-airport_name      0.818     0.310     0.450      29.0
               B-fare_amount      0.667     1.000     0.800       2.0
               I-fare_amount      1.000     1.000     1.000       2.0
                 I-city_name      0.714     0.167     0.270      30.0
        I-toloc.airport_name      1.000     0.333     0.500       3.0
            B-transport_type      1.000     0.700     0.824      10.0
    B-arrive_date.month_name      0.000     0.000     0.000       6.0
    B-arrive_date.day_number      0.000     0.000     0.000       6.0
         I-stoploc.city_name      0.800     0.400     0.533      10.0
                      B-meal      1.000     1.000     1.000      16.0
        B-fromloc.state_code      1.000     0.913     0.955      23.0
    B-depart_time.period_mod      0.833     1.000     0.909       5.0
                   B-connect      1.000     1.000     1.000       6.0
               B-flight_days      1.000     1.000     1.000      10.0
        B-toloc.airport_name      1.000     0.333     0.500       3.0
        B-fromloc.state_name      0.000     0.000     0.000      17.0
              B-airport_name      0.833     0.238     0.370      21.0
                   B-economy      1.000     1.000     1.000       6.0
               I-flight_time      1.000     1.000     1.000       1.0
             B-aircraft_code      0.950     0.576     0.717      33.0
                       B-mod      0.000     0.000     0.000       2.0
              B-airport_code      1.000     0.111     0.200       9.0
    B-depart_time.start_time      0.000     0.000     0.000       3.0
      B-depart_time.end_time      0.000     0.000     0.000       3.0
          B-depart_date.year      1.000     0.667     0.800       3.0
            I-transport_type      0.250     1.000     0.400       1.0
          B-restriction_code      0.800     1.000     0.889       4.0
    B-arrive_time.start_time      0.000     0.000     0.000       8.0
        B-toloc.airport_code      0.000     0.000     0.000       4.0
      B-arrive_time.end_time      0.000     0.000     0.000       8.0
      I-arrive_time.end_time      0.000     0.000     0.000       8.0
      I-depart_time.end_time      0.000     0.000     0.000       3.0
               I-flight_stop      0.000     0.000     0.000       0.0
      B-fromloc.airport_code      0.000     0.000     0.000       5.0
          I-restriction_code      0.000     0.000     0.000       3.0
    I-depart_time.start_time      0.000     0.000     0.000       1.0
          I-toloc.state_name      0.000     0.000     0.000       1.0
I-depart_date.today_relative      0.000     0.000     0.000       0.0
 B-arrive_date.date_relative      0.000     0.000     0.000       2.0
                I-flight_mod      0.000     0.000     0.000       6.0
                   I-economy      0.000     0.000     0.000       0.0
 B-return_date.date_relative      0.000     0.000     0.000       3.0
        I-fromloc.state_name      0.000     0.000     0.000       1.0
                B-state_code      0.000     0.000     0.000       1.0
    I-arrive_time.start_time      0.000     0.000     0.000       1.0
    I-arrive_date.day_number      0.000     0.000     0.000       0.0
                 B-meal_code      0.000     0.000     0.000       1.0
 I-depart_time.period_of_day      0.000     0.000     0.000       1.0
                  B-day_name      0.000     0.000     0.000       2.0
             B-period_of_day      0.000     0.000     0.000       4.0
        B-stoploc.state_code      0.000     0.000     0.000       0.0
    B-return_date.month_name      0.000     0.000     0.000       0.0
    B-return_date.day_number      0.000     0.000     0.000       0.0
    B-arrive_time.period_mod      0.000     0.000     0.000       0.0
                 I-meal_code      0.000     0.000     0.000       0.0
        B-toloc.country_name      0.000     0.000     0.000       1.0
                 B-days_code      0.000     0.000     0.000       1.0
 I-arrive_time.period_of_day      0.000     0.000     0.000       0.0
            I-today_relative      0.000     0.000     0.000       0.0
 B-return_time.period_of_day      0.000     0.000     0.000       0.0
                      B-time      0.000     0.000     0.000       0.0
           I-fare_basis_code      0.000     0.000     0.000       0.0
 I-arrive_time.time_relative      0.000     0.000     0.000       4.0
 I-depart_time.time_relative      0.000     0.000     0.000       1.0
            B-today_relative      0.000     0.000     0.000       0.0
                B-state_name      0.000     0.000     0.000       9.0
B-arrive_date.today_relative      0.000     0.000     0.000       0.0
    B-return_time.period_mod      0.000     0.000     0.000       0.0
                B-month_name      0.000     0.000     0.000       0.0
                B-day_number      0.000     0.000     0.000       0.0
 I-return_date.date_relative      0.000     0.000     0.000       3.0
I-return_date.today_relative      0.000     0.000     0.000       0.0
      B-stoploc.airport_name      0.000     0.000     0.000       0.0
             B-time_relative      0.000     0.000     0.000       0.0
                      I-time      0.000     0.000     0.000       0.0
    I-return_date.day_number      0.000     0.000     0.000       0.0
          I-meal_description      0.000     0.000     0.000       0.0
B-return_date.today_relative      0.000     0.000     0.000       0.0
      B-return_date.day_name      0.000     0.000     0.000       2.0

                   micro avg      0.891     0.893     0.892    3657.0
                   macro avg      0.430     0.413     0.404    3657.0
                weighted avg      0.883     0.893     0.878    3657.0

I0715 05:50:52.876182 139833336047488 interactiveshell.py:2882] Best Slot F1: 0.892
Reading ../data/atis.train.w-intent.iob
I0715 05:50:54.110882 139833336047488 interactiveshell.py:2882] Step 950 | Loss: 0.1547 | Loss_intent: 0.0353 | Loss_slots: 0.1194 | Spent: 3.3 secs | LR: 0.000271
I0715 05:50:55.985314 139833336047488 interactiveshell.py:2882] Step 1000 | Loss: 0.0938 | Loss_intent: 0.0145 | Loss_slots: 0.0793 | Spent: 1.9 secs | LR: 0.000270
I0715 05:50:57.844733 139833336047488 interactiveshell.py:2882] Step 1050 | Loss: 0.3027 | Loss_intent: 0.2371 | Loss_slots: 0.0656 | Spent: 1.9 secs | LR: 0.000269
I0715 05:50:59.725263 139833336047488 interactiveshell.py:2882] Step 1100 | Loss: 0.1286 | Loss_intent: 0.0171 | Loss_slots: 0.1115 | Spent: 1.9 secs | LR: 0.000267
I0715 05:51:01.602135 139833336047488 interactiveshell.py:2882] Step 1150 | Loss: 0.0966 | Loss_intent: 0.0617 | Loss_slots: 0.0349 | Spent: 1.9 secs | LR: 0.000266
I0715 05:51:03.480500 139833336047488 interactiveshell.py:2882] Step 1200 | Loss: 0.0608 | Loss_intent: 0.0412 | Loss_slots: 0.0196 | Spent: 1.9 secs | LR: 0.000264
Reading ../data/atis.test.w-intent.iob
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.
  'precision', 'predicted', average, warn_for)
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1439: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples.
  'recall', 'true', average, warn_for)
I0715 05:51:05.950017 139833336047488 interactiveshell.py:2882] 
                                          precision    recall  f1-score   support

                             atis_flight      0.977     0.992     0.984       632
                            atis_airfare      0.889     1.000     0.941        48
                     atis_ground_service      1.000     1.000     1.000        36
                            atis_airline      1.000     1.000     1.000        38
                       atis_abbreviation      1.000     1.000     1.000        33
                           atis_aircraft      1.000     1.000     1.000         9
                        atis_flight_time      0.500     1.000     0.667         1
                           atis_quantity      0.429     1.000     0.600         3
                atis_flight#atis_airfare      0.875     0.583     0.700        12
                            atis_airport      0.944     0.944     0.944        18
                           atis_distance      1.000     0.800     0.889        10
                               atis_city      1.000     0.500     0.667         6
                        atis_ground_fare      1.000     0.714     0.833         7
                           atis_capacity      1.000     0.952     0.976        21
                          atis_flight_no      0.889     1.000     0.941         8
                               atis_meal      1.000     0.167     0.286         6
                        atis_restriction      0.000     0.000     0.000         0
             atis_airline#atis_flight_no      0.000     0.000     0.000         0
    atis_ground_service#atis_ground_fare      0.000     0.000     0.000         0
           atis_airfare#atis_flight_time      0.000     0.000     0.000         0
                           atis_cheapest      0.000     0.000     0.000         0
atis_aircraft#atis_flight#atis_flight_no      0.000     0.000     0.000         0

                               micro avg      0.968     0.973     0.970       888
                               macro avg      0.659     0.621     0.610       888
                            weighted avg      0.971     0.973     0.968       888

I0715 05:51:05.990044 139833336047488 interactiveshell.py:2882] 
                              precision    recall  f1-score   support

                           O      0.000     0.000     0.000       0.0
           B-toloc.city_name      0.971     0.990     0.981     716.0
         B-fromloc.city_name      0.975     0.999     0.987     704.0
           I-toloc.city_name      0.957     0.996     0.976     265.0
      B-depart_date.day_name      0.946     0.991     0.968     212.0
              B-airline_name      0.962     1.000     0.981     101.0
         I-fromloc.city_name      0.956     0.983     0.969     177.0
 B-depart_time.period_of_day      0.912     0.877     0.894     130.0
              I-airline_name      1.000     0.969     0.984      65.0
    B-depart_date.day_number      0.917     1.000     0.957      55.0
    B-depart_date.month_name      0.902     0.982     0.940      56.0
          B-depart_time.time      0.760     1.000     0.864      57.0
                B-round_trip      1.000     0.973     0.986      73.0
             B-cost_relative      1.000     0.973     0.986      37.0
                I-round_trip      1.000     1.000     1.000      71.0
                B-flight_mod      0.767     0.958     0.852      24.0
 B-depart_time.time_relative      0.939     0.954     0.947      65.0
          I-depart_time.time      0.803     0.942     0.867      52.0
         B-stoploc.city_name      0.905     0.950     0.927      20.0
                 B-city_name      0.773     0.596     0.673      57.0
                B-class_type      0.960     1.000     0.980      24.0
          B-arrive_time.time      0.968     0.882     0.923      34.0
 B-arrive_time.time_relative      0.824     0.903     0.862      31.0
                I-class_type      1.000     1.000     1.000      17.0
               B-flight_stop      1.000     1.000     1.000      21.0
          I-arrive_time.time      0.689     0.886     0.775      35.0
              B-airline_code      0.970     0.941     0.955      34.0
    I-depart_date.day_number      1.000     1.000     1.000      15.0
      I-fromloc.airport_name      0.452     0.933     0.609      15.0
      B-fromloc.airport_name      0.364     0.667     0.471      12.0
      B-arrive_date.day_name      0.600     0.273     0.375      11.0
          B-toloc.state_code      0.947     1.000     0.973      18.0
B-depart_date.today_relative      1.000     0.889     0.941       9.0
             B-flight_number      0.647     1.000     0.786      11.0
 B-depart_date.date_relative      0.810     1.000     0.895      17.0
          B-toloc.state_name      0.735     0.893     0.806      28.0
           B-fare_basis_code      0.810     1.000     0.895      17.0
               B-flight_time      1.000     1.000     1.000       1.0
                        B-or      1.000     1.000     1.000       3.0
 B-arrive_time.period_of_day      0.000     0.000     0.000       6.0
          B-meal_description      1.000     0.900     0.947      10.0
             I-cost_relative      1.000     0.667     0.800       3.0
              I-airport_name      0.900     0.310     0.462      29.0
               B-fare_amount      0.667     1.000     0.800       2.0
               I-fare_amount      1.000     1.000     1.000       2.0
                 I-city_name      0.889     0.533     0.667      30.0
        I-toloc.airport_name      1.000     1.000     1.000       3.0
            B-transport_type      1.000     0.700     0.824      10.0
    B-arrive_date.month_name      0.000     0.000     0.000       6.0
    B-arrive_date.day_number      1.000     0.167     0.286       6.0
         I-stoploc.city_name      0.800     0.400     0.533      10.0
                      B-meal      1.000     1.000     1.000      16.0
        B-fromloc.state_code      1.000     0.957     0.978      23.0
    B-depart_time.period_mod      0.714     1.000     0.833       5.0
                   B-connect      1.000     1.000     1.000       6.0
               B-flight_days      1.000     1.000     1.000      10.0
        B-toloc.airport_name      1.000     0.333     0.500       3.0
        B-fromloc.state_name      0.923     0.706     0.800      17.0
              B-airport_name      0.750     0.286     0.414      21.0
                   B-economy      1.000     1.000     1.000       6.0
               I-flight_time      1.000     1.000     1.000       1.0
             B-aircraft_code      0.870     0.606     0.714      33.0
                       B-mod      0.000     0.000     0.000       2.0
              B-airport_code      0.500     0.111     0.182       9.0
    B-depart_time.start_time      0.000     0.000     0.000       3.0
      B-depart_time.end_time      0.000     0.000     0.000       3.0
          B-depart_date.year      1.000     0.667     0.800       3.0
            I-transport_type      0.250     1.000     0.400       1.0
          B-restriction_code      0.667     1.000     0.800       4.0
    B-arrive_time.start_time      0.875     0.875     0.875       8.0
        B-toloc.airport_code      0.667     0.500     0.571       4.0
      B-arrive_time.end_time      0.800     1.000     0.889       8.0
      I-arrive_time.end_time      0.000     0.000     0.000       8.0
      I-depart_time.end_time      0.000     0.000     0.000       3.0
               I-flight_stop      0.000     0.000     0.000       0.0
      B-fromloc.airport_code      0.667     0.400     0.500       5.0
          I-restriction_code      0.429     1.000     0.600       3.0
    I-depart_time.start_time      0.000     0.000     0.000       1.0
          I-toloc.state_name      0.000     0.000     0.000       1.0
I-depart_date.today_relative      0.000     0.000     0.000       0.0
 B-arrive_date.date_relative      0.000     0.000     0.000       2.0
                I-flight_mod      0.000     0.000     0.000       6.0
                   I-economy      0.000     0.000     0.000       0.0
 B-return_date.date_relative      0.000     0.000     0.000       3.0
        I-fromloc.state_name      0.000     0.000     0.000       1.0
                B-state_code      1.000     1.000     1.000       1.0
    I-arrive_time.start_time      0.000     0.000     0.000       1.0
    I-arrive_date.day_number      0.000     0.000     0.000       0.0
                 B-meal_code      0.000     0.000     0.000       1.0
 I-depart_time.period_of_day      0.000     0.000     0.000       1.0
                  B-day_name      0.000     0.000     0.000       2.0
             B-period_of_day      0.000     0.000     0.000       4.0
        B-stoploc.state_code      0.000     0.000     0.000       0.0
    B-return_date.month_name      0.000     0.000     0.000       0.0
    B-return_date.day_number      0.000     0.000     0.000       0.0
    B-arrive_time.period_mod      0.000     0.000     0.000       0.0
                 I-meal_code      0.000     0.000     0.000       0.0
        B-toloc.country_name      0.000     0.000     0.000       1.0
                 B-days_code      0.000     0.000     0.000       1.0
 I-arrive_time.period_of_day      0.000     0.000     0.000       0.0
            I-today_relative      0.000     0.000     0.000       0.0
 B-return_time.period_of_day      0.000     0.000     0.000       0.0
                      B-time      0.000     0.000     0.000       0.0
           I-fare_basis_code      0.000     0.000     0.000       0.0
 I-arrive_time.time_relative      0.000     0.000     0.000       4.0
 I-depart_time.time_relative      0.000     0.000     0.000       1.0
            B-today_relative      0.000     0.000     0.000       0.0
                B-state_name      0.000     0.000     0.000       9.0
B-arrive_date.today_relative      0.000     0.000     0.000       0.0
    B-return_time.period_mod      0.000     0.000     0.000       0.0
                B-month_name      0.000     0.000     0.000       0.0
                B-day_number      0.000     0.000     0.000       0.0
 I-return_date.date_relative      0.000     0.000     0.000       3.0
I-return_date.today_relative      0.000     0.000     0.000       0.0
      B-stoploc.airport_name      0.000     0.000     0.000       0.0
             B-time_relative      0.000     0.000     0.000       0.0
                      I-time      0.000     0.000     0.000       0.0
    I-return_date.day_number      0.000     0.000     0.000       0.0
          I-meal_description      0.000     0.000     0.000       0.0
B-return_date.today_relative      0.000     0.000     0.000       0.0
      B-return_date.day_name      0.000     0.000     0.000       2.0

                   micro avg      0.924     0.925     0.925    3657.0
                   macro avg      0.492     0.484     0.472    3657.0
                weighted avg      0.915     0.925     0.914    3657.0

I0715 05:51:05.990868 139833336047488 interactiveshell.py:2882] Best Slot F1: 0.925
Reading ../data/atis.train.w-intent.iob
I0715 05:51:06.793634 139833336047488 interactiveshell.py:2882] Step 1250 | Loss: 0.0865 | Loss_intent: 0.0189 | Loss_slots: 0.0676 | Spent: 3.3 secs | LR: 0.000263
I0715 05:51:08.691933 139833336047488 interactiveshell.py:2882] Step 1300 | Loss: 0.0811 | Loss_intent: 0.0557 | Loss_slots: 0.0253 | Spent: 1.9 secs | LR: 0.000262
I0715 05:51:10.596405 139833336047488 interactiveshell.py:2882] Step 1350 | Loss: 0.0421 | Loss_intent: 0.0164 | Loss_slots: 0.0257 | Spent: 1.9 secs | LR: 0.000260
I0715 05:51:12.495470 139833336047488 interactiveshell.py:2882] Step 1400 | Loss: 0.0328 | Loss_intent: 0.0012 | Loss_slots: 0.0316 | Spent: 1.9 secs | LR: 0.000259
I0715 05:51:14.390663 139833336047488 interactiveshell.py:2882] Step 1450 | Loss: 0.0536 | Loss_intent: 0.0035 | Loss_slots: 0.0501 | Spent: 1.9 secs | LR: 0.000257
I0715 05:51:16.402510 139833336047488 interactiveshell.py:2882] Step 1500 | Loss: 0.0296 | Loss_intent: 0.0037 | Loss_slots: 0.0260 | Spent: 2.0 secs | LR: 0.000256
I0715 05:51:18.461431 139833336047488 interactiveshell.py:2882] Step 1550 | Loss: 0.0843 | Loss_intent: 0.0125 | Loss_slots: 0.0717 | Spent: 2.1 secs | LR: 0.000255
Reading ../data/atis.test.w-intent.iob
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.
  'precision', 'predicted', average, warn_for)
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1439: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples.
  'recall', 'true', average, warn_for)
I0715 05:51:19.610587 139833336047488 interactiveshell.py:2882] 
                                          precision    recall  f1-score   support

                             atis_flight      0.978     0.992     0.985       632
                            atis_airfare      0.873     1.000     0.932        48
                     atis_ground_service      0.947     1.000     0.973        36
                            atis_airline      1.000     1.000     1.000        38
                       atis_abbreviation      1.000     1.000     1.000        33
                           atis_aircraft      1.000     1.000     1.000         9
                        atis_flight_time      0.500     1.000     0.667         1
                           atis_quantity      0.429     1.000     0.600         3
                atis_flight#atis_airfare      0.667     0.167     0.267        12
                            atis_airport      1.000     1.000     1.000        18
                           atis_distance      1.000     0.900     0.947        10
                               atis_city      0.800     0.667     0.727         6
                        atis_ground_fare      1.000     0.714     0.833         7
                           atis_capacity      1.000     0.952     0.976        21
                          atis_flight_no      0.889     1.000     0.941         8
                               atis_meal      1.000     0.167     0.286         6
                        atis_restriction      0.000     0.000     0.000         0
             atis_airline#atis_flight_no      0.000     0.000     0.000         0
    atis_ground_service#atis_ground_fare      0.000     0.000     0.000         0
           atis_airfare#atis_flight_time      0.000     0.000     0.000         0
                           atis_cheapest      0.000     0.000     0.000         0
atis_aircraft#atis_flight#atis_flight_no      0.000     0.000     0.000         0

                               micro avg      0.965     0.971     0.968       888
                               macro avg      0.640     0.616     0.597       888
                            weighted avg      0.966     0.971     0.963       888

I0715 05:51:19.655464 139833336047488 interactiveshell.py:2882] 
                              precision    recall  f1-score   support

                           O      0.000     0.000     0.000       0.0
           B-toloc.city_name      0.970     0.990     0.980     716.0
         B-fromloc.city_name      0.985     0.999     0.992     704.0
           I-toloc.city_name      0.960     0.996     0.978     265.0
      B-depart_date.day_name      0.955     1.000     0.977     212.0
              B-airline_name      0.981     1.000     0.990     101.0
         I-fromloc.city_name      0.972     0.989     0.980     177.0
 B-depart_time.period_of_day      0.919     0.877     0.898     130.0
              I-airline_name      1.000     0.969     0.984      65.0
    B-depart_date.day_number      0.902     1.000     0.948      55.0
    B-depart_date.month_name      0.903     1.000     0.949      56.0
          B-depart_time.time      0.792     1.000     0.884      57.0
                B-round_trip      1.000     0.986     0.993      73.0
             B-cost_relative      1.000     0.973     0.986      37.0
                I-round_trip      1.000     0.986     0.993      71.0
                B-flight_mod      0.793     0.958     0.868      24.0
 B-depart_time.time_relative      0.985     0.985     0.985      65.0
          I-depart_time.time      0.820     0.962     0.885      52.0
         B-stoploc.city_name      0.900     0.900     0.900      20.0
                 B-city_name      0.868     0.579     0.695      57.0
                B-class_type      0.960     1.000     0.980      24.0
          B-arrive_time.time      0.868     0.971     0.917      34.0
 B-arrive_time.time_relative      0.886     1.000     0.939      31.0
                I-class_type      1.000     1.000     1.000      17.0
               B-flight_stop      1.000     1.000     1.000      21.0
          I-arrive_time.time      0.733     0.943     0.825      35.0
              B-airline_code      0.970     0.941     0.955      34.0
    I-depart_date.day_number      1.000     1.000     1.000      15.0
      I-fromloc.airport_name      0.484     1.000     0.652      15.0
      B-fromloc.airport_name      0.370     0.833     0.513      12.0
      B-arrive_date.day_name      1.000     0.364     0.533      11.0
          B-toloc.state_code      1.000     1.000     1.000      18.0
B-depart_date.today_relative      1.000     0.889     0.941       9.0
             B-flight_number      0.550     1.000     0.710      11.0
 B-depart_date.date_relative      0.810     1.000     0.895      17.0
          B-toloc.state_name      0.758     0.893     0.820      28.0
           B-fare_basis_code      0.850     1.000     0.919      17.0
               B-flight_time      1.000     1.000     1.000       1.0
                        B-or      1.000     1.000     1.000       3.0
 B-arrive_time.period_of_day      0.000     0.000     0.000       6.0
          B-meal_description      1.000     0.900     0.947      10.0
             I-cost_relative      1.000     0.667     0.800       3.0
              I-airport_name      0.909     0.345     0.500      29.0
               B-fare_amount      0.500     1.000     0.667       2.0
               I-fare_amount      1.000     1.000     1.000       2.0
                 I-city_name      0.913     0.700     0.792      30.0
        I-toloc.airport_name      1.000     1.000     1.000       3.0
            B-transport_type      1.000     0.700     0.824      10.0
    B-arrive_date.month_name      0.000     0.000     0.000       6.0
    B-arrive_date.day_number      0.000     0.000     0.000       6.0
         I-stoploc.city_name      1.000     0.400     0.571      10.0
                      B-meal      1.000     1.000     1.000      16.0
        B-fromloc.state_code      1.000     0.957     0.978      23.0
    B-depart_time.period_mod      0.833     1.000     0.909       5.0
                   B-connect      1.000     1.000     1.000       6.0
               B-flight_days      1.000     1.000     1.000      10.0
        B-toloc.airport_name      0.333     0.333     0.333       3.0
        B-fromloc.state_name      0.938     0.882     0.909      17.0
              B-airport_name      0.667     0.286     0.400      21.0
                   B-economy      1.000     1.000     1.000       6.0
               I-flight_time      1.000     1.000     1.000       1.0
             B-aircraft_code      0.893     0.758     0.820      33.0
                       B-mod      0.000     0.000     0.000       2.0
              B-airport_code      0.500     0.111     0.182       9.0
    B-depart_time.start_time      0.667     0.667     0.667       3.0
      B-depart_time.end_time      1.000     0.667     0.800       3.0
          B-depart_date.year      1.000     0.667     0.800       3.0
            I-transport_type      0.250     1.000     0.400       1.0
          B-restriction_code      1.000     0.750     0.857       4.0
    B-arrive_time.start_time      1.000     0.750     0.857       8.0
        B-toloc.airport_code      0.750     0.750     0.750       4.0
      B-arrive_time.end_time      1.000     0.375     0.545       8.0
      I-arrive_time.end_time      0.000     0.000     0.000       8.0
      I-depart_time.end_time      0.000     0.000     0.000       3.0
               I-flight_stop      0.000     0.000     0.000       0.0
      B-fromloc.airport_code      0.800     0.800     0.800       5.0
          I-restriction_code      0.333     1.000     0.500       3.0
    I-depart_time.start_time      0.000     0.000     0.000       1.0
          I-toloc.state_name      1.000     1.000     1.000       1.0
I-depart_date.today_relative      0.000     0.000     0.000       0.0
 B-arrive_date.date_relative      0.000     0.000     0.000       2.0
                I-flight_mod      0.000     0.000     0.000       6.0
                   I-economy      0.000     0.000     0.000       0.0
 B-return_date.date_relative      0.500     0.333     0.400       3.0
        I-fromloc.state_name      0.000     0.000     0.000       1.0
                B-state_code      1.000     1.000     1.000       1.0
    I-arrive_time.start_time      0.000     0.000     0.000       1.0
    I-arrive_date.day_number      0.000     0.000     0.000       0.0
                 B-meal_code      0.000     0.000     0.000       1.0
 I-depart_time.period_of_day      0.000     0.000     0.000       1.0
                  B-day_name      0.500     0.500     0.500       2.0
             B-period_of_day      0.000     0.000     0.000       4.0
        B-stoploc.state_code      0.000     0.000     0.000       0.0
    B-return_date.month_name      0.000     0.000     0.000       0.0
    B-return_date.day_number      0.000     0.000     0.000       0.0
    B-arrive_time.period_mod      0.000     0.000     0.000       0.0
                 I-meal_code      0.000     0.000     0.000       0.0
        B-toloc.country_name      0.000     0.000     0.000       1.0
                 B-days_code      0.000     0.000     0.000       1.0
 I-arrive_time.period_of_day      0.000     0.000     0.000       0.0
            I-today_relative      0.000     0.000     0.000       0.0
 B-return_time.period_of_day      0.000     0.000     0.000       0.0
                      B-time      0.000     0.000     0.000       0.0
           I-fare_basis_code      0.000     0.000     0.000       0.0
 I-arrive_time.time_relative      0.000     0.000     0.000       4.0
 I-depart_time.time_relative      0.000     0.000     0.000       1.0
            B-today_relative      0.000     0.000     0.000       0.0
                B-state_name      0.000     0.000     0.000       9.0
B-arrive_date.today_relative      0.000     0.000     0.000       0.0
    B-return_time.period_mod      0.000     0.000     0.000       0.0
                B-month_name      0.000     0.000     0.000       0.0
                B-day_number      0.000     0.000     0.000       0.0
 I-return_date.date_relative      0.000     0.000     0.000       3.0
I-return_date.today_relative      0.000     0.000     0.000       0.0
      B-stoploc.airport_name      0.000     0.000     0.000       0.0
             B-time_relative      0.000     0.000     0.000       0.0
                      I-time      0.000     0.000     0.000       0.0
    I-return_date.day_number      0.000     0.000     0.000       0.0
          I-meal_description      0.000     0.000     0.000       0.0
B-return_date.today_relative      0.000     0.000     0.000       0.0
      B-return_date.day_name      0.000     0.000     0.000       2.0

                   micro avg      0.933     0.934     0.934    3657.0
                   macro avg      0.523     0.515     0.504    3657.0
                weighted avg      0.926     0.934     0.924    3657.0

I0715 05:51:19.656380 139833336047488 interactiveshell.py:2882] Best Slot F1: 0.934
Reading ../data/atis.train.w-intent.iob
I0715 05:51:22.036737 139833336047488 interactiveshell.py:2882] Step 1600 | Loss: 0.1837 | Loss_intent: 0.1507 | Loss_slots: 0.0330 | Spent: 3.6 secs | LR: 0.000253
I0715 05:51:24.092976 139833336047488 interactiveshell.py:2882] Step 1650 | Loss: 0.0363 | Loss_intent: 0.0090 | Loss_slots: 0.0273 | Spent: 2.1 secs | LR: 0.000252
I0715 05:51:26.124994 139833336047488 interactiveshell.py:2882] Step 1700 | Loss: 0.0484 | Loss_intent: 0.0041 | Loss_slots: 0.0443 | Spent: 2.0 secs | LR: 0.000251
I0715 05:51:27.999428 139833336047488 interactiveshell.py:2882] Step 1750 | Loss: 0.0365 | Loss_intent: 0.0039 | Loss_slots: 0.0326 | Spent: 1.9 secs | LR: 0.000249
I0715 05:51:29.855452 139833336047488 interactiveshell.py:2882] Step 1800 | Loss: 0.0825 | Loss_intent: 0.0036 | Loss_slots: 0.0790 | Spent: 1.9 secs | LR: 0.000248
I0715 05:51:31.739198 139833336047488 interactiveshell.py:2882] Step 1850 | Loss: 0.0783 | Loss_intent: 0.0114 | Loss_slots: 0.0669 | Spent: 1.9 secs | LR: 0.000247
Reading ../data/atis.test.w-intent.iob
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.
  'precision', 'predicted', average, warn_for)
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1439: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples.
  'recall', 'true', average, warn_for)
I0715 05:51:33.283348 139833336047488 interactiveshell.py:2882] 
                                          precision    recall  f1-score   support

                             atis_flight      0.971     0.992     0.981       632
                            atis_airfare      0.959     0.979     0.969        48
                     atis_ground_service      1.000     1.000     1.000        36
                            atis_airline      1.000     1.000     1.000        38
                       atis_abbreviation      1.000     1.000     1.000        33
                           atis_aircraft      1.000     1.000     1.000         9
                        atis_flight_time      1.000     1.000     1.000         1
                           atis_quantity      0.429     1.000     0.600         3
                atis_flight#atis_airfare      0.833     0.417     0.556        12
                            atis_airport      1.000     0.944     0.971        18
                           atis_distance      1.000     0.900     0.947        10
                               atis_city      1.000     0.500     0.667         6
                        atis_ground_fare      1.000     0.857     0.923         7
                           atis_capacity      1.000     0.952     0.976        21
                          atis_flight_no      0.889     1.000     0.941         8
                               atis_meal      1.000     0.667     0.800         6
                        atis_restriction      0.000     0.000     0.000         0
             atis_airline#atis_flight_no      0.000     0.000     0.000         0
    atis_ground_service#atis_ground_fare      0.000     0.000     0.000         0
           atis_airfare#atis_flight_time      0.000     0.000     0.000         0
                           atis_cheapest      0.000     0.000     0.000         0
atis_aircraft#atis_flight#atis_flight_no      0.000     0.000     0.000         0

                               micro avg      0.970     0.975     0.972       888
                               macro avg      0.685     0.646     0.651       888
                            weighted avg      0.972     0.975     0.971       888

I0715 05:51:33.322489 139833336047488 interactiveshell.py:2882] 
                              precision    recall  f1-score   support

                           O      0.000     0.000     0.000       0.0
           B-toloc.city_name      0.971     0.993     0.982     716.0
         B-fromloc.city_name      0.986     0.999     0.992     704.0
           I-toloc.city_name      0.960     1.000     0.980     265.0
      B-depart_date.day_name      0.959     1.000     0.979     212.0
              B-airline_name      0.981     1.000     0.990     101.0
         I-fromloc.city_name      0.972     0.989     0.980     177.0
 B-depart_time.period_of_day      0.943     0.885     0.913     130.0
              I-airline_name      1.000     0.969     0.984      65.0
    B-depart_date.day_number      0.932     1.000     0.965      55.0
    B-depart_date.month_name      0.965     0.982     0.973      56.0
          B-depart_time.time      0.814     1.000     0.898      57.0
                B-round_trip      1.000     0.986     0.993      73.0
             B-cost_relative      1.000     0.973     0.986      37.0
                I-round_trip      1.000     0.986     0.993      71.0
                B-flight_mod      0.786     0.917     0.846      24.0
 B-depart_time.time_relative      0.941     0.985     0.962      65.0
          I-depart_time.time      0.867     1.000     0.929      52.0
         B-stoploc.city_name      0.905     0.950     0.927      20.0
                 B-city_name      0.857     0.526     0.652      57.0
                B-class_type      0.960     1.000     0.980      24.0
          B-arrive_time.time      1.000     0.971     0.985      34.0
 B-arrive_time.time_relative      1.000     0.968     0.984      31.0
                I-class_type      1.000     1.000     1.000      17.0
               B-flight_stop      1.000     1.000     1.000      21.0
          I-arrive_time.time      0.895     0.971     0.932      35.0
              B-airline_code      1.000     0.794     0.885      34.0
    I-depart_date.day_number      1.000     1.000     1.000      15.0
      I-fromloc.airport_name      0.484     1.000     0.652      15.0
      B-fromloc.airport_name      0.367     0.917     0.524      12.0
      B-arrive_date.day_name      1.000     0.455     0.625      11.0
          B-toloc.state_code      1.000     1.000     1.000      18.0
B-depart_date.today_relative      1.000     0.889     0.941       9.0
             B-flight_number      0.500     1.000     0.667      11.0
 B-depart_date.date_relative      0.810     1.000     0.895      17.0
          B-toloc.state_name      0.758     0.893     0.820      28.0
           B-fare_basis_code      0.850     1.000     0.919      17.0
               B-flight_time      1.000     1.000     1.000       1.0
                        B-or      1.000     1.000     1.000       3.0
 B-arrive_time.period_of_day      0.250     0.167     0.200       6.0
          B-meal_description      1.000     0.900     0.947      10.0
             I-cost_relative      1.000     0.667     0.800       3.0
              I-airport_name      0.917     0.379     0.537      29.0
               B-fare_amount      0.667     1.000     0.800       2.0
               I-fare_amount      1.000     1.000     1.000       2.0
                 I-city_name      0.909     0.667     0.769      30.0
        I-toloc.airport_name      1.000     1.000     1.000       3.0
            B-transport_type      1.000     1.000     1.000      10.0
    B-arrive_date.month_name      0.800     0.667     0.727       6.0
    B-arrive_date.day_number      1.000     0.333     0.500       6.0
         I-stoploc.city_name      1.000     0.400     0.571      10.0
                      B-meal      1.000     1.000     1.000      16.0
        B-fromloc.state_code      1.000     1.000     1.000      23.0
    B-depart_time.period_mod      1.000     1.000     1.000       5.0
                   B-connect      1.000     1.000     1.000       6.0
               B-flight_days      1.000     1.000     1.000      10.0
        B-toloc.airport_name      1.000     0.333     0.500       3.0
        B-fromloc.state_name      0.938     0.882     0.909      17.0
              B-airport_name      0.750     0.286     0.414      21.0
                   B-economy      1.000     1.000     1.000       6.0
               I-flight_time      1.000     1.000     1.000       1.0
             B-aircraft_code      0.893     0.758     0.820      33.0
                       B-mod      0.000     0.000     0.000       2.0
              B-airport_code      0.500     0.111     0.182       9.0
    B-depart_time.start_time      0.600     1.000     0.750       3.0
      B-depart_time.end_time      0.400     0.667     0.500       3.0
          B-depart_date.year      1.000     0.667     0.800       3.0
            I-transport_type      1.000     1.000     1.000       1.0
          B-restriction_code      1.000     1.000     1.000       4.0
    B-arrive_time.start_time      0.750     0.750     0.750       8.0
        B-toloc.airport_code      0.750     0.750     0.750       4.0
      B-arrive_time.end_time      1.000     0.625     0.769       8.0
      I-arrive_time.end_time      0.000     0.000     0.000       8.0
      I-depart_time.end_time      0.333     0.667     0.444       3.0
               I-flight_stop      0.000     0.000     0.000       0.0
      B-fromloc.airport_code      1.000     1.000     1.000       5.0
          I-restriction_code      0.375     1.000     0.545       3.0
    I-depart_time.start_time      0.500     1.000     0.667       1.0
          I-toloc.state_name      1.000     1.000     1.000       1.0
I-depart_date.today_relative      0.000     0.000     0.000       0.0
 B-arrive_date.date_relative      1.000     0.500     0.667       2.0
                I-flight_mod      0.000     0.000     0.000       6.0
                   I-economy      0.000     0.000     0.000       0.0
 B-return_date.date_relative      0.000     0.000     0.000       3.0
        I-fromloc.state_name      0.000     0.000     0.000       1.0
                B-state_code      1.000     1.000     1.000       1.0
    I-arrive_time.start_time      0.000     0.000     0.000       1.0
    I-arrive_date.day_number      0.000     0.000     0.000       0.0
                 B-meal_code      0.000     0.000     0.000       1.0
 I-depart_time.period_of_day      0.000     0.000     0.000       1.0
                  B-day_name      1.000     0.500     0.667       2.0
             B-period_of_day      1.000     0.250     0.400       4.0
        B-stoploc.state_code      0.000     0.000     0.000       0.0
    B-return_date.month_name      0.000     0.000     0.000       0.0
    B-return_date.day_number      0.000     0.000     0.000       0.0
    B-arrive_time.period_mod      0.000     0.000     0.000       0.0
                 I-meal_code      0.000     0.000     0.000       0.0
        B-toloc.country_name      0.000     0.000     0.000       1.0
                 B-days_code      0.000     0.000     0.000       1.0
 I-arrive_time.period_of_day      0.000     0.000     0.000       0.0
            I-today_relative      0.000     0.000     0.000       0.0
 B-return_time.period_of_day      0.000     0.000     0.000       0.0
                      B-time      0.000     0.000     0.000       0.0
           I-fare_basis_code      0.000     0.000     0.000       0.0
 I-arrive_time.time_relative      0.000     0.000     0.000       4.0
 I-depart_time.time_relative      0.000     0.000     0.000       1.0
            B-today_relative      0.000     0.000     0.000       0.0
                B-state_name      0.000     0.000     0.000       9.0
B-arrive_date.today_relative      0.000     0.000     0.000       0.0
    B-return_time.period_mod      0.000     0.000     0.000       0.0
                B-month_name      0.000     0.000     0.000       0.0
                B-day_number      0.000     0.000     0.000       0.0
 I-return_date.date_relative      0.000     0.000     0.000       3.0
I-return_date.today_relative      0.000     0.000     0.000       0.0
      B-stoploc.airport_name      0.000     0.000     0.000       0.0
             B-time_relative      0.000     0.000     0.000       0.0
                      I-time      0.000     0.000     0.000       0.0
    I-return_date.day_number      0.000     0.000     0.000       0.0
          I-meal_description      0.000     0.000     0.000       0.0
B-return_date.today_relative      0.000     0.000     0.000       0.0
      B-return_date.day_name      0.000     0.000     0.000       2.0

                   micro avg      0.938     0.940     0.939    3657.0
                   macro avg      0.577     0.553     0.546    3657.0
                weighted avg      0.939     0.940     0.933    3657.0

I0715 05:51:33.323501 139833336047488 interactiveshell.py:2882] Best Slot F1: 0.939
Reading ../data/atis.train.w-intent.iob
I0715 05:51:35.087854 139833336047488 interactiveshell.py:2882] Step 1900 | Loss: 0.0191 | Loss_intent: 0.0085 | Loss_slots: 0.0106 | Spent: 3.3 secs | LR: 0.000246
I0715 05:51:36.960785 139833336047488 interactiveshell.py:2882] Step 1950 | Loss: 0.0585 | Loss_intent: 0.0077 | Loss_slots: 0.0508 | Spent: 1.9 secs | LR: 0.000244
I0715 05:51:38.823240 139833336047488 interactiveshell.py:2882] Step 2000 | Loss: 0.0507 | Loss_intent: 0.0330 | Loss_slots: 0.0177 | Spent: 1.9 secs | LR: 0.000243
I0715 05:51:40.696264 139833336047488 interactiveshell.py:2882] Step 2050 | Loss: 0.0592 | Loss_intent: 0.0417 | Loss_slots: 0.0174 | Spent: 1.9 secs | LR: 0.000242
I0715 05:51:42.571961 139833336047488 interactiveshell.py:2882] Step 2100 | Loss: 0.0444 | Loss_intent: 0.0222 | Loss_slots: 0.0222 | Spent: 1.9 secs | LR: 0.000240
I0715 05:51:44.443181 139833336047488 interactiveshell.py:2882] Step 2150 | Loss: 0.0241 | Loss_intent: 0.0074 | Loss_slots: 0.0166 | Spent: 1.9 secs | LR: 0.000239
Reading ../data/atis.test.w-intent.iob
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.
  'precision', 'predicted', average, warn_for)
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1439: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples.
  'recall', 'true', average, warn_for)
I0715 05:51:46.397082 139833336047488 interactiveshell.py:2882] 
                                          precision    recall  f1-score   support

                             atis_flight      0.978     0.992     0.985       632
                            atis_airfare      0.906     1.000     0.950        48
                     atis_ground_service      1.000     1.000     1.000        36
                            atis_airline      1.000     1.000     1.000        38
                       atis_abbreviation      1.000     1.000     1.000        33
                           atis_aircraft      1.000     1.000     1.000         9
                        atis_flight_time      1.000     1.000     1.000         1
                           atis_quantity      0.429     1.000     0.600         3
                atis_flight#atis_airfare      0.833     0.417     0.556        12
                            atis_airport      1.000     1.000     1.000        18
                           atis_distance      1.000     1.000     1.000        10
                               atis_city      1.000     0.500     0.667         6
                        atis_ground_fare      1.000     0.714     0.833         7
                           atis_capacity      1.000     0.952     0.976        21
                          atis_flight_no      0.889     1.000     0.941         8
                               atis_meal      1.000     0.667     0.800         6
                        atis_restriction      0.000     0.000     0.000         0
             atis_airline#atis_flight_no      0.000     0.000     0.000         0
    atis_ground_service#atis_ground_fare      0.000     0.000     0.000         0
           atis_airfare#atis_flight_time      0.000     0.000     0.000         0
                           atis_cheapest      0.000     0.000     0.000         0
atis_aircraft#atis_flight#atis_flight_no      0.000     0.000     0.000         0

                               micro avg      0.972     0.977     0.975       888
                               macro avg      0.683     0.647     0.650       888
                            weighted avg      0.974     0.977     0.973       888

I0715 05:51:46.436014 139833336047488 interactiveshell.py:2882] 
                              precision    recall  f1-score   support

                           O      0.000     0.000     0.000       0.0
           B-toloc.city_name      0.971     0.993     0.982     716.0
         B-fromloc.city_name      0.986     0.999     0.992     704.0
           I-toloc.city_name      0.964     0.996     0.980     265.0
      B-depart_date.day_name      0.986     0.991     0.988     212.0
              B-airline_name      0.990     1.000     0.995     101.0
         I-fromloc.city_name      0.967     0.994     0.981     177.0
 B-depart_time.period_of_day      0.991     0.869     0.926     130.0
              I-airline_name      1.000     0.985     0.992      65.0
    B-depart_date.day_number      0.982     0.982     0.982      55.0
    B-depart_date.month_name      0.982     0.982     0.982      56.0
          B-depart_time.time      0.786     0.965     0.866      57.0
                B-round_trip      1.000     0.986     0.993      73.0
             B-cost_relative      1.000     0.973     0.986      37.0
                I-round_trip      0.986     0.986     0.986      71.0
                B-flight_mod      0.786     0.917     0.846      24.0
 B-depart_time.time_relative      0.969     0.969     0.969      65.0
          I-depart_time.time      0.862     0.962     0.909      52.0
         B-stoploc.city_name      0.905     0.950     0.927      20.0
                 B-city_name      0.857     0.526     0.652      57.0
                B-class_type      0.960     1.000     0.980      24.0
          B-arrive_time.time      0.943     0.971     0.957      34.0
 B-arrive_time.time_relative      0.882     0.968     0.923      31.0
                I-class_type      1.000     1.000     1.000      17.0
               B-flight_stop      1.000     1.000     1.000      21.0
          I-arrive_time.time      0.850     0.971     0.907      35.0
              B-airline_code      1.000     0.912     0.954      34.0
    I-depart_date.day_number      1.000     1.000     1.000      15.0
      I-fromloc.airport_name      0.469     1.000     0.638      15.0
      B-fromloc.airport_name      0.379     0.917     0.537      12.0
      B-arrive_date.day_name      0.769     0.909     0.833      11.0
          B-toloc.state_code      1.000     1.000     1.000      18.0
B-depart_date.today_relative      1.000     0.889     0.941       9.0
             B-flight_number      0.556     0.909     0.690      11.0
 B-depart_date.date_relative      0.810     1.000     0.895      17.0
          B-toloc.state_name      0.758     0.893     0.820      28.0
           B-fare_basis_code      0.944     1.000     0.971      17.0
               B-flight_time      1.000     1.000     1.000       1.0
                        B-or      1.000     1.000     1.000       3.0
 B-arrive_time.period_of_day      0.545     1.000     0.706       6.0
          B-meal_description      1.000     0.900     0.947      10.0
             I-cost_relative      1.000     0.667     0.800       3.0
              I-airport_name      0.917     0.379     0.537      29.0
               B-fare_amount      1.000     1.000     1.000       2.0
               I-fare_amount      1.000     1.000     1.000       2.0
                 I-city_name      0.895     0.567     0.694      30.0
        I-toloc.airport_name      1.000     1.000     1.000       3.0
            B-transport_type      1.000     1.000     1.000      10.0
    B-arrive_date.month_name      0.833     0.833     0.833       6.0
    B-arrive_date.day_number      0.833     0.833     0.833       6.0
         I-stoploc.city_name      0.750     0.600     0.667      10.0
                      B-meal      1.000     1.000     1.000      16.0
        B-fromloc.state_code      1.000     1.000     1.000      23.0
    B-depart_time.period_mod      0.714     1.000     0.833       5.0
                   B-connect      1.000     1.000     1.000       6.0
               B-flight_days      1.000     1.000     1.000      10.0
        B-toloc.airport_name      1.000     0.333     0.500       3.0
        B-fromloc.state_name      0.941     0.941     0.941      17.0
              B-airport_name      0.750     0.286     0.414      21.0
                   B-economy      1.000     1.000     1.000       6.0
               I-flight_time      1.000     1.000     1.000       1.0
             B-aircraft_code      0.778     0.848     0.812      33.0
                       B-mod      0.000     0.000     0.000       2.0
              B-airport_code      0.000     0.000     0.000       9.0
    B-depart_time.start_time      1.000     0.333     0.500       3.0
      B-depart_time.end_time      1.000     0.333     0.500       3.0
          B-depart_date.year      1.000     1.000     1.000       3.0
            I-transport_type      1.000     1.000     1.000       1.0
          B-restriction_code      1.000     0.750     0.857       4.0
    B-arrive_time.start_time      0.889     1.000     0.941       8.0
        B-toloc.airport_code      1.000     0.750     0.857       4.0
      B-arrive_time.end_time      0.889     1.000     0.941       8.0
      I-arrive_time.end_time      0.833     0.625     0.714       8.0
      I-depart_time.end_time      1.000     0.333     0.500       3.0
               I-flight_stop      0.000     0.000     0.000       0.0
      B-fromloc.airport_code      0.714     1.000     0.833       5.0
          I-restriction_code      0.600     1.000     0.750       3.0
    I-depart_time.start_time      1.000     1.000     1.000       1.0
          I-toloc.state_name      1.000     1.000     1.000       1.0
I-depart_date.today_relative      0.000     0.000     0.000       0.0
 B-arrive_date.date_relative      1.000     0.500     0.667       2.0
                I-flight_mod      0.000     0.000     0.000       6.0
                   I-economy      0.000     0.000     0.000       0.0
 B-return_date.date_relative      0.500     0.333     0.400       3.0
        I-fromloc.state_name      0.000     0.000     0.000       1.0
                B-state_code      1.000     1.000     1.000       1.0
    I-arrive_time.start_time      0.000     0.000     0.000       1.0
    I-arrive_date.day_number      0.000     0.000     0.000       0.0
                 B-meal_code      0.000     0.000     0.000       1.0
 I-depart_time.period_of_day      0.000     0.000     0.000       1.0
                  B-day_name      1.000     0.500     0.667       2.0
             B-period_of_day      0.000     0.000     0.000       4.0
        B-stoploc.state_code      0.000     0.000     0.000       0.0
    B-return_date.month_name      0.000     0.000     0.000       0.0
    B-return_date.day_number      0.000     0.000     0.000       0.0
    B-arrive_time.period_mod      0.000     0.000     0.000       0.0
                 I-meal_code      0.000     0.000     0.000       0.0
        B-toloc.country_name      0.000     0.000     0.000       1.0
                 B-days_code      1.000     1.000     1.000       1.0
 I-arrive_time.period_of_day      0.000     0.000     0.000       0.0
            I-today_relative      0.000     0.000     0.000       0.0
 B-return_time.period_of_day      0.000     0.000     0.000       0.0
                      B-time      0.000     0.000     0.000       0.0
           I-fare_basis_code      0.000     0.000     0.000       0.0
 I-arrive_time.time_relative      0.000     0.000     0.000       4.0
 I-depart_time.time_relative      0.000     0.000     0.000       1.0
            B-today_relative      0.000     0.000     0.000       0.0
                B-state_name      0.000     0.000     0.000       9.0
B-arrive_date.today_relative      0.000     0.000     0.000       0.0
    B-return_time.period_mod      0.000     0.000     0.000       0.0
                B-month_name      0.000     0.000     0.000       0.0
                B-day_number      0.000     0.000     0.000       0.0
 I-return_date.date_relative      0.000     0.000     0.000       3.0
I-return_date.today_relative      0.000     0.000     0.000       0.0
      B-stoploc.airport_name      0.000     0.000     0.000       0.0
             B-time_relative      0.000     0.000     0.000       0.0
                      I-time      0.000     0.000     0.000       0.0
    I-return_date.day_number      0.000     0.000     0.000       0.0
          I-meal_description      0.000     0.000     0.000       0.0
B-return_date.today_relative      0.000     0.000     0.000       0.0
      B-return_date.day_name      0.000     0.000     0.000       2.0

                   micro avg      0.943     0.944     0.943    3657.0
                   macro avg      0.601     0.579     0.575    3657.0
                weighted avg      0.941     0.944     0.938    3657.0

I0715 05:51:46.437025 139833336047488 interactiveshell.py:2882] Best Slot F1: 0.943
Reading ../data/atis.train.w-intent.iob
I0715 05:51:47.768914 139833336047488 interactiveshell.py:2882] Step 2200 | Loss: 0.0390 | Loss_intent: 0.0067 | Loss_slots: 0.0324 | Spent: 3.3 secs | LR: 0.000238
I0715 05:51:49.646832 139833336047488 interactiveshell.py:2882] Step 2250 | Loss: 0.0191 | Loss_intent: 0.0051 | Loss_slots: 0.0141 | Spent: 1.9 secs | LR: 0.000237
I0715 05:51:51.517327 139833336047488 interactiveshell.py:2882] Step 2300 | Loss: 0.0050 | Loss_intent: 0.0020 | Loss_slots: 0.0030 | Spent: 1.9 secs | LR: 0.000235
I0715 05:51:53.379412 139833336047488 interactiveshell.py:2882] Step 2350 | Loss: 0.0318 | Loss_intent: 0.0050 | Loss_slots: 0.0268 | Spent: 1.9 secs | LR: 0.000234
I0715 05:51:55.268158 139833336047488 interactiveshell.py:2882] Step 2400 | Loss: 0.0515 | Loss_intent: 0.0065 | Loss_slots: 0.0450 | Spent: 1.9 secs | LR: 0.000233
I0715 05:51:57.137461 139833336047488 interactiveshell.py:2882] Step 2450 | Loss: 0.0536 | Loss_intent: 0.0435 | Loss_slots: 0.0101 | Spent: 1.9 secs | LR: 0.000232
Reading ../data/atis.test.w-intent.iob
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.
  'precision', 'predicted', average, warn_for)
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1439: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples.
  'recall', 'true', average, warn_for)
I0715 05:51:59.540464 139833336047488 interactiveshell.py:2882] 
                                          precision    recall  f1-score   support

                             atis_flight      0.966     0.994     0.980       632
                            atis_airfare      0.940     0.979     0.959        48
                     atis_ground_service      1.000     1.000     1.000        36
                            atis_airline      1.000     1.000     1.000        38
                       atis_abbreviation      1.000     1.000     1.000        33
                           atis_aircraft      1.000     1.000     1.000         9
                        atis_flight_time      1.000     1.000     1.000         1
                           atis_quantity      0.429     1.000     0.600         3
                atis_flight#atis_airfare      0.000     0.000     0.000        12
                            atis_airport      1.000     0.944     0.971        18
                           atis_distance      1.000     1.000     1.000        10
                               atis_city      1.000     0.500     0.667         6
                        atis_ground_fare      1.000     0.714     0.833         7
                           atis_capacity      1.000     0.952     0.976        21
                          atis_flight_no      0.889     1.000     0.941         8
                               atis_meal      1.000     0.833     0.909         6
                        atis_restriction      0.000     0.000     0.000         0
             atis_airline#atis_flight_no      0.000     0.000     0.000         0
    atis_ground_service#atis_ground_fare      0.000     0.000     0.000         0
           atis_airfare#atis_flight_time      0.000     0.000     0.000         0
                           atis_cheapest      0.000     0.000     0.000         0
atis_aircraft#atis_flight#atis_flight_no      0.000     0.000     0.000         0

                               micro avg      0.966     0.972     0.969       888
                               macro avg      0.647     0.633     0.629       888
                            weighted avg      0.956     0.972     0.963       888

I0715 05:51:59.578812 139833336047488 interactiveshell.py:2882] 
                              precision    recall  f1-score   support

                           O      0.000     0.000     0.000       0.0
           B-toloc.city_name      0.973     0.993     0.983     716.0
         B-fromloc.city_name      0.980     0.999     0.989     704.0
           I-toloc.city_name      0.967     0.996     0.981     265.0
      B-depart_date.day_name      0.986     0.995     0.991     212.0
              B-airline_name      0.990     1.000     0.995     101.0
         I-fromloc.city_name      0.967     0.994     0.981     177.0
 B-depart_time.period_of_day      0.991     0.892     0.939     130.0
              I-airline_name      1.000     0.985     0.992      65.0
    B-depart_date.day_number      0.982     0.982     0.982      55.0
    B-depart_date.month_name      0.964     0.964     0.964      56.0
          B-depart_time.time      0.814     1.000     0.898      57.0
                B-round_trip      1.000     0.986     0.993      73.0
             B-cost_relative      1.000     0.973     0.986      37.0
                I-round_trip      1.000     1.000     1.000      71.0
                B-flight_mod      0.786     0.917     0.846      24.0
 B-depart_time.time_relative      0.928     0.985     0.955      65.0
          I-depart_time.time      0.881     1.000     0.937      52.0
         B-stoploc.city_name      0.900     0.900     0.900      20.0
                 B-city_name      0.906     0.509     0.652      57.0
                B-class_type      0.960     1.000     0.980      24.0
          B-arrive_time.time      1.000     0.971     0.985      34.0
 B-arrive_time.time_relative      0.935     0.935     0.935      31.0
                I-class_type      1.000     1.000     1.000      17.0
               B-flight_stop      1.000     1.000     1.000      21.0
          I-arrive_time.time      1.000     0.971     0.986      35.0
              B-airline_code      0.969     0.912     0.939      34.0
    I-depart_date.day_number      1.000     1.000     1.000      15.0
      I-fromloc.airport_name      0.429     1.000     0.600      15.0
      B-fromloc.airport_name      0.367     0.917     0.524      12.0
      B-arrive_date.day_name      0.833     0.909     0.870      11.0
          B-toloc.state_code      1.000     1.000     1.000      18.0
B-depart_date.today_relative      1.000     0.889     0.941       9.0
             B-flight_number      0.500     1.000     0.667      11.0
 B-depart_date.date_relative      0.773     1.000     0.872      17.0
          B-toloc.state_name      0.833     0.893     0.862      28.0
           B-fare_basis_code      0.944     1.000     0.971      17.0
               B-flight_time      1.000     1.000     1.000       1.0
                        B-or      1.000     1.000     1.000       3.0
 B-arrive_time.period_of_day      0.857     1.000     0.923       6.0
          B-meal_description      1.000     0.900     0.947      10.0
             I-cost_relative      1.000     0.667     0.800       3.0
              I-airport_name      0.900     0.310     0.462      29.0
               B-fare_amount      1.000     1.000     1.000       2.0
               I-fare_amount      1.000     1.000     1.000       2.0
                 I-city_name      0.905     0.633     0.745      30.0
        I-toloc.airport_name      1.000     1.000     1.000       3.0
            B-transport_type      1.000     1.000     1.000      10.0
    B-arrive_date.month_name      0.667     0.667     0.667       6.0
    B-arrive_date.day_number      0.833     0.833     0.833       6.0
         I-stoploc.city_name      0.667     0.400     0.500      10.0
                      B-meal      1.000     1.000     1.000      16.0
        B-fromloc.state_code      1.000     1.000     1.000      23.0
    B-depart_time.period_mod      1.000     1.000     1.000       5.0
                   B-connect      1.000     1.000     1.000       6.0
               B-flight_days      1.000     1.000     1.000      10.0
        B-toloc.airport_name      1.000     0.667     0.800       3.0
        B-fromloc.state_name      0.941     0.941     0.941      17.0
              B-airport_name      0.600     0.286     0.387      21.0
                   B-economy      1.000     1.000     1.000       6.0
               I-flight_time      1.000     1.000     1.000       1.0
             B-aircraft_code      0.788     0.788     0.788      33.0
                       B-mod      0.000     0.000     0.000       2.0
              B-airport_code      0.000     0.000     0.000       9.0
    B-depart_time.start_time      0.600     1.000     0.750       3.0
      B-depart_time.end_time      0.400     0.667     0.500       3.0
          B-depart_date.year      1.000     1.000     1.000       3.0
            I-transport_type      1.000     1.000     1.000       1.0
          B-restriction_code      1.000     1.000     1.000       4.0
    B-arrive_time.start_time      1.000     0.750     0.857       8.0
        B-toloc.airport_code      1.000     0.750     0.857       4.0
      B-arrive_time.end_time      1.000     0.625     0.769       8.0
      I-arrive_time.end_time      1.000     0.625     0.769       8.0
      I-depart_time.end_time      0.400     0.667     0.500       3.0
               I-flight_stop      0.000     0.000     0.000       0.0
      B-fromloc.airport_code      0.714     1.000     0.833       5.0
          I-restriction_code      1.000     1.000     1.000       3.0
    I-depart_time.start_time      0.500     1.000     0.667       1.0
          I-toloc.state_name      1.000     1.000     1.000       1.0
I-depart_date.today_relative      0.000     0.000     0.000       0.0
 B-arrive_date.date_relative      1.000     0.500     0.667       2.0
                I-flight_mod      0.000     0.000     0.000       6.0
                   I-economy      0.000     0.000     0.000       0.0
 B-return_date.date_relative      0.500     0.333     0.400       3.0
        I-fromloc.state_name      0.000     0.000     0.000       1.0
                B-state_code      1.000     1.000     1.000       1.0
    I-arrive_time.start_time      0.000     0.000     0.000       1.0
    I-arrive_date.day_number      0.000     0.000     0.000       0.0
                 B-meal_code      0.000     0.000     0.000       1.0
 I-depart_time.period_of_day      0.000     0.000     0.000       1.0
                  B-day_name      1.000     0.500     0.667       2.0
             B-period_of_day      1.000     0.750     0.857       4.0
        B-stoploc.state_code      0.000     0.000     0.000       0.0
    B-return_date.month_name      0.000     0.000     0.000       0.0
    B-return_date.day_number      0.000     0.000     0.000       0.0
    B-arrive_time.period_mod      0.000     0.000     0.000       0.0
                 I-meal_code      0.000     0.000     0.000       0.0
        B-toloc.country_name      1.000     1.000     1.000       1.0
                 B-days_code      1.000     1.000     1.000       1.0
 I-arrive_time.period_of_day      0.000     0.000     0.000       0.0
            I-today_relative      0.000     0.000     0.000       0.0
 B-return_time.period_of_day      0.000     0.000     0.000       0.0
                      B-time      0.000     0.000     0.000       0.0
           I-fare_basis_code      0.000     0.000     0.000       0.0
 I-arrive_time.time_relative      0.000     0.000     0.000       4.0
 I-depart_time.time_relative      0.000     0.000     0.000       1.0
            B-today_relative      0.000     0.000     0.000       0.0
                B-state_name      0.000     0.000     0.000       9.0
B-arrive_date.today_relative      0.000     0.000     0.000       0.0
    B-return_time.period_mod      0.000     0.000     0.000       0.0
                B-month_name      0.000     0.000     0.000       0.0
                B-day_number      0.000     0.000     0.000       0.0
 I-return_date.date_relative      0.000     0.000     0.000       3.0
I-return_date.today_relative      0.000     0.000     0.000       0.0
      B-stoploc.airport_name      0.000     0.000     0.000       0.0
             B-time_relative      0.000     0.000     0.000       0.0
                      I-time      0.000     0.000     0.000       0.0
    I-return_date.day_number      0.000     0.000     0.000       0.0
          I-meal_description      0.000     0.000     0.000       0.0
B-return_date.today_relative      0.000     0.000     0.000       0.0
      B-return_date.day_name      0.000     0.000     0.000       2.0

                   micro avg      0.945     0.946     0.945    3657.0
                   macro avg      0.610     0.601     0.595    3657.0
                weighted avg      0.944     0.946     0.940    3657.0

I0715 05:51:59.579784 139833336047488 interactiveshell.py:2882] Best Slot F1: 0.945
Reading ../data/atis.train.w-intent.iob
I0715 05:52:00.457273 139833336047488 interactiveshell.py:2882] Step 2500 | Loss: 0.0207 | Loss_intent: 0.0016 | Loss_slots: 0.0191 | Spent: 3.3 secs | LR: 0.000231
I0715 05:52:02.344679 139833336047488 interactiveshell.py:2882] Step 2550 | Loss: 0.0146 | Loss_intent: 0.0032 | Loss_slots: 0.0114 | Spent: 1.9 secs | LR: 0.000229
I0715 05:52:04.209347 139833336047488 interactiveshell.py:2882] Step 2600 | Loss: 0.0399 | Loss_intent: 0.0024 | Loss_slots: 0.0375 | Spent: 1.9 secs | LR: 0.000228
I0715 05:52:06.097216 139833336047488 interactiveshell.py:2882] Step 2650 | Loss: 0.0782 | Loss_intent: 0.0130 | Loss_slots: 0.0652 | Spent: 1.9 secs | LR: 0.000227
I0715 05:52:07.977402 139833336047488 interactiveshell.py:2882] Step 2700 | Loss: 0.0660 | Loss_intent: 0.0530 | Loss_slots: 0.0131 | Spent: 1.9 secs | LR: 0.000226
I0715 05:52:09.856759 139833336047488 interactiveshell.py:2882] Step 2750 | Loss: 0.0242 | Loss_intent: 0.0172 | Loss_slots: 0.0070 | Spent: 1.9 secs | LR: 0.000225
I0715 05:52:11.725517 139833336047488 interactiveshell.py:2882] Step 2800 | Loss: 0.0332 | Loss_intent: 0.0029 | Loss_slots: 0.0303 | Spent: 1.9 secs | LR: 0.000223
Reading ../data/atis.test.w-intent.iob
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.
  'precision', 'predicted', average, warn_for)
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1439: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples.
  'recall', 'true', average, warn_for)
I0715 05:52:12.722597 139833336047488 interactiveshell.py:2882] 
                                          precision    recall  f1-score   support

                             atis_flight      0.980     0.992     0.986       632
                            atis_airfare      0.960     1.000     0.980        48
                     atis_ground_service      1.000     1.000     1.000        36
                            atis_airline      1.000     1.000     1.000        38
                       atis_abbreviation      1.000     1.000     1.000        33
                           atis_aircraft      1.000     1.000     1.000         9
                        atis_flight_time      1.000     1.000     1.000         1
                           atis_quantity      0.429     1.000     0.600         3
                atis_flight#atis_airfare      0.875     0.583     0.700        12
                            atis_airport      1.000     0.944     0.971        18
                           atis_distance      1.000     1.000     1.000        10
                               atis_city      1.000     0.500     0.667         6
                        atis_ground_fare      1.000     0.857     0.923         7
                           atis_capacity      1.000     1.000     1.000        21
                          atis_flight_no      0.889     1.000     0.941         8
                               atis_meal      1.000     0.833     0.909         6
                        atis_restriction      0.000     0.000     0.000         0
             atis_airline#atis_flight_no      0.000     0.000     0.000         0
    atis_ground_service#atis_ground_fare      0.000     0.000     0.000         0
           atis_airfare#atis_flight_time      0.000     0.000     0.000         0
                           atis_cheapest      0.000     0.000     0.000         0
atis_aircraft#atis_flight#atis_flight_no      0.000     0.000     0.000         0

                               micro avg      0.976     0.982     0.979       888
                               macro avg      0.688     0.669     0.667       888
                            weighted avg      0.979     0.982     0.979       888

I0715 05:52:12.761290 139833336047488 interactiveshell.py:2882] 
                              precision    recall  f1-score   support

                           O      0.000     0.000     0.000       0.0
           B-toloc.city_name      0.973     0.994     0.983     716.0
         B-fromloc.city_name      0.986     0.999     0.992     704.0
           I-toloc.city_name      0.970     0.992     0.981     265.0
      B-depart_date.day_name      0.986     0.991     0.988     212.0
              B-airline_name      0.990     1.000     0.995     101.0
         I-fromloc.city_name      0.972     0.994     0.983     177.0
 B-depart_time.period_of_day      0.991     0.892     0.939     130.0
              I-airline_name      1.000     0.985     0.992      65.0
    B-depart_date.day_number      0.981     0.945     0.963      55.0
    B-depart_date.month_name      0.981     0.946     0.964      56.0
          B-depart_time.time      0.814     1.000     0.898      57.0
                B-round_trip      1.000     0.986     0.993      73.0
             B-cost_relative      1.000     0.973     0.986      37.0
                I-round_trip      1.000     0.986     0.993      71.0
                B-flight_mod      0.786     0.917     0.846      24.0
 B-depart_time.time_relative      0.955     0.985     0.970      65.0
          I-depart_time.time      0.912     1.000     0.954      52.0
         B-stoploc.city_name      0.905     0.950     0.927      20.0
                 B-city_name      0.853     0.509     0.637      57.0
                B-class_type      0.960     1.000     0.980      24.0
          B-arrive_time.time      1.000     0.971     0.985      34.0
 B-arrive_time.time_relative      0.879     0.935     0.906      31.0
                I-class_type      1.000     1.000     1.000      17.0
               B-flight_stop      1.000     1.000     1.000      21.0
          I-arrive_time.time      1.000     0.971     0.986      35.0
              B-airline_code      1.000     0.912     0.954      34.0
    I-depart_date.day_number      1.000     1.000     1.000      15.0
      I-fromloc.airport_name      0.455     1.000     0.625      15.0
      B-fromloc.airport_name      0.379     0.917     0.537      12.0
      B-arrive_date.day_name      0.769     0.909     0.833      11.0
          B-toloc.state_code      0.947     1.000     0.973      18.0
B-depart_date.today_relative      1.000     0.889     0.941       9.0
             B-flight_number      0.556     0.909     0.690      11.0
 B-depart_date.date_relative      0.895     1.000     0.944      17.0
          B-toloc.state_name      0.781     0.893     0.833      28.0
           B-fare_basis_code      0.944     1.000     0.971      17.0
               B-flight_time      1.000     1.000     1.000       1.0
                        B-or      1.000     1.000     1.000       3.0
 B-arrive_time.period_of_day      0.667     1.000     0.800       6.0
          B-meal_description      1.000     0.900     0.947      10.0
             I-cost_relative      1.000     0.667     0.800       3.0
              I-airport_name      0.917     0.379     0.537      29.0
               B-fare_amount      0.667     1.000     0.800       2.0
               I-fare_amount      1.000     1.000     1.000       2.0
                 I-city_name      0.900     0.600     0.720      30.0
        I-toloc.airport_name      1.000     1.000     1.000       3.0
            B-transport_type      1.000     1.000     1.000      10.0
    B-arrive_date.month_name      0.625     0.833     0.714       6.0
    B-arrive_date.day_number      0.625     0.833     0.714       6.0
         I-stoploc.city_name      0.700     0.700     0.700      10.0
                      B-meal      1.000     1.000     1.000      16.0
        B-fromloc.state_code      1.000     0.957     0.978      23.0
    B-depart_time.period_mod      1.000     1.000     1.000       5.0
                   B-connect      1.000     1.000     1.000       6.0
               B-flight_days      1.000     1.000     1.000      10.0
        B-toloc.airport_name      1.000     1.000     1.000       3.0
        B-fromloc.state_name      0.941     0.941     0.941      17.0
              B-airport_name      0.750     0.286     0.414      21.0
                   B-economy      1.000     1.000     1.000       6.0
               I-flight_time      1.000     1.000     1.000       1.0
             B-aircraft_code      0.778     0.848     0.812      33.0
                       B-mod      0.000     0.000     0.000       2.0
              B-airport_code      0.000     0.000     0.000       9.0
    B-depart_time.start_time      1.000     1.000     1.000       3.0
      B-depart_time.end_time      1.000     0.667     0.800       3.0
          B-depart_date.year      1.000     0.667     0.800       3.0
            I-transport_type      1.000     1.000     1.000       1.0
          B-restriction_code      0.800     1.000     0.889       4.0
    B-arrive_time.start_time      1.000     1.000     1.000       8.0
        B-toloc.airport_code      0.750     0.750     0.750       4.0
      B-arrive_time.end_time      1.000     1.000     1.000       8.0
      I-arrive_time.end_time      1.000     1.000     1.000       8.0
      I-depart_time.end_time      0.500     0.667     0.571       3.0
               I-flight_stop      0.000     0.000     0.000       0.0
      B-fromloc.airport_code      0.833     1.000     0.909       5.0
          I-restriction_code      0.750     1.000     0.857       3.0
    I-depart_time.start_time      1.000     1.000     1.000       1.0
          I-toloc.state_name      1.000     1.000     1.000       1.0
I-depart_date.today_relative      0.000     0.000     0.000       0.0
 B-arrive_date.date_relative      1.000     1.000     1.000       2.0
                I-flight_mod      0.000     0.000     0.000       6.0
                   I-economy      0.000     0.000     0.000       0.0
 B-return_date.date_relative      0.000     0.000     0.000       3.0
        I-fromloc.state_name      0.000     0.000     0.000       1.0
                B-state_code      1.000     1.000     1.000       1.0
    I-arrive_time.start_time      1.000     1.000     1.000       1.0
    I-arrive_date.day_number      0.000     0.000     0.000       0.0
                 B-meal_code      0.000     0.000     0.000       1.0
 I-depart_time.period_of_day      0.000     0.000     0.000       1.0
                  B-day_name      1.000     0.500     0.667       2.0
             B-period_of_day      1.000     0.500     0.667       4.0
        B-stoploc.state_code      0.000     0.000     0.000       0.0
    B-return_date.month_name      0.000     0.000     0.000       0.0
    B-return_date.day_number      0.000     0.000     0.000       0.0
    B-arrive_time.period_mod      0.000     0.000     0.000       0.0
                 I-meal_code      0.000     0.000     0.000       0.0
        B-toloc.country_name      1.000     1.000     1.000       1.0
                 B-days_code      1.000     1.000     1.000       1.0
 I-arrive_time.period_of_day      0.000     0.000     0.000       0.0
            I-today_relative      0.000     0.000     0.000       0.0
 B-return_time.period_of_day      0.000     0.000     0.000       0.0
                      B-time      0.000     0.000     0.000       0.0
           I-fare_basis_code      0.000     0.000     0.000       0.0
 I-arrive_time.time_relative      0.000     0.000     0.000       4.0
 I-depart_time.time_relative      0.000     0.000     0.000       1.0
            B-today_relative      0.000     0.000     0.000       0.0
                B-state_name      0.000     0.000     0.000       9.0
B-arrive_date.today_relative      0.000     0.000     0.000       0.0
    B-return_time.period_mod      0.000     0.000     0.000       0.0
                B-month_name      0.000     0.000     0.000       0.0
                B-day_number      0.000     0.000     0.000       0.0
 I-return_date.date_relative      0.000     0.000     0.000       3.0
I-return_date.today_relative      0.000     0.000     0.000       0.0
      B-stoploc.airport_name      0.000     0.000     0.000       0.0
             B-time_relative      0.000     0.000     0.000       0.0
                      I-time      0.000     0.000     0.000       0.0
    I-return_date.day_number      0.000     0.000     0.000       0.0
          I-meal_description      0.000     0.000     0.000       0.0
B-return_date.today_relative      0.000     0.000     0.000       0.0
      B-return_date.day_name      0.000     0.000     0.000       2.0

                   micro avg      0.947     0.949     0.948    3657.0
                   macro avg      0.618     0.620     0.611    3657.0
                weighted avg      0.946     0.949     0.943    3657.0

I0715 05:52:12.762137 139833336047488 interactiveshell.py:2882] Best Slot F1: 0.948
Reading ../data/atis.train.w-intent.iob
I0715 05:52:15.042876 139833336047488 interactiveshell.py:2882] Step 2850 | Loss: 0.0331 | Loss_intent: 0.0236 | Loss_slots: 0.0095 | Spent: 3.3 secs | LR: 0.000222
I0715 05:52:16.947185 139833336047488 interactiveshell.py:2882] Step 2900 | Loss: 0.0217 | Loss_intent: 0.0007 | Loss_slots: 0.0210 | Spent: 1.9 secs | LR: 0.000221
I0715 05:52:18.824805 139833336047488 interactiveshell.py:2882] Step 2950 | Loss: 0.0263 | Loss_intent: 0.0037 | Loss_slots: 0.0227 | Spent: 1.9 secs | LR: 0.000220
I0715 05:52:20.687183 139833336047488 interactiveshell.py:2882] Step 3000 | Loss: 0.0418 | Loss_intent: 0.0053 | Loss_slots: 0.0365 | Spent: 1.9 secs | LR: 0.000219
I0715 05:52:22.572074 139833336047488 interactiveshell.py:2882] Step 3050 | Loss: 0.0399 | Loss_intent: 0.0020 | Loss_slots: 0.0380 | Spent: 1.9 secs | LR: 0.000218
I0715 05:52:24.438899 139833336047488 interactiveshell.py:2882] Step 3100 | Loss: 0.0140 | Loss_intent: 0.0038 | Loss_slots: 0.0101 | Spent: 1.9 secs | LR: 0.000216
Reading ../data/atis.test.w-intent.iob
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.
  'precision', 'predicted', average, warn_for)
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1439: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples.
  'recall', 'true', average, warn_for)
I0715 05:52:25.885400 139833336047488 interactiveshell.py:2882] 
                                          precision    recall  f1-score   support

                             atis_flight      0.978     0.991     0.984       632
                            atis_airfare      0.959     0.979     0.969        48
                     atis_ground_service      1.000     1.000     1.000        36
                            atis_airline      1.000     1.000     1.000        38
                       atis_abbreviation      1.000     1.000     1.000        33
                           atis_aircraft      1.000     1.000     1.000         9
                        atis_flight_time      1.000     1.000     1.000         1
                           atis_quantity      0.429     1.000     0.600         3
                atis_flight#atis_airfare      0.875     0.583     0.700        12
                            atis_airport      0.944     0.944     0.944        18
                           atis_distance      1.000     1.000     1.000        10
                               atis_city      1.000     0.500     0.667         6
                        atis_ground_fare      1.000     0.857     0.923         7
                           atis_capacity      1.000     1.000     1.000        21
                          atis_flight_no      0.889     1.000     0.941         8
                               atis_meal      1.000     0.833     0.909         6
                        atis_restriction      0.000     0.000     0.000         0
             atis_airline#atis_flight_no      0.000     0.000     0.000         0
    atis_ground_service#atis_ground_fare      0.000     0.000     0.000         0
           atis_airfare#atis_flight_time      0.000     0.000     0.000         0
                           atis_cheapest      0.000     0.000     0.000         0
atis_aircraft#atis_flight#atis_flight_no      0.000     0.000     0.000         0

                               micro avg      0.974     0.980     0.977       888
                               macro avg      0.685     0.668     0.665       888
                            weighted avg      0.976     0.980     0.977       888

I0715 05:52:25.935689 139833336047488 interactiveshell.py:2882] 
                              precision    recall  f1-score   support

                           O      0.000     0.000     0.000       0.0
           B-toloc.city_name      0.973     0.993     0.983     716.0
         B-fromloc.city_name      0.980     0.999     0.989     704.0
           I-toloc.city_name      0.970     0.989     0.979     265.0
      B-depart_date.day_name      0.991     0.991     0.991     212.0
              B-airline_name      1.000     1.000     1.000     101.0
         I-fromloc.city_name      0.972     0.994     0.983     177.0
 B-depart_time.period_of_day      0.992     0.915     0.952     130.0
              I-airline_name      1.000     1.000     1.000      65.0
    B-depart_date.day_number      0.981     0.945     0.963      55.0
    B-depart_date.month_name      0.981     0.946     0.964      56.0
          B-depart_time.time      0.826     1.000     0.905      57.0
                B-round_trip      1.000     0.986     0.993      73.0
             B-cost_relative      1.000     0.973     0.986      37.0
                I-round_trip      1.000     0.986     0.993      71.0
                B-flight_mod      0.786     0.917     0.846      24.0
 B-depart_time.time_relative      0.970     0.985     0.977      65.0
          I-depart_time.time      0.929     1.000     0.963      52.0
         B-stoploc.city_name      0.950     0.950     0.950      20.0
                 B-city_name      0.879     0.509     0.644      57.0
                B-class_type      0.960     1.000     0.980      24.0
          B-arrive_time.time      1.000     0.971     0.985      34.0
 B-arrive_time.time_relative      0.882     0.968     0.923      31.0
                I-class_type      1.000     1.000     1.000      17.0
               B-flight_stop      1.000     1.000     1.000      21.0
          I-arrive_time.time      1.000     0.971     0.986      35.0
              B-airline_code      0.969     0.912     0.939      34.0
    I-depart_date.day_number      1.000     1.000     1.000      15.0
      I-fromloc.airport_name      0.417     1.000     0.588      15.0
      B-fromloc.airport_name      0.393     0.917     0.550      12.0
      B-arrive_date.day_name      0.786     1.000     0.880      11.0
          B-toloc.state_code      1.000     1.000     1.000      18.0
B-depart_date.today_relative      1.000     0.889     0.941       9.0
             B-flight_number      0.524     1.000     0.688      11.0
 B-depart_date.date_relative      0.850     1.000     0.919      17.0
          B-toloc.state_name      0.781     0.893     0.833      28.0
           B-fare_basis_code      0.944     1.000     0.971      17.0
               B-flight_time      1.000     1.000     1.000       1.0
                        B-or      1.000     1.000     1.000       3.0
 B-arrive_time.period_of_day      0.667     1.000     0.800       6.0
          B-meal_description      1.000     0.900     0.947      10.0
             I-cost_relative      1.000     0.667     0.800       3.0
              I-airport_name      0.900     0.310     0.462      29.0
               B-fare_amount      0.667     1.000     0.800       2.0
               I-fare_amount      1.000     1.000     1.000       2.0
                 I-city_name      0.900     0.600     0.720      30.0
        I-toloc.airport_name      1.000     1.000     1.000       3.0
            B-transport_type      1.000     1.000     1.000      10.0
    B-arrive_date.month_name      0.625     0.833     0.714       6.0
    B-arrive_date.day_number      0.625     0.833     0.714       6.0
         I-stoploc.city_name      0.700     0.700     0.700      10.0
                      B-meal      1.000     1.000     1.000      16.0
        B-fromloc.state_code      1.000     1.000     1.000      23.0
    B-depart_time.period_mod      1.000     0.800     0.889       5.0
                   B-connect      1.000     1.000     1.000       6.0
               B-flight_days      1.000     1.000     1.000      10.0
        B-toloc.airport_name      1.000     1.000     1.000       3.0
        B-fromloc.state_name      1.000     0.941     0.970      17.0
              B-airport_name      0.750     0.286     0.414      21.0
                   B-economy      1.000     1.000     1.000       6.0
               I-flight_time      1.000     1.000     1.000       1.0
             B-aircraft_code      0.800     0.848     0.824      33.0
                       B-mod      1.000     0.500     0.667       2.0
              B-airport_code      0.000     0.000     0.000       9.0
    B-depart_time.start_time      1.000     0.667     0.800       3.0
      B-depart_time.end_time      1.000     0.333     0.500       3.0
          B-depart_date.year      1.000     0.667     0.800       3.0
            I-transport_type      1.000     1.000     1.000       1.0
          B-restriction_code      1.000     1.000     1.000       4.0
    B-arrive_time.start_time      0.889     1.000     0.941       8.0
        B-toloc.airport_code      1.000     0.750     0.857       4.0
      B-arrive_time.end_time      0.889     1.000     0.941       8.0
      I-arrive_time.end_time      0.889     1.000     0.941       8.0
      I-depart_time.end_time      1.000     0.333     0.500       3.0
               I-flight_stop      0.000     0.000     0.000       0.0
      B-fromloc.airport_code      0.714     1.000     0.833       5.0
          I-restriction_code      1.000     1.000     1.000       3.0
    I-depart_time.start_time      1.000     1.000     1.000       1.0
          I-toloc.state_name      1.000     1.000     1.000       1.0
I-depart_date.today_relative      0.000     0.000     0.000       0.0
 B-arrive_date.date_relative      1.000     1.000     1.000       2.0
                I-flight_mod      0.000     0.000     0.000       6.0
                   I-economy      0.000     0.000     0.000       0.0
 B-return_date.date_relative      0.500     0.333     0.400       3.0
        I-fromloc.state_name      1.000     1.000     1.000       1.0
                B-state_code      1.000     1.000     1.000       1.0
    I-arrive_time.start_time      1.000     1.000     1.000       1.0
    I-arrive_date.day_number      0.000     0.000     0.000       0.0
                 B-meal_code      0.000     0.000     0.000       1.0
 I-depart_time.period_of_day      0.000     0.000     0.000       1.0
                  B-day_name      1.000     0.500     0.667       2.0
             B-period_of_day      1.000     0.500     0.667       4.0
        B-stoploc.state_code      0.000     0.000     0.000       0.0
    B-return_date.month_name      0.000     0.000     0.000       0.0
    B-return_date.day_number      0.000     0.000     0.000       0.0
    B-arrive_time.period_mod      0.000     0.000     0.000       0.0
                 I-meal_code      0.000     0.000     0.000       0.0
        B-toloc.country_name      1.000     1.000     1.000       1.0
                 B-days_code      1.000     1.000     1.000       1.0
 I-arrive_time.period_of_day      0.000     0.000     0.000       0.0
            I-today_relative      0.000     0.000     0.000       0.0
 B-return_time.period_of_day      0.000     0.000     0.000       0.0
                      B-time      0.000     0.000     0.000       0.0
           I-fare_basis_code      0.000     0.000     0.000       0.0
 I-arrive_time.time_relative      0.000     0.000     0.000       4.0
 I-depart_time.time_relative      0.000     0.000     0.000       1.0
            B-today_relative      0.000     0.000     0.000       0.0
                B-state_name      0.000     0.000     0.000       9.0
B-arrive_date.today_relative      0.000     0.000     0.000       0.0
    B-return_time.period_mod      0.000     0.000     0.000       0.0
                B-month_name      0.000     0.000     0.000       0.0
                B-day_number      0.000     0.000     0.000       0.0
 I-return_date.date_relative      0.000     0.000     0.000       3.0
I-return_date.today_relative      0.000     0.000     0.000       0.0
      B-stoploc.airport_name      0.000     0.000     0.000       0.0
             B-time_relative      0.000     0.000     0.000       0.0
                      I-time      0.000     0.000     0.000       0.0
    I-return_date.day_number      0.000     0.000     0.000       0.0
          I-meal_description      0.000     0.000     0.000       0.0
B-return_date.today_relative      0.000     0.000     0.000       0.0
      B-return_date.day_name      0.000     0.000     0.000       2.0

                   micro avg      0.948     0.949     0.949    3657.0
                   macro avg      0.646     0.627     0.624    3657.0
                weighted avg      0.949     0.949     0.944    3657.0

I0715 05:52:25.936770 139833336047488 interactiveshell.py:2882] Best Slot F1: 0.949
Reading ../data/atis.train.w-intent.iob
I0715 05:52:27.809744 139833336047488 interactiveshell.py:2882] Step 3150 | Loss: 0.0156 | Loss_intent: 0.0013 | Loss_slots: 0.0143 | Spent: 3.4 secs | LR: 0.000215
I0715 05:52:29.721324 139833336047488 interactiveshell.py:2882] Step 3200 | Loss: 0.0135 | Loss_intent: 0.0072 | Loss_slots: 0.0063 | Spent: 1.9 secs | LR: 0.000214
I0715 05:52:31.615401 139833336047488 interactiveshell.py:2882] Step 3250 | Loss: 0.0079 | Loss_intent: 0.0010 | Loss_slots: 0.0069 | Spent: 1.9 secs | LR: 0.000213
I0715 05:52:33.539676 139833336047488 interactiveshell.py:2882] Step 3300 | Loss: 0.0194 | Loss_intent: 0.0135 | Loss_slots: 0.0059 | Spent: 1.9 secs | LR: 0.000212
I0715 05:52:35.442972 139833336047488 interactiveshell.py:2882] Step 3350 | Loss: 0.0088 | Loss_intent: 0.0005 | Loss_slots: 0.0083 | Spent: 1.9 secs | LR: 0.000211
I0715 05:52:37.330175 139833336047488 interactiveshell.py:2882] Step 3400 | Loss: 0.0270 | Loss_intent: 0.0121 | Loss_slots: 0.0149 | Spent: 1.9 secs | LR: 0.000210
Reading ../data/atis.test.w-intent.iob
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.
  'precision', 'predicted', average, warn_for)
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1439: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples.
  'recall', 'true', average, warn_for)
I0715 05:52:39.221972 139833336047488 interactiveshell.py:2882] 
                                          precision    recall  f1-score   support

                             atis_flight      0.972     0.992     0.982       632
                            atis_airfare      0.960     1.000     0.980        48
                     atis_ground_service      1.000     1.000     1.000        36
                            atis_airline      1.000     1.000     1.000        38
                       atis_abbreviation      1.000     1.000     1.000        33
                           atis_aircraft      1.000     1.000     1.000         9
                        atis_flight_time      1.000     1.000     1.000         1
                           atis_quantity      0.429     1.000     0.600         3
                atis_flight#atis_airfare      0.667     0.167     0.267        12
                            atis_airport      1.000     0.944     0.971        18
                           atis_distance      1.000     1.000     1.000        10
                               atis_city      1.000     0.500     0.667         6
                        atis_ground_fare      1.000     0.857     0.923         7
                           atis_capacity      1.000     1.000     1.000        21
                          atis_flight_no      0.889     1.000     0.941         8
                               atis_meal      1.000     0.833     0.909         6
                        atis_restriction      0.000     0.000     0.000         0
             atis_airline#atis_flight_no      0.000     0.000     0.000         0
    atis_ground_service#atis_ground_fare      0.000     0.000     0.000         0
           atis_airfare#atis_flight_time      0.000     0.000     0.000         0
                           atis_cheapest      0.000     0.000     0.000         0
atis_aircraft#atis_flight#atis_flight_no      0.000     0.000     0.000         0

                               micro avg      0.971     0.976     0.974       888
                               macro avg      0.678     0.650     0.647       888
                            weighted avg      0.971     0.976     0.970       888

I0715 05:52:39.261209 139833336047488 interactiveshell.py:2882] 
                              precision    recall  f1-score   support

                           O      0.000     0.000     0.000       0.0
           B-toloc.city_name      0.973     0.993     0.983     716.0
         B-fromloc.city_name      0.985     1.000     0.992     704.0
           I-toloc.city_name      0.963     0.992     0.978     265.0
      B-depart_date.day_name      0.986     0.991     0.988     212.0
              B-airline_name      1.000     1.000     1.000     101.0
         I-fromloc.city_name      0.978     0.994     0.986     177.0
 B-depart_time.period_of_day      0.992     0.915     0.952     130.0
              I-airline_name      1.000     1.000     1.000      65.0
    B-depart_date.day_number      0.981     0.964     0.972      55.0
    B-depart_date.month_name      0.981     0.946     0.964      56.0
          B-depart_time.time      0.826     1.000     0.905      57.0
                B-round_trip      1.000     0.986     0.993      73.0
             B-cost_relative      1.000     0.973     0.986      37.0
                I-round_trip      1.000     1.000     1.000      71.0
                B-flight_mod      0.786     0.917     0.846      24.0
 B-depart_time.time_relative      0.955     0.985     0.970      65.0
          I-depart_time.time      0.945     1.000     0.972      52.0
         B-stoploc.city_name      0.909     1.000     0.952      20.0
                 B-city_name      0.879     0.509     0.644      57.0
                B-class_type      0.960     1.000     0.980      24.0
          B-arrive_time.time      1.000     0.971     0.985      34.0
 B-arrive_time.time_relative      0.879     0.935     0.906      31.0
                I-class_type      1.000     1.000     1.000      17.0
               B-flight_stop      1.000     1.000     1.000      21.0
          I-arrive_time.time      1.000     0.971     0.986      35.0
              B-airline_code      1.000     0.912     0.954      34.0
    I-depart_date.day_number      1.000     1.000     1.000      15.0
      I-fromloc.airport_name      0.417     1.000     0.588      15.0
      B-fromloc.airport_name      0.429     1.000     0.600      12.0
      B-arrive_date.day_name      0.769     0.909     0.833      11.0
          B-toloc.state_code      1.000     1.000     1.000      18.0
B-depart_date.today_relative      0.889     0.889     0.889       9.0
             B-flight_number      0.524     1.000     0.688      11.0
 B-depart_date.date_relative      0.895     1.000     0.944      17.0
          B-toloc.state_name      0.758     0.893     0.820      28.0
           B-fare_basis_code      0.944     1.000     0.971      17.0
               B-flight_time      1.000     1.000     1.000       1.0
                        B-or      1.000     1.000     1.000       3.0
 B-arrive_time.period_of_day      0.750     1.000     0.857       6.0
          B-meal_description      1.000     0.900     0.947      10.0
             I-cost_relative      1.000     0.667     0.800       3.0
              I-airport_name      0.900     0.310     0.462      29.0
               B-fare_amount      0.667     1.000     0.800       2.0
               I-fare_amount      1.000     1.000     1.000       2.0
                 I-city_name      0.941     0.533     0.681      30.0
        I-toloc.airport_name      1.000     1.000     1.000       3.0
            B-transport_type      1.000     1.000     1.000      10.0
    B-arrive_date.month_name      0.625     0.833     0.714       6.0
    B-arrive_date.day_number      0.714     0.833     0.769       6.0
         I-stoploc.city_name      0.778     0.700     0.737      10.0
                      B-meal      1.000     1.000     1.000      16.0
        B-fromloc.state_code      1.000     1.000     1.000      23.0
    B-depart_time.period_mod      1.000     1.000     1.000       5.0
                   B-connect      1.000     1.000     1.000       6.0
               B-flight_days      1.000     1.000     1.000      10.0
        B-toloc.airport_name      1.000     1.000     1.000       3.0
        B-fromloc.state_name      1.000     1.000     1.000      17.0
              B-airport_name      0.750     0.286     0.414      21.0
                   B-economy      1.000     1.000     1.000       6.0
               I-flight_time      1.000     1.000     1.000       1.0
             B-aircraft_code      0.811     0.909     0.857      33.0
                       B-mod      1.000     0.500     0.667       2.0
              B-airport_code      0.000     0.000     0.000       9.0
    B-depart_time.start_time      1.000     1.000     1.000       3.0
      B-depart_time.end_time      1.000     0.667     0.800       3.0
          B-depart_date.year      1.000     1.000     1.000       3.0
            I-transport_type      1.000     1.000     1.000       1.0
          B-restriction_code      0.800     1.000     0.889       4.0
    B-arrive_time.start_time      1.000     1.000     1.000       8.0
        B-toloc.airport_code      1.000     0.750     0.857       4.0
      B-arrive_time.end_time      1.000     1.000     1.000       8.0
      I-arrive_time.end_time      0.889     1.000     0.941       8.0
      I-depart_time.end_time      0.500     0.333     0.400       3.0
               I-flight_stop      0.000     0.000     0.000       0.0
      B-fromloc.airport_code      0.833     1.000     0.909       5.0
          I-restriction_code      1.000     1.000     1.000       3.0
    I-depart_time.start_time      1.000     1.000     1.000       1.0
          I-toloc.state_name      1.000     1.000     1.000       1.0
I-depart_date.today_relative      0.000     0.000     0.000       0.0
 B-arrive_date.date_relative      1.000     1.000     1.000       2.0
                I-flight_mod      0.000     0.000     0.000       6.0
                   I-economy      0.000     0.000     0.000       0.0
 B-return_date.date_relative      0.500     0.333     0.400       3.0
        I-fromloc.state_name      1.000     1.000     1.000       1.0
                B-state_code      1.000     1.000     1.000       1.0
    I-arrive_time.start_time      1.000     1.000     1.000       1.0
    I-arrive_date.day_number      0.000     0.000     0.000       0.0
                 B-meal_code      0.000     0.000     0.000       1.0
 I-depart_time.period_of_day      0.000     0.000     0.000       1.0
                  B-day_name      1.000     0.500     0.667       2.0
             B-period_of_day      1.000     0.750     0.857       4.0
        B-stoploc.state_code      0.000     0.000     0.000       0.0
    B-return_date.month_name      0.000     0.000     0.000       0.0
    B-return_date.day_number      0.000     0.000     0.000       0.0
    B-arrive_time.period_mod      0.000     0.000     0.000       0.0
                 I-meal_code      0.000     0.000     0.000       0.0
        B-toloc.country_name      1.000     1.000     1.000       1.0
                 B-days_code      1.000     1.000     1.000       1.0
 I-arrive_time.period_of_day      0.000     0.000     0.000       0.0
            I-today_relative      0.000     0.000     0.000       0.0
 B-return_time.period_of_day      0.000     0.000     0.000       0.0
                      B-time      0.000     0.000     0.000       0.0
           I-fare_basis_code      0.000     0.000     0.000       0.0
 I-arrive_time.time_relative      0.000     0.000     0.000       4.0
 I-depart_time.time_relative      0.000     0.000     0.000       1.0
            B-today_relative      0.000     0.000     0.000       0.0
                B-state_name      0.000     0.000     0.000       9.0
B-arrive_date.today_relative      0.000     0.000     0.000       0.0
    B-return_time.period_mod      0.000     0.000     0.000       0.0
                B-month_name      0.000     0.000     0.000       0.0
                B-day_number      0.000     0.000     0.000       0.0
 I-return_date.date_relative      0.000     0.000     0.000       3.0
I-return_date.today_relative      0.000     0.000     0.000       0.0
      B-stoploc.airport_name      0.000     0.000     0.000       0.0
             B-time_relative      0.000     0.000     0.000       0.0
                      I-time      0.000     0.000     0.000       0.0
    I-return_date.day_number      0.000     0.000     0.000       0.0
          I-meal_description      0.000     0.000     0.000       0.0
B-return_date.today_relative      0.000     0.000     0.000       0.0
      B-return_date.day_name      0.000     0.000     0.000       2.0

                   micro avg      0.951     0.952     0.951    3657.0
                   macro avg      0.645     0.640     0.633    3657.0
                weighted avg      0.950     0.952     0.946    3657.0

I0715 05:52:39.262105 139833336047488 interactiveshell.py:2882] Best Slot F1: 0.951
Reading ../data/atis.train.w-intent.iob
I0715 05:52:40.661109 139833336047488 interactiveshell.py:2882] Step 3450 | Loss: 0.0046 | Loss_intent: 0.0002 | Loss_slots: 0.0044 | Spent: 3.3 secs | LR: 0.000209
I0715 05:52:42.531146 139833336047488 interactiveshell.py:2882] Step 3500 | Loss: 0.0327 | Loss_intent: 0.0012 | Loss_slots: 0.0315 | Spent: 1.9 secs | LR: 0.000207
I0715 05:52:44.390250 139833336047488 interactiveshell.py:2882] Step 3550 | Loss: 0.0120 | Loss_intent: 0.0033 | Loss_slots: 0.0087 | Spent: 1.9 secs | LR: 0.000206
I0715 05:52:46.257851 139833336047488 interactiveshell.py:2882] Step 3600 | Loss: 0.0155 | Loss_intent: 0.0097 | Loss_slots: 0.0058 | Spent: 1.9 secs | LR: 0.000205
I0715 05:52:48.148509 139833336047488 interactiveshell.py:2882] Step 3650 | Loss: 0.0074 | Loss_intent: 0.0005 | Loss_slots: 0.0069 | Spent: 1.9 secs | LR: 0.000204
I0715 05:52:50.056491 139833336047488 interactiveshell.py:2882] Step 3700 | Loss: 0.0071 | Loss_intent: 0.0010 | Loss_slots: 0.0061 | Spent: 1.9 secs | LR: 0.000203
Reading ../data/atis.test.w-intent.iob
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.
  'precision', 'predicted', average, warn_for)
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1439: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples.
  'recall', 'true', average, warn_for)
I0715 05:52:52.414182 139833336047488 interactiveshell.py:2882] 
                                          precision    recall  f1-score   support

                             atis_flight      0.971     0.992     0.981       632
                            atis_airfare      0.960     1.000     0.980        48
                     atis_ground_service      0.973     1.000     0.986        36
                            atis_airline      1.000     1.000     1.000        38
                       atis_abbreviation      1.000     1.000     1.000        33
                           atis_aircraft      1.000     1.000     1.000         9
                        atis_flight_time      1.000     1.000     1.000         1
                           atis_quantity      0.429     1.000     0.600         3
                atis_flight#atis_airfare      0.500     0.083     0.143        12
                            atis_airport      1.000     0.944     0.971        18
                           atis_distance      1.000     1.000     1.000        10
                               atis_city      1.000     0.500     0.667         6
                        atis_ground_fare      1.000     0.857     0.923         7
                           atis_capacity      1.000     0.952     0.976        21
                          atis_flight_no      0.889     1.000     0.941         8
                               atis_meal      1.000     0.833     0.909         6
                        atis_restriction      0.000     0.000     0.000         0
             atis_airline#atis_flight_no      0.000     0.000     0.000         0
    atis_ground_service#atis_ground_fare      0.000     0.000     0.000         0
           atis_airfare#atis_flight_time      0.000     0.000     0.000         0
                           atis_cheapest      0.000     0.000     0.000         0
atis_aircraft#atis_flight#atis_flight_no      0.000     0.000     0.000         0

                               micro avg      0.969     0.974     0.971       888
                               macro avg      0.669     0.644     0.640       888
                            weighted avg      0.966     0.974     0.967       888

I0715 05:52:52.453293 139833336047488 interactiveshell.py:2882] 
                              precision    recall  f1-score   support

                           O      0.000     0.000     0.000       0.0
           B-toloc.city_name      0.974     0.993     0.983     716.0
         B-fromloc.city_name      0.985     0.999     0.992     704.0
           I-toloc.city_name      0.967     0.996     0.981     265.0
      B-depart_date.day_name      0.986     0.991     0.988     212.0
              B-airline_name      1.000     1.000     1.000     101.0
         I-fromloc.city_name      0.978     0.994     0.986     177.0
 B-depart_time.period_of_day      0.992     0.908     0.948     130.0
              I-airline_name      1.000     1.000     1.000      65.0
    B-depart_date.day_number      0.981     0.945     0.963      55.0
    B-depart_date.month_name      0.981     0.946     0.964      56.0
          B-depart_time.time      0.826     1.000     0.905      57.0
                B-round_trip      1.000     0.986     0.993      73.0
             B-cost_relative      1.000     0.973     0.986      37.0
                I-round_trip      1.000     1.000     1.000      71.0
                B-flight_mod      0.793     0.958     0.868      24.0
 B-depart_time.time_relative      0.955     0.969     0.962      65.0
          I-depart_time.time      0.945     1.000     0.972      52.0
         B-stoploc.city_name      0.909     1.000     0.952      20.0
                 B-city_name      0.882     0.526     0.659      57.0
                B-class_type      0.960     1.000     0.980      24.0
          B-arrive_time.time      1.000     0.971     0.985      34.0
 B-arrive_time.time_relative      0.879     0.935     0.906      31.0
                I-class_type      1.000     1.000     1.000      17.0
               B-flight_stop      1.000     1.000     1.000      21.0
          I-arrive_time.time      1.000     0.971     0.986      35.0
              B-airline_code      1.000     0.912     0.954      34.0
    I-depart_date.day_number      1.000     1.000     1.000      15.0
      I-fromloc.airport_name      0.417     1.000     0.588      15.0
      B-fromloc.airport_name      0.414     1.000     0.585      12.0
      B-arrive_date.day_name      0.769     0.909     0.833      11.0
          B-toloc.state_code      0.947     1.000     0.973      18.0
B-depart_date.today_relative      0.889     0.889     0.889       9.0
             B-flight_number      0.550     1.000     0.710      11.0
 B-depart_date.date_relative      0.944     1.000     0.971      17.0
          B-toloc.state_name      0.781     0.893     0.833      28.0
           B-fare_basis_code      0.944     1.000     0.971      17.0
               B-flight_time      1.000     1.000     1.000       1.0
                        B-or      1.000     1.000     1.000       3.0
 B-arrive_time.period_of_day      0.750     1.000     0.857       6.0
          B-meal_description      1.000     0.900     0.947      10.0
             I-cost_relative      1.000     0.667     0.800       3.0
              I-airport_name      0.900     0.310     0.462      29.0
               B-fare_amount      0.667     1.000     0.800       2.0
               I-fare_amount      1.000     1.000     1.000       2.0
                 I-city_name      0.889     0.533     0.667      30.0
        I-toloc.airport_name      1.000     1.000     1.000       3.0
            B-transport_type      1.000     1.000     1.000      10.0
    B-arrive_date.month_name      0.625     0.833     0.714       6.0
    B-arrive_date.day_number      0.625     0.833     0.714       6.0
         I-stoploc.city_name      0.889     0.800     0.842      10.0
                      B-meal      1.000     1.000     1.000      16.0
        B-fromloc.state_code      1.000     0.957     0.978      23.0
    B-depart_time.period_mod      0.833     1.000     0.909       5.0
                   B-connect      1.000     1.000     1.000       6.0
               B-flight_days      1.000     1.000     1.000      10.0
        B-toloc.airport_name      1.000     1.000     1.000       3.0
        B-fromloc.state_name      1.000     1.000     1.000      17.0
              B-airport_name      0.750     0.286     0.414      21.0
                   B-economy      1.000     1.000     1.000       6.0
               I-flight_time      1.000     1.000     1.000       1.0
             B-aircraft_code      0.806     0.879     0.841      33.0
                       B-mod      1.000     0.500     0.667       2.0
              B-airport_code      0.000     0.000     0.000       9.0
    B-depart_time.start_time      1.000     1.000     1.000       3.0
      B-depart_time.end_time      1.000     0.667     0.800       3.0
          B-depart_date.year      1.000     1.000     1.000       3.0
            I-transport_type      0.000     0.000     0.000       1.0
          B-restriction_code      0.800     1.000     0.889       4.0
    B-arrive_time.start_time      1.000     1.000     1.000       8.0
        B-toloc.airport_code      1.000     0.750     0.857       4.0
      B-arrive_time.end_time      1.000     1.000     1.000       8.0
      I-arrive_time.end_time      1.000     1.000     1.000       8.0
      I-depart_time.end_time      0.500     0.667     0.571       3.0
               I-flight_stop      0.000     0.000     0.000       0.0
      B-fromloc.airport_code      0.833     1.000     0.909       5.0
          I-restriction_code      1.000     1.000     1.000       3.0
    I-depart_time.start_time      1.000     1.000     1.000       1.0
          I-toloc.state_name      1.000     1.000     1.000       1.0
I-depart_date.today_relative      0.000     0.000     0.000       0.0
 B-arrive_date.date_relative      1.000     1.000     1.000       2.0
                I-flight_mod      0.000     0.000     0.000       6.0
                   I-economy      0.000     0.000     0.000       0.0
 B-return_date.date_relative      0.000     0.000     0.000       3.0
        I-fromloc.state_name      1.000     1.000     1.000       1.0
                B-state_code      1.000     1.000     1.000       1.0
    I-arrive_time.start_time      1.000     1.000     1.000       1.0
    I-arrive_date.day_number      0.000     0.000     0.000       0.0
                 B-meal_code      0.000     0.000     0.000       1.0
 I-depart_time.period_of_day      0.000     0.000     0.000       1.0
                  B-day_name      1.000     0.500     0.667       2.0
             B-period_of_day      1.000     0.750     0.857       4.0
        B-stoploc.state_code      0.000     0.000     0.000       0.0
    B-return_date.month_name      0.000     0.000     0.000       0.0
    B-return_date.day_number      0.000     0.000     0.000       0.0
    B-arrive_time.period_mod      0.000     0.000     0.000       0.0
                 I-meal_code      0.000     0.000     0.000       0.0
        B-toloc.country_name      1.000     1.000     1.000       1.0
                 B-days_code      1.000     1.000     1.000       1.0
 I-arrive_time.period_of_day      0.000     0.000     0.000       0.0
            I-today_relative      0.000     0.000     0.000       0.0
 B-return_time.period_of_day      0.000     0.000     0.000       0.0
                      B-time      0.000     0.000     0.000       0.0
           I-fare_basis_code      0.000     0.000     0.000       0.0
 I-arrive_time.time_relative      0.000     0.000     0.000       4.0
 I-depart_time.time_relative      0.000     0.000     0.000       1.0
            B-today_relative      0.000     0.000     0.000       0.0
                B-state_name      0.000     0.000     0.000       9.0
B-arrive_date.today_relative      0.000     0.000     0.000       0.0
    B-return_time.period_mod      0.000     0.000     0.000       0.0
                B-month_name      0.000     0.000     0.000       0.0
                B-day_number      0.000     0.000     0.000       0.0
 I-return_date.date_relative      0.000     0.000     0.000       3.0
I-return_date.today_relative      0.000     0.000     0.000       0.0
      B-stoploc.airport_name      0.000     0.000     0.000       0.0
             B-time_relative      0.000     0.000     0.000       0.0
                      I-time      0.000     0.000     0.000       0.0
    I-return_date.day_number      0.000     0.000     0.000       0.0
          I-meal_description      0.000     0.000     0.000       0.0
B-return_date.today_relative      0.000     0.000     0.000       0.0
      B-return_date.day_name      0.000     0.000     0.000       2.0

                   micro avg      0.950     0.951     0.951    3657.0
                   macro avg      0.632     0.632     0.623    3657.0
                weighted avg      0.949     0.951     0.946    3657.0

I0715 05:52:52.454120 139833336047488 interactiveshell.py:2882] Best Slot F1: 0.951
Reading ../data/atis.train.w-intent.iob
I0715 05:52:53.393849 139833336047488 interactiveshell.py:2882] Step 3750 | Loss: 0.0328 | Loss_intent: 0.0014 | Loss_slots: 0.0315 | Spent: 3.3 secs | LR: 0.000202
I0715 05:52:55.278124 139833336047488 interactiveshell.py:2882] Step 3800 | Loss: 0.0073 | Loss_intent: 0.0034 | Loss_slots: 0.0038 | Spent: 1.9 secs | LR: 0.000201
I0715 05:52:57.159537 139833336047488 interactiveshell.py:2882] Step 3850 | Loss: 0.0125 | Loss_intent: 0.0072 | Loss_slots: 0.0053 | Spent: 1.9 secs | LR: 0.000200
I0715 05:52:59.030962 139833336047488 interactiveshell.py:2882] Step 3900 | Loss: 0.0050 | Loss_intent: 0.0009 | Loss_slots: 0.0040 | Spent: 1.9 secs | LR: 0.000199
I0715 05:53:01.066010 139833336047488 interactiveshell.py:2882] Step 3950 | Loss: 0.0199 | Loss_intent: 0.0016 | Loss_slots: 0.0182 | Spent: 2.0 secs | LR: 0.000198
I0715 05:53:02.953169 139833336047488 interactiveshell.py:2882] Step 4000 | Loss: 0.0040 | Loss_intent: 0.0012 | Loss_slots: 0.0028 | Spent: 1.9 secs | LR: 0.000197
I0715 05:53:04.837635 139833336047488 interactiveshell.py:2882] Step 4050 | Loss: 0.0088 | Loss_intent: 0.0005 | Loss_slots: 0.0083 | Spent: 1.9 secs | LR: 0.000196
Reading ../data/atis.test.w-intent.iob
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.
  'precision', 'predicted', average, warn_for)
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1439: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples.
  'recall', 'true', average, warn_for)
I0715 05:53:05.758448 139833336047488 interactiveshell.py:2882] 
                                          precision    recall  f1-score   support

                             atis_flight      0.981     0.991     0.986       632
                            atis_airfare      0.941     1.000     0.970        48
                     atis_ground_service      1.000     1.000     1.000        36
                            atis_airline      1.000     1.000     1.000        38
                       atis_abbreviation      1.000     1.000     1.000        33
                           atis_aircraft      0.900     1.000     0.947         9
                        atis_flight_time      1.000     1.000     1.000         1
                           atis_quantity      0.429     1.000     0.600         3
                atis_flight#atis_airfare      0.875     0.583     0.700        12
                            atis_airport      1.000     0.944     0.971        18
                           atis_distance      1.000     1.000     1.000        10
                               atis_city      1.000     0.500     0.667         6
                        atis_ground_fare      1.000     0.857     0.923         7
                           atis_capacity      1.000     1.000     1.000        21
                          atis_flight_no      0.889     1.000     0.941         8
                               atis_meal      1.000     0.833     0.909         6
                        atis_restriction      0.000     0.000     0.000         0
             atis_airline#atis_flight_no      0.000     0.000     0.000         0
    atis_ground_service#atis_ground_fare      0.000     0.000     0.000         0
           atis_airfare#atis_flight_time      0.000     0.000     0.000         0
                           atis_cheapest      0.000     0.000     0.000         0
atis_aircraft#atis_flight#atis_flight_no      0.000     0.000     0.000         0

                               micro avg      0.975     0.981     0.978       888
                               macro avg      0.682     0.669     0.664       888
                            weighted avg      0.978     0.981     0.978       888

I0715 05:53:05.797506 139833336047488 interactiveshell.py:2882] 
                              precision    recall  f1-score   support

                           O      0.000     0.000     0.000       0.0
           B-toloc.city_name      0.975     0.993     0.984     716.0
         B-fromloc.city_name      0.983     0.999     0.991     704.0
           I-toloc.city_name      0.964     0.996     0.980     265.0
      B-depart_date.day_name      0.991     0.991     0.991     212.0
              B-airline_name      1.000     1.000     1.000     101.0
         I-fromloc.city_name      0.983     0.994     0.989     177.0
 B-depart_time.period_of_day      1.000     0.908     0.952     130.0
              I-airline_name      1.000     1.000     1.000      65.0
    B-depart_date.day_number      0.981     0.945     0.963      55.0
    B-depart_date.month_name      0.981     0.946     0.964      56.0
          B-depart_time.time      0.838     1.000     0.912      57.0
                B-round_trip      1.000     0.986     0.993      73.0
             B-cost_relative      1.000     0.973     0.986      37.0
                I-round_trip      1.000     1.000     1.000      71.0
                B-flight_mod      0.800     1.000     0.889      24.0
 B-depart_time.time_relative      0.955     0.969     0.962      65.0
          I-depart_time.time      0.945     1.000     0.972      52.0
         B-stoploc.city_name      0.952     1.000     0.976      20.0
                 B-city_name      0.889     0.561     0.688      57.0
                B-class_type      0.960     1.000     0.980      24.0
          B-arrive_time.time      0.971     0.971     0.971      34.0
 B-arrive_time.time_relative      0.853     0.935     0.892      31.0
                I-class_type      1.000     1.000     1.000      17.0
               B-flight_stop      1.000     1.000     1.000      21.0
          I-arrive_time.time      1.000     0.971     0.986      35.0
              B-airline_code      1.000     0.912     0.954      34.0
    I-depart_date.day_number      1.000     1.000     1.000      15.0
      I-fromloc.airport_name      0.429     1.000     0.600      15.0
      B-fromloc.airport_name      0.429     1.000     0.600      12.0
      B-arrive_date.day_name      0.786     1.000     0.880      11.0
          B-toloc.state_code      0.947     1.000     0.973      18.0
B-depart_date.today_relative      1.000     0.889     0.941       9.0
             B-flight_number      0.550     1.000     0.710      11.0
 B-depart_date.date_relative      0.944     1.000     0.971      17.0
          B-toloc.state_name      0.758     0.893     0.820      28.0
           B-fare_basis_code      0.944     1.000     0.971      17.0
               B-flight_time      1.000     1.000     1.000       1.0
                        B-or      1.000     1.000     1.000       3.0
 B-arrive_time.period_of_day      0.750     1.000     0.857       6.0
          B-meal_description      1.000     0.900     0.947      10.0
             I-cost_relative      1.000     0.667     0.800       3.0
              I-airport_name      0.917     0.379     0.537      29.0
               B-fare_amount      1.000     1.000     1.000       2.0
               I-fare_amount      1.000     1.000     1.000       2.0
                 I-city_name      1.000     0.533     0.696      30.0
        I-toloc.airport_name      1.000     1.000     1.000       3.0
            B-transport_type      1.000     1.000     1.000      10.0
    B-arrive_date.month_name      0.625     0.833     0.714       6.0
    B-arrive_date.day_number      0.625     0.833     0.714       6.0
         I-stoploc.city_name      0.778     0.700     0.737      10.0
                      B-meal      1.000     1.000     1.000      16.0
        B-fromloc.state_code      1.000     0.957     0.978      23.0
    B-depart_time.period_mod      1.000     1.000     1.000       5.0
                   B-connect      1.000     1.000     1.000       6.0
               B-flight_days      1.000     1.000     1.000      10.0
        B-toloc.airport_name      1.000     1.000     1.000       3.0
        B-fromloc.state_name      1.000     1.000     1.000      17.0
              B-airport_name      0.778     0.333     0.467      21.0
                   B-economy      1.000     1.000     1.000       6.0
               I-flight_time      1.000     1.000     1.000       1.0
             B-aircraft_code      0.794     0.818     0.806      33.0
                       B-mod      1.000     0.500     0.667       2.0
              B-airport_code      0.000     0.000     0.000       9.0
    B-depart_time.start_time      1.000     0.667     0.800       3.0
      B-depart_time.end_time      1.000     0.333     0.500       3.0
          B-depart_date.year      1.000     1.000     1.000       3.0
            I-transport_type      1.000     1.000     1.000       1.0
          B-restriction_code      0.571     1.000     0.727       4.0
    B-arrive_time.start_time      0.889     1.000     0.941       8.0
        B-toloc.airport_code      1.000     0.750     0.857       4.0
      B-arrive_time.end_time      0.889     1.000     0.941       8.0
      I-arrive_time.end_time      0.889     1.000     0.941       8.0
      I-depart_time.end_time      0.333     0.333     0.333       3.0
               I-flight_stop      0.000     0.000     0.000       0.0
      B-fromloc.airport_code      0.833     1.000     0.909       5.0
          I-restriction_code      0.750     1.000     0.857       3.0
    I-depart_time.start_time      1.000     1.000     1.000       1.0
          I-toloc.state_name      1.000     1.000     1.000       1.0
I-depart_date.today_relative      0.000     0.000     0.000       0.0
 B-arrive_date.date_relative      1.000     1.000     1.000       2.0
                I-flight_mod      0.000     0.000     0.000       6.0
                   I-economy      0.000     0.000     0.000       0.0
 B-return_date.date_relative      0.667     0.667     0.667       3.0
        I-fromloc.state_name      1.000     1.000     1.000       1.0
                B-state_code      1.000     1.000     1.000       1.0
    I-arrive_time.start_time      1.000     1.000     1.000       1.0
    I-arrive_date.day_number      0.000     0.000     0.000       0.0
                 B-meal_code      0.000     0.000     0.000       1.0
 I-depart_time.period_of_day      1.000     1.000     1.000       1.0
                  B-day_name      1.000     0.500     0.667       2.0
             B-period_of_day      1.000     0.750     0.857       4.0
        B-stoploc.state_code      0.000     0.000     0.000       0.0
    B-return_date.month_name      0.000     0.000     0.000       0.0
    B-return_date.day_number      0.000     0.000     0.000       0.0
    B-arrive_time.period_mod      0.000     0.000     0.000       0.0
                 I-meal_code      0.000     0.000     0.000       0.0
        B-toloc.country_name      1.000     1.000     1.000       1.0
                 B-days_code      1.000     1.000     1.000       1.0
 I-arrive_time.period_of_day      0.000     0.000     0.000       0.0
            I-today_relative      0.000     0.000     0.000       0.0
 B-return_time.period_of_day      0.000     0.000     0.000       0.0
                      B-time      0.000     0.000     0.000       0.0
           I-fare_basis_code      0.000     0.000     0.000       0.0
 I-arrive_time.time_relative      0.000     0.000     0.000       4.0
 I-depart_time.time_relative      0.000     0.000     0.000       1.0
            B-today_relative      0.000     0.000     0.000       0.0
                B-state_name      0.000     0.000     0.000       9.0
B-arrive_date.today_relative      0.000     0.000     0.000       0.0
    B-return_time.period_mod      0.000     0.000     0.000       0.0
                B-month_name      0.000     0.000     0.000       0.0
                B-day_number      0.000     0.000     0.000       0.0
 I-return_date.date_relative      1.000     0.333     0.500       3.0
I-return_date.today_relative      0.000     0.000     0.000       0.0
      B-stoploc.airport_name      0.000     0.000     0.000       0.0
             B-time_relative      0.000     0.000     0.000       0.0
                      I-time      0.000     0.000     0.000       0.0
    I-return_date.day_number      0.000     0.000     0.000       0.0
          I-meal_description      0.000     0.000     0.000       0.0
B-return_date.today_relative      0.000     0.000     0.000       0.0
      B-return_date.day_name      0.000     0.000     0.000       2.0

                   micro avg      0.951     0.953     0.952    3657.0
                   macro avg      0.660     0.650     0.643    3657.0
                weighted avg      0.952     0.953     0.948    3657.0

I0715 05:53:05.798247 139833336047488 interactiveshell.py:2882] Best Slot F1: 0.952
Reading ../data/atis.train.w-intent.iob
I0715 05:53:08.166302 139833336047488 interactiveshell.py:2882] Step 4100 | Loss: 0.0047 | Loss_intent: 0.0010 | Loss_slots: 0.0038 | Spent: 3.3 secs | LR: 0.000195
I0715 05:53:10.049460 139833336047488 interactiveshell.py:2882] Step 4150 | Loss: 0.0087 | Loss_intent: 0.0011 | Loss_slots: 0.0076 | Spent: 1.9 secs | LR: 0.000194
I0715 05:53:11.942860 139833336047488 interactiveshell.py:2882] Step 4200 | Loss: 0.0233 | Loss_intent: 0.0006 | Loss_slots: 0.0227 | Spent: 1.9 secs | LR: 0.000193
I0715 05:53:13.830464 139833336047488 interactiveshell.py:2882] Step 4250 | Loss: 0.0085 | Loss_intent: 0.0024 | Loss_slots: 0.0061 | Spent: 1.9 secs | LR: 0.000192
I0715 05:53:15.706782 139833336047488 interactiveshell.py:2882] Step 4300 | Loss: 0.0124 | Loss_intent: 0.0042 | Loss_slots: 0.0082 | Spent: 1.9 secs | LR: 0.000191
I0715 05:53:17.577881 139833336047488 interactiveshell.py:2882] Step 4350 | Loss: 0.0392 | Loss_intent: 0.0093 | Loss_slots: 0.0299 | Spent: 1.9 secs | LR: 0.000190
Reading ../data/atis.test.w-intent.iob
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.
  'precision', 'predicted', average, warn_for)
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1439: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples.
  'recall', 'true', average, warn_for)
I0715 05:53:18.950169 139833336047488 interactiveshell.py:2882] 
                                          precision    recall  f1-score   support

                             atis_flight      0.971     0.992     0.981       632
                            atis_airfare      0.940     0.979     0.959        48
                     atis_ground_service      1.000     1.000     1.000        36
                            atis_airline      1.000     1.000     1.000        38
                       atis_abbreviation      1.000     1.000     1.000        33
                           atis_aircraft      1.000     1.000     1.000         9
                        atis_flight_time      1.000     1.000     1.000         1
                           atis_quantity      0.429     1.000     0.600         3
                atis_flight#atis_airfare      0.667     0.167     0.267        12
                            atis_airport      1.000     0.944     0.971        18
                           atis_distance      1.000     1.000     1.000        10
                               atis_city      1.000     0.500     0.667         6
                        atis_ground_fare      1.000     0.714     0.833         7
                           atis_capacity      1.000     1.000     1.000        21
                          atis_flight_no      0.889     1.000     0.941         8
                               atis_meal      1.000     0.833     0.909         6
                        atis_restriction      0.000     0.000     0.000         0
             atis_airline#atis_flight_no      0.000     0.000     0.000         0
    atis_ground_service#atis_ground_fare      0.000     0.000     0.000         0
           atis_airfare#atis_flight_time      0.000     0.000     0.000         0
                           atis_cheapest      0.000     0.000     0.000         0
atis_aircraft#atis_flight#atis_flight_no      0.000     0.000     0.000         0

                               micro avg      0.969     0.974     0.971       888
                               macro avg      0.677     0.642     0.642       888
                            weighted avg      0.968     0.974     0.968       888

I0715 05:53:18.990012 139833336047488 interactiveshell.py:2882] 
                              precision    recall  f1-score   support

                           O      0.000     0.000     0.000       0.0
           B-toloc.city_name      0.974     0.993     0.983     716.0
         B-fromloc.city_name      0.985     1.000     0.992     704.0
           I-toloc.city_name      0.967     0.985     0.976     265.0
      B-depart_date.day_name      0.991     0.991     0.991     212.0
              B-airline_name      1.000     1.000     1.000     101.0
         I-fromloc.city_name      0.989     0.994     0.992     177.0
 B-depart_time.period_of_day      0.983     0.908     0.944     130.0
              I-airline_name      1.000     1.000     1.000      65.0
    B-depart_date.day_number      0.981     0.945     0.963      55.0
    B-depart_date.month_name      0.981     0.946     0.964      56.0
          B-depart_time.time      0.851     1.000     0.919      57.0
                B-round_trip      1.000     0.986     0.993      73.0
             B-cost_relative      1.000     0.973     0.986      37.0
                I-round_trip      1.000     1.000     1.000      71.0
                B-flight_mod      0.793     0.958     0.868      24.0
 B-depart_time.time_relative      0.955     0.969     0.962      65.0
          I-depart_time.time      0.944     0.981     0.962      52.0
         B-stoploc.city_name      0.909     1.000     0.952      20.0
                 B-city_name      0.886     0.544     0.674      57.0
                B-class_type      0.960     1.000     0.980      24.0
          B-arrive_time.time      0.943     0.971     0.957      34.0
 B-arrive_time.time_relative      0.853     0.935     0.892      31.0
                I-class_type      1.000     1.000     1.000      17.0
               B-flight_stop      1.000     1.000     1.000      21.0
          I-arrive_time.time      0.971     0.971     0.971      35.0
              B-airline_code      1.000     0.912     0.954      34.0
    I-depart_date.day_number      1.000     1.000     1.000      15.0
      I-fromloc.airport_name      0.417     1.000     0.588      15.0
      B-fromloc.airport_name      0.429     1.000     0.600      12.0
      B-arrive_date.day_name      0.786     1.000     0.880      11.0
          B-toloc.state_code      0.947     1.000     0.973      18.0
B-depart_date.today_relative      0.889     0.889     0.889       9.0
             B-flight_number      0.407     1.000     0.579      11.0
 B-depart_date.date_relative      0.895     1.000     0.944      17.0
          B-toloc.state_name      0.758     0.893     0.820      28.0
           B-fare_basis_code      0.917     0.647     0.759      17.0
               B-flight_time      1.000     1.000     1.000       1.0
                        B-or      1.000     1.000     1.000       3.0
 B-arrive_time.period_of_day      0.750     1.000     0.857       6.0
          B-meal_description      1.000     0.900     0.947      10.0
             I-cost_relative      1.000     0.667     0.800       3.0
              I-airport_name      0.909     0.345     0.500      29.0
               B-fare_amount      0.667     1.000     0.800       2.0
               I-fare_amount      1.000     1.000     1.000       2.0
                 I-city_name      1.000     0.533     0.696      30.0
        I-toloc.airport_name      1.000     1.000     1.000       3.0
            B-transport_type      1.000     1.000     1.000      10.0
    B-arrive_date.month_name      0.625     0.833     0.714       6.0
    B-arrive_date.day_number      0.625     0.833     0.714       6.0
         I-stoploc.city_name      0.714     1.000     0.833      10.0
                      B-meal      1.000     1.000     1.000      16.0
        B-fromloc.state_code      1.000     0.957     0.978      23.0
    B-depart_time.period_mod      0.833     1.000     0.909       5.0
                   B-connect      1.000     1.000     1.000       6.0
               B-flight_days      1.000     1.000     1.000      10.0
        B-toloc.airport_name      1.000     1.000     1.000       3.0
        B-fromloc.state_name      1.000     1.000     1.000      17.0
              B-airport_name      0.750     0.286     0.414      21.0
                   B-economy      1.000     1.000     1.000       6.0
               I-flight_time      1.000     1.000     1.000       1.0
             B-aircraft_code      0.771     0.818     0.794      33.0
                       B-mod      1.000     0.500     0.667       2.0
              B-airport_code      0.000     0.000     0.000       9.0
    B-depart_time.start_time      1.000     0.667     0.800       3.0
      B-depart_time.end_time      1.000     0.333     0.500       3.0
          B-depart_date.year      1.000     1.000     1.000       3.0
            I-transport_type      1.000     1.000     1.000       1.0
          B-restriction_code      0.571     1.000     0.727       4.0
    B-arrive_time.start_time      0.889     1.000     0.941       8.0
        B-toloc.airport_code      1.000     0.750     0.857       4.0
      B-arrive_time.end_time      0.875     0.875     0.875       8.0
      I-arrive_time.end_time      0.875     0.875     0.875       8.0
      I-depart_time.end_time      0.500     0.333     0.400       3.0
               I-flight_stop      0.000     0.000     0.000       0.0
      B-fromloc.airport_code      0.833     1.000     0.909       5.0
          I-restriction_code      1.000     1.000     1.000       3.0
    I-depart_time.start_time      1.000     1.000     1.000       1.0
          I-toloc.state_name      1.000     1.000     1.000       1.0
I-depart_date.today_relative      0.000     0.000     0.000       0.0
 B-arrive_date.date_relative      1.000     1.000     1.000       2.0
                I-flight_mod      0.000     0.000     0.000       6.0
                   I-economy      0.000     0.000     0.000       0.0
 B-return_date.date_relative      0.500     0.333     0.400       3.0
        I-fromloc.state_name      1.000     1.000     1.000       1.0
                B-state_code      1.000     1.000     1.000       1.0
    I-arrive_time.start_time      1.000     1.000     1.000       1.0
    I-arrive_date.day_number      0.000     0.000     0.000       0.0
                 B-meal_code      0.000     0.000     0.000       1.0
 I-depart_time.period_of_day      0.000     0.000     0.000       1.0
                  B-day_name      1.000     0.500     0.667       2.0
             B-period_of_day      1.000     0.750     0.857       4.0
        B-stoploc.state_code      0.000     0.000     0.000       0.0
    B-return_date.month_name      0.000     0.000     0.000       0.0
    B-return_date.day_number      0.000     0.000     0.000       0.0
    B-arrive_time.period_mod      0.000     0.000     0.000       0.0
                 I-meal_code      0.000     0.000     0.000       0.0
        B-toloc.country_name      1.000     1.000     1.000       1.0
                 B-days_code      1.000     1.000     1.000       1.0
 I-arrive_time.period_of_day      0.000     0.000     0.000       0.0
            I-today_relative      0.000     0.000     0.000       0.0
 B-return_time.period_of_day      0.000     0.000     0.000       0.0
                      B-time      0.000     0.000     0.000       0.0
           I-fare_basis_code      0.000     0.000     0.000       0.0
 I-arrive_time.time_relative      0.000     0.000     0.000       4.0
 I-depart_time.time_relative      0.000     0.000     0.000       1.0
            B-today_relative      0.000     0.000     0.000       0.0
                B-state_name      0.000     0.000     0.000       9.0
B-arrive_date.today_relative      0.000     0.000     0.000       0.0
    B-return_time.period_mod      0.000     0.000     0.000       0.0
                B-month_name      0.000     0.000     0.000       0.0
                B-day_number      0.000     0.000     0.000       0.0
 I-return_date.date_relative      1.000     0.333     0.500       3.0
I-return_date.today_relative      0.000     0.000     0.000       0.0
      B-stoploc.airport_name      0.000     0.000     0.000       0.0
             B-time_relative      0.000     0.000     0.000       0.0
                      I-time      0.000     0.000     0.000       0.0
    I-return_date.day_number      0.000     0.000     0.000       0.0
          I-meal_description      0.000     0.000     0.000       0.0
B-return_date.today_relative      0.000     0.000     0.000       0.0
      B-return_date.day_name      0.000     0.000     0.000       2.0

                   micro avg      0.948     0.949     0.948    3657.0
                   macro avg      0.645     0.635     0.627    3657.0
                weighted avg      0.949     0.949     0.944    3657.0

I0715 05:53:18.990850 139833336047488 interactiveshell.py:2882] Best Slot F1: 0.952
Reading ../data/atis.train.w-intent.iob
I0715 05:53:20.918143 139833336047488 interactiveshell.py:2882] Step 4400 | Loss: 0.0051 | Loss_intent: 0.0020 | Loss_slots: 0.0031 | Spent: 3.3 secs | LR: 0.000189
I0715 05:53:22.824635 139833336047488 interactiveshell.py:2882] Step 4450 | Loss: 0.0072 | Loss_intent: 0.0017 | Loss_slots: 0.0055 | Spent: 1.9 secs | LR: 0.000188
I0715 05:53:24.699034 139833336047488 interactiveshell.py:2882] Step 4500 | Loss: 0.0269 | Loss_intent: 0.0008 | Loss_slots: 0.0261 | Spent: 1.9 secs | LR: 0.000187
I0715 05:53:26.574381 139833336047488 interactiveshell.py:2882] Step 4550 | Loss: 0.0088 | Loss_intent: 0.0012 | Loss_slots: 0.0076 | Spent: 1.9 secs | LR: 0.000186
I0715 05:53:28.466720 139833336047488 interactiveshell.py:2882] Step 4600 | Loss: 0.0027 | Loss_intent: 0.0010 | Loss_slots: 0.0017 | Spent: 1.9 secs | LR: 0.000185
I0715 05:53:30.366808 139833336047488 interactiveshell.py:2882] Step 4650 | Loss: 0.0064 | Loss_intent: 0.0009 | Loss_slots: 0.0055 | Spent: 1.9 secs | LR: 0.000184
Reading ../data/atis.test.w-intent.iob
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.
  'precision', 'predicted', average, warn_for)
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1439: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples.
  'recall', 'true', average, warn_for)
I0715 05:53:32.175656 139833336047488 interactiveshell.py:2882] 
                                          precision    recall  f1-score   support

                             atis_flight      0.978     0.992     0.985       632
                            atis_airfare      0.960     1.000     0.980        48
                     atis_ground_service      1.000     1.000     1.000        36
                            atis_airline      1.000     1.000     1.000        38
                       atis_abbreviation      1.000     1.000     1.000        33
                           atis_aircraft      1.000     1.000     1.000         9
                        atis_flight_time      1.000     1.000     1.000         1
                           atis_quantity      0.429     1.000     0.600         3
                atis_flight#atis_airfare      0.857     0.500     0.632        12
                            atis_airport      1.000     0.944     0.971        18
                           atis_distance      1.000     1.000     1.000        10
                               atis_city      1.000     0.500     0.667         6
                        atis_ground_fare      1.000     0.857     0.923         7
                           atis_capacity      1.000     1.000     1.000        21
                          atis_flight_no      0.889     1.000     0.941         8
                               atis_meal      1.000     0.833     0.909         6
                        atis_restriction      0.000     0.000     0.000         0
             atis_airline#atis_flight_no      0.000     0.000     0.000         0
    atis_ground_service#atis_ground_fare      0.000     0.000     0.000         0
           atis_airfare#atis_flight_time      0.000     0.000     0.000         0
                           atis_cheapest      0.000     0.000     0.000         0
atis_aircraft#atis_flight#atis_flight_no      0.000     0.000     0.000         0

                               micro avg      0.975     0.981     0.978       888
                               macro avg      0.687     0.665     0.664       888
                            weighted avg      0.977     0.981     0.977       888

I0715 05:53:32.218976 139833336047488 interactiveshell.py:2882] 
                              precision    recall  f1-score   support

                           O      0.000     0.000     0.000       0.0
           B-toloc.city_name      0.974     0.993     0.983     716.0
         B-fromloc.city_name      0.985     1.000     0.992     704.0
           I-toloc.city_name      0.967     0.989     0.978     265.0
      B-depart_date.day_name      0.991     0.991     0.991     212.0
              B-airline_name      1.000     1.000     1.000     101.0
         I-fromloc.city_name      0.983     0.994     0.989     177.0
 B-depart_time.period_of_day      1.000     0.915     0.956     130.0
              I-airline_name      1.000     1.000     1.000      65.0
    B-depart_date.day_number      0.981     0.964     0.972      55.0
    B-depart_date.month_name      0.981     0.946     0.964      56.0
          B-depart_time.time      0.838     1.000     0.912      57.0
                B-round_trip      1.000     0.986     0.993      73.0
             B-cost_relative      1.000     0.973     0.986      37.0
                I-round_trip      1.000     1.000     1.000      71.0
                B-flight_mod      0.793     0.958     0.868      24.0
 B-depart_time.time_relative      0.955     0.969     0.962      65.0
          I-depart_time.time      0.945     1.000     0.972      52.0
         B-stoploc.city_name      0.909     1.000     0.952      20.0
                 B-city_name      0.886     0.544     0.674      57.0
                B-class_type      0.960     1.000     0.980      24.0
          B-arrive_time.time      0.943     0.971     0.957      34.0
 B-arrive_time.time_relative      0.906     0.935     0.921      31.0
                I-class_type      1.000     1.000     1.000      17.0
               B-flight_stop      1.000     1.000     1.000      21.0
          I-arrive_time.time      0.971     0.971     0.971      35.0
              B-airline_code      1.000     0.912     0.954      34.0
    I-depart_date.day_number      1.000     0.933     0.966      15.0
      I-fromloc.airport_name      0.469     1.000     0.638      15.0
      B-fromloc.airport_name      0.462     1.000     0.632      12.0
      B-arrive_date.day_name      0.786     1.000     0.880      11.0
          B-toloc.state_code      1.000     1.000     1.000      18.0
B-depart_date.today_relative      1.000     0.889     0.941       9.0
             B-flight_number      0.550     1.000     0.710      11.0
 B-depart_date.date_relative      0.895     1.000     0.944      17.0
          B-toloc.state_name      0.758     0.893     0.820      28.0
           B-fare_basis_code      0.944     1.000     0.971      17.0
               B-flight_time      1.000     1.000     1.000       1.0
                        B-or      1.000     1.000     1.000       3.0
 B-arrive_time.period_of_day      0.750     1.000     0.857       6.0
          B-meal_description      1.000     0.900     0.947      10.0
             I-cost_relative      1.000     0.667     0.800       3.0
              I-airport_name      0.929     0.448     0.605      29.0
               B-fare_amount      0.667     1.000     0.800       2.0
               I-fare_amount      1.000     1.000     1.000       2.0
                 I-city_name      0.941     0.533     0.681      30.0
        I-toloc.airport_name      1.000     1.000     1.000       3.0
            B-transport_type      1.000     1.000     1.000      10.0
    B-arrive_date.month_name      0.625     0.833     0.714       6.0
    B-arrive_date.day_number      0.714     0.833     0.769       6.0
         I-stoploc.city_name      0.750     0.900     0.818      10.0
                      B-meal      1.000     1.000     1.000      16.0
        B-fromloc.state_code      1.000     1.000     1.000      23.0
    B-depart_time.period_mod      1.000     1.000     1.000       5.0
                   B-connect      1.000     1.000     1.000       6.0
               B-flight_days      1.000     1.000     1.000      10.0
        B-toloc.airport_name      1.000     1.000     1.000       3.0
        B-fromloc.state_name      1.000     1.000     1.000      17.0
              B-airport_name      0.800     0.381     0.516      21.0
                   B-economy      1.000     1.000     1.000       6.0
               I-flight_time      1.000     1.000     1.000       1.0
             B-aircraft_code      0.816     0.939     0.873      33.0
                       B-mod      1.000     0.500     0.667       2.0
              B-airport_code      0.000     0.000     0.000       9.0
    B-depart_time.start_time      1.000     1.000     1.000       3.0
      B-depart_time.end_time      1.000     0.667     0.800       3.0
          B-depart_date.year      1.000     1.000     1.000       3.0
            I-transport_type      1.000     1.000     1.000       1.0
          B-restriction_code      0.800     1.000     0.889       4.0
    B-arrive_time.start_time      1.000     1.000     1.000       8.0
        B-toloc.airport_code      1.000     0.750     0.857       4.0
      B-arrive_time.end_time      1.000     0.875     0.933       8.0
      I-arrive_time.end_time      0.875     0.875     0.875       8.0
      I-depart_time.end_time      0.500     0.333     0.400       3.0
               I-flight_stop      0.000     0.000     0.000       0.0
      B-fromloc.airport_code      0.833     1.000     0.909       5.0
          I-restriction_code      1.000     1.000     1.000       3.0
    I-depart_time.start_time      1.000     1.000     1.000       1.0
          I-toloc.state_name      1.000     1.000     1.000       1.0
I-depart_date.today_relative      0.000     0.000     0.000       0.0
 B-arrive_date.date_relative      1.000     1.000     1.000       2.0
                I-flight_mod      0.000     0.000     0.000       6.0
                   I-economy      0.000     0.000     0.000       0.0
 B-return_date.date_relative      0.667     0.667     0.667       3.0
        I-fromloc.state_name      1.000     1.000     1.000       1.0
                B-state_code      1.000     1.000     1.000       1.0
    I-arrive_time.start_time      1.000     1.000     1.000       1.0
    I-arrive_date.day_number      0.000     0.000     0.000       0.0
                 B-meal_code      0.000     0.000     0.000       1.0
 I-depart_time.period_of_day      1.000     1.000     1.000       1.0
                  B-day_name      1.000     0.500     0.667       2.0
             B-period_of_day      1.000     0.750     0.857       4.0
        B-stoploc.state_code      0.000     0.000     0.000       0.0
    B-return_date.month_name      0.000     0.000     0.000       0.0
    B-return_date.day_number      0.000     0.000     0.000       0.0
    B-arrive_time.period_mod      0.000     0.000     0.000       0.0
                 I-meal_code      0.000     0.000     0.000       0.0
        B-toloc.country_name      1.000     1.000     1.000       1.0
                 B-days_code      1.000     1.000     1.000       1.0
 I-arrive_time.period_of_day      0.000     0.000     0.000       0.0
            I-today_relative      0.000     0.000     0.000       0.0
 B-return_time.period_of_day      0.000     0.000     0.000       0.0
                      B-time      0.000     0.000     0.000       0.0
           I-fare_basis_code      0.000     0.000     0.000       0.0
 I-arrive_time.time_relative      1.000     0.500     0.667       4.0
 I-depart_time.time_relative      0.000     0.000     0.000       1.0
            B-today_relative      0.000     0.000     0.000       0.0
                B-state_name      0.000     0.000     0.000       9.0
B-arrive_date.today_relative      0.000     0.000     0.000       0.0
    B-return_time.period_mod      0.000     0.000     0.000       0.0
                B-month_name      0.000     0.000     0.000       0.0
                B-day_number      0.000     0.000     0.000       0.0
 I-return_date.date_relative      0.000     0.000     0.000       3.0
I-return_date.today_relative      0.000     0.000     0.000       0.0
      B-stoploc.airport_name      0.000     0.000     0.000       0.0
             B-time_relative      0.000     0.000     0.000       0.0
                      I-time      0.000     0.000     0.000       0.0
    I-return_date.day_number      0.000     0.000     0.000       0.0
          I-meal_description      0.000     0.000     0.000       0.0
B-return_date.today_relative      0.000     0.000     0.000       0.0
      B-return_date.day_name      0.000     0.000     0.000       2.0

                   micro avg      0.954     0.955     0.955    3657.0
                   macro avg      0.665     0.658     0.653    3657.0
                weighted avg      0.953     0.955     0.950    3657.0

I0715 05:53:32.219907 139833336047488 interactiveshell.py:2882] Best Slot F1: 0.955
Reading ../data/atis.train.w-intent.iob
I0715 05:53:33.721035 139833336047488 interactiveshell.py:2882] Step 4700 | Loss: 0.0134 | Loss_intent: 0.0068 | Loss_slots: 0.0065 | Spent: 3.4 secs | LR: 0.000183
I0715 05:53:35.601190 139833336047488 interactiveshell.py:2882] Step 4750 | Loss: 0.0026 | Loss_intent: 0.0008 | Loss_slots: 0.0018 | Spent: 1.9 secs | LR: 0.000182
I0715 05:53:37.476715 139833336047488 interactiveshell.py:2882] Step 4800 | Loss: 0.0026 | Loss_intent: 0.0001 | Loss_slots: 0.0025 | Spent: 1.9 secs | LR: 0.000181
I0715 05:53:39.349852 139833336047488 interactiveshell.py:2882] Step 4850 | Loss: 0.0152 | Loss_intent: 0.0012 | Loss_slots: 0.0140 | Spent: 1.9 secs | LR: 0.000180
I0715 05:53:41.230113 139833336047488 interactiveshell.py:2882] Step 4900 | Loss: 0.0741 | Loss_intent: 0.0700 | Loss_slots: 0.0041 | Spent: 1.9 secs | LR: 0.000179
I0715 05:53:43.122516 139833336047488 interactiveshell.py:2882] Step 4950 | Loss: 0.0093 | Loss_intent: 0.0054 | Loss_slots: 0.0039 | Spent: 1.9 secs | LR: 0.000178
Reading ../data/atis.test.w-intent.iob
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.
  'precision', 'predicted', average, warn_for)
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1439: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples.
  'recall', 'true', average, warn_for)
I0715 05:53:45.421993 139833336047488 interactiveshell.py:2882] 
                                          precision    recall  f1-score   support

                             atis_flight      0.977     0.992     0.984       632
                            atis_airfare      0.959     0.979     0.969        48
                     atis_ground_service      1.000     1.000     1.000        36
                            atis_airline      1.000     1.000     1.000        38
                       atis_abbreviation      1.000     1.000     1.000        33
                           atis_aircraft      1.000     1.000     1.000         9
                        atis_flight_time      1.000     1.000     1.000         1
                           atis_quantity      0.429     1.000     0.600         3
                atis_flight#atis_airfare      0.857     0.500     0.632        12
                            atis_airport      1.000     0.944     0.971        18
                           atis_distance      1.000     1.000     1.000        10
                               atis_city      1.000     0.500     0.667         6
                        atis_ground_fare      1.000     0.857     0.923         7
                           atis_capacity      1.000     1.000     1.000        21
                          atis_flight_no      0.889     1.000     0.941         8
                               atis_meal      1.000     0.833     0.909         6
                        atis_restriction      0.000     0.000     0.000         0
             atis_airline#atis_flight_no      0.000     0.000     0.000         0
    atis_ground_service#atis_ground_fare      0.000     0.000     0.000         0
           atis_airfare#atis_flight_time      0.000     0.000     0.000         0
                           atis_cheapest      0.000     0.000     0.000         0
atis_aircraft#atis_flight#atis_flight_no      0.000     0.000     0.000         0

                               micro avg      0.974     0.980     0.977       888
                               macro avg      0.687     0.664     0.663       888
                            weighted avg      0.976     0.980     0.976       888

I0715 05:53:45.460754 139833336047488 interactiveshell.py:2882] 
                              precision    recall  f1-score   support

                           O      0.000     0.000     0.000       0.0
           B-toloc.city_name      0.974     0.993     0.983     716.0
         B-fromloc.city_name      0.985     1.000     0.992     704.0
           I-toloc.city_name      0.967     0.985     0.976     265.0
      B-depart_date.day_name      0.991     0.991     0.991     212.0
              B-airline_name      1.000     1.000     1.000     101.0
         I-fromloc.city_name      0.972     0.994     0.983     177.0
 B-depart_time.period_of_day      0.992     0.923     0.956     130.0
              I-airline_name      1.000     1.000     1.000      65.0
    B-depart_date.day_number      0.981     0.945     0.963      55.0
    B-depart_date.month_name      0.981     0.946     0.964      56.0
          B-depart_time.time      0.851     1.000     0.919      57.0
                B-round_trip      1.000     0.986     0.993      73.0
             B-cost_relative      1.000     0.973     0.986      37.0
                I-round_trip      1.000     1.000     1.000      71.0
                B-flight_mod      0.800     1.000     0.889      24.0
 B-depart_time.time_relative      0.954     0.954     0.954      65.0
          I-depart_time.time      0.944     0.981     0.962      52.0
         B-stoploc.city_name      0.909     1.000     0.952      20.0
                 B-city_name      0.912     0.544     0.681      57.0
                B-class_type      0.960     1.000     0.980      24.0
          B-arrive_time.time      0.943     0.971     0.957      34.0
 B-arrive_time.time_relative      0.906     0.935     0.921      31.0
                I-class_type      1.000     1.000     1.000      17.0
               B-flight_stop      1.000     1.000     1.000      21.0
          I-arrive_time.time      0.971     0.971     0.971      35.0
              B-airline_code      1.000     0.912     0.954      34.0
    I-depart_date.day_number      1.000     0.933     0.966      15.0
      I-fromloc.airport_name      0.429     1.000     0.600      15.0
      B-fromloc.airport_name      0.480     1.000     0.649      12.0
      B-arrive_date.day_name      0.786     1.000     0.880      11.0
          B-toloc.state_code      1.000     1.000     1.000      18.0
B-depart_date.today_relative      1.000     0.889     0.941       9.0
             B-flight_number      0.379     1.000     0.550      11.0
 B-depart_date.date_relative      0.944     1.000     0.971      17.0
          B-toloc.state_name      0.758     0.893     0.820      28.0
           B-fare_basis_code      0.917     0.647     0.759      17.0
               B-flight_time      1.000     1.000     1.000       1.0
                        B-or      1.000     1.000     1.000       3.0
 B-arrive_time.period_of_day      0.750     1.000     0.857       6.0
          B-meal_description      1.000     0.900     0.947      10.0
             I-cost_relative      1.000     0.667     0.800       3.0
              I-airport_name      0.909     0.345     0.500      29.0
               B-fare_amount      0.667     1.000     0.800       2.0
               I-fare_amount      1.000     1.000     1.000       2.0
                 I-city_name      0.933     0.467     0.622      30.0
        I-toloc.airport_name      1.000     1.000     1.000       3.0
            B-transport_type      1.000     1.000     1.000      10.0
    B-arrive_date.month_name      0.625     0.833     0.714       6.0
    B-arrive_date.day_number      0.625     0.833     0.714       6.0
         I-stoploc.city_name      0.714     1.000     0.833      10.0
                      B-meal      1.000     1.000     1.000      16.0
        B-fromloc.state_code      1.000     1.000     1.000      23.0
    B-depart_time.period_mod      0.833     1.000     0.909       5.0
                   B-connect      1.000     1.000     1.000       6.0
               B-flight_days      1.000     1.000     1.000      10.0
        B-toloc.airport_name      1.000     1.000     1.000       3.0
        B-fromloc.state_name      1.000     1.000     1.000      17.0
              B-airport_name      0.818     0.429     0.562      21.0
                   B-economy      1.000     1.000     1.000       6.0
               I-flight_time      1.000     1.000     1.000       1.0
             B-aircraft_code      0.806     0.879     0.841      33.0
                       B-mod      1.000     0.500     0.667       2.0
              B-airport_code      0.000     0.000     0.000       9.0
    B-depart_time.start_time      1.000     0.667     0.800       3.0
      B-depart_time.end_time      1.000     0.333     0.500       3.0
          B-depart_date.year      1.000     1.000     1.000       3.0
            I-transport_type      1.000     1.000     1.000       1.0
          B-restriction_code      0.800     1.000     0.889       4.0
    B-arrive_time.start_time      0.889     1.000     0.941       8.0
        B-toloc.airport_code      1.000     0.750     0.857       4.0
      B-arrive_time.end_time      0.875     0.875     0.875       8.0
      I-arrive_time.end_time      0.875     0.875     0.875       8.0
      I-depart_time.end_time      1.000     0.333     0.500       3.0
               I-flight_stop      0.000     0.000     0.000       0.0
      B-fromloc.airport_code      0.833     1.000     0.909       5.0
          I-restriction_code      1.000     1.000     1.000       3.0
    I-depart_time.start_time      1.000     1.000     1.000       1.0
          I-toloc.state_name      1.000     1.000     1.000       1.0
I-depart_date.today_relative      0.000     0.000     0.000       0.0
 B-arrive_date.date_relative      1.000     1.000     1.000       2.0
                I-flight_mod      0.000     0.000     0.000       6.0
                   I-economy      0.000     0.000     0.000       0.0
 B-return_date.date_relative      0.667     0.667     0.667       3.0
        I-fromloc.state_name      1.000     1.000     1.000       1.0
                B-state_code      1.000     1.000     1.000       1.0
    I-arrive_time.start_time      1.000     1.000     1.000       1.0
    I-arrive_date.day_number      0.000     0.000     0.000       0.0
                 B-meal_code      0.000     0.000     0.000       1.0
 I-depart_time.period_of_day      1.000     1.000     1.000       1.0
                  B-day_name      1.000     0.500     0.667       2.0
             B-period_of_day      1.000     0.750     0.857       4.0
        B-stoploc.state_code      0.000     0.000     0.000       0.0
    B-return_date.month_name      0.000     0.000     0.000       0.0
    B-return_date.day_number      0.000     0.000     0.000       0.0
    B-arrive_time.period_mod      0.000     0.000     0.000       0.0
                 I-meal_code      0.000     0.000     0.000       0.0
        B-toloc.country_name      1.000     1.000     1.000       1.0
                 B-days_code      1.000     1.000     1.000       1.0
 I-arrive_time.period_of_day      0.000     0.000     0.000       0.0
            I-today_relative      0.000     0.000     0.000       0.0
 B-return_time.period_of_day      0.000     0.000     0.000       0.0
                      B-time      0.000     0.000     0.000       0.0
           I-fare_basis_code      0.000     0.000     0.000       0.0
 I-arrive_time.time_relative      1.000     0.500     0.667       4.0
 I-depart_time.time_relative      0.000     0.000     0.000       1.0
            B-today_relative      0.000     0.000     0.000       0.0
                B-state_name      0.000     0.000     0.000       9.0
B-arrive_date.today_relative      0.000     0.000     0.000       0.0
    B-return_time.period_mod      0.000     0.000     0.000       0.0
                B-month_name      0.000     0.000     0.000       0.0
                B-day_number      0.000     0.000     0.000       0.0
 I-return_date.date_relative      1.000     0.333     0.500       3.0
I-return_date.today_relative      0.000     0.000     0.000       0.0
      B-stoploc.airport_name      0.000     0.000     0.000       0.0
             B-time_relative      0.000     0.000     0.000       0.0
                      I-time      0.000     0.000     0.000       0.0
    I-return_date.day_number      0.000     0.000     0.000       0.0
          I-meal_description      0.000     0.000     0.000       0.0
B-return_date.today_relative      0.000     0.000     0.000       0.0
      B-return_date.day_name      0.000     0.000     0.000       2.0

                   micro avg      0.950     0.952     0.951    3657.0
                   macro avg      0.672     0.651     0.647    3657.0
                weighted avg      0.953     0.952     0.947    3657.0

I0715 05:53:45.461728 139833336047488 interactiveshell.py:2882] Best Slot F1: 0.955
Reading ../data/atis.train.w-intent.iob
I0715 05:53:46.481125 139833336047488 interactiveshell.py:2882] Step 5000 | Loss: 0.0113 | Loss_intent: 0.0075 | Loss_slots: 0.0038 | Spent: 3.4 secs | LR: 0.000177
I0715 05:53:48.406789 139833336047488 interactiveshell.py:2882] Step 5050 | Loss: 0.0074 | Loss_intent: 0.0047 | Loss_slots: 0.0027 | Spent: 1.9 secs | LR: 0.000176
I0715 05:53:50.318863 139833336047488 interactiveshell.py:2882] Step 5100 | Loss: 0.0060 | Loss_intent: 0.0016 | Loss_slots: 0.0044 | Spent: 1.9 secs | LR: 0.000175
I0715 05:53:52.237299 139833336047488 interactiveshell.py:2882] Step 5150 | Loss: 0.0080 | Loss_intent: 0.0005 | Loss_slots: 0.0075 | Spent: 1.9 secs | LR: 0.000174
I0715 05:53:54.176798 139833336047488 interactiveshell.py:2882] Step 5200 | Loss: 0.0032 | Loss_intent: 0.0005 | Loss_slots: 0.0027 | Spent: 1.9 secs | LR: 0.000173
I0715 05:53:56.123248 139833336047488 interactiveshell.py:2882] Step 5250 | Loss: 0.0093 | Loss_intent: 0.0016 | Loss_slots: 0.0077 | Spent: 1.9 secs | LR: 0.000173
I0715 05:53:58.006026 139833336047488 interactiveshell.py:2882] Step 5300 | Loss: 0.0104 | Loss_intent: 0.0036 | Loss_slots: 0.0068 | Spent: 1.9 secs | LR: 0.000172
Reading ../data/atis.test.w-intent.iob
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.
  'precision', 'predicted', average, warn_for)
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1439: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples.
  'recall', 'true', average, warn_for)
I0715 05:53:58.846474 139833336047488 interactiveshell.py:2882] 
                                          precision    recall  f1-score   support

                             atis_flight      0.975     0.992     0.984       632
                            atis_airfare      0.959     0.979     0.969        48
                     atis_ground_service      1.000     1.000     1.000        36
                            atis_airline      1.000     1.000     1.000        38
                       atis_abbreviation      1.000     1.000     1.000        33
                           atis_aircraft      1.000     1.000     1.000         9
                        atis_flight_time      1.000     1.000     1.000         1
                           atis_quantity      0.429     1.000     0.600         3
                atis_flight#atis_airfare      0.833     0.417     0.556        12
                            atis_airport      1.000     0.944     0.971        18
                           atis_distance      1.000     1.000     1.000        10
                               atis_city      1.000     0.500     0.667         6
                        atis_ground_fare      1.000     0.857     0.923         7
                           atis_capacity      1.000     1.000     1.000        21
                          atis_flight_no      0.889     1.000     0.941         8
                               atis_meal      1.000     0.833     0.909         6
                        atis_restriction      0.000     0.000     0.000         0
             atis_airline#atis_flight_no      0.000     0.000     0.000         0
    atis_ground_service#atis_ground_fare      0.000     0.000     0.000         0
           atis_airfare#atis_flight_time      0.000     0.000     0.000         0
                           atis_cheapest      0.000     0.000     0.000         0
atis_aircraft#atis_flight#atis_flight_no      0.000     0.000     0.000         0

                               micro avg      0.973     0.979     0.976       888
                               macro avg      0.686     0.660     0.660       888
                            weighted avg      0.975     0.979     0.975       888

I0715 05:53:58.885052 139833336047488 interactiveshell.py:2882] 
                              precision    recall  f1-score   support

                           O      0.000     0.000     0.000       0.0
           B-toloc.city_name      0.974     0.990     0.982     716.0
         B-fromloc.city_name      0.982     1.000     0.991     704.0
           I-toloc.city_name      0.967     0.981     0.974     265.0
      B-depart_date.day_name      0.991     0.991     0.991     212.0
              B-airline_name      1.000     1.000     1.000     101.0
         I-fromloc.city_name      0.972     0.994     0.983     177.0
 B-depart_time.period_of_day      0.992     0.923     0.956     130.0
              I-airline_name      1.000     0.985     0.992      65.0
    B-depart_date.day_number      0.981     0.964     0.972      55.0
    B-depart_date.month_name      0.982     0.964     0.973      56.0
          B-depart_time.time      0.838     1.000     0.912      57.0
                B-round_trip      1.000     0.986     0.993      73.0
             B-cost_relative      1.000     0.973     0.986      37.0
                I-round_trip      1.000     1.000     1.000      71.0
                B-flight_mod      0.793     0.958     0.868      24.0
 B-depart_time.time_relative      0.954     0.954     0.954      65.0
          I-depart_time.time      0.944     0.981     0.962      52.0
         B-stoploc.city_name      0.909     1.000     0.952      20.0
                 B-city_name      0.909     0.526     0.667      57.0
                B-class_type      0.960     1.000     0.980      24.0
          B-arrive_time.time      0.943     0.971     0.957      34.0
 B-arrive_time.time_relative      0.906     0.935     0.921      31.0
                I-class_type      1.000     1.000     1.000      17.0
               B-flight_stop      1.000     1.000     1.000      21.0
          I-arrive_time.time      1.000     0.971     0.986      35.0
              B-airline_code      1.000     0.912     0.954      34.0
    I-depart_date.day_number      1.000     0.933     0.966      15.0
      I-fromloc.airport_name      0.417     1.000     0.588      15.0
      B-fromloc.airport_name      0.444     1.000     0.615      12.0
      B-arrive_date.day_name      0.786     1.000     0.880      11.0
          B-toloc.state_code      1.000     1.000     1.000      18.0
B-depart_date.today_relative      1.000     0.889     0.941       9.0
             B-flight_number      0.367     1.000     0.537      11.0
 B-depart_date.date_relative      0.895     1.000     0.944      17.0
          B-toloc.state_name      0.781     0.893     0.833      28.0
           B-fare_basis_code      0.917     0.647     0.759      17.0
               B-flight_time      1.000     1.000     1.000       1.0
                        B-or      1.000     1.000     1.000       3.0
 B-arrive_time.period_of_day      0.750     1.000     0.857       6.0
          B-meal_description      1.000     0.900     0.947      10.0
             I-cost_relative      1.000     0.667     0.800       3.0
              I-airport_name      0.909     0.345     0.500      29.0
               B-fare_amount      0.500     1.000     0.667       2.0
               I-fare_amount      1.000     1.000     1.000       2.0
                 I-city_name      0.867     0.433     0.578      30.0
        I-toloc.airport_name      1.000     1.000     1.000       3.0
            B-transport_type      1.000     1.000     1.000      10.0
    B-arrive_date.month_name      0.714     0.833     0.769       6.0
    B-arrive_date.day_number      0.714     0.833     0.769       6.0
         I-stoploc.city_name      0.714     1.000     0.833      10.0
                      B-meal      1.000     1.000     1.000      16.0
        B-fromloc.state_code      1.000     1.000     1.000      23.0
    B-depart_time.period_mod      0.833     1.000     0.909       5.0
                   B-connect      1.000     1.000     1.000       6.0
               B-flight_days      1.000     1.000     1.000      10.0
        B-toloc.airport_name      1.000     1.000     1.000       3.0
        B-fromloc.state_name      1.000     1.000     1.000      17.0
              B-airport_name      0.800     0.381     0.516      21.0
                   B-economy      1.000     1.000     1.000       6.0
               I-flight_time      1.000     1.000     1.000       1.0
             B-aircraft_code      0.824     0.848     0.836      33.0
                       B-mod      1.000     0.500     0.667       2.0
              B-airport_code      0.000     0.000     0.000       9.0
    B-depart_time.start_time      1.000     0.667     0.800       3.0
      B-depart_time.end_time      1.000     0.333     0.500       3.0
          B-depart_date.year      1.000     1.000     1.000       3.0
            I-transport_type      0.000     0.000     0.000       1.0
          B-restriction_code      0.800     1.000     0.889       4.0
    B-arrive_time.start_time      0.889     1.000     0.941       8.0
        B-toloc.airport_code      1.000     0.750     0.857       4.0
      B-arrive_time.end_time      0.875     0.875     0.875       8.0
      I-arrive_time.end_time      0.889     1.000     0.941       8.0
      I-depart_time.end_time      1.000     0.333     0.500       3.0
               I-flight_stop      0.000     0.000     0.000       0.0
      B-fromloc.airport_code      0.833     1.000     0.909       5.0
          I-restriction_code      1.000     1.000     1.000       3.0
    I-depart_time.start_time      1.000     1.000     1.000       1.0
          I-toloc.state_name      1.000     1.000     1.000       1.0
I-depart_date.today_relative      0.000     0.000     0.000       0.0
 B-arrive_date.date_relative      1.000     1.000     1.000       2.0
                I-flight_mod      0.000     0.000     0.000       6.0
                   I-economy      0.000     0.000     0.000       0.0
 B-return_date.date_relative      1.000     0.333     0.500       3.0
        I-fromloc.state_name      1.000     1.000     1.000       1.0
                B-state_code      1.000     1.000     1.000       1.0
    I-arrive_time.start_time      1.000     1.000     1.000       1.0
    I-arrive_date.day_number      0.000     0.000     0.000       0.0
                 B-meal_code      0.000     0.000     0.000       1.0
 I-depart_time.period_of_day      1.000     1.000     1.000       1.0
                  B-day_name      1.000     0.500     0.667       2.0
             B-period_of_day      1.000     0.750     0.857       4.0
        B-stoploc.state_code      0.000     0.000     0.000       0.0
    B-return_date.month_name      0.000     0.000     0.000       0.0
    B-return_date.day_number      0.000     0.000     0.000       0.0
    B-arrive_time.period_mod      0.000     0.000     0.000       0.0
                 I-meal_code      0.000     0.000     0.000       0.0
        B-toloc.country_name      1.000     1.000     1.000       1.0
                 B-days_code      1.000     1.000     1.000       1.0
 I-arrive_time.period_of_day      0.000     0.000     0.000       0.0
            I-today_relative      0.000     0.000     0.000       0.0
 B-return_time.period_of_day      0.000     0.000     0.000       0.0
                      B-time      0.000     0.000     0.000       0.0
           I-fare_basis_code      0.000     0.000     0.000       0.0
 I-arrive_time.time_relative      1.000     0.500     0.667       4.0
 I-depart_time.time_relative      0.000     0.000     0.000       1.0
            B-today_relative      0.000     0.000     0.000       0.0
                B-state_name      0.000     0.000     0.000       9.0
B-arrive_date.today_relative      0.000     0.000     0.000       0.0
    B-return_time.period_mod      0.000     0.000     0.000       0.0
                B-month_name      0.000     0.000     0.000       0.0
                B-day_number      0.000     0.000     0.000       0.0
 I-return_date.date_relative      0.000     0.000     0.000       3.0
I-return_date.today_relative      0.000     0.000     0.000       0.0
      B-stoploc.airport_name      0.000     0.000     0.000       0.0
             B-time_relative      0.000     0.000     0.000       0.0
                      I-time      0.000     0.000     0.000       0.0
    I-return_date.day_number      0.000     0.000     0.000       0.0
          I-meal_description      0.000     0.000     0.000       0.0
B-return_date.today_relative      0.000     0.000     0.000       0.0
      B-return_date.day_name      0.000     0.000     0.000       2.0

                   micro avg      0.948     0.949     0.948    3657.0
                   macro avg      0.657     0.637     0.632    3657.0
                weighted avg      0.951     0.949     0.944    3657.0

I0715 05:53:58.885811 139833336047488 interactiveshell.py:2882] Best Slot F1: 0.955
Reading ../data/atis.train.w-intent.iob
I0715 05:54:01.345040 139833336047488 interactiveshell.py:2882] Step 5350 | Loss: 0.0022 | Loss_intent: 0.0002 | Loss_slots: 0.0020 | Spent: 3.3 secs | LR: 0.000171
I0715 05:54:03.219426 139833336047488 interactiveshell.py:2882] Step 5400 | Loss: 0.0035 | Loss_intent: 0.0018 | Loss_slots: 0.0017 | Spent: 1.9 secs | LR: 0.000170
I0715 05:54:05.148188 139833336047488 interactiveshell.py:2882] Step 5450 | Loss: 0.0042 | Loss_intent: 0.0014 | Loss_slots: 0.0028 | Spent: 1.9 secs | LR: 0.000169
I0715 05:54:07.059718 139833336047488 interactiveshell.py:2882] Step 5500 | Loss: 0.0090 | Loss_intent: 0.0037 | Loss_slots: 0.0053 | Spent: 1.9 secs | LR: 0.000168
I0715 05:54:08.947798 139833336047488 interactiveshell.py:2882] Step 5550 | Loss: 0.0044 | Loss_intent: 0.0012 | Loss_slots: 0.0031 | Spent: 1.9 secs | LR: 0.000167
I0715 05:54:10.855414 139833336047488 interactiveshell.py:2882] Step 5600 | Loss: 0.0025 | Loss_intent: 0.0002 | Loss_slots: 0.0023 | Spent: 1.9 secs | LR: 0.000166
Reading ../data/atis.test.w-intent.iob
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.
  'precision', 'predicted', average, warn_for)
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1439: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples.
  'recall', 'true', average, warn_for)
I0715 05:54:12.144621 139833336047488 interactiveshell.py:2882] 
                                          precision    recall  f1-score   support

                             atis_flight      0.977     0.992     0.984       632
                            atis_airfare      0.959     0.979     0.969        48
                     atis_ground_service      1.000     1.000     1.000        36
                            atis_airline      1.000     1.000     1.000        38
                       atis_abbreviation      1.000     1.000     1.000        33
                           atis_aircraft      1.000     1.000     1.000         9
                        atis_flight_time      1.000     1.000     1.000         1
                           atis_quantity      0.429     1.000     0.600         3
                atis_flight#atis_airfare      0.857     0.500     0.632        12
                            atis_airport      1.000     0.944     0.971        18
                           atis_distance      1.000     1.000     1.000        10
                               atis_city      1.000     0.500     0.667         6
                        atis_ground_fare      1.000     0.857     0.923         7
                           atis_capacity      1.000     1.000     1.000        21
                          atis_flight_no      0.889     1.000     0.941         8
                               atis_meal      1.000     0.833     0.909         6
                        atis_restriction      0.000     0.000     0.000         0
             atis_airline#atis_flight_no      0.000     0.000     0.000         0
    atis_ground_service#atis_ground_fare      0.000     0.000     0.000         0
           atis_airfare#atis_flight_time      0.000     0.000     0.000         0
                           atis_cheapest      0.000     0.000     0.000         0
atis_aircraft#atis_flight#atis_flight_no      0.000     0.000     0.000         0

                               micro avg      0.974     0.980     0.977       888
                               macro avg      0.687     0.664     0.663       888
                            weighted avg      0.976     0.980     0.976       888

I0715 05:54:12.183968 139833336047488 interactiveshell.py:2882] 
                              precision    recall  f1-score   support

                           O      0.000     0.000     0.000       0.0
           B-toloc.city_name      0.974     0.993     0.983     716.0
         B-fromloc.city_name      0.985     0.999     0.992     704.0
           I-toloc.city_name      0.967     0.985     0.976     265.0
      B-depart_date.day_name      0.991     0.991     0.991     212.0
              B-airline_name      1.000     1.000     1.000     101.0
         I-fromloc.city_name      0.978     0.994     0.986     177.0
 B-depart_time.period_of_day      1.000     0.923     0.960     130.0
              I-airline_name      1.000     0.985     0.992      65.0
    B-depart_date.day_number      0.981     0.945     0.963      55.0
    B-depart_date.month_name      0.981     0.946     0.964      56.0
          B-depart_time.time      0.864     1.000     0.927      57.0
                B-round_trip      1.000     0.986     0.993      73.0
             B-cost_relative      1.000     0.973     0.986      37.0
                I-round_trip      1.000     1.000     1.000      71.0
                B-flight_mod      0.793     0.958     0.868      24.0
 B-depart_time.time_relative      0.955     0.969     0.962      65.0
          I-depart_time.time      0.945     1.000     0.972      52.0
         B-stoploc.city_name      0.870     1.000     0.930      20.0
                 B-city_name      0.912     0.544     0.681      57.0
                B-class_type      0.960     1.000     0.980      24.0
          B-arrive_time.time      0.917     0.971     0.943      34.0
 B-arrive_time.time_relative      0.906     0.935     0.921      31.0
                I-class_type      1.000     1.000     1.000      17.0
               B-flight_stop      1.000     1.000     1.000      21.0
          I-arrive_time.time      0.971     0.971     0.971      35.0
              B-airline_code      1.000     0.912     0.954      34.0
    I-depart_date.day_number      1.000     0.933     0.966      15.0
      I-fromloc.airport_name      0.417     1.000     0.588      15.0
      B-fromloc.airport_name      0.480     1.000     0.649      12.0
      B-arrive_date.day_name      0.786     1.000     0.880      11.0
          B-toloc.state_code      1.000     1.000     1.000      18.0
B-depart_date.today_relative      1.000     0.889     0.941       9.0
             B-flight_number      0.393     1.000     0.564      11.0
 B-depart_date.date_relative      0.895     1.000     0.944      17.0
          B-toloc.state_name      0.781     0.893     0.833      28.0
           B-fare_basis_code      0.917     0.647     0.759      17.0
               B-flight_time      1.000     1.000     1.000       1.0
                        B-or      1.000     1.000     1.000       3.0
 B-arrive_time.period_of_day      0.750     1.000     0.857       6.0
          B-meal_description      1.000     0.900     0.947      10.0
             I-cost_relative      1.000     0.667     0.800       3.0
              I-airport_name      0.909     0.345     0.500      29.0
               B-fare_amount      0.667     1.000     0.800       2.0
               I-fare_amount      1.000     1.000     1.000       2.0
                 I-city_name      0.882     0.500     0.638      30.0
        I-toloc.airport_name      1.000     1.000     1.000       3.0
            B-transport_type      1.000     1.000     1.000      10.0
    B-arrive_date.month_name      0.625     0.833     0.714       6.0
    B-arrive_date.day_number      0.625     0.833     0.714       6.0
         I-stoploc.city_name      0.714     1.000     0.833      10.0
                      B-meal      1.000     1.000     1.000      16.0
        B-fromloc.state_code      1.000     1.000     1.000      23.0
    B-depart_time.period_mod      1.000     1.000     1.000       5.0
                   B-connect      1.000     1.000     1.000       6.0
               B-flight_days      1.000     1.000     1.000      10.0
        B-toloc.airport_name      1.000     1.000     1.000       3.0
        B-fromloc.state_name      1.000     1.000     1.000      17.0
              B-airport_name      0.818     0.429     0.562      21.0
                   B-economy      1.000     1.000     1.000       6.0
               I-flight_time      1.000     1.000     1.000       1.0
             B-aircraft_code      0.800     0.848     0.824      33.0
                       B-mod      1.000     0.500     0.667       2.0
              B-airport_code      0.000     0.000     0.000       9.0
    B-depart_time.start_time      1.000     0.667     0.800       3.0
      B-depart_time.end_time      1.000     0.333     0.500       3.0
          B-depart_date.year      1.000     1.000     1.000       3.0
            I-transport_type      0.000     0.000     0.000       1.0
          B-restriction_code      0.571     1.000     0.727       4.0
    B-arrive_time.start_time      0.889     1.000     0.941       8.0
        B-toloc.airport_code      1.000     0.750     0.857       4.0
      B-arrive_time.end_time      0.875     0.875     0.875       8.0
      I-arrive_time.end_time      0.875     0.875     0.875       8.0
      I-depart_time.end_time      1.000     0.333     0.500       3.0
               I-flight_stop      0.000     0.000     0.000       0.0
      B-fromloc.airport_code      0.833     1.000     0.909       5.0
          I-restriction_code      1.000     1.000     1.000       3.0
    I-depart_time.start_time      1.000     1.000     1.000       1.0
          I-toloc.state_name      1.000     1.000     1.000       1.0
I-depart_date.today_relative      0.000     0.000     0.000       0.0
 B-arrive_date.date_relative      1.000     1.000     1.000       2.0
                I-flight_mod      0.000     0.000     0.000       6.0
                   I-economy      0.000     0.000     0.000       0.0
 B-return_date.date_relative      1.000     0.667     0.800       3.0
        I-fromloc.state_name      1.000     1.000     1.000       1.0
                B-state_code      1.000     1.000     1.000       1.0
    I-arrive_time.start_time      1.000     1.000     1.000       1.0
    I-arrive_date.day_number      0.000     0.000     0.000       0.0
                 B-meal_code      0.000     0.000     0.000       1.0
 I-depart_time.period_of_day      1.000     1.000     1.000       1.0
                  B-day_name      1.000     0.500     0.667       2.0
             B-period_of_day      1.000     0.750     0.857       4.0
        B-stoploc.state_code      0.000     0.000     0.000       0.0
    B-return_date.month_name      0.000     0.000     0.000       0.0
    B-return_date.day_number      0.000     0.000     0.000       0.0
    B-arrive_time.period_mod      0.000     0.000     0.000       0.0
                 I-meal_code      0.000     0.000     0.000       0.0
        B-toloc.country_name      1.000     1.000     1.000       1.0
                 B-days_code      1.000     1.000     1.000       1.0
 I-arrive_time.period_of_day      0.000     0.000     0.000       0.0
            I-today_relative      0.000     0.000     0.000       0.0
 B-return_time.period_of_day      0.000     0.000     0.000       0.0
                      B-time      0.000     0.000     0.000       0.0
           I-fare_basis_code      0.000     0.000     0.000       0.0
 I-arrive_time.time_relative      1.000     0.500     0.667       4.0
 I-depart_time.time_relative      0.000     0.000     0.000       1.0
            B-today_relative      0.000     0.000     0.000       0.0
                B-state_name      0.000     0.000     0.000       9.0
B-arrive_date.today_relative      0.000     0.000     0.000       0.0
    B-return_time.period_mod      0.000     0.000     0.000       0.0
                B-month_name      0.000     0.000     0.000       0.0
                B-day_number      0.000     0.000     0.000       0.0
 I-return_date.date_relative      1.000     0.667     0.800       3.0
I-return_date.today_relative      0.000     0.000     0.000       0.0
      B-stoploc.airport_name      0.000     0.000     0.000       0.0
             B-time_relative      0.000     0.000     0.000       0.0
                      I-time      0.000     0.000     0.000       0.0
    I-return_date.day_number      0.000     0.000     0.000       0.0
          I-meal_description      0.000     0.000     0.000       0.0
B-return_date.today_relative      0.000     0.000     0.000       0.0
      B-return_date.day_name      0.000     0.000     0.000       2.0

                   micro avg      0.950     0.951     0.951    3657.0
                   macro avg      0.665     0.646     0.642    3657.0
                weighted avg      0.952     0.951     0.947    3657.0

I0715 05:54:12.184844 139833336047488 interactiveshell.py:2882] Best Slot F1: 0.955
Reading ../data/atis.train.w-intent.iob
I0715 05:54:14.177740 139833336047488 interactiveshell.py:2882] Step 5650 | Loss: 0.0034 | Loss_intent: 0.0012 | Loss_slots: 0.0022 | Spent: 3.3 secs | LR: 0.000165
I0715 05:54:16.070000 139833336047488 interactiveshell.py:2882] Step 5700 | Loss: 0.0086 | Loss_intent: 0.0007 | Loss_slots: 0.0079 | Spent: 1.9 secs | LR: 0.000165
I0715 05:54:17.966001 139833336047488 interactiveshell.py:2882] Step 5750 | Loss: 0.0042 | Loss_intent: 0.0004 | Loss_slots: 0.0039 | Spent: 1.9 secs | LR: 0.000164
I0715 05:54:19.870420 139833336047488 interactiveshell.py:2882] Step 5800 | Loss: 0.0049 | Loss_intent: 0.0011 | Loss_slots: 0.0038 | Spent: 1.9 secs | LR: 0.000163
I0715 05:54:21.750899 139833336047488 interactiveshell.py:2882] Step 5850 | Loss: 0.0147 | Loss_intent: 0.0133 | Loss_slots: 0.0014 | Spent: 1.9 secs | LR: 0.000162
I0715 05:54:23.668395 139833336047488 interactiveshell.py:2882] Step 5900 | Loss: 0.0062 | Loss_intent: 0.0010 | Loss_slots: 0.0053 | Spent: 1.9 secs | LR: 0.000161
Reading ../data/atis.test.w-intent.iob
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.
  'precision', 'predicted', average, warn_for)
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1439: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples.
  'recall', 'true', average, warn_for)
I0715 05:54:25.419917 139833336047488 interactiveshell.py:2882] 
                                          precision    recall  f1-score   support

                             atis_flight      0.972     0.992     0.982       632
                            atis_airfare      0.940     0.979     0.959        48
                     atis_ground_service      1.000     1.000     1.000        36
                            atis_airline      1.000     1.000     1.000        38
                       atis_abbreviation      1.000     1.000     1.000        33
                           atis_aircraft      1.000     1.000     1.000         9
                        atis_flight_time      1.000     1.000     1.000         1
                           atis_quantity      0.429     1.000     0.600         3
                atis_flight#atis_airfare      0.750     0.250     0.375        12
                            atis_airport      1.000     0.944     0.971        18
                           atis_distance      1.000     1.000     1.000        10
                               atis_city      1.000     0.500     0.667         6
                        atis_ground_fare      1.000     0.857     0.923         7
                           atis_capacity      1.000     0.952     0.976        21
                          atis_flight_no      0.889     1.000     0.941         8
                               atis_meal      1.000     0.833     0.909         6
                        atis_restriction      0.000     0.000     0.000         0
             atis_airline#atis_flight_no      0.000     0.000     0.000         0
    atis_ground_service#atis_ground_fare      0.000     0.000     0.000         0
           atis_airfare#atis_flight_time      0.000     0.000     0.000         0
                           atis_cheapest      0.000     0.000     0.000         0
atis_aircraft#atis_flight#atis_flight_no      0.000     0.000     0.000         0

                               micro avg      0.970     0.975     0.972       888
                               macro avg      0.681     0.650     0.650       888
                            weighted avg      0.971     0.975     0.970       888

I0715 05:54:25.459209 139833336047488 interactiveshell.py:2882] 
                              precision    recall  f1-score   support

                           O      0.000     0.000     0.000       0.0
           B-toloc.city_name      0.973     0.993     0.983     716.0
         B-fromloc.city_name      0.986     0.997     0.992     704.0
           I-toloc.city_name      0.967     0.989     0.978     265.0
      B-depart_date.day_name      0.995     0.991     0.993     212.0
              B-airline_name      1.000     1.000     1.000     101.0
         I-fromloc.city_name      0.983     0.994     0.989     177.0
 B-depart_time.period_of_day      0.992     0.915     0.952     130.0
              I-airline_name      1.000     1.000     1.000      65.0
    B-depart_date.day_number      0.981     0.964     0.972      55.0
    B-depart_date.month_name      0.982     0.964     0.973      56.0
          B-depart_time.time      0.838     1.000     0.912      57.0
                B-round_trip      1.000     0.986     0.993      73.0
             B-cost_relative      1.000     0.973     0.986      37.0
                I-round_trip      1.000     1.000     1.000      71.0
                B-flight_mod      0.800     1.000     0.889      24.0
 B-depart_time.time_relative      0.954     0.954     0.954      65.0
          I-depart_time.time      0.944     0.981     0.962      52.0
         B-stoploc.city_name      0.870     1.000     0.930      20.0
                 B-city_name      0.914     0.561     0.696      57.0
                B-class_type      0.960     1.000     0.980      24.0
          B-arrive_time.time      0.917     0.971     0.943      34.0
 B-arrive_time.time_relative      0.906     0.935     0.921      31.0
                I-class_type      1.000     1.000     1.000      17.0
               B-flight_stop      1.000     1.000     1.000      21.0
          I-arrive_time.time      0.971     0.971     0.971      35.0
              B-airline_code      1.000     0.912     0.954      34.0
    I-depart_date.day_number      1.000     0.933     0.966      15.0
      I-fromloc.airport_name      0.429     1.000     0.600      15.0
      B-fromloc.airport_name      0.480     1.000     0.649      12.0
      B-arrive_date.day_name      0.733     1.000     0.846      11.0
          B-toloc.state_code      0.947     1.000     0.973      18.0
B-depart_date.today_relative      1.000     0.889     0.941       9.0
             B-flight_number      0.393     1.000     0.564      11.0
 B-depart_date.date_relative      0.944     1.000     0.971      17.0
          B-toloc.state_name      0.781     0.893     0.833      28.0
           B-fare_basis_code      0.917     0.647     0.759      17.0
               B-flight_time      1.000     1.000     1.000       1.0
                        B-or      1.000     1.000     1.000       3.0
 B-arrive_time.period_of_day      0.750     1.000     0.857       6.0
          B-meal_description      1.000     0.900     0.947      10.0
             I-cost_relative      1.000     0.667     0.800       3.0
              I-airport_name      0.909     0.345     0.500      29.0
               B-fare_amount      0.667     1.000     0.800       2.0
               I-fare_amount      1.000     1.000     1.000       2.0
                 I-city_name      0.889     0.533     0.667      30.0
        I-toloc.airport_name      1.000     1.000     1.000       3.0
            B-transport_type      1.000     1.000     1.000      10.0
    B-arrive_date.month_name      0.714     0.833     0.769       6.0
    B-arrive_date.day_number      0.714     0.833     0.769       6.0
         I-stoploc.city_name      0.769     1.000     0.870      10.0
                      B-meal      1.000     1.000     1.000      16.0
        B-fromloc.state_code      1.000     0.957     0.978      23.0
    B-depart_time.period_mod      0.833     1.000     0.909       5.0
                   B-connect      1.000     1.000     1.000       6.0
               B-flight_days      1.000     1.000     1.000      10.0
        B-toloc.airport_name      1.000     1.000     1.000       3.0
        B-fromloc.state_name      1.000     1.000     1.000      17.0
              B-airport_name      0.818     0.429     0.562      21.0
                   B-economy      1.000     1.000     1.000       6.0
               I-flight_time      1.000     1.000     1.000       1.0
             B-aircraft_code      0.824     0.848     0.836      33.0
                       B-mod      1.000     0.500     0.667       2.0
              B-airport_code      0.000     0.000     0.000       9.0
    B-depart_time.start_time      1.000     0.667     0.800       3.0
      B-depart_time.end_time      1.000     0.333     0.500       3.0
          B-depart_date.year      1.000     1.000     1.000       3.0
            I-transport_type      0.000     0.000     0.000       1.0
          B-restriction_code      0.571     1.000     0.727       4.0
    B-arrive_time.start_time      0.875     0.875     0.875       8.0
        B-toloc.airport_code      1.000     0.750     0.857       4.0
      B-arrive_time.end_time      0.875     0.875     0.875       8.0
      I-arrive_time.end_time      0.875     0.875     0.875       8.0
      I-depart_time.end_time      0.500     0.333     0.400       3.0
               I-flight_stop      0.000     0.000     0.000       0.0
      B-fromloc.airport_code      0.833     1.000     0.909       5.0
          I-restriction_code      1.000     1.000     1.000       3.0
    I-depart_time.start_time      1.000     1.000     1.000       1.0
          I-toloc.state_name      1.000     1.000     1.000       1.0
I-depart_date.today_relative      0.000     0.000     0.000       0.0
 B-arrive_date.date_relative      1.000     1.000     1.000       2.0
                I-flight_mod      0.000     0.000     0.000       6.0
                   I-economy      0.000     0.000     0.000       0.0
 B-return_date.date_relative      1.000     0.667     0.800       3.0
        I-fromloc.state_name      1.000     1.000     1.000       1.0
                B-state_code      1.000     1.000     1.000       1.0
    I-arrive_time.start_time      1.000     1.000     1.000       1.0
    I-arrive_date.day_number      0.000     0.000     0.000       0.0
                 B-meal_code      0.000     0.000     0.000       1.0
 I-depart_time.period_of_day      1.000     1.000     1.000       1.0
                  B-day_name      1.000     0.500     0.667       2.0
             B-period_of_day      1.000     0.750     0.857       4.0
        B-stoploc.state_code      0.000     0.000     0.000       0.0
    B-return_date.month_name      0.000     0.000     0.000       0.0
    B-return_date.day_number      0.000     0.000     0.000       0.0
    B-arrive_time.period_mod      0.000     0.000     0.000       0.0
                 I-meal_code      0.000     0.000     0.000       0.0
        B-toloc.country_name      1.000     1.000     1.000       1.0
                 B-days_code      1.000     1.000     1.000       1.0
 I-arrive_time.period_of_day      0.000     0.000     0.000       0.0
            I-today_relative      0.000     0.000     0.000       0.0
 B-return_time.period_of_day      0.000     0.000     0.000       0.0
                      B-time      0.000     0.000     0.000       0.0
           I-fare_basis_code      0.000     0.000     0.000       0.0
 I-arrive_time.time_relative      1.000     0.500     0.667       4.0
 I-depart_time.time_relative      0.000     0.000     0.000       1.0
            B-today_relative      0.000     0.000     0.000       0.0
                B-state_name      0.000     0.000     0.000       9.0
B-arrive_date.today_relative      0.000     0.000     0.000       0.0
    B-return_time.period_mod      0.000     0.000     0.000       0.0
                B-month_name      0.000     0.000     0.000       0.0
                B-day_number      0.000     0.000     0.000       0.0
 I-return_date.date_relative      1.000     0.667     0.800       3.0
I-return_date.today_relative      0.000     0.000     0.000       0.0
      B-stoploc.airport_name      0.000     0.000     0.000       0.0
             B-time_relative      0.000     0.000     0.000       0.0
                      I-time      0.000     0.000     0.000       0.0
    I-return_date.day_number      0.000     0.000     0.000       0.0
          I-meal_description      0.000     0.000     0.000       0.0
B-return_date.today_relative      0.000     0.000     0.000       0.0
      B-return_date.day_name      0.000     0.000     0.000       2.0

                   micro avg      0.950     0.952     0.951    3657.0
                   macro avg      0.661     0.645     0.641    3657.0
                weighted avg      0.952     0.952     0.947    3657.0

I0715 05:54:25.460139 139833336047488 interactiveshell.py:2882] Best Slot F1: 0.955
Reading ../data/atis.train.w-intent.iob
I0715 05:54:26.993900 139833336047488 interactiveshell.py:2882] Step 5950 | Loss: 0.0064 | Loss_intent: 0.0011 | Loss_slots: 0.0054 | Spent: 3.3 secs | LR: 0.000160
I0715 05:54:28.875462 139833336047488 interactiveshell.py:2882] Step 6000 | Loss: 0.0019 | Loss_intent: 0.0001 | Loss_slots: 0.0017 | Spent: 1.9 secs | LR: 0.000159
I0715 05:54:30.788919 139833336047488 interactiveshell.py:2882] Step 6050 | Loss: 0.0044 | Loss_intent: 0.0007 | Loss_slots: 0.0037 | Spent: 1.9 secs | LR: 0.000159
I0715 05:54:32.679121 139833336047488 interactiveshell.py:2882] Step 6100 | Loss: 0.0026 | Loss_intent: 0.0004 | Loss_slots: 0.0021 | Spent: 1.9 secs | LR: 0.000158
I0715 05:54:34.588469 139833336047488 interactiveshell.py:2882] Step 6150 | Loss: 0.0019 | Loss_intent: 0.0007 | Loss_slots: 0.0012 | Spent: 1.9 secs | LR: 0.000157
I0715 05:54:36.488010 139833336047488 interactiveshell.py:2882] Step 6200 | Loss: 0.0022 | Loss_intent: 0.0003 | Loss_slots: 0.0019 | Spent: 1.9 secs | LR: 0.000156
Reading ../data/atis.test.w-intent.iob
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.
  'precision', 'predicted', average, warn_for)
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1439: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples.
  'recall', 'true', average, warn_for)
I0715 05:54:38.722012 139833336047488 interactiveshell.py:2882] 
                                          precision    recall  f1-score   support

                             atis_flight      0.978     0.992     0.985       632
                            atis_airfare      0.959     0.979     0.969        48
                     atis_ground_service      1.000     1.000     1.000        36
                            atis_airline      1.000     1.000     1.000        38
                       atis_abbreviation      1.000     1.000     1.000        33
                           atis_aircraft      1.000     1.000     1.000         9
                        atis_flight_time      1.000     1.000     1.000         1
                           atis_quantity      0.429     1.000     0.600         3
                atis_flight#atis_airfare      0.875     0.583     0.700        12
                            atis_airport      1.000     0.944     0.971        18
                           atis_distance      1.000     1.000     1.000        10
                               atis_city      1.000     0.500     0.667         6
                        atis_ground_fare      1.000     0.857     0.923         7
                           atis_capacity      1.000     1.000     1.000        21
                          atis_flight_no      0.889     1.000     0.941         8
                               atis_meal      1.000     0.833     0.909         6
                        atis_restriction      0.000     0.000     0.000         0
             atis_airline#atis_flight_no      0.000     0.000     0.000         0
    atis_ground_service#atis_ground_fare      0.000     0.000     0.000         0
           atis_airfare#atis_flight_time      0.000     0.000     0.000         0
                           atis_cheapest      0.000     0.000     0.000         0
atis_aircraft#atis_flight#atis_flight_no      0.000     0.000     0.000         0

                               micro avg      0.975     0.981     0.978       888
                               macro avg      0.688     0.668     0.667       888
                            weighted avg      0.978     0.981     0.978       888

I0715 05:54:38.760716 139833336047488 interactiveshell.py:2882] 
                              precision    recall  f1-score   support

                           O      0.000     0.000     0.000       0.0
           B-toloc.city_name      0.977     0.993     0.985     716.0
         B-fromloc.city_name      0.985     0.999     0.992     704.0
           I-toloc.city_name      0.967     0.981     0.974     265.0
      B-depart_date.day_name      0.991     0.991     0.991     212.0
              B-airline_name      1.000     1.000     1.000     101.0
         I-fromloc.city_name      0.978     0.994     0.986     177.0
 B-depart_time.period_of_day      1.000     0.915     0.956     130.0
              I-airline_name      1.000     1.000     1.000      65.0
    B-depart_date.day_number      0.981     0.945     0.963      55.0
    B-depart_date.month_name      0.982     0.964     0.973      56.0
          B-depart_time.time      0.838     1.000     0.912      57.0
                B-round_trip      1.000     0.986     0.993      73.0
             B-cost_relative      1.000     0.973     0.986      37.0
                I-round_trip      1.000     1.000     1.000      71.0
                B-flight_mod      0.828     1.000     0.906      24.0
 B-depart_time.time_relative      0.954     0.954     0.954      65.0
          I-depart_time.time      0.945     1.000     0.972      52.0
         B-stoploc.city_name      0.870     1.000     0.930      20.0
                 B-city_name      0.889     0.561     0.688      57.0
                B-class_type      0.960     1.000     0.980      24.0
          B-arrive_time.time      0.943     0.971     0.957      34.0
 B-arrive_time.time_relative      0.906     0.935     0.921      31.0
                I-class_type      1.000     1.000     1.000      17.0
               B-flight_stop      1.000     1.000     1.000      21.0
          I-arrive_time.time      0.971     0.971     0.971      35.0
              B-airline_code      1.000     0.912     0.954      34.0
    I-depart_date.day_number      1.000     0.933     0.966      15.0
      I-fromloc.airport_name      0.429     1.000     0.600      15.0
      B-fromloc.airport_name      0.480     1.000     0.649      12.0
      B-arrive_date.day_name      0.786     1.000     0.880      11.0
          B-toloc.state_code      1.000     1.000     1.000      18.0
B-depart_date.today_relative      1.000     0.889     0.941       9.0
             B-flight_number      0.367     1.000     0.537      11.0
 B-depart_date.date_relative      0.944     1.000     0.971      17.0
          B-toloc.state_name      0.765     0.929     0.839      28.0
           B-fare_basis_code      0.917     0.647     0.759      17.0
               B-flight_time      1.000     1.000     1.000       1.0
                        B-or      1.000     1.000     1.000       3.0
 B-arrive_time.period_of_day      0.750     1.000     0.857       6.0
          B-meal_description      1.000     0.900     0.947      10.0
             I-cost_relative      1.000     0.667     0.800       3.0
              I-airport_name      0.909     0.345     0.500      29.0
               B-fare_amount      0.667     1.000     0.800       2.0
               I-fare_amount      1.000     1.000     1.000       2.0
                 I-city_name      0.938     0.500     0.652      30.0
        I-toloc.airport_name      1.000     1.000     1.000       3.0
            B-transport_type      1.000     1.000     1.000      10.0
    B-arrive_date.month_name      0.714     0.833     0.769       6.0
    B-arrive_date.day_number      0.625     0.833     0.714       6.0
         I-stoploc.city_name      0.667     1.000     0.800      10.0
                      B-meal      1.000     1.000     1.000      16.0
        B-fromloc.state_code      1.000     1.000     1.000      23.0
    B-depart_time.period_mod      0.833     1.000     0.909       5.0
                   B-connect      1.000     1.000     1.000       6.0
               B-flight_days      1.000     1.000     1.000      10.0
        B-toloc.airport_name      1.000     1.000     1.000       3.0
        B-fromloc.state_name      1.000     1.000     1.000      17.0
              B-airport_name      0.800     0.381     0.516      21.0
                   B-economy      1.000     1.000     1.000       6.0
               I-flight_time      1.000     1.000     1.000       1.0
             B-aircraft_code      0.812     0.788     0.800      33.0
                       B-mod      1.000     0.500     0.667       2.0
              B-airport_code      0.000     0.000     0.000       9.0
    B-depart_time.start_time      1.000     0.667     0.800       3.0
      B-depart_time.end_time      1.000     0.333     0.500       3.0
          B-depart_date.year      1.000     1.000     1.000       3.0
            I-transport_type      0.000     0.000     0.000       1.0
          B-restriction_code      0.571     1.000     0.727       4.0
    B-arrive_time.start_time      0.889     1.000     0.941       8.0
        B-toloc.airport_code      1.000     0.750     0.857       4.0
      B-arrive_time.end_time      0.875     0.875     0.875       8.0
      I-arrive_time.end_time      0.875     0.875     0.875       8.0
      I-depart_time.end_time      0.500     0.333     0.400       3.0
               I-flight_stop      0.000     0.000     0.000       0.0
      B-fromloc.airport_code      0.833     1.000     0.909       5.0
          I-restriction_code      1.000     1.000     1.000       3.0
    I-depart_time.start_time      1.000     1.000     1.000       1.0
          I-toloc.state_name      1.000     1.000     1.000       1.0
I-depart_date.today_relative      0.000     0.000     0.000       0.0
 B-arrive_date.date_relative      1.000     1.000     1.000       2.0
                I-flight_mod      1.000     0.167     0.286       6.0
                   I-economy      0.000     0.000     0.000       0.0
 B-return_date.date_relative      1.000     0.333     0.500       3.0
        I-fromloc.state_name      1.000     1.000     1.000       1.0
                B-state_code      1.000     1.000     1.000       1.0
    I-arrive_time.start_time      1.000     1.000     1.000       1.0
    I-arrive_date.day_number      0.000     0.000     0.000       0.0
                 B-meal_code      0.000     0.000     0.000       1.0
 I-depart_time.period_of_day      1.000     1.000     1.000       1.0
                  B-day_name      1.000     0.500     0.667       2.0
             B-period_of_day      1.000     0.750     0.857       4.0
        B-stoploc.state_code      0.000     0.000     0.000       0.0
    B-return_date.month_name      0.000     0.000     0.000       0.0
    B-return_date.day_number      0.000     0.000     0.000       0.0
    B-arrive_time.period_mod      0.000     0.000     0.000       0.0
                 I-meal_code      0.000     0.000     0.000       0.0
        B-toloc.country_name      1.000     1.000     1.000       1.0
                 B-days_code      1.000     1.000     1.000       1.0
 I-arrive_time.period_of_day      0.000     0.000     0.000       0.0
            I-today_relative      0.000     0.000     0.000       0.0
 B-return_time.period_of_day      0.000     0.000     0.000       0.0
                      B-time      0.000     0.000     0.000       0.0
           I-fare_basis_code      0.000     0.000     0.000       0.0
 I-arrive_time.time_relative      1.000     0.500     0.667       4.0
 I-depart_time.time_relative      0.000     0.000     0.000       1.0
            B-today_relative      0.000     0.000     0.000       0.0
                B-state_name      0.000     0.000     0.000       9.0
B-arrive_date.today_relative      0.000     0.000     0.000       0.0
    B-return_time.period_mod      0.000     0.000     0.000       0.0
                B-month_name      0.000     0.000     0.000       0.0
                B-day_number      0.000     0.000     0.000       0.0
 I-return_date.date_relative      0.667     0.667     0.667       3.0
I-return_date.today_relative      0.000     0.000     0.000       0.0
      B-stoploc.airport_name      0.000     0.000     0.000       0.0
             B-time_relative      0.000     0.000     0.000       0.0
                      I-time      0.000     0.000     0.000       0.0
    I-return_date.day_number      0.000     0.000     0.000       0.0
          I-meal_description      0.000     0.000     0.000       0.0
B-return_date.today_relative      0.000     0.000     0.000       0.0
      B-return_date.day_name      0.000     0.000     0.000       2.0

                   micro avg      0.949     0.951     0.950    3657.0
                   macro avg      0.666     0.644     0.639    3657.0
                weighted avg      0.954     0.951     0.947    3657.0

I0715 05:54:38.761739 139833336047488 interactiveshell.py:2882] Best Slot F1: 0.955
Reading ../data/atis.train.w-intent.iob
I0715 05:54:39.868201 139833336047488 interactiveshell.py:2882] Step 6250 | Loss: 0.0038 | Loss_intent: 0.0010 | Loss_slots: 0.0027 | Spent: 3.4 secs | LR: 0.000155
I0715 05:54:41.768856 139833336047488 interactiveshell.py:2882] Step 6300 | Loss: 0.0044 | Loss_intent: 0.0023 | Loss_slots: 0.0021 | Spent: 1.9 secs | LR: 0.000154
I0715 05:54:43.662760 139833336047488 interactiveshell.py:2882] Step 6350 | Loss: 0.0087 | Loss_intent: 0.0013 | Loss_slots: 0.0074 | Spent: 1.9 secs | LR: 0.000154
I0715 05:54:45.563884 139833336047488 interactiveshell.py:2882] Step 6400 | Loss: 0.0029 | Loss_intent: 0.0002 | Loss_slots: 0.0028 | Spent: 1.9 secs | LR: 0.000153
I0715 05:54:47.527995 139833336047488 interactiveshell.py:2882] Step 6450 | Loss: 0.0046 | Loss_intent: 0.0003 | Loss_slots: 0.0043 | Spent: 2.0 secs | LR: 0.000152
I0715 05:54:49.552820 139833336047488 interactiveshell.py:2882] Step 6500 | Loss: 0.0040 | Loss_intent: 0.0010 | Loss_slots: 0.0031 | Spent: 2.0 secs | LR: 0.000151
I0715 05:54:51.557841 139833336047488 interactiveshell.py:2882] Step 6550 | Loss: 0.0004 | Loss_intent: 0.0001 | Loss_slots: 0.0003 | Spent: 2.0 secs | LR: 0.000150
Reading ../data/atis.test.w-intent.iob
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.
  'precision', 'predicted', average, warn_for)
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1439: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples.
  'recall', 'true', average, warn_for)
I0715 05:54:52.337574 139833336047488 interactiveshell.py:2882] 
                                          precision    recall  f1-score   support

                             atis_flight      0.978     0.992     0.985       632
                            atis_airfare      0.959     0.979     0.969        48
                     atis_ground_service      1.000     1.000     1.000        36
                            atis_airline      1.000     1.000     1.000        38
                       atis_abbreviation      1.000     0.970     0.985        33
                           atis_aircraft      1.000     1.000     1.000         9
                        atis_flight_time      1.000     1.000     1.000         1
                           atis_quantity      0.429     1.000     0.600         3
                atis_flight#atis_airfare      0.875     0.583     0.700        12
                            atis_airport      1.000     0.944     0.971        18
                           atis_distance      1.000     1.000     1.000        10
                               atis_city      1.000     0.500     0.667         6
                        atis_ground_fare      1.000     0.857     0.923         7
                           atis_capacity      1.000     1.000     1.000        21
                          atis_flight_no      0.889     1.000     0.941         8
                               atis_meal      1.000     0.833     0.909         6
                        atis_restriction      0.000     0.000     0.000         0
             atis_airline#atis_flight_no      0.000     0.000     0.000         0
    atis_ground_service#atis_ground_fare      0.000     0.000     0.000         0
           atis_airfare#atis_flight_time      0.000     0.000     0.000         0
                           atis_cheapest      0.000     0.000     0.000         0
atis_aircraft#atis_flight#atis_flight_no      0.000     0.000     0.000         0

                               micro avg      0.974     0.980     0.977       888
                               macro avg      0.688     0.666     0.666       888
                            weighted avg      0.978     0.980     0.977       888

I0715 05:54:52.378910 139833336047488 interactiveshell.py:2882] 
                              precision    recall  f1-score   support

                           O      0.000     0.000     0.000       0.0
           B-toloc.city_name      0.973     0.993     0.983     716.0
         B-fromloc.city_name      0.986     0.999     0.992     704.0
           I-toloc.city_name      0.967     0.985     0.976     265.0
      B-depart_date.day_name      0.986     0.991     0.988     212.0
              B-airline_name      1.000     1.000     1.000     101.0
         I-fromloc.city_name      0.972     0.994     0.983     177.0
 B-depart_time.period_of_day      1.000     0.915     0.956     130.0
              I-airline_name      1.000     1.000     1.000      65.0
    B-depart_date.day_number      0.981     0.964     0.972      55.0
    B-depart_date.month_name      0.982     0.964     0.973      56.0
          B-depart_time.time      0.851     1.000     0.919      57.0
                B-round_trip      1.000     0.986     0.993      73.0
             B-cost_relative      1.000     0.973     0.986      37.0
                I-round_trip      1.000     1.000     1.000      71.0
                B-flight_mod      0.821     0.958     0.885      24.0
 B-depart_time.time_relative      0.955     0.969     0.962      65.0
          I-depart_time.time      0.945     1.000     0.972      52.0
         B-stoploc.city_name      0.870     1.000     0.930      20.0
                 B-city_name      0.912     0.544     0.681      57.0
                B-class_type      0.960     1.000     0.980      24.0
          B-arrive_time.time      0.971     0.971     0.971      34.0
 B-arrive_time.time_relative      0.906     0.935     0.921      31.0
                I-class_type      1.000     1.000     1.000      17.0
               B-flight_stop      1.000     1.000     1.000      21.0
          I-arrive_time.time      1.000     0.971     0.986      35.0
              B-airline_code      1.000     0.912     0.954      34.0
    I-depart_date.day_number      1.000     1.000     1.000      15.0
      I-fromloc.airport_name      0.429     1.000     0.600      15.0
      B-fromloc.airport_name      0.480     1.000     0.649      12.0
      B-arrive_date.day_name      0.769     0.909     0.833      11.0
          B-toloc.state_code      1.000     1.000     1.000      18.0
B-depart_date.today_relative      1.000     0.889     0.941       9.0
             B-flight_number      0.379     1.000     0.550      11.0
 B-depart_date.date_relative      0.895     1.000     0.944      17.0
          B-toloc.state_name      0.781     0.893     0.833      28.0
           B-fare_basis_code      0.917     0.647     0.759      17.0
               B-flight_time      1.000     1.000     1.000       1.0
                        B-or      1.000     1.000     1.000       3.0
 B-arrive_time.period_of_day      0.750     1.000     0.857       6.0
          B-meal_description      1.000     0.900     0.947      10.0
             I-cost_relative      1.000     0.667     0.800       3.0
              I-airport_name      0.917     0.379     0.537      29.0
               B-fare_amount      0.500     1.000     0.667       2.0
               I-fare_amount      1.000     1.000     1.000       2.0
                 I-city_name      0.941     0.533     0.681      30.0
        I-toloc.airport_name      1.000     1.000     1.000       3.0
            B-transport_type      1.000     1.000     1.000      10.0
    B-arrive_date.month_name      0.714     0.833     0.769       6.0
    B-arrive_date.day_number      0.714     0.833     0.769       6.0
         I-stoploc.city_name      0.818     0.900     0.857      10.0
                      B-meal      1.000     1.000     1.000      16.0
        B-fromloc.state_code      1.000     1.000     1.000      23.0
    B-depart_time.period_mod      1.000     1.000     1.000       5.0
                   B-connect      1.000     1.000     1.000       6.0
               B-flight_days      1.000     1.000     1.000      10.0
        B-toloc.airport_name      1.000     1.000     1.000       3.0
        B-fromloc.state_name      1.000     1.000     1.000      17.0
              B-airport_name      0.818     0.429     0.562      21.0
                   B-economy      1.000     1.000     1.000       6.0
               I-flight_time      1.000     1.000     1.000       1.0
             B-aircraft_code      0.794     0.818     0.806      33.0
                       B-mod      1.000     0.500     0.667       2.0
              B-airport_code      0.000     0.000     0.000       9.0
    B-depart_time.start_time      1.000     0.667     0.800       3.0
      B-depart_time.end_time      1.000     0.333     0.500       3.0
          B-depart_date.year      1.000     1.000     1.000       3.0
            I-transport_type      0.000     0.000     0.000       1.0
          B-restriction_code      0.571     1.000     0.727       4.0
    B-arrive_time.start_time      0.889     1.000     0.941       8.0
        B-toloc.airport_code      1.000     0.750     0.857       4.0
      B-arrive_time.end_time      0.889     1.000     0.941       8.0
      I-arrive_time.end_time      0.889     1.000     0.941       8.0
      I-depart_time.end_time      0.500     0.333     0.400       3.0
               I-flight_stop      0.000     0.000     0.000       0.0
      B-fromloc.airport_code      0.833     1.000     0.909       5.0
          I-restriction_code      1.000     1.000     1.000       3.0
    I-depart_time.start_time      1.000     1.000     1.000       1.0
          I-toloc.state_name      1.000     1.000     1.000       1.0
I-depart_date.today_relative      0.000     0.000     0.000       0.0
 B-arrive_date.date_relative      1.000     1.000     1.000       2.0
                I-flight_mod      1.000     0.167     0.286       6.0
                   I-economy      0.000     0.000     0.000       0.0
 B-return_date.date_relative      1.000     0.333     0.500       3.0
        I-fromloc.state_name      1.000     1.000     1.000       1.0
                B-state_code      1.000     1.000     1.000       1.0
    I-arrive_time.start_time      1.000     1.000     1.000       1.0
    I-arrive_date.day_number      0.000     0.000     0.000       0.0
                 B-meal_code      0.000     0.000     0.000       1.0
 I-depart_time.period_of_day      1.000     1.000     1.000       1.0
                  B-day_name      1.000     0.500     0.667       2.0
             B-period_of_day      1.000     0.750     0.857       4.0
        B-stoploc.state_code      0.000     0.000     0.000       0.0
    B-return_date.month_name      0.000     0.000     0.000       0.0
    B-return_date.day_number      0.000     0.000     0.000       0.0
    B-arrive_time.period_mod      0.000     0.000     0.000       0.0
                 I-meal_code      0.000     0.000     0.000       0.0
        B-toloc.country_name      1.000     1.000     1.000       1.0
                 B-days_code      1.000     1.000     1.000       1.0
 I-arrive_time.period_of_day      0.000     0.000     0.000       0.0
            I-today_relative      0.000     0.000     0.000       0.0
 B-return_time.period_of_day      0.000     0.000     0.000       0.0
                      B-time      0.000     0.000     0.000       0.0
           I-fare_basis_code      0.000     0.000     0.000       0.0
 I-arrive_time.time_relative      1.000     0.500     0.667       4.0
 I-depart_time.time_relative      0.000     0.000     0.000       1.0
            B-today_relative      0.000     0.000     0.000       0.0
                B-state_name      0.000     0.000     0.000       9.0
B-arrive_date.today_relative      0.000     0.000     0.000       0.0
    B-return_time.period_mod      0.000     0.000     0.000       0.0
                B-month_name      0.000     0.000     0.000       0.0
                B-day_number      0.000     0.000     0.000       0.0
 I-return_date.date_relative      1.000     0.667     0.800       3.0
I-return_date.today_relative      0.000     0.000     0.000       0.0
      B-stoploc.airport_name      0.000     0.000     0.000       0.0
             B-time_relative      0.000     0.000     0.000       0.0
                      I-time      0.000     0.000     0.000       0.0
    I-return_date.day_number      0.000     0.000     0.000       0.0
          I-meal_description      0.000     0.000     0.000       0.0
B-return_date.today_relative      0.000     0.000     0.000       0.0
      B-return_date.day_name      0.000     0.000     0.000       2.0

                   micro avg      0.951     0.952     0.952    3657.0
                   macro avg      0.671     0.646     0.643    3657.0
                weighted avg      0.955     0.952     0.948    3657.0

I0715 05:54:52.379759 139833336047488 interactiveshell.py:2882] Best Slot F1: 0.955
Reading ../data/atis.train.w-intent.iob
I0715 05:54:54.945136 139833336047488 interactiveshell.py:2882] Step 6600 | Loss: 0.0017 | Loss_intent: 0.0007 | Loss_slots: 0.0010 | Spent: 3.4 secs | LR: 0.000150
I0715 05:54:56.846164 139833336047488 interactiveshell.py:2882] Step 6650 | Loss: 0.0030 | Loss_intent: 0.0002 | Loss_slots: 0.0028 | Spent: 1.9 secs | LR: 0.000149
I0715 05:54:58.744682 139833336047488 interactiveshell.py:2882] Step 6700 | Loss: 0.0042 | Loss_intent: 0.0007 | Loss_slots: 0.0036 | Spent: 1.9 secs | LR: 0.000148
I0715 05:55:00.672721 139833336047488 interactiveshell.py:2882] Step 6750 | Loss: 0.0098 | Loss_intent: 0.0025 | Loss_slots: 0.0073 | Spent: 1.9 secs | LR: 0.000147
I0715 05:55:02.570484 139833336047488 interactiveshell.py:2882] Step 6800 | Loss: 0.0007 | Loss_intent: 0.0000 | Loss_slots: 0.0006 | Spent: 1.9 secs | LR: 0.000147
I0715 05:55:04.453252 139833336047488 interactiveshell.py:2882] Step 6850 | Loss: 0.0071 | Loss_intent: 0.0006 | Loss_slots: 0.0065 | Spent: 1.9 secs | LR: 0.000146
Reading ../data/atis.test.w-intent.iob
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.
  'precision', 'predicted', average, warn_for)
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1439: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples.
  'recall', 'true', average, warn_for)
I0715 05:55:05.677547 139833336047488 interactiveshell.py:2882] 
                                          precision    recall  f1-score   support

                             atis_flight      0.975     0.992     0.984       632
                            atis_airfare      0.959     0.979     0.969        48
                     atis_ground_service      1.000     1.000     1.000        36
                            atis_airline      1.000     1.000     1.000        38
                       atis_abbreviation      1.000     1.000     1.000        33
                           atis_aircraft      1.000     1.000     1.000         9
                        atis_flight_time      1.000     1.000     1.000         1
                           atis_quantity      0.429     1.000     0.600         3
                atis_flight#atis_airfare      0.833     0.417     0.556        12
                            atis_airport      1.000     0.944     0.971        18
                           atis_distance      1.000     1.000     1.000        10
                               atis_city      1.000     0.500     0.667         6
                        atis_ground_fare      1.000     0.857     0.923         7
                           atis_capacity      1.000     1.000     1.000        21
                          atis_flight_no      0.889     1.000     0.941         8
                               atis_meal      1.000     0.833     0.909         6
                        atis_restriction      0.000     0.000     0.000         0
             atis_airline#atis_flight_no      0.000     0.000     0.000         0
    atis_ground_service#atis_ground_fare      0.000     0.000     0.000         0
           atis_airfare#atis_flight_time      0.000     0.000     0.000         0
                           atis_cheapest      0.000     0.000     0.000         0
atis_aircraft#atis_flight#atis_flight_no      0.000     0.000     0.000         0

                               micro avg      0.973     0.979     0.976       888
                               macro avg      0.686     0.660     0.660       888
                            weighted avg      0.975     0.979     0.975       888

I0715 05:55:05.716372 139833336047488 interactiveshell.py:2882] 
                              precision    recall  f1-score   support

                           O      0.000     0.000     0.000       0.0
           B-toloc.city_name      0.974     0.993     0.983     716.0
         B-fromloc.city_name      0.985     0.999     0.992     704.0
           I-toloc.city_name      0.967     0.981     0.974     265.0
      B-depart_date.day_name      0.995     0.991     0.993     212.0
              B-airline_name      1.000     1.000     1.000     101.0
         I-fromloc.city_name      0.983     0.994     0.989     177.0
 B-depart_time.period_of_day      1.000     0.908     0.952     130.0
              I-airline_name      1.000     1.000     1.000      65.0
    B-depart_date.day_number      0.981     0.964     0.972      55.0
    B-depart_date.month_name      0.982     0.964     0.973      56.0
          B-depart_time.time      0.864     1.000     0.927      57.0
                B-round_trip      1.000     0.986     0.993      73.0
             B-cost_relative      1.000     0.973     0.986      37.0
                I-round_trip      1.000     1.000     1.000      71.0
                B-flight_mod      0.821     0.958     0.885      24.0
 B-depart_time.time_relative      0.954     0.954     0.954      65.0
          I-depart_time.time      0.945     1.000     0.972      52.0
         B-stoploc.city_name      0.870     1.000     0.930      20.0
                 B-city_name      0.912     0.544     0.681      57.0
                B-class_type      0.960     1.000     0.980      24.0
          B-arrive_time.time      0.971     0.971     0.971      34.0
 B-arrive_time.time_relative      0.906     0.935     0.921      31.0
                I-class_type      1.000     1.000     1.000      17.0
               B-flight_stop      1.000     1.000     1.000      21.0
          I-arrive_time.time      0.971     0.971     0.971      35.0
              B-airline_code      1.000     0.912     0.954      34.0
    I-depart_date.day_number      1.000     0.933     0.966      15.0
      I-fromloc.airport_name      0.455     1.000     0.625      15.0
      B-fromloc.airport_name      0.500     1.000     0.667      12.0
      B-arrive_date.day_name      0.733     1.000     0.846      11.0
          B-toloc.state_code      1.000     1.000     1.000      18.0
B-depart_date.today_relative      1.000     0.889     0.941       9.0
             B-flight_number      0.379     1.000     0.550      11.0
 B-depart_date.date_relative      0.895     1.000     0.944      17.0
          B-toloc.state_name      0.781     0.893     0.833      28.0
           B-fare_basis_code      0.917     0.647     0.759      17.0
               B-flight_time      1.000     1.000     1.000       1.0
                        B-or      1.000     1.000     1.000       3.0
 B-arrive_time.period_of_day      0.750     1.000     0.857       6.0
          B-meal_description      1.000     0.900     0.947      10.0
             I-cost_relative      1.000     0.667     0.800       3.0
              I-airport_name      0.923     0.414     0.571      29.0
               B-fare_amount      0.667     1.000     0.800       2.0
               I-fare_amount      1.000     1.000     1.000       2.0
                 I-city_name      0.889     0.533     0.667      30.0
        I-toloc.airport_name      1.000     1.000     1.000       3.0
            B-transport_type      1.000     1.000     1.000      10.0
    B-arrive_date.month_name      0.714     0.833     0.769       6.0
    B-arrive_date.day_number      0.714     0.833     0.769       6.0
         I-stoploc.city_name      0.667     1.000     0.800      10.0
                      B-meal      1.000     1.000     1.000      16.0
        B-fromloc.state_code      1.000     1.000     1.000      23.0
    B-depart_time.period_mod      0.714     1.000     0.833       5.0
                   B-connect      1.000     1.000     1.000       6.0
               B-flight_days      1.000     1.000     1.000      10.0
        B-toloc.airport_name      1.000     1.000     1.000       3.0
        B-fromloc.state_name      1.000     1.000     1.000      17.0
              B-airport_name      0.833     0.476     0.606      21.0
                   B-economy      1.000     1.000     1.000       6.0
               I-flight_time      1.000     1.000     1.000       1.0
             B-aircraft_code      0.771     0.818     0.794      33.0
                       B-mod      1.000     0.500     0.667       2.0
              B-airport_code      0.000     0.000     0.000       9.0
    B-depart_time.start_time      1.000     0.667     0.800       3.0
      B-depart_time.end_time      1.000     0.333     0.500       3.0
          B-depart_date.year      1.000     1.000     1.000       3.0
            I-transport_type      0.000     0.000     0.000       1.0
          B-restriction_code      0.571     1.000     0.727       4.0
    B-arrive_time.start_time      0.889     1.000     0.941       8.0
        B-toloc.airport_code      1.000     0.750     0.857       4.0
      B-arrive_time.end_time      0.889     1.000     0.941       8.0
      I-arrive_time.end_time      0.875     0.875     0.875       8.0
      I-depart_time.end_time      0.500     0.333     0.400       3.0
               I-flight_stop      0.000     0.000     0.000       0.0
      B-fromloc.airport_code      0.833     1.000     0.909       5.0
          I-restriction_code      1.000     1.000     1.000       3.0
    I-depart_time.start_time      1.000     1.000     1.000       1.0
          I-toloc.state_name      1.000     1.000     1.000       1.0
I-depart_date.today_relative      0.000     0.000     0.000       0.0
 B-arrive_date.date_relative      1.000     1.000     1.000       2.0
                I-flight_mod      1.000     0.167     0.286       6.0
                   I-economy      0.000     0.000     0.000       0.0
 B-return_date.date_relative      1.000     0.333     0.500       3.0
        I-fromloc.state_name      1.000     1.000     1.000       1.0
                B-state_code      1.000     1.000     1.000       1.0
    I-arrive_time.start_time      1.000     1.000     1.000       1.0
    I-arrive_date.day_number      0.000     0.000     0.000       0.0
                 B-meal_code      0.000     0.000     0.000       1.0
 I-depart_time.period_of_day      1.000     1.000     1.000       1.0
                  B-day_name      1.000     0.500     0.667       2.0
             B-period_of_day      1.000     0.750     0.857       4.0
        B-stoploc.state_code      0.000     0.000     0.000       0.0
    B-return_date.month_name      0.000     0.000     0.000       0.0
    B-return_date.day_number      0.000     0.000     0.000       0.0
    B-arrive_time.period_mod      0.000     0.000     0.000       0.0
                 I-meal_code      0.000     0.000     0.000       0.0
        B-toloc.country_name      1.000     1.000     1.000       1.0
                 B-days_code      1.000     1.000     1.000       1.0
 I-arrive_time.period_of_day      0.000     0.000     0.000       0.0
            I-today_relative      0.000     0.000     0.000       0.0
 B-return_time.period_of_day      0.000     0.000     0.000       0.0
                      B-time      0.000     0.000     0.000       0.0
           I-fare_basis_code      0.000     0.000     0.000       0.0
 I-arrive_time.time_relative      1.000     0.500     0.667       4.0
 I-depart_time.time_relative      0.000     0.000     0.000       1.0
            B-today_relative      0.000     0.000     0.000       0.0
                B-state_name      0.000     0.000     0.000       9.0
B-arrive_date.today_relative      0.000     0.000     0.000       0.0
    B-return_time.period_mod      0.000     0.000     0.000       0.0
                B-month_name      0.000     0.000     0.000       0.0
                B-day_number      0.000     0.000     0.000       0.0
 I-return_date.date_relative      0.667     0.667     0.667       3.0
I-return_date.today_relative      0.000     0.000     0.000       0.0
      B-stoploc.airport_name      0.000     0.000     0.000       0.0
             B-time_relative      0.000     0.000     0.000       0.0
                      I-time      0.000     0.000     0.000       0.0
    I-return_date.day_number      0.000     0.000     0.000       0.0
          I-meal_description      0.000     0.000     0.000       0.0
B-return_date.today_relative      0.000     0.000     0.000       0.0
      B-return_date.day_name      0.000     0.000     0.000       2.0

                   micro avg      0.951     0.952     0.951    3657.0
                   macro avg      0.666     0.646     0.641    3657.0
                weighted avg      0.954     0.952     0.948    3657.0

I0715 05:55:05.717260 139833336047488 interactiveshell.py:2882] Best Slot F1: 0.955
Reading ../data/atis.train.w-intent.iob
I0715 05:55:07.827859 139833336047488 interactiveshell.py:2882] Step 6900 | Loss: 0.0035 | Loss_intent: 0.0019 | Loss_slots: 0.0016 | Spent: 3.4 secs | LR: 0.000145
I0715 05:55:09.754088 139833336047488 interactiveshell.py:2882] Step 6950 | Loss: 0.0017 | Loss_intent: 0.0002 | Loss_slots: 0.0015 | Spent: 1.9 secs | LR: 0.000144
I0715 05:55:11.708011 139833336047488 interactiveshell.py:2882] Step 7000 | Loss: 0.0052 | Loss_intent: 0.0004 | Loss_slots: 0.0048 | Spent: 2.0 secs | LR: 0.000143
I0715 05:55:13.605139 139833336047488 interactiveshell.py:2882] Step 7050 | Loss: 0.0011 | Loss_intent: 0.0001 | Loss_slots: 0.0011 | Spent: 1.9 secs | LR: 0.000143
I0715 05:55:15.536360 139833336047488 interactiveshell.py:2882] Step 7100 | Loss: 0.0007 | Loss_intent: 0.0001 | Loss_slots: 0.0006 | Spent: 1.9 secs | LR: 0.000142
I0715 05:55:17.460630 139833336047488 interactiveshell.py:2882] Step 7150 | Loss: 0.0050 | Loss_intent: 0.0033 | Loss_slots: 0.0017 | Spent: 1.9 secs | LR: 0.000141
Reading ../data/atis.test.w-intent.iob
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.
  'precision', 'predicted', average, warn_for)
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1439: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples.
  'recall', 'true', average, warn_for)
I0715 05:55:19.142204 139833336047488 interactiveshell.py:2882] 
                                          precision    recall  f1-score   support

                             atis_flight      0.978     0.992     0.985       632
                            atis_airfare      0.959     0.979     0.969        48
                     atis_ground_service      1.000     1.000     1.000        36
                            atis_airline      1.000     1.000     1.000        38
                       atis_abbreviation      1.000     1.000     1.000        33
                           atis_aircraft      1.000     1.000     1.000         9
                        atis_flight_time      1.000     1.000     1.000         1
                           atis_quantity      0.429     1.000     0.600         3
                atis_flight#atis_airfare      0.875     0.583     0.700        12
                            atis_airport      1.000     0.944     0.971        18
                           atis_distance      1.000     1.000     1.000        10
                               atis_city      1.000     0.500     0.667         6
                        atis_ground_fare      1.000     0.857     0.923         7
                           atis_capacity      1.000     1.000     1.000        21
                          atis_flight_no      0.889     1.000     0.941         8
                               atis_meal      1.000     0.833     0.909         6
                        atis_restriction      0.000     0.000     0.000         0
             atis_airline#atis_flight_no      0.000     0.000     0.000         0
    atis_ground_service#atis_ground_fare      0.000     0.000     0.000         0
           atis_airfare#atis_flight_time      0.000     0.000     0.000         0
                           atis_cheapest      0.000     0.000     0.000         0
atis_aircraft#atis_flight#atis_flight_no      0.000     0.000     0.000         0

                               micro avg      0.975     0.981     0.978       888
                               macro avg      0.688     0.668     0.667       888
                            weighted avg      0.978     0.981     0.978       888

I0715 05:55:19.180337 139833336047488 interactiveshell.py:2882] 
                              precision    recall  f1-score   support

                           O      0.000     0.000     0.000       0.0
           B-toloc.city_name      0.974     0.993     0.983     716.0
         B-fromloc.city_name      0.985     0.999     0.992     704.0
           I-toloc.city_name      0.967     0.981     0.974     265.0
      B-depart_date.day_name      0.991     0.991     0.991     212.0
              B-airline_name      1.000     1.000     1.000     101.0
         I-fromloc.city_name      0.978     0.994     0.986     177.0
 B-depart_time.period_of_day      1.000     0.915     0.956     130.0
              I-airline_name      1.000     1.000     1.000      65.0
    B-depart_date.day_number      0.981     0.964     0.972      55.0
    B-depart_date.month_name      0.982     0.964     0.973      56.0
          B-depart_time.time      0.851     1.000     0.919      57.0
                B-round_trip      1.000     0.986     0.993      73.0
             B-cost_relative      1.000     0.973     0.986      37.0
                I-round_trip      1.000     1.000     1.000      71.0
                B-flight_mod      0.821     0.958     0.885      24.0
 B-depart_time.time_relative      0.955     0.969     0.962      65.0
          I-depart_time.time      0.945     1.000     0.972      52.0
         B-stoploc.city_name      0.870     1.000     0.930      20.0
                 B-city_name      0.886     0.544     0.674      57.0
                B-class_type      0.960     1.000     0.980      24.0
          B-arrive_time.time      0.943     0.971     0.957      34.0
 B-arrive_time.time_relative      0.906     0.935     0.921      31.0
                I-class_type      1.000     1.000     1.000      17.0
               B-flight_stop      1.000     1.000     1.000      21.0
          I-arrive_time.time      0.971     0.971     0.971      35.0
              B-airline_code      1.000     0.912     0.954      34.0
    I-depart_date.day_number      1.000     0.933     0.966      15.0
      I-fromloc.airport_name      0.441     1.000     0.612      15.0
      B-fromloc.airport_name      0.500     1.000     0.667      12.0
      B-arrive_date.day_name      0.786     1.000     0.880      11.0
          B-toloc.state_code      1.000     1.000     1.000      18.0
B-depart_date.today_relative      1.000     0.889     0.941       9.0
             B-flight_number      0.367     1.000     0.537      11.0
 B-depart_date.date_relative      0.895     1.000     0.944      17.0
          B-toloc.state_name      0.781     0.893     0.833      28.0
           B-fare_basis_code      0.917     0.647     0.759      17.0
               B-flight_time      1.000     1.000     1.000       1.0
                        B-or      1.000     1.000     1.000       3.0
 B-arrive_time.period_of_day      0.750     1.000     0.857       6.0
          B-meal_description      1.000     0.900     0.947      10.0
             I-cost_relative      1.000     0.667     0.800       3.0
              I-airport_name      0.923     0.414     0.571      29.0
               B-fare_amount      0.667     1.000     0.800       2.0
               I-fare_amount      1.000     1.000     1.000       2.0
                 I-city_name      0.938     0.500     0.652      30.0
        I-toloc.airport_name      1.000     1.000     1.000       3.0
            B-transport_type      1.000     1.000     1.000      10.0
    B-arrive_date.month_name      0.714     0.833     0.769       6.0
    B-arrive_date.day_number      0.714     0.833     0.769       6.0
         I-stoploc.city_name      0.667     1.000     0.800      10.0
                      B-meal      1.000     1.000     1.000      16.0
        B-fromloc.state_code      1.000     1.000     1.000      23.0
    B-depart_time.period_mod      1.000     1.000     1.000       5.0
                   B-connect      1.000     1.000     1.000       6.0
               B-flight_days      1.000     1.000     1.000      10.0
        B-toloc.airport_name      1.000     1.000     1.000       3.0
        B-fromloc.state_name      1.000     1.000     1.000      17.0
              B-airport_name      0.818     0.429     0.562      21.0
                   B-economy      1.000     1.000     1.000       6.0
               I-flight_time      1.000     1.000     1.000       1.0
             B-aircraft_code      0.800     0.848     0.824      33.0
                       B-mod      1.000     0.500     0.667       2.0
              B-airport_code      0.000     0.000     0.000       9.0
    B-depart_time.start_time      1.000     0.667     0.800       3.0
      B-depart_time.end_time      1.000     0.333     0.500       3.0
          B-depart_date.year      1.000     1.000     1.000       3.0
            I-transport_type      0.000     0.000     0.000       1.0
          B-restriction_code      0.800     1.000     0.889       4.0
    B-arrive_time.start_time      0.889     1.000     0.941       8.0
        B-toloc.airport_code      1.000     0.750     0.857       4.0
      B-arrive_time.end_time      0.875     0.875     0.875       8.0
      I-arrive_time.end_time      0.875     0.875     0.875       8.0
      I-depart_time.end_time      0.500     0.333     0.400       3.0
               I-flight_stop      0.000     0.000     0.000       0.0
      B-fromloc.airport_code      0.833     1.000     0.909       5.0
          I-restriction_code      1.000     1.000     1.000       3.0
    I-depart_time.start_time      1.000     1.000     1.000       1.0
          I-toloc.state_name      1.000     1.000     1.000       1.0
I-depart_date.today_relative      0.000     0.000     0.000       0.0
 B-arrive_date.date_relative      1.000     1.000     1.000       2.0
                I-flight_mod      1.000     0.167     0.286       6.0
                   I-economy      0.000     0.000     0.000       0.0
 B-return_date.date_relative      1.000     0.333     0.500       3.0
        I-fromloc.state_name      1.000     1.000     1.000       1.0
                B-state_code      1.000     1.000     1.000       1.0
    I-arrive_time.start_time      1.000     1.000     1.000       1.0
    I-arrive_date.day_number      0.000     0.000     0.000       0.0
                 B-meal_code      0.000     0.000     0.000       1.0
 I-depart_time.period_of_day      1.000     1.000     1.000       1.0
                  B-day_name      1.000     0.500     0.667       2.0
             B-period_of_day      1.000     0.750     0.857       4.0
        B-stoploc.state_code      0.000     0.000     0.000       0.0
    B-return_date.month_name      0.000     0.000     0.000       0.0
    B-return_date.day_number      0.000     0.000     0.000       0.0
    B-arrive_time.period_mod      0.000     0.000     0.000       0.0
                 I-meal_code      0.000     0.000     0.000       0.0
        B-toloc.country_name      1.000     1.000     1.000       1.0
                 B-days_code      1.000     1.000     1.000       1.0
 I-arrive_time.period_of_day      0.000     0.000     0.000       0.0
            I-today_relative      0.000     0.000     0.000       0.0
 B-return_time.period_of_day      0.000     0.000     0.000       0.0
                      B-time      0.000     0.000     0.000       0.0
           I-fare_basis_code      0.000     0.000     0.000       0.0
 I-arrive_time.time_relative      1.000     0.500     0.667       4.0
 I-depart_time.time_relative      0.000     0.000     0.000       1.0
            B-today_relative      0.000     0.000     0.000       0.0
                B-state_name      0.000     0.000     0.000       9.0
B-arrive_date.today_relative      0.000     0.000     0.000       0.0
    B-return_time.period_mod      0.000     0.000     0.000       0.0
                B-month_name      0.000     0.000     0.000       0.0
                B-day_number      0.000     0.000     0.000       0.0
 I-return_date.date_relative      0.667     0.667     0.667       3.0
I-return_date.today_relative      0.000     0.000     0.000       0.0
      B-stoploc.airport_name      0.000     0.000     0.000       0.0
             B-time_relative      0.000     0.000     0.000       0.0
                      I-time      0.000     0.000     0.000       0.0
    I-return_date.day_number      0.000     0.000     0.000       0.0
          I-meal_description      0.000     0.000     0.000       0.0
B-return_date.today_relative      0.000     0.000     0.000       0.0
      B-return_date.day_name      0.000     0.000     0.000       2.0

                   micro avg      0.951     0.952     0.951    3657.0
                   macro avg      0.670     0.645     0.643    3657.0
                weighted avg      0.954     0.952     0.948    3657.0

I0715 05:55:19.181339 139833336047488 interactiveshell.py:2882] Best Slot F1: 0.955
Reading ../data/atis.train.w-intent.iob
I0715 05:55:20.833000 139833336047488 interactiveshell.py:2882] Step 7200 | Loss: 0.0029 | Loss_intent: 0.0018 | Loss_slots: 0.0012 | Spent: 3.4 secs | LR: 0.000140
I0715 05:55:22.734451 139833336047488 interactiveshell.py:2882] Step 7250 | Loss: 0.0135 | Loss_intent: 0.0022 | Loss_slots: 0.0112 | Spent: 1.9 secs | LR: 0.000140
I0715 05:55:24.632305 139833336047488 interactiveshell.py:2882] Step 7300 | Loss: 0.0035 | Loss_intent: 0.0016 | Loss_slots: 0.0019 | Spent: 1.9 secs | LR: 0.000139
I0715 05:55:26.544250 139833336047488 interactiveshell.py:2882] Step 7350 | Loss: 0.0010 | Loss_intent: 0.0003 | Loss_slots: 0.0007 | Spent: 1.9 secs | LR: 0.000138
I0715 05:55:28.441694 139833336047488 interactiveshell.py:2882] Step 7400 | Loss: 0.0080 | Loss_intent: 0.0051 | Loss_slots: 0.0029 | Spent: 1.9 secs | LR: 0.000138
I0715 05:55:30.346471 139833336047488 interactiveshell.py:2882] Step 7450 | Loss: 0.0033 | Loss_intent: 0.0007 | Loss_slots: 0.0026 | Spent: 1.9 secs | LR: 0.000137
Reading ../data/atis.test.w-intent.iob
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.
  'precision', 'predicted', average, warn_for)
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1439: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples.
  'recall', 'true', average, warn_for)
I0715 05:55:32.498428 139833336047488 interactiveshell.py:2882] 
                                          precision    recall  f1-score   support

                             atis_flight      0.974     0.992     0.983       632
                            atis_airfare      0.960     1.000     0.980        48
                     atis_ground_service      1.000     1.000     1.000        36
                            atis_airline      1.000     1.000     1.000        38
                       atis_abbreviation      1.000     1.000     1.000        33
                           atis_aircraft      1.000     1.000     1.000         9
                        atis_flight_time      1.000     1.000     1.000         1
                           atis_quantity      0.429     1.000     0.600         3
                atis_flight#atis_airfare      0.667     0.167     0.267        12
                            atis_airport      1.000     0.944     0.971        18
                           atis_distance      1.000     1.000     1.000        10
                               atis_city      1.000     0.667     0.800         6
                        atis_ground_fare      1.000     0.857     0.923         7
                           atis_capacity      1.000     1.000     1.000        21
                          atis_flight_no      0.889     1.000     0.941         8
                               atis_meal      1.000     0.833     0.909         6
                        atis_restriction      0.000     0.000     0.000         0
             atis_airline#atis_flight_no      0.000     0.000     0.000         0
    atis_ground_service#atis_ground_fare      0.000     0.000     0.000         0
           atis_airfare#atis_flight_time      0.000     0.000     0.000         0
                           atis_cheapest      0.000     0.000     0.000         0
atis_aircraft#atis_flight#atis_flight_no      0.000     0.000     0.000         0

                               micro avg      0.972     0.977     0.975       888
                               macro avg      0.678     0.657     0.653       888
                            weighted avg      0.972     0.977     0.972       888

I0715 05:55:32.537544 139833336047488 interactiveshell.py:2882] 
                              precision    recall  f1-score   support

                           O      0.000     0.000     0.000       0.0
           B-toloc.city_name      0.974     0.993     0.983     716.0
         B-fromloc.city_name      0.986     0.999     0.992     704.0
           I-toloc.city_name      0.967     0.981     0.974     265.0
      B-depart_date.day_name      0.986     0.991     0.988     212.0
              B-airline_name      1.000     1.000     1.000     101.0
         I-fromloc.city_name      0.983     0.994     0.989     177.0
 B-depart_time.period_of_day      1.000     0.908     0.952     130.0
              I-airline_name      1.000     1.000     1.000      65.0
    B-depart_date.day_number      0.981     0.964     0.972      55.0
    B-depart_date.month_name      0.982     0.964     0.973      56.0
          B-depart_time.time      0.851     1.000     0.919      57.0
                B-round_trip      1.000     0.986     0.993      73.0
             B-cost_relative      1.000     0.973     0.986      37.0
                I-round_trip      1.000     1.000     1.000      71.0
                B-flight_mod      0.821     0.958     0.885      24.0
 B-depart_time.time_relative      0.955     0.969     0.962      65.0
          I-depart_time.time      0.945     1.000     0.972      52.0
         B-stoploc.city_name      0.870     1.000     0.930      20.0
                 B-city_name      0.912     0.544     0.681      57.0
                B-class_type      0.960     1.000     0.980      24.0
          B-arrive_time.time      0.971     0.971     0.971      34.0
 B-arrive_time.time_relative      0.906     0.935     0.921      31.0
                I-class_type      1.000     1.000     1.000      17.0
               B-flight_stop      1.000     1.000     1.000      21.0
          I-arrive_time.time      0.971     0.971     0.971      35.0
              B-airline_code      1.000     0.912     0.954      34.0
    I-depart_date.day_number      1.000     1.000     1.000      15.0
      I-fromloc.airport_name      0.429     1.000     0.600      15.0
      B-fromloc.airport_name      0.462     1.000     0.632      12.0
      B-arrive_date.day_name      0.846     1.000     0.917      11.0
          B-toloc.state_code      1.000     1.000     1.000      18.0
B-depart_date.today_relative      1.000     0.889     0.941       9.0
             B-flight_number      0.393     1.000     0.564      11.0
 B-depart_date.date_relative      0.895     1.000     0.944      17.0
          B-toloc.state_name      0.788     0.929     0.852      28.0
           B-fare_basis_code      0.917     0.647     0.759      17.0
               B-flight_time      1.000     1.000     1.000       1.0
                        B-or      1.000     1.000     1.000       3.0
 B-arrive_time.period_of_day      0.750     1.000     0.857       6.0
          B-meal_description      1.000     0.900     0.947      10.0
             I-cost_relative      1.000     0.667     0.800       3.0
              I-airport_name      0.917     0.379     0.537      29.0
               B-fare_amount      0.500     1.000     0.667       2.0
               I-fare_amount      1.000     1.000     1.000       2.0
                 I-city_name      0.941     0.533     0.681      30.0
        I-toloc.airport_name      1.000     1.000     1.000       3.0
            B-transport_type      1.000     1.000     1.000      10.0
    B-arrive_date.month_name      0.714     0.833     0.769       6.0
    B-arrive_date.day_number      0.714     0.833     0.769       6.0
         I-stoploc.city_name      0.714     1.000     0.833      10.0
                      B-meal      1.000     1.000     1.000      16.0
        B-fromloc.state_code      1.000     1.000     1.000      23.0
    B-depart_time.period_mod      1.000     1.000     1.000       5.0
                   B-connect      1.000     1.000     1.000       6.0
               B-flight_days      1.000     1.000     1.000      10.0
        B-toloc.airport_name      1.000     1.000     1.000       3.0
        B-fromloc.state_name      1.000     1.000     1.000      17.0
              B-airport_name      0.800     0.381     0.516      21.0
                   B-economy      1.000     1.000     1.000       6.0
               I-flight_time      1.000     1.000     1.000       1.0
             B-aircraft_code      0.794     0.818     0.806      33.0
                       B-mod      1.000     0.500     0.667       2.0
              B-airport_code      0.000     0.000     0.000       9.0
    B-depart_time.start_time      1.000     0.667     0.800       3.0
      B-depart_time.end_time      1.000     0.333     0.500       3.0
          B-depart_date.year      1.000     1.000     1.000       3.0
            I-transport_type      0.000     0.000     0.000       1.0
          B-restriction_code      0.500     1.000     0.667       4.0
    B-arrive_time.start_time      0.889     1.000     0.941       8.0
        B-toloc.airport_code      1.000     0.750     0.857       4.0
      B-arrive_time.end_time      0.889     1.000     0.941       8.0
      I-arrive_time.end_time      0.875     0.875     0.875       8.0
      I-depart_time.end_time      0.333     0.333     0.333       3.0
               I-flight_stop      0.000     0.000     0.000       0.0
      B-fromloc.airport_code      0.833     1.000     0.909       5.0
          I-restriction_code      1.000     1.000     1.000       3.0
    I-depart_time.start_time      1.000     1.000     1.000       1.0
          I-toloc.state_name      1.000     1.000     1.000       1.0
I-depart_date.today_relative      0.000     0.000     0.000       0.0
 B-arrive_date.date_relative      1.000     1.000     1.000       2.0
                I-flight_mod      1.000     0.167     0.286       6.0
                   I-economy      0.000     0.000     0.000       0.0
 B-return_date.date_relative      1.000     0.667     0.800       3.0
        I-fromloc.state_name      1.000     1.000     1.000       1.0
                B-state_code      1.000     1.000     1.000       1.0
    I-arrive_time.start_time      1.000     1.000     1.000       1.0
    I-arrive_date.day_number      0.000     0.000     0.000       0.0
                 B-meal_code      0.000     0.000     0.000       1.0
 I-depart_time.period_of_day      1.000     1.000     1.000       1.0
                  B-day_name      1.000     0.500     0.667       2.0
             B-period_of_day      1.000     0.750     0.857       4.0
        B-stoploc.state_code      0.000     0.000     0.000       0.0
    B-return_date.month_name      0.000     0.000     0.000       0.0
    B-return_date.day_number      0.000     0.000     0.000       0.0
    B-arrive_time.period_mod      0.000     0.000     0.000       0.0
                 I-meal_code      0.000     0.000     0.000       0.0
        B-toloc.country_name      1.000     1.000     1.000       1.0
                 B-days_code      1.000     1.000     1.000       1.0
 I-arrive_time.period_of_day      0.000     0.000     0.000       0.0
            I-today_relative      0.000     0.000     0.000       0.0
 B-return_time.period_of_day      0.000     0.000     0.000       0.0
                      B-time      0.000     0.000     0.000       0.0
           I-fare_basis_code      0.000     0.000     0.000       0.0
 I-arrive_time.time_relative      1.000     0.500     0.667       4.0
 I-depart_time.time_relative      0.000     0.000     0.000       1.0
            B-today_relative      0.000     0.000     0.000       0.0
                B-state_name      0.000     0.000     0.000       9.0
B-arrive_date.today_relative      0.000     0.000     0.000       0.0
    B-return_time.period_mod      0.000     0.000     0.000       0.0
                B-month_name      0.000     0.000     0.000       0.0
                B-day_number      0.000     0.000     0.000       0.0
 I-return_date.date_relative      1.000     0.667     0.800       3.0
I-return_date.today_relative      0.000     0.000     0.000       0.0
      B-stoploc.airport_name      0.000     0.000     0.000       0.0
             B-time_relative      0.000     0.000     0.000       0.0
                      I-time      0.000     0.000     0.000       0.0
    I-return_date.day_number      0.000     0.000     0.000       0.0
          I-meal_description      0.000     0.000     0.000       0.0
B-return_date.today_relative      0.000     0.000     0.000       0.0
      B-return_date.day_name      0.000     0.000     0.000       2.0

                   micro avg      0.951     0.952     0.952    3657.0
                   macro avg      0.669     0.649     0.644    3657.0
                weighted avg      0.955     0.952     0.948    3657.0

I0715 05:55:32.538353 139833336047488 interactiveshell.py:2882] Best Slot F1: 0.955
Reading ../data/atis.train.w-intent.iob
I0715 05:55:33.742354 139833336047488 interactiveshell.py:2882] Step 7500 | Loss: 0.0036 | Loss_intent: 0.0013 | Loss_slots: 0.0024 | Spent: 3.4 secs | LR: 0.000136
I0715 05:55:35.635952 139833336047488 interactiveshell.py:2882] Step 7550 | Loss: 0.0028 | Loss_intent: 0.0004 | Loss_slots: 0.0024 | Spent: 1.9 secs | LR: 0.000135
I0715 05:55:37.557732 139833336047488 interactiveshell.py:2882] Step 7600 | Loss: 0.0028 | Loss_intent: 0.0006 | Loss_slots: 0.0022 | Spent: 1.9 secs | LR: 0.000135
I0715 05:55:39.439792 139833336047488 interactiveshell.py:2882] Step 7650 | Loss: 0.0022 | Loss_intent: 0.0012 | Loss_slots: 0.0010 | Spent: 1.9 secs | LR: 0.000134
I0715 05:55:41.348175 139833336047488 interactiveshell.py:2882] Step 7700 | Loss: 0.0020 | Loss_intent: 0.0006 | Loss_slots: 0.0014 | Spent: 1.9 secs | LR: 0.000133
I0715 05:55:43.249029 139833336047488 interactiveshell.py:2882] Step 7750 | Loss: 0.0028 | Loss_intent: 0.0002 | Loss_slots: 0.0026 | Spent: 1.9 secs | LR: 0.000133
Reading ../data/atis.test.w-intent.iob
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.
  'precision', 'predicted', average, warn_for)
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1439: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples.
  'recall', 'true', average, warn_for)
I0715 05:55:45.857776 139833336047488 interactiveshell.py:2882] 
                                          precision    recall  f1-score   support

                             atis_flight      0.974     0.992     0.983       632
                            atis_airfare      0.960     1.000     0.980        48
                     atis_ground_service      1.000     1.000     1.000        36
                            atis_airline      1.000     1.000     1.000        38
                       atis_abbreviation      1.000     1.000     1.000        33
                           atis_aircraft      1.000     1.000     1.000         9
                        atis_flight_time      1.000     1.000     1.000         1
                           atis_quantity      0.429     1.000     0.600         3
                atis_flight#atis_airfare      0.667     0.167     0.267        12
                            atis_airport      1.000     0.944     0.971        18
                           atis_distance      1.000     1.000     1.000        10
                               atis_city      1.000     0.667     0.800         6
                        atis_ground_fare      1.000     0.857     0.923         7
                           atis_capacity      1.000     1.000     1.000        21
                          atis_flight_no      0.889     1.000     0.941         8
                               atis_meal      1.000     0.833     0.909         6
                        atis_restriction      0.000     0.000     0.000         0
             atis_airline#atis_flight_no      0.000     0.000     0.000         0
    atis_ground_service#atis_ground_fare      0.000     0.000     0.000         0
           atis_airfare#atis_flight_time      0.000     0.000     0.000         0
                           atis_cheapest      0.000     0.000     0.000         0
atis_aircraft#atis_flight#atis_flight_no      0.000     0.000     0.000         0

                               micro avg      0.972     0.977     0.975       888
                               macro avg      0.678     0.657     0.653       888
                            weighted avg      0.972     0.977     0.972       888

I0715 05:55:45.897861 139833336047488 interactiveshell.py:2882] 
                              precision    recall  f1-score   support

                           O      0.000     0.000     0.000       0.0
           B-toloc.city_name      0.973     0.993     0.983     716.0
         B-fromloc.city_name      0.989     0.999     0.994     704.0
           I-toloc.city_name      0.967     0.981     0.974     265.0
      B-depart_date.day_name      0.986     0.991     0.988     212.0
              B-airline_name      1.000     1.000     1.000     101.0
         I-fromloc.city_name      0.983     0.994     0.989     177.0
 B-depart_time.period_of_day      0.992     0.915     0.952     130.0
              I-airline_name      1.000     1.000     1.000      65.0
    B-depart_date.day_number      0.981     0.964     0.972      55.0
    B-depart_date.month_name      0.982     0.964     0.973      56.0
          B-depart_time.time      0.851     1.000     0.919      57.0
                B-round_trip      1.000     0.986     0.993      73.0
             B-cost_relative      1.000     0.973     0.986      37.0
                I-round_trip      1.000     1.000     1.000      71.0
                B-flight_mod      0.821     0.958     0.885      24.0
 B-depart_time.time_relative      0.954     0.954     0.954      65.0
          I-depart_time.time      0.944     0.981     0.962      52.0
         B-stoploc.city_name      0.870     1.000     0.930      20.0
                 B-city_name      0.909     0.526     0.667      57.0
                B-class_type      0.960     1.000     0.980      24.0
          B-arrive_time.time      0.971     0.971     0.971      34.0
 B-arrive_time.time_relative      0.906     0.935     0.921      31.0
                I-class_type      1.000     1.000     1.000      17.0
               B-flight_stop      1.000     1.000     1.000      21.0
          I-arrive_time.time      1.000     0.971     0.986      35.0
              B-airline_code      1.000     0.912     0.954      34.0
    I-depart_date.day_number      1.000     1.000     1.000      15.0
      I-fromloc.airport_name      0.429     1.000     0.600      15.0
      B-fromloc.airport_name      0.480     1.000     0.649      12.0
      B-arrive_date.day_name      0.846     1.000     0.917      11.0
          B-toloc.state_code      1.000     1.000     1.000      18.0
B-depart_date.today_relative      1.000     0.889     0.941       9.0
             B-flight_number      0.367     1.000     0.537      11.0
 B-depart_date.date_relative      0.895     1.000     0.944      17.0
          B-toloc.state_name      0.800     1.000     0.889      28.0
           B-fare_basis_code      0.917     0.647     0.759      17.0
               B-flight_time      1.000     1.000     1.000       1.0
                        B-or      1.000     1.000     1.000       3.0
 B-arrive_time.period_of_day      0.750     1.000     0.857       6.0
          B-meal_description      1.000     0.900     0.947      10.0
             I-cost_relative      1.000     0.667     0.800       3.0
              I-airport_name      0.909     0.345     0.500      29.0
               B-fare_amount      0.667     1.000     0.800       2.0
               I-fare_amount      1.000     1.000     1.000       2.0
                 I-city_name      0.889     0.533     0.667      30.0
        I-toloc.airport_name      1.000     1.000     1.000       3.0
            B-transport_type      1.000     1.000     1.000      10.0
    B-arrive_date.month_name      0.714     0.833     0.769       6.0
    B-arrive_date.day_number      0.714     0.833     0.769       6.0
         I-stoploc.city_name      0.667     1.000     0.800      10.0
                      B-meal      1.000     1.000     1.000      16.0
        B-fromloc.state_code      1.000     1.000     1.000      23.0
    B-depart_time.period_mod      0.833     1.000     0.909       5.0
                   B-connect      1.000     1.000     1.000       6.0
               B-flight_days      1.000     1.000     1.000      10.0
        B-toloc.airport_name      1.000     1.000     1.000       3.0
        B-fromloc.state_name      1.000     1.000     1.000      17.0
              B-airport_name      0.818     0.429     0.562      21.0
                   B-economy      1.000     1.000     1.000       6.0
               I-flight_time      1.000     1.000     1.000       1.0
             B-aircraft_code      0.788     0.788     0.788      33.0
                       B-mod      1.000     0.500     0.667       2.0
              B-airport_code      0.000     0.000     0.000       9.0
    B-depart_time.start_time      1.000     0.667     0.800       3.0
      B-depart_time.end_time      1.000     0.333     0.500       3.0
          B-depart_date.year      1.000     1.000     1.000       3.0
            I-transport_type      0.000     0.000     0.000       1.0
          B-restriction_code      0.571     1.000     0.727       4.0
    B-arrive_time.start_time      0.889     1.000     0.941       8.0
        B-toloc.airport_code      1.000     0.750     0.857       4.0
      B-arrive_time.end_time      0.889     1.000     0.941       8.0
      I-arrive_time.end_time      0.889     1.000     0.941       8.0
      I-depart_time.end_time      0.500     0.333     0.400       3.0
               I-flight_stop      0.000     0.000     0.000       0.0
      B-fromloc.airport_code      0.833     1.000     0.909       5.0
          I-restriction_code      1.000     1.000     1.000       3.0
    I-depart_time.start_time      1.000     1.000     1.000       1.0
          I-toloc.state_name      1.000     1.000     1.000       1.0
I-depart_date.today_relative      0.000     0.000     0.000       0.0
 B-arrive_date.date_relative      1.000     1.000     1.000       2.0
                I-flight_mod      1.000     0.167     0.286       6.0
                   I-economy      0.000     0.000     0.000       0.0
 B-return_date.date_relative      1.000     0.333     0.500       3.0
        I-fromloc.state_name      1.000     1.000     1.000       1.0
                B-state_code      1.000     1.000     1.000       1.0
    I-arrive_time.start_time      1.000     1.000     1.000       1.0
    I-arrive_date.day_number      0.000     0.000     0.000       0.0
                 B-meal_code      0.000     0.000     0.000       1.0
 I-depart_time.period_of_day      1.000     1.000     1.000       1.0
                  B-day_name      1.000     0.500     0.667       2.0
             B-period_of_day      1.000     0.750     0.857       4.0
        B-stoploc.state_code      0.000     0.000     0.000       0.0
    B-return_date.month_name      0.000     0.000     0.000       0.0
    B-return_date.day_number      0.000     0.000     0.000       0.0
    B-arrive_time.period_mod      0.000     0.000     0.000       0.0
                 I-meal_code      0.000     0.000     0.000       0.0
        B-toloc.country_name      1.000     1.000     1.000       1.0
                 B-days_code      1.000     1.000     1.000       1.0
 I-arrive_time.period_of_day      0.000     0.000     0.000       0.0
            I-today_relative      0.000     0.000     0.000       0.0
 B-return_time.period_of_day      0.000     0.000     0.000       0.0
                      B-time      0.000     0.000     0.000       0.0
           I-fare_basis_code      0.000     0.000     0.000       0.0
 I-arrive_time.time_relative      1.000     0.500     0.667       4.0
 I-depart_time.time_relative      0.000     0.000     0.000       1.0
            B-today_relative      0.000     0.000     0.000       0.0
                B-state_name      0.000     0.000     0.000       9.0
B-arrive_date.today_relative      0.000     0.000     0.000       0.0
    B-return_time.period_mod      0.000     0.000     0.000       0.0
                B-month_name      0.000     0.000     0.000       0.0
                B-day_number      0.000     0.000     0.000       0.0
 I-return_date.date_relative      0.667     0.667     0.667       3.0
I-return_date.today_relative      0.000     0.000     0.000       0.0
      B-stoploc.airport_name      0.000     0.000     0.000       0.0
             B-time_relative      0.000     0.000     0.000       0.0
                      I-time      0.000     0.000     0.000       0.0
    I-return_date.day_number      0.000     0.000     0.000       0.0
          I-meal_description      0.000     0.000     0.000       0.0
B-return_date.today_relative      0.000     0.000     0.000       0.0
      B-return_date.day_name      0.000     0.000     0.000       2.0

                   micro avg      0.951     0.952     0.951    3657.0
                   macro avg      0.667     0.647     0.642    3657.0
                weighted avg      0.955     0.952     0.948    3657.0

I0715 05:55:45.898730 139833336047488 interactiveshell.py:2882] Best Slot F1: 0.955
Reading ../data/atis.train.w-intent.iob
I0715 05:55:46.613506 139833336047488 interactiveshell.py:2882] Step 7800 | Loss: 0.0017 | Loss_intent: 0.0005 | Loss_slots: 0.0012 | Spent: 3.4 secs | LR: 0.000132
I0715 05:55:48.532293 139833336047488 interactiveshell.py:2882] Step 7850 | Loss: 0.0004 | Loss_intent: 0.0001 | Loss_slots: 0.0003 | Spent: 1.9 secs | LR: 0.000131
I0715 05:55:50.427450 139833336047488 interactiveshell.py:2882] Step 7900 | Loss: 0.0011 | Loss_intent: 0.0006 | Loss_slots: 0.0005 | Spent: 1.9 secs | LR: 0.000131
I0715 05:55:52.319225 139833336047488 interactiveshell.py:2882] Step 7950 | Loss: 0.0020 | Loss_intent: 0.0009 | Loss_slots: 0.0010 | Spent: 1.9 secs | LR: 0.000130
I0715 05:55:54.232952 139833336047488 interactiveshell.py:2882] Step 8000 | Loss: 0.0026 | Loss_intent: 0.0002 | Loss_slots: 0.0024 | Spent: 1.9 secs | LR: 0.000129
I0715 05:55:56.175089 139833336047488 interactiveshell.py:2882] Step 8050 | Loss: 0.0013 | Loss_intent: 0.0003 | Loss_slots: 0.0010 | Spent: 1.9 secs | LR: 0.000128
I0715 05:55:58.062583 139833336047488 interactiveshell.py:2882] Step 8100 | Loss: 0.0009 | Loss_intent: 0.0002 | Loss_slots: 0.0007 | Spent: 1.9 secs | LR: 0.000128
Reading ../data/atis.test.w-intent.iob
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.
  'precision', 'predicted', average, warn_for)
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1439: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples.
  'recall', 'true', average, warn_for)
I0715 05:55:59.209125 139833336047488 interactiveshell.py:2882] 
                                          precision    recall  f1-score   support

                             atis_flight      0.971     0.992     0.981       632
                            atis_airfare      0.959     0.979     0.969        48
                     atis_ground_service      1.000     1.000     1.000        36
                            atis_airline      1.000     1.000     1.000        38
                       atis_abbreviation      1.000     1.000     1.000        33
                           atis_aircraft      1.000     1.000     1.000         9
                        atis_flight_time      1.000     1.000     1.000         1
                           atis_quantity      0.429     1.000     0.600         3
                atis_flight#atis_airfare      0.667     0.167     0.267        12
                            atis_airport      1.000     0.944     0.971        18
                           atis_distance      1.000     1.000     1.000        10
                               atis_city      1.000     0.500     0.667         6
                        atis_ground_fare      1.000     0.857     0.923         7
                           atis_capacity      1.000     1.000     1.000        21
                          atis_flight_no      0.889     1.000     0.941         8
                               atis_meal      1.000     0.833     0.909         6
                        atis_restriction      0.000     0.000     0.000         0
             atis_airline#atis_flight_no      0.000     0.000     0.000         0
    atis_ground_service#atis_ground_fare      0.000     0.000     0.000         0
           atis_airfare#atis_flight_time      0.000     0.000     0.000         0
                           atis_cheapest      0.000     0.000     0.000         0
atis_aircraft#atis_flight#atis_flight_no      0.000     0.000     0.000         0

                               micro avg      0.970     0.975     0.972       888
                               macro avg      0.678     0.649     0.647       888
                            weighted avg      0.969     0.975     0.969       888

I0715 05:55:59.254539 139833336047488 interactiveshell.py:2882] 
                              precision    recall  f1-score   support

                           O      0.000     0.000     0.000       0.0
           B-toloc.city_name      0.971     0.993     0.982     716.0
         B-fromloc.city_name      0.987     0.999     0.993     704.0
           I-toloc.city_name      0.967     0.981     0.974     265.0
      B-depart_date.day_name      0.991     0.991     0.991     212.0
              B-airline_name      1.000     1.000     1.000     101.0
         I-fromloc.city_name      0.983     0.994     0.989     177.0
 B-depart_time.period_of_day      0.992     0.915     0.952     130.0
              I-airline_name      1.000     1.000     1.000      65.0
    B-depart_date.day_number      0.981     0.964     0.972      55.0
    B-depart_date.month_name      0.982     0.964     0.973      56.0
          B-depart_time.time      0.851     1.000     0.919      57.0
                B-round_trip      1.000     0.986     0.993      73.0
             B-cost_relative      1.000     0.973     0.986      37.0
                I-round_trip      1.000     1.000     1.000      71.0
                B-flight_mod      0.828     1.000     0.906      24.0
 B-depart_time.time_relative      0.955     0.969     0.962      65.0
          I-depart_time.time      0.944     0.981     0.962      52.0
         B-stoploc.city_name      0.870     1.000     0.930      20.0
                 B-city_name      0.886     0.544     0.674      57.0
                B-class_type      0.960     1.000     0.980      24.0
          B-arrive_time.time      0.943     0.971     0.957      34.0
 B-arrive_time.time_relative      0.906     0.935     0.921      31.0
                I-class_type      1.000     1.000     1.000      17.0
               B-flight_stop      1.000     1.000     1.000      21.0
          I-arrive_time.time      0.971     0.971     0.971      35.0
              B-airline_code      1.000     0.912     0.954      34.0
    I-depart_date.day_number      1.000     0.933     0.966      15.0
      I-fromloc.airport_name      0.441     1.000     0.612      15.0
      B-fromloc.airport_name      0.500     1.000     0.667      12.0
      B-arrive_date.day_name      0.786     1.000     0.880      11.0
          B-toloc.state_code      1.000     1.000     1.000      18.0
B-depart_date.today_relative      1.000     0.889     0.941       9.0
             B-flight_number      0.379     1.000     0.550      11.0
 B-depart_date.date_relative      0.944     1.000     0.971      17.0
          B-toloc.state_name      0.781     0.893     0.833      28.0
           B-fare_basis_code      0.917     0.647     0.759      17.0
               B-flight_time      1.000     1.000     1.000       1.0
                        B-or      1.000     1.000     1.000       3.0
 B-arrive_time.period_of_day      0.750     1.000     0.857       6.0
          B-meal_description      1.000     0.900     0.947      10.0
             I-cost_relative      1.000     0.667     0.800       3.0
              I-airport_name      0.917     0.379     0.537      29.0
               B-fare_amount      0.667     1.000     0.800       2.0
               I-fare_amount      1.000     1.000     1.000       2.0
                 I-city_name      0.889     0.533     0.667      30.0
        I-toloc.airport_name      1.000     1.000     1.000       3.0
            B-transport_type      1.000     1.000     1.000      10.0
    B-arrive_date.month_name      0.714     0.833     0.769       6.0
    B-arrive_date.day_number      0.714     0.833     0.769       6.0
         I-stoploc.city_name      0.667     1.000     0.800      10.0
                      B-meal      1.000     1.000     1.000      16.0
        B-fromloc.state_code      1.000     1.000     1.000      23.0
    B-depart_time.period_mod      1.000     1.000     1.000       5.0
                   B-connect      1.000     1.000     1.000       6.0
               B-flight_days      1.000     1.000     1.000      10.0
        B-toloc.airport_name      1.000     1.000     1.000       3.0
        B-fromloc.state_name      1.000     1.000     1.000      17.0
              B-airport_name      0.818     0.429     0.562      21.0
                   B-economy      1.000     1.000     1.000       6.0
               I-flight_time      1.000     1.000     1.000       1.0
             B-aircraft_code      0.794     0.818     0.806      33.0
                       B-mod      1.000     0.500     0.667       2.0
              B-airport_code      0.000     0.000     0.000       9.0
    B-depart_time.start_time      1.000     0.667     0.800       3.0
      B-depart_time.end_time      1.000     0.333     0.500       3.0
          B-depart_date.year      1.000     1.000     1.000       3.0
            I-transport_type      0.000     0.000     0.000       1.0
          B-restriction_code      0.571     1.000     0.727       4.0
    B-arrive_time.start_time      0.889     1.000     0.941       8.0
        B-toloc.airport_code      1.000     0.750     0.857       4.0
      B-arrive_time.end_time      0.875     0.875     0.875       8.0
      I-arrive_time.end_time      0.875     0.875     0.875       8.0
      I-depart_time.end_time      0.500     0.333     0.400       3.0
               I-flight_stop      0.000     0.000     0.000       0.0
      B-fromloc.airport_code      0.833     1.000     0.909       5.0
          I-restriction_code      1.000     1.000     1.000       3.0
    I-depart_time.start_time      1.000     1.000     1.000       1.0
          I-toloc.state_name      1.000     1.000     1.000       1.0
I-depart_date.today_relative      0.000     0.000     0.000       0.0
 B-arrive_date.date_relative      1.000     1.000     1.000       2.0
                I-flight_mod      1.000     0.167     0.286       6.0
                   I-economy      0.000     0.000     0.000       0.0
 B-return_date.date_relative      1.000     0.333     0.500       3.0
        I-fromloc.state_name      1.000     1.000     1.000       1.0
                B-state_code      1.000     1.000     1.000       1.0
    I-arrive_time.start_time      1.000     1.000     1.000       1.0
    I-arrive_date.day_number      0.000     0.000     0.000       0.0
                 B-meal_code      0.000     0.000     0.000       1.0
 I-depart_time.period_of_day      1.000     1.000     1.000       1.0
                  B-day_name      1.000     0.500     0.667       2.0
             B-period_of_day      1.000     0.750     0.857       4.0
        B-stoploc.state_code      0.000     0.000     0.000       0.0
    B-return_date.month_name      0.000     0.000     0.000       0.0
    B-return_date.day_number      0.000     0.000     0.000       0.0
    B-arrive_time.period_mod      0.000     0.000     0.000       0.0
                 I-meal_code      0.000     0.000     0.000       0.0
        B-toloc.country_name      1.000     1.000     1.000       1.0
                 B-days_code      1.000     1.000     1.000       1.0
 I-arrive_time.period_of_day      0.000     0.000     0.000       0.0
            I-today_relative      0.000     0.000     0.000       0.0
 B-return_time.period_of_day      0.000     0.000     0.000       0.0
                      B-time      0.000     0.000     0.000       0.0
           I-fare_basis_code      0.000     0.000     0.000       0.0
 I-arrive_time.time_relative      1.000     0.500     0.667       4.0
 I-depart_time.time_relative      0.000     0.000     0.000       1.0
            B-today_relative      0.000     0.000     0.000       0.0
                B-state_name      0.000     0.000     0.000       9.0
B-arrive_date.today_relative      0.000     0.000     0.000       0.0
    B-return_time.period_mod      0.000     0.000     0.000       0.0
                B-month_name      0.000     0.000     0.000       0.0
                B-day_number      0.000     0.000     0.000       0.0
 I-return_date.date_relative      0.667     0.667     0.667       3.0
I-return_date.today_relative      0.000     0.000     0.000       0.0
      B-stoploc.airport_name      0.000     0.000     0.000       0.0
             B-time_relative      0.000     0.000     0.000       0.0
                      I-time      0.000     0.000     0.000       0.0
    I-return_date.day_number      0.000     0.000     0.000       0.0
          I-meal_description      0.000     0.000     0.000       0.0
B-return_date.today_relative      0.000     0.000     0.000       0.0
      B-return_date.day_name      0.000     0.000     0.000       2.0

                   micro avg      0.950     0.952     0.951    3657.0
                   macro avg      0.668     0.645     0.642    3657.0
                weighted avg      0.954     0.952     0.948    3657.0

I0715 05:55:59.255562 139833336047488 interactiveshell.py:2882] Best Slot F1: 0.955
Reading ../data/atis.train.w-intent.iob
I0715 05:56:01.414052 139833336047488 interactiveshell.py:2882] Step 8150 | Loss: 0.0032 | Loss_intent: 0.0005 | Loss_slots: 0.0027 | Spent: 3.4 secs | LR: 0.000127
I0715 05:56:03.314813 139833336047488 interactiveshell.py:2882] Step 8200 | Loss: 0.0011 | Loss_intent: 0.0004 | Loss_slots: 0.0008 | Spent: 1.9 secs | LR: 0.000126
I0715 05:56:05.206096 139833336047488 interactiveshell.py:2882] Step 8250 | Loss: 0.0095 | Loss_intent: 0.0091 | Loss_slots: 0.0004 | Spent: 1.9 secs | LR: 0.000126
I0715 05:56:07.142755 139833336047488 interactiveshell.py:2882] Step 8300 | Loss: 0.0014 | Loss_intent: 0.0006 | Loss_slots: 0.0008 | Spent: 1.9 secs | LR: 0.000125
I0715 05:56:09.052573 139833336047488 interactiveshell.py:2882] Step 8350 | Loss: 0.0013 | Loss_intent: 0.0002 | Loss_slots: 0.0011 | Spent: 1.9 secs | LR: 0.000124
I0715 05:56:10.954452 139833336047488 interactiveshell.py:2882] Step 8400 | Loss: 0.0020 | Loss_intent: 0.0001 | Loss_slots: 0.0019 | Spent: 1.9 secs | LR: 0.000124
Reading ../data/atis.test.w-intent.iob
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.
  'precision', 'predicted', average, warn_for)
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1439: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples.
  'recall', 'true', average, warn_for)
I0715 05:56:12.543276 139833336047488 interactiveshell.py:2882] 
                                          precision    recall  f1-score   support

                             atis_flight      0.974     0.992     0.983       632
                            atis_airfare      0.959     0.979     0.969        48
                     atis_ground_service      1.000     1.000     1.000        36
                            atis_airline      1.000     1.000     1.000        38
                       atis_abbreviation      1.000     1.000     1.000        33
                           atis_aircraft      1.000     1.000     1.000         9
                        atis_flight_time      1.000     1.000     1.000         1
                           atis_quantity      0.429     1.000     0.600         3
                atis_flight#atis_airfare      0.800     0.333     0.471        12
                            atis_airport      1.000     0.944     0.971        18
                           atis_distance      1.000     1.000     1.000        10
                               atis_city      1.000     0.500     0.667         6
                        atis_ground_fare      1.000     0.857     0.923         7
                           atis_capacity      1.000     1.000     1.000        21
                          atis_flight_no      0.889     1.000     0.941         8
                               atis_meal      1.000     0.833     0.909         6
                        atis_restriction      0.000     0.000     0.000         0
             atis_airline#atis_flight_no      0.000     0.000     0.000         0
    atis_ground_service#atis_ground_fare      0.000     0.000     0.000         0
           atis_airfare#atis_flight_time      0.000     0.000     0.000         0
                           atis_cheapest      0.000     0.000     0.000         0
atis_aircraft#atis_flight#atis_flight_no      0.000     0.000     0.000         0

                               micro avg      0.972     0.977     0.975       888
                               macro avg      0.684     0.656     0.656       888
                            weighted avg      0.973     0.977     0.973       888

I0715 05:56:12.582518 139833336047488 interactiveshell.py:2882] 
                              precision    recall  f1-score   support

                           O      0.000     0.000     0.000       0.0
           B-toloc.city_name      0.975     0.993     0.984     716.0
         B-fromloc.city_name      0.985     0.999     0.992     704.0
           I-toloc.city_name      0.967     0.981     0.974     265.0
      B-depart_date.day_name      0.991     0.991     0.991     212.0
              B-airline_name      1.000     1.000     1.000     101.0
         I-fromloc.city_name      0.983     0.994     0.989     177.0
 B-depart_time.period_of_day      0.992     0.915     0.952     130.0
              I-airline_name      1.000     1.000     1.000      65.0
    B-depart_date.day_number      0.981     0.964     0.972      55.0
    B-depart_date.month_name      0.982     0.964     0.973      56.0
          B-depart_time.time      0.851     1.000     0.919      57.0
                B-round_trip      1.000     0.986     0.993      73.0
             B-cost_relative      1.000     0.973     0.986      37.0
                I-round_trip      1.000     1.000     1.000      71.0
                B-flight_mod      0.828     1.000     0.906      24.0
 B-depart_time.time_relative      0.954     0.954     0.954      65.0
          I-depart_time.time      0.944     0.981     0.962      52.0
         B-stoploc.city_name      0.870     1.000     0.930      20.0
                 B-city_name      0.886     0.544     0.674      57.0
                B-class_type      0.960     1.000     0.980      24.0
          B-arrive_time.time      0.943     0.971     0.957      34.0
 B-arrive_time.time_relative      0.906     0.935     0.921      31.0
                I-class_type      1.000     1.000     1.000      17.0
               B-flight_stop      1.000     1.000     1.000      21.0
          I-arrive_time.time      1.000     0.971     0.986      35.0
              B-airline_code      1.000     0.912     0.954      34.0
    I-depart_date.day_number      1.000     1.000     1.000      15.0
      I-fromloc.airport_name      0.429     1.000     0.600      15.0
      B-fromloc.airport_name      0.480     1.000     0.649      12.0
      B-arrive_date.day_name      0.786     1.000     0.880      11.0
          B-toloc.state_code      1.000     1.000     1.000      18.0
B-depart_date.today_relative      1.000     0.889     0.941       9.0
             B-flight_number      0.367     1.000     0.537      11.0
 B-depart_date.date_relative      0.944     1.000     0.971      17.0
          B-toloc.state_name      0.788     0.929     0.852      28.0
           B-fare_basis_code      0.917     0.647     0.759      17.0
               B-flight_time      1.000     1.000     1.000       1.0
                        B-or      1.000     1.000     1.000       3.0
 B-arrive_time.period_of_day      0.750     1.000     0.857       6.0
          B-meal_description      1.000     0.900     0.947      10.0
             I-cost_relative      1.000     0.667     0.800       3.0
              I-airport_name      0.909     0.345     0.500      29.0
               B-fare_amount      0.667     1.000     0.800       2.0
               I-fare_amount      1.000     1.000     1.000       2.0
                 I-city_name      0.882     0.500     0.638      30.0
        I-toloc.airport_name      1.000     1.000     1.000       3.0
            B-transport_type      1.000     1.000     1.000      10.0
    B-arrive_date.month_name      0.714     0.833     0.769       6.0
    B-arrive_date.day_number      0.714     0.833     0.769       6.0
         I-stoploc.city_name      0.667     1.000     0.800      10.0
                      B-meal      1.000     1.000     1.000      16.0
        B-fromloc.state_code      1.000     1.000     1.000      23.0
    B-depart_time.period_mod      1.000     1.000     1.000       5.0
                   B-connect      1.000     1.000     1.000       6.0
               B-flight_days      1.000     1.000     1.000      10.0
        B-toloc.airport_name      1.000     1.000     1.000       3.0
        B-fromloc.state_name      1.000     1.000     1.000      17.0
              B-airport_name      0.800     0.381     0.516      21.0
                   B-economy      1.000     1.000     1.000       6.0
               I-flight_time      1.000     1.000     1.000       1.0
             B-aircraft_code      0.788     0.788     0.788      33.0
                       B-mod      1.000     0.500     0.667       2.0
              B-airport_code      0.000     0.000     0.000       9.0
    B-depart_time.start_time      1.000     0.667     0.800       3.0
      B-depart_time.end_time      1.000     0.333     0.500       3.0
          B-depart_date.year      1.000     1.000     1.000       3.0
            I-transport_type      0.000     0.000     0.000       1.0
          B-restriction_code      0.571     1.000     0.727       4.0
    B-arrive_time.start_time      0.889     1.000     0.941       8.0
        B-toloc.airport_code      1.000     0.750     0.857       4.0
      B-arrive_time.end_time      0.875     0.875     0.875       8.0
      I-arrive_time.end_time      0.889     1.000     0.941       8.0
      I-depart_time.end_time      0.500     0.333     0.400       3.0
               I-flight_stop      0.000     0.000     0.000       0.0
      B-fromloc.airport_code      0.833     1.000     0.909       5.0
          I-restriction_code      1.000     1.000     1.000       3.0
    I-depart_time.start_time      1.000     1.000     1.000       1.0
          I-toloc.state_name      1.000     1.000     1.000       1.0
I-depart_date.today_relative      0.000     0.000     0.000       0.0
 B-arrive_date.date_relative      1.000     1.000     1.000       2.0
                I-flight_mod      1.000     0.167     0.286       6.0
                   I-economy      0.000     0.000     0.000       0.0
 B-return_date.date_relative      1.000     0.333     0.500       3.0
        I-fromloc.state_name      1.000     1.000     1.000       1.0
                B-state_code      1.000     1.000     1.000       1.0
    I-arrive_time.start_time      1.000     1.000     1.000       1.0
    I-arrive_date.day_number      0.000     0.000     0.000       0.0
                 B-meal_code      0.000     0.000     0.000       1.0
 I-depart_time.period_of_day      1.000     1.000     1.000       1.0
                  B-day_name      1.000     0.500     0.667       2.0
             B-period_of_day      1.000     0.750     0.857       4.0
        B-stoploc.state_code      0.000     0.000     0.000       0.0
    B-return_date.month_name      0.000     0.000     0.000       0.0
    B-return_date.day_number      0.000     0.000     0.000       0.0
    B-arrive_time.period_mod      0.000     0.000     0.000       0.0
                 I-meal_code      0.000     0.000     0.000       0.0
        B-toloc.country_name      1.000     1.000     1.000       1.0
                 B-days_code      1.000     1.000     1.000       1.0
 I-arrive_time.period_of_day      0.000     0.000     0.000       0.0
            I-today_relative      0.000     0.000     0.000       0.0
 B-return_time.period_of_day      0.000     0.000     0.000       0.0
                      B-time      0.000     0.000     0.000       0.0
           I-fare_basis_code      0.000     0.000     0.000       0.0
 I-arrive_time.time_relative      1.000     0.500     0.667       4.0
 I-depart_time.time_relative      0.000     0.000     0.000       1.0
            B-today_relative      0.000     0.000     0.000       0.0
                B-state_name      0.000     0.000     0.000       9.0
B-arrive_date.today_relative      0.000     0.000     0.000       0.0
    B-return_time.period_mod      0.000     0.000     0.000       0.0
                B-month_name      0.000     0.000     0.000       0.0
                B-day_number      0.000     0.000     0.000       0.0
 I-return_date.date_relative      0.667     0.667     0.667       3.0
I-return_date.today_relative      0.000     0.000     0.000       0.0
      B-stoploc.airport_name      0.000     0.000     0.000       0.0
             B-time_relative      0.000     0.000     0.000       0.0
                      I-time      0.000     0.000     0.000       0.0
    I-return_date.day_number      0.000     0.000     0.000       0.0
          I-meal_description      0.000     0.000     0.000       0.0
B-return_date.today_relative      0.000     0.000     0.000       0.0
      B-return_date.day_name      0.000     0.000     0.000       2.0

                   micro avg      0.950     0.951     0.951    3657.0
                   macro avg      0.668     0.646     0.641    3657.0
                weighted avg      0.954     0.951     0.947    3657.0

I0715 05:56:12.583443 139833336047488 interactiveshell.py:2882] Best Slot F1: 0.955
I0715 05:56:12.584650 139833336047488 interactiveshell.py:2882] Testing Slot Accuracy not improved over 3 epochs, Early Stop