1. finch
  2. finch
  3. tensorflow2
  4. spoken_language_understanding
  5. atis
  6. main
In [ ]:
from google.colab import drive
drive.mount('/content/gdrive')
import os
os.chdir('/content/gdrive/My Drive/finch/tensorflow2/spoken_language_understanding/atis/main')
In [2]:
%tensorflow_version 2.x
!pip install tensorflow-addons

Requirement already satisfied: tensorflow-addons in /usr/local/lib/python3.6/dist-packages (0.8.3)
Requirement already satisfied: typeguard in /usr/local/lib/python3.6/dist-packages (from tensorflow-addons) (2.7.1)
In [3]:
from tensorflow_addons.optimizers.cyclical_learning_rate import Triangular2CyclicalLearningRate
from sklearn.metrics import classification_report, f1_score, accuracy_score

import tensorflow as tf
import pprint
import logging
import time
import numpy as np

print("TensorFlow Version", tf.__version__)
print('GPU Enabled:', tf.test.is_gpu_available())

TensorFlow Version 2.3.0
WARNING:tensorflow:From <ipython-input-3-27b53cc52b5d>:11: is_gpu_available (from tensorflow.python.framework.test_util) is deprecated and will be removed in a future version.
Instructions for updating:
Use `tf.config.list_physical_devices('GPU')` instead.
GPU Enabled: True
In [4]:
def get_vocab(vocab_path):
  word2idx = {}
  with open(vocab_path) as f:
    for i, line in enumerate(f):
      line = line.rstrip()
      word2idx[line] = i
  return word2idx
In [5]:
def data_generator(f_path, params):
  print('Reading', f_path)
  with open(f_path) as f:
    for line in f:
      line = line.rstrip()
      text, slot_intent = line.split('\t')
      words = text.split()[1:-1]
      slot_intent = slot_intent.split()
      slots, intent = slot_intent[1:-1], slot_intent[-1]
      assert len(words) == len(slots)
      
      words = [params['word2idx'].get(w, len(params['word2idx'])) for w in words]
      intent = params['intent2idx'].get(intent, len(params['intent2idx']))
      slots = [params['slot2idx'].get(s, len(params['slot2idx'])) for s in slots]
      
      yield (words, (intent, slots))
In [6]:
def dataset(is_training, params):
  _shapes = ([None], ((), [None]))
  _types = (tf.int32, (tf.int32, tf.int32))
  _pads = (0, (-1, 0))
  
  if is_training:
    ds = tf.data.Dataset.from_generator(
      lambda: data_generator(params['train_path'], params),
      output_shapes = _shapes,
      output_types = _types,)
    ds = ds.shuffle(params['num_samples'])
    ds = ds.padded_batch(params['batch_size'], _shapes, _pads)
    ds = ds.prefetch(tf.data.experimental.AUTOTUNE)
  else:
    ds = tf.data.Dataset.from_generator(
      lambda: data_generator(params['test_path'], params),
      output_shapes = _shapes,
      output_types = _types,)
    ds = ds.padded_batch(params['batch_size'], _shapes, _pads)
    ds = ds.prefetch(tf.data.experimental.AUTOTUNE)
  
  return ds
In [7]:
def get_timing_signal_1d(length,
                         channels,
                         min_timescale=1.0,
                         max_timescale=1.0e4,
                         start_index=0):
  to_float = lambda x: tf.cast(x, tf.float32)
  position = to_float(tf.range(length) + start_index)
  num_timescales = channels // 2
  log_timescale_increment = (
      tf.math.log(float(max_timescale) / float(min_timescale)) /
      tf.maximum(to_float(num_timescales) - 1, 1))
  inv_timescales = min_timescale * tf.exp(
      to_float(tf.range(num_timescales)) * -log_timescale_increment)
  scaled_time = tf.expand_dims(position, 1) * tf.expand_dims(inv_timescales, 0)
  signal = tf.concat([tf.sin(scaled_time), tf.cos(scaled_time)], axis=1)
  signal = tf.pad(signal, [[0, 0], [0, tf.compat.v1.mod(channels, 2)]])
  signal = tf.reshape(signal, [1, length, channels])
  return signal
In [8]:
class LayerNorm(tf.keras.layers.Layer):
  def __init__(self, params):
    super().__init__()
    self._epsilon = params['epsilon']
    self._hidden_units = params['global_units']
  
  def build(self, input_shape):
    self.scale = self.add_weight(name='scale',
                                 shape=[self._hidden_units],
                                 initializer=tf.ones_initializer(),
                                 trainable=True)
    self.bias = self.add_weight(name='bias',
                                shape=[self._hidden_units],
                                initializer=tf.zeros_initializer(),
                                trainable=True)
    super().build(input_shape)
  
  def call(self, inputs):
    mean, variance = tf.nn.moments(inputs, [-1], keepdims=True)
    norm_x = (inputs - mean) * tf.math.rsqrt(variance + self._epsilon)
    return norm_x * self.scale + self.bias
  
  def compute_output_shape(self, input_shape):
    return input_shape


class EncoderBlock(tf.keras.Model):
  def __init__(self, SubModel, params, name):
    super().__init__(name = name)
    self.layer_norm = LayerNorm(params)
    self.sub_model = SubModel(params)
    self.dropout = tf.keras.layers.Dropout(params['dropout_rate'])
  
  def call(self, inputs, training):
    inputs, masks = inputs
    x = self.layer_norm(inputs)
    x = self.sub_model((x, masks), training=training)
    x = self.dropout(x, training=training)
    x += inputs
    return x


class MultiheadSelfAttention(tf.keras.Model):
  def __init__(self, params):
    super().__init__()
    self.qkv_linear = tf.keras.layers.Dense(3*params['hidden_units'], name='qkv_linear')
    self.dropout = tf.keras.layers.Dropout(params['dropout_rate'])
    self.out_linear = tf.keras.layers.Dense(params['global_units'], params['activation'], name='out_linear')
    self.num_heads = params['num_heads']
    self.is_bidirectional = params['is_bidirectional']
  
  def call(self, inputs, training):
    x, masks = inputs
    batch_sz = tf.shape(x)[0]
    timesteps = tf.shape(x)[1]
    
    x_shift = tf.concat((tf.zeros((batch_sz, 1, params['global_units'])), x[:, :-1, :]), axis=1)
    thres = params['global_units'] // 2
    x = tf.concat((x_shift[:, :, :thres], x[:, :, thres:]), axis=-1)

    q_k_v = self.qkv_linear(x)
    q, k, v = tf.split(q_k_v, 3, axis=-1)
    
    if self.num_heads > 1:
      q = tf.concat(tf.split(q, self.num_heads, axis=2), axis=0)                        
      k = tf.concat(tf.split(k, self.num_heads, axis=2), axis=0)                        
      v = tf.concat(tf.split(v, self.num_heads, axis=2), axis=0)
    
    align = tf.matmul(q, k, transpose_b=True)
    align *= tf.math.rsqrt(tf.cast(k.shape[-1], tf.float32))
    
    if (masks is not None) or (not self.is_bidirectional):
      paddings = tf.fill(tf.shape(align), float('-inf'))
    
    if masks is not None:
      c_masks = tf.tile(masks, [params['num_heads'], 1])
      c_masks = tf.tile(tf.expand_dims(c_masks, 1), [1, timesteps, 1])
      align = tf.where(tf.equal(c_masks, 0), paddings, align)
    
    if not self.is_bidirectional:
      lower_tri = tf.ones((timesteps, timesteps))                                       
      lower_tri = tf.linalg.LinearOperatorLowerTriangular(lower_tri).to_dense()      
      t_masks = tf.tile(tf.expand_dims(lower_tri, 0), [tf.shape(align)[0], 1, 1])     
      align = tf.where(tf.equal(t_masks, 0), paddings, align)
    
    align = tf.nn.softmax(align)
    align = self.dropout(align, training=training)
    
    if masks is not None:
      q_masks = tf.tile(masks, [params['num_heads'], 1])
      q_masks = tf.tile(tf.expand_dims(q_masks, 2), [1, 1, timesteps])
      align *= tf.cast(q_masks, tf.float32)
    
    x = tf.matmul(align, v)
    if self.num_heads > 1:
      x = tf.concat(tf.split(x, self.num_heads, axis=0), axis=2)
    x = self.out_linear(x)
    
    return x
  

class PointwiseFFN(tf.keras.Model):
  def __init__(self, params):
    super().__init__()
    self.dense_1 = tf.keras.layers.Dense(params['multiplier']*params['global_units'], params['activation'], name='fc')
    self.dropout = tf.keras.layers.Dropout(params['dropout_rate'])
    self.dense_2 = tf.keras.layers.Dense(params['global_units'], name='linear')
  
  def call(self, inputs, training):
    x, masks = inputs
    return self.dense_2(self.dropout(self.dense_1(x), training=training))
In [9]:
class Model(tf.keras.Model):
  def __init__(self, params: dict):
    super().__init__()
    self.embedding = tf.Variable(np.load(params['vocab_path']),
                                 dtype=tf.float32,
                                 name='pretrained_embedding')
    self.input_dropout = tf.keras.layers.Dropout(params['dropout_rate'])
    
    self.blocks = []
    for i in range(params['num_layers']):
      self.blocks.append(EncoderBlock(
          MultiheadSelfAttention, params, name='layer{}.1'.format(i+1)))
      self.blocks.append(EncoderBlock(
          PointwiseFFN, params, name='layer{}.2'.format(i+1)))
    
    self.intent_dropout = tf.keras.layers.Dropout(params['dropout_rate'])
    self.fc_intent = tf.keras.layers.Dense(params['global_units'], params['activation'], name='fc_intent')
    self.out_linear_intent = tf.keras.layers.Dense(params['intent_size'], name='output_intent')
    self.out_linear_slot = tf.keras.layers.Dense(params['slot_size'], name='output_slot')
    
  
  def call(self, inputs, training):
    if inputs.dtype != tf.int32:
      inputs = tf.cast(inputs, tf.int32)
    masks = tf.sign(inputs)
    
    x = tf.nn.embedding_lookup(self.embedding, inputs)
    if params['is_embedding_scaled']:
      x *= tf.sqrt(tf.cast(params['global_units'], tf.float32))
    x += get_timing_signal_1d(tf.shape(x)[1], params['global_units'])
    x = self.input_dropout(x, training=training)
    
    for block in self.blocks:
      x = block((x, masks), training=training)
    
    x_intent = tf.reduce_max(x, 1)
    x_intent = self.intent_dropout(x_intent, training=training)
    x_intent = self.out_linear_intent(self.fc_intent(x_intent))
    x_slot = self.out_linear_slot(x)
    return (x_intent, x_slot)
In [10]:
params = {
  'train_path': '../data/atis.train.w-intent.iob',
  'test_path': '../data/atis.test.w-intent.iob',
  'word_path': '../vocab/word.txt',
  'vocab_path': '../vocab/word.npy',
  'intent_path': '../vocab/intent.txt',
  'slot_path': '../vocab/slot.txt',
  'batch_size': 16,
  'num_samples': 4978,
  'num_layers': 2,
  'global_units': 300,
  'hidden_units': 512,
  'activation': tf.nn.elu,
  'num_heads': 8,
  'multiplier': 2,
  'dropout_rate': .1,
  'epsilon': 1e-6,
  'is_bidirectional': True,
  'is_embedding_scaled': False,
  'clip_norm': 5.,
}
In [11]:
params['word2idx'] = get_vocab(params['word_path'])
params['intent2idx'] = get_vocab(params['intent_path'])
params['slot2idx'] = get_vocab(params['slot_path'])

params['word_size'] = len(params['word2idx']) + 1
params['intent_size'] = len(params['intent2idx']) + 1
params['slot_size'] = len(params['slot2idx']) + 1
In [12]:
model = Model(params)
model.build(input_shape=(None, None))
pprint.pprint([(v.name, v.shape) for v in model.trainable_variables])

decay_lr = Triangular2CyclicalLearningRate(
  initial_learning_rate = 1e-4,
  maximal_learning_rate = 8e-4,
  step_size = 8 * params['num_samples'] // params['batch_size'],
)
optim = tf.optimizers.Adam(1e-4)
global_step = 0

slot_best_f1 = .0
intent_acc_with_that = .0

t0 = time.time()
logger = logging.getLogger('tensorflow')
logger.setLevel(logging.INFO)

for n_epoch in range(1, 64+1):
  # TRAINING
  for (words, (intent, slots)) in dataset(is_training=True, params=params):
    with tf.GradientTape() as tape:
      y_intent, y_slots = model(words, training=True)
      loss_intent = tf.compat.v1.losses.softmax_cross_entropy(
        onehot_labels = tf.one_hot(intent, len(params['intent2idx'])+1),
        logits = y_intent,
        label_smoothing = .2)
      # weight of 'O' is set to be small
      weights = tf.cast(tf.sign(slots), tf.float32)
      padding = tf.constant(1e-2, tf.float32, weights.shape)
      weights = tf.where(tf.equal(weights, 0.), padding, weights)

      loss_slots = tf.compat.v1.losses.softmax_cross_entropy(
        onehot_labels = tf.one_hot(slots, len(params['slot2idx'])+1),
        logits = y_slots,
        weights = tf.cast(weights, tf.float32),
        label_smoothing = .2)
      # joint loss
      loss = loss_intent + loss_slots
    
    optim.lr.assign(decay_lr(global_step))
    grads = tape.gradient(loss, model.trainable_variables)
    grads, _ = tf.clip_by_global_norm(grads, params['clip_norm'])
    optim.apply_gradients(zip(grads, model.trainable_variables))

    if global_step % 50 == 0:
      logger.info("Step {} | Loss: {:.4f} | Loss_intent: {:.4f} | Loss_slots: {:.4f} | Spent: {:.1f} secs | LR: {:.6f}".format(
          global_step, loss.numpy().item(), loss_intent.numpy().item(), loss_slots.numpy().item(), time.time()-t0, optim.lr.numpy().item()))
      t0 = time.time()
    global_step += 1
    
  # EVALUATION
  intent_true = []
  intent_pred = []
  slot_true = []
  slot_pred = []

  for (words, (intent, slots)) in dataset(is_training=False, params=params):
    y_intent, y_slots = model(words, training=False)
    y_intent = tf.argmax(y_intent, -1)
    y_slots = tf.argmax(y_slots, -1)
    
    intent_true += intent.numpy().flatten().tolist()
    intent_pred += y_intent.numpy().flatten().tolist()
    slot_true += slots.numpy().flatten().tolist()
    slot_pred += y_slots.numpy().flatten().tolist()
    
  f1_slots = f1_score(y_true = slot_true,
                      y_pred = slot_pred,
                      labels = list(params['slot2idx'].values()),
                      sample_weight = np.sign(slot_true),
                      average='micro',)
  
  acc_intent = accuracy_score(intent_true, intent_pred)

  logger.info("Slot F1: {:.3f}, Intent Acc: {:.3f}".format(f1_slots, acc_intent))

  if n_epoch != 1 and n_epoch % 8 == 0:
    logger.info('\n'+classification_report(y_true = intent_true,
                                          y_pred = intent_pred,
                                          labels = list(params['intent2idx'].values()),
                                          target_names = list(params['intent2idx'].keys()),
                                          digits=3))
    logger.info('\n'+classification_report(y_true = slot_true,
                                          y_pred = slot_pred,
                                          labels = list(params['slot2idx'].values()),
                                          target_names = list(params['slot2idx'].keys()),
                                          sample_weight = np.sign(slot_true),
                                          digits=3))
  
  if f1_slots > slot_best_f1:
    slot_best_f1 = f1_slots
    intent_acc_with_that = acc_intent
    # you can save model here
  logger.info("Best Slot F1: {:.3f}, Intent Acc: {:.3f}".format(slot_best_f1, intent_acc_with_that))

[('layer1.1/layer_norm/scale:0', TensorShape([300])),
 ('layer1.1/layer_norm/bias:0', TensorShape([300])),
 ('layer1.1/multihead_self_attention/qkv_linear/kernel:0',
  TensorShape([300, 1536])),
 ('layer1.1/multihead_self_attention/qkv_linear/bias:0', TensorShape([1536])),
 ('layer1.1/multihead_self_attention/out_linear/kernel:0',
  TensorShape([512, 300])),
 ('layer1.1/multihead_self_attention/out_linear/bias:0', TensorShape([300])),
 ('layer1.2/layer_norm_1/scale:0', TensorShape([300])),
 ('layer1.2/layer_norm_1/bias:0', TensorShape([300])),
 ('layer1.2/pointwise_ffn/fc/kernel:0', TensorShape([300, 600])),
 ('layer1.2/pointwise_ffn/fc/bias:0', TensorShape([600])),
 ('layer1.2/pointwise_ffn/linear/kernel:0', TensorShape([600, 300])),
 ('layer1.2/pointwise_ffn/linear/bias:0', TensorShape([300])),
 ('layer2.1/layer_norm_2/scale:0', TensorShape([300])),
 ('layer2.1/layer_norm_2/bias:0', TensorShape([300])),
 ('layer2.1/multihead_self_attention_1/qkv_linear/kernel:0',
  TensorShape([300, 1536])),
 ('layer2.1/multihead_self_attention_1/qkv_linear/bias:0', TensorShape([1536])),
 ('layer2.1/multihead_self_attention_1/out_linear/kernel:0',
  TensorShape([512, 300])),
 ('layer2.1/multihead_self_attention_1/out_linear/bias:0', TensorShape([300])),
 ('layer2.2/layer_norm_3/scale:0', TensorShape([300])),
 ('layer2.2/layer_norm_3/bias:0', TensorShape([300])),
 ('layer2.2/pointwise_ffn_1/fc/kernel:0', TensorShape([300, 600])),
 ('layer2.2/pointwise_ffn_1/fc/bias:0', TensorShape([600])),
 ('layer2.2/pointwise_ffn_1/linear/kernel:0', TensorShape([600, 300])),
 ('layer2.2/pointwise_ffn_1/linear/bias:0', TensorShape([300])),
 ('fc_intent/kernel:0', TensorShape([300, 300])),
 ('fc_intent/bias:0', TensorShape([300])),
 ('output_intent/kernel:0', TensorShape([300, 23])),
 ('output_intent/bias:0', TensorShape([23])),
 ('output_slot/kernel:0', TensorShape([300, 122])),
 ('output_slot/bias:0', TensorShape([122])),
 ('pretrained_embedding:0', TensorShape([750, 300]))]
Reading ../data/atis.train.w-intent.iob
INFO:tensorflow:Step 0 | Loss: 7.3096 | Loss_intent: 5.7735 | Loss_slots: 1.5361 | Spent: 3.0 secs | LR: 0.000100
INFO:tensorflow:Step 50 | Loss: 2.9308 | Loss_intent: 2.1498 | Loss_slots: 0.7810 | Spent: 2.9 secs | LR: 0.000114
INFO:tensorflow:Step 100 | Loss: 2.6540 | Loss_intent: 1.8339 | Loss_slots: 0.8201 | Spent: 2.9 secs | LR: 0.000128
INFO:tensorflow:Step 150 | Loss: 2.2284 | Loss_intent: 1.7676 | Loss_slots: 0.4608 | Spent: 2.9 secs | LR: 0.000142
INFO:tensorflow:Step 200 | Loss: 2.5896 | Loss_intent: 1.7489 | Loss_slots: 0.8408 | Spent: 2.9 secs | LR: 0.000156
INFO:tensorflow:Step 250 | Loss: 1.8302 | Loss_intent: 1.2908 | Loss_slots: 0.5394 | Spent: 2.9 secs | LR: 0.000170
INFO:tensorflow:Step 300 | Loss: 2.3105 | Loss_intent: 1.7389 | Loss_slots: 0.5716 | Spent: 2.9 secs | LR: 0.000184
Reading ../data/atis.test.w-intent.iob
INFO:tensorflow:Slot F1: 0.735, Intent Acc: 0.875
INFO:tensorflow:Best Slot F1: 0.735, Intent Acc: 0.875
Reading ../data/atis.train.w-intent.iob
INFO:tensorflow:Step 350 | Loss: 1.8233 | Loss_intent: 1.4247 | Loss_slots: 0.3985 | Spent: 5.2 secs | LR: 0.000198
INFO:tensorflow:Step 400 | Loss: 1.8213 | Loss_intent: 1.3725 | Loss_slots: 0.4488 | Spent: 2.9 secs | LR: 0.000212
INFO:tensorflow:Step 450 | Loss: 1.8354 | Loss_intent: 1.3543 | Loss_slots: 0.4811 | Spent: 2.9 secs | LR: 0.000227
INFO:tensorflow:Step 500 | Loss: 1.9175 | Loss_intent: 1.3177 | Loss_slots: 0.5998 | Spent: 2.9 secs | LR: 0.000241
INFO:tensorflow:Step 550 | Loss: 1.6561 | Loss_intent: 1.3196 | Loss_slots: 0.3366 | Spent: 2.9 secs | LR: 0.000255
INFO:tensorflow:Step 600 | Loss: 1.8324 | Loss_intent: 1.3612 | Loss_slots: 0.4712 | Spent: 2.9 secs | LR: 0.000269
Reading ../data/atis.test.w-intent.iob
INFO:tensorflow:Slot F1: 0.801, Intent Acc: 0.912
INFO:tensorflow:Best Slot F1: 0.801, Intent Acc: 0.912
Reading ../data/atis.train.w-intent.iob
INFO:tensorflow:Step 650 | Loss: 1.7042 | Loss_intent: 1.2906 | Loss_slots: 0.4136 | Spent: 5.1 secs | LR: 0.000283
INFO:tensorflow:Step 700 | Loss: 1.7549 | Loss_intent: 1.2290 | Loss_slots: 0.5259 | Spent: 2.9 secs | LR: 0.000297
INFO:tensorflow:Step 750 | Loss: 1.5586 | Loss_intent: 1.1831 | Loss_slots: 0.3755 | Spent: 2.9 secs | LR: 0.000311
INFO:tensorflow:Step 800 | Loss: 1.5360 | Loss_intent: 1.1404 | Loss_slots: 0.3956 | Spent: 3.0 secs | LR: 0.000325
INFO:tensorflow:Step 850 | Loss: 1.6226 | Loss_intent: 1.1295 | Loss_slots: 0.4931 | Spent: 2.9 secs | LR: 0.000339
INFO:tensorflow:Step 900 | Loss: 1.5773 | Loss_intent: 1.1334 | Loss_slots: 0.4439 | Spent: 2.9 secs | LR: 0.000353
Reading ../data/atis.test.w-intent.iob
INFO:tensorflow:Slot F1: 0.859, Intent Acc: 0.938
INFO:tensorflow:Best Slot F1: 0.859, Intent Acc: 0.938
Reading ../data/atis.train.w-intent.iob
INFO:tensorflow:Step 950 | Loss: 1.7405 | Loss_intent: 1.2303 | Loss_slots: 0.5102 | Spent: 5.1 secs | LR: 0.000367
INFO:tensorflow:Step 1000 | Loss: 1.5616 | Loss_intent: 1.1849 | Loss_slots: 0.3767 | Spent: 2.9 secs | LR: 0.000381
INFO:tensorflow:Step 1050 | Loss: 1.8015 | Loss_intent: 1.2129 | Loss_slots: 0.5887 | Spent: 2.9 secs | LR: 0.000395
INFO:tensorflow:Step 1100 | Loss: 1.6677 | Loss_intent: 1.1529 | Loss_slots: 0.5148 | Spent: 3.0 secs | LR: 0.000409
INFO:tensorflow:Step 1150 | Loss: 1.4375 | Loss_intent: 1.1291 | Loss_slots: 0.3084 | Spent: 2.9 secs | LR: 0.000423
INFO:tensorflow:Step 1200 | Loss: 1.6223 | Loss_intent: 1.1139 | Loss_slots: 0.5085 | Spent: 2.9 secs | LR: 0.000437
Reading ../data/atis.test.w-intent.iob
INFO:tensorflow:Slot F1: 0.862, Intent Acc: 0.928
INFO:tensorflow:Best Slot F1: 0.862, Intent Acc: 0.928
Reading ../data/atis.train.w-intent.iob
INFO:tensorflow:Step 1250 | Loss: 1.7519 | Loss_intent: 1.2602 | Loss_slots: 0.4917 | Spent: 5.0 secs | LR: 0.000452
INFO:tensorflow:Step 1300 | Loss: 1.6168 | Loss_intent: 1.1346 | Loss_slots: 0.4822 | Spent: 2.9 secs | LR: 0.000466
INFO:tensorflow:Step 1350 | Loss: 1.5600 | Loss_intent: 1.2026 | Loss_slots: 0.3574 | Spent: 2.9 secs | LR: 0.000480
INFO:tensorflow:Step 1400 | Loss: 1.4480 | Loss_intent: 1.2365 | Loss_slots: 0.2115 | Spent: 2.9 secs | LR: 0.000494
INFO:tensorflow:Step 1450 | Loss: 1.5832 | Loss_intent: 1.1147 | Loss_slots: 0.4684 | Spent: 2.9 secs | LR: 0.000508
INFO:tensorflow:Step 1500 | Loss: 1.6212 | Loss_intent: 1.2098 | Loss_slots: 0.4114 | Spent: 2.9 secs | LR: 0.000522
INFO:tensorflow:Step 1550 | Loss: 1.6087 | Loss_intent: 1.1409 | Loss_slots: 0.4679 | Spent: 2.9 secs | LR: 0.000536
Reading ../data/atis.test.w-intent.iob
INFO:tensorflow:Slot F1: 0.901, Intent Acc: 0.968
INFO:tensorflow:Best Slot F1: 0.901, Intent Acc: 0.968
Reading ../data/atis.train.w-intent.iob
INFO:tensorflow:Step 1600 | Loss: 1.5528 | Loss_intent: 1.2188 | Loss_slots: 0.3340 | Spent: 5.1 secs | LR: 0.000550
INFO:tensorflow:Step 1650 | Loss: 1.5240 | Loss_intent: 1.1248 | Loss_slots: 0.3992 | Spent: 2.9 secs | LR: 0.000564
INFO:tensorflow:Step 1700 | Loss: 1.5675 | Loss_intent: 1.1502 | Loss_slots: 0.4173 | Spent: 2.9 secs | LR: 0.000578
INFO:tensorflow:Step 1750 | Loss: 1.4530 | Loss_intent: 1.2361 | Loss_slots: 0.2169 | Spent: 2.9 secs | LR: 0.000592
INFO:tensorflow:Step 1800 | Loss: 1.5940 | Loss_intent: 1.2323 | Loss_slots: 0.3616 | Spent: 2.9 secs | LR: 0.000606
INFO:tensorflow:Step 1850 | Loss: 1.4237 | Loss_intent: 1.1163 | Loss_slots: 0.3074 | Spent: 2.9 secs | LR: 0.000620
Reading ../data/atis.test.w-intent.iob
INFO:tensorflow:Slot F1: 0.903, Intent Acc: 0.969
INFO:tensorflow:Best Slot F1: 0.903, Intent Acc: 0.969
Reading ../data/atis.train.w-intent.iob
INFO:tensorflow:Step 1900 | Loss: 1.6110 | Loss_intent: 1.1194 | Loss_slots: 0.4916 | Spent: 5.2 secs | LR: 0.000634
INFO:tensorflow:Step 1950 | Loss: 1.5508 | Loss_intent: 1.1725 | Loss_slots: 0.3783 | Spent: 2.9 secs | LR: 0.000648
INFO:tensorflow:Step 2000 | Loss: 1.4204 | Loss_intent: 1.1450 | Loss_slots: 0.2754 | Spent: 2.9 secs | LR: 0.000662
INFO:tensorflow:Step 2050 | Loss: 1.3889 | Loss_intent: 1.1146 | Loss_slots: 0.2743 | Spent: 3.0 secs | LR: 0.000677
INFO:tensorflow:Step 2100 | Loss: 1.5934 | Loss_intent: 1.1162 | Loss_slots: 0.4771 | Spent: 3.0 secs | LR: 0.000691
INFO:tensorflow:Step 2150 | Loss: 1.4733 | Loss_intent: 1.0952 | Loss_slots: 0.3781 | Spent: 2.9 secs | LR: 0.000705
Reading ../data/atis.test.w-intent.iob
INFO:tensorflow:Slot F1: 0.923, Intent Acc: 0.965
INFO:tensorflow:Best Slot F1: 0.923, Intent Acc: 0.965
Reading ../data/atis.train.w-intent.iob
INFO:tensorflow:Step 2200 | Loss: 1.5176 | Loss_intent: 1.1200 | Loss_slots: 0.3975 | Spent: 5.1 secs | LR: 0.000719
INFO:tensorflow:Step 2250 | Loss: 1.7210 | Loss_intent: 1.2479 | Loss_slots: 0.4731 | Spent: 2.9 secs | LR: 0.000733
INFO:tensorflow:Step 2300 | Loss: 1.5647 | Loss_intent: 1.1045 | Loss_slots: 0.4602 | Spent: 2.9 secs | LR: 0.000747
INFO:tensorflow:Step 2350 | Loss: 1.7809 | Loss_intent: 1.3192 | Loss_slots: 0.4617 | Spent: 2.9 secs | LR: 0.000761
INFO:tensorflow:Step 2400 | Loss: 1.5953 | Loss_intent: 1.1467 | Loss_slots: 0.4487 | Spent: 2.9 secs | LR: 0.000775
INFO:tensorflow:Step 2450 | Loss: 1.5938 | Loss_intent: 1.1307 | Loss_slots: 0.4631 | Spent: 2.9 secs | LR: 0.000789
Reading ../data/atis.test.w-intent.iob
INFO:tensorflow:Slot F1: 0.906, Intent Acc: 0.970
INFO:tensorflow:
                                          precision    recall  f1-score   support

                             atis_flight      0.983     0.991     0.987       632
                            atis_airfare      0.940     0.979     0.959        48
                     atis_ground_service      1.000     1.000     1.000        36
                            atis_airline      1.000     1.000     1.000        38
                       atis_abbreviation      1.000     0.970     0.985        33
                           atis_aircraft      0.889     0.889     0.889         9
                        atis_flight_time      0.500     1.000     0.667         1
                           atis_quantity      0.429     1.000     0.600         3
                atis_flight#atis_airfare      0.889     0.667     0.762        12
                            atis_airport      0.818     1.000     0.900        18
                           atis_distance      1.000     1.000     1.000        10
                               atis_city      1.000     0.667     0.800         6
                        atis_ground_fare      1.000     0.714     0.833         7
                           atis_capacity      1.000     0.952     0.976        21
                          atis_flight_no      1.000     0.750     0.857         8
                               atis_meal      1.000     0.667     0.800         6
                        atis_restriction      0.000     0.000     0.000         0
             atis_airline#atis_flight_no      0.000     0.000     0.000         0
    atis_ground_service#atis_ground_fare      0.000     0.000     0.000         0
           atis_airfare#atis_flight_time      0.000     0.000     0.000         0
                           atis_cheapest      0.000     0.000     0.000         0
atis_aircraft#atis_flight#atis_flight_no      0.000     0.000     0.000         0

                               micro avg      0.970     0.975     0.972       888
                               macro avg      0.657     0.647     0.637       888
                            weighted avg      0.976     0.975     0.974       888

INFO:tensorflow:
                              precision    recall  f1-score   support

                           O      0.000     0.000     0.000       0.0
           B-toloc.city_name      0.954     0.996     0.975     716.0
         B-fromloc.city_name      0.983     0.986     0.984     704.0
           I-toloc.city_name      0.923     0.989     0.954     265.0
      B-depart_date.day_name      0.972     0.981     0.977     212.0
              B-airline_name      0.943     0.980     0.961     101.0
         I-fromloc.city_name      0.901     0.972     0.935     177.0
 B-depart_time.period_of_day      0.921     0.892     0.906     130.0
              I-airline_name      0.984     0.938     0.961      65.0
    B-depart_date.day_number      0.980     0.891     0.933      55.0
    B-depart_date.month_name      0.964     0.964     0.964      56.0
          B-depart_time.time      0.667     0.982     0.794      57.0
                B-round_trip      1.000     0.973     0.986      73.0
             B-cost_relative      1.000     0.973     0.986      37.0
                I-round_trip      0.986     1.000     0.993      71.0
                B-flight_mod      0.958     0.958     0.958      24.0
 B-depart_time.time_relative      0.926     0.969     0.947      65.0
          I-depart_time.time      0.839     1.000     0.912      52.0
         B-stoploc.city_name      0.905     0.950     0.927      20.0
                 B-city_name      0.649     0.649     0.649      57.0
                B-class_type      0.774     1.000     0.873      24.0
          B-arrive_time.time      0.717     0.971     0.825      34.0
 B-arrive_time.time_relative      0.811     0.968     0.882      31.0
                I-class_type      1.000     1.000     1.000      17.0
               B-flight_stop      0.955     1.000     0.977      21.0
          I-arrive_time.time      0.773     0.971     0.861      35.0
              B-airline_code      0.952     0.588     0.727      34.0
    I-depart_date.day_number      1.000     1.000     1.000      15.0
      I-fromloc.airport_name      0.311     0.933     0.467      15.0
      B-fromloc.airport_name      0.292     0.583     0.389      12.0
      B-arrive_date.day_name      0.583     0.636     0.609      11.0
          B-toloc.state_code      0.947     1.000     0.973      18.0
B-depart_date.today_relative      1.000     0.889     0.941       9.0
             B-flight_number      0.172     0.455     0.250      11.0
 B-depart_date.date_relative      0.850     1.000     0.919      17.0
          B-toloc.state_name      0.957     0.786     0.863      28.0
           B-fare_basis_code      0.739     1.000     0.850      17.0
               B-flight_time      0.500     1.000     0.667       1.0
                        B-or      1.000     1.000     1.000       3.0
 B-arrive_time.period_of_day      0.000     0.000     0.000       6.0
          B-meal_description      1.000     0.900     0.947      10.0
             I-cost_relative      1.000     0.667     0.800       3.0
              I-airport_name      0.000     0.000     0.000      29.0
               B-fare_amount      1.000     0.500     0.667       2.0
               I-fare_amount      1.000     1.000     1.000       2.0
                 I-city_name      1.000     0.067     0.125      30.0
        I-toloc.airport_name      1.000     1.000     1.000       3.0
            B-transport_type      1.000     0.900     0.947      10.0
    B-arrive_date.month_name      0.600     0.500     0.545       6.0
    B-arrive_date.day_number      0.667     0.333     0.444       6.0
         I-stoploc.city_name      1.000     0.300     0.462      10.0
                      B-meal      1.000     1.000     1.000      16.0
        B-fromloc.state_code      1.000     0.957     0.978      23.0
    B-depart_time.period_mod      0.800     0.800     0.800       5.0
                   B-connect      1.000     0.833     0.909       6.0
               B-flight_days      1.000     1.000     1.000      10.0
        B-toloc.airport_name      1.000     0.333     0.500       3.0
        B-fromloc.state_name      0.682     0.882     0.769      17.0
              B-airport_name      0.000     0.000     0.000      21.0
                   B-economy      1.000     1.000     1.000       6.0
               I-flight_time      0.000     0.000     0.000       1.0
             B-aircraft_code      1.000     0.515     0.680      33.0
                       B-mod      0.091     0.500     0.154       2.0
              B-airport_code      0.000     0.000     0.000       9.0
    B-depart_time.start_time      1.000     0.333     0.500       3.0
      B-depart_time.end_time      0.000     0.000     0.000       3.0
          B-depart_date.year      0.000     0.000     0.000       3.0
            I-transport_type      0.000     0.000     0.000       1.0
          B-restriction_code      0.000     0.000     0.000       4.0
    B-arrive_time.start_time      1.000     0.625     0.769       8.0
        B-toloc.airport_code      0.000     0.000     0.000       4.0
      B-arrive_time.end_time      1.000     0.125     0.222       8.0
      I-arrive_time.end_time      0.000     0.000     0.000       8.0
      I-depart_time.end_time      0.000     0.000     0.000       3.0
               I-flight_stop      0.000     0.000     0.000       0.0
      B-fromloc.airport_code      1.000     1.000     1.000       5.0
          I-restriction_code      0.500     0.333     0.400       3.0
    I-depart_time.start_time      0.000     0.000     0.000       1.0
          I-toloc.state_name      0.500     1.000     0.667       1.0
I-depart_date.today_relative      0.000     0.000     0.000       0.0
 B-arrive_date.date_relative      0.000     0.000     0.000       2.0
                I-flight_mod      0.000     0.000     0.000       6.0
                   I-economy      0.000     0.000     0.000       0.0
 B-return_date.date_relative      0.000     0.000     0.000       3.0
        I-fromloc.state_name      0.000     0.000     0.000       1.0
                B-state_code      1.000     1.000     1.000       1.0
    I-arrive_time.start_time      0.000     0.000     0.000       1.0
    I-arrive_date.day_number      0.000     0.000     0.000       0.0
                 B-meal_code      0.000     0.000     0.000       1.0
 I-depart_time.period_of_day      0.000     0.000     0.000       1.0
                  B-day_name      0.000     0.000     0.000       2.0
             B-period_of_day      0.000     0.000     0.000       4.0
        B-stoploc.state_code      0.000     0.000     0.000       0.0
    B-return_date.month_name      0.000     0.000     0.000       0.0
    B-return_date.day_number      0.000     0.000     0.000       0.0
    B-arrive_time.period_mod      0.000     0.000     0.000       0.0
                 I-meal_code      0.000     0.000     0.000       0.0
        B-toloc.country_name      0.000     0.000     0.000       1.0
                 B-days_code      0.000     0.000     0.000       1.0
 I-arrive_time.period_of_day      0.000     0.000     0.000       0.0
            I-today_relative      0.000     0.000     0.000       0.0
 B-return_time.period_of_day      0.000     0.000     0.000       0.0
                      B-time      0.000     0.000     0.000       0.0
           I-fare_basis_code      0.000     0.000     0.000       0.0
 I-arrive_time.time_relative      0.000     0.000     0.000       4.0
 I-depart_time.time_relative      0.000     0.000     0.000       1.0
            B-today_relative      0.000     0.000     0.000       0.0
                B-state_name      0.000     0.000     0.000       9.0
B-arrive_date.today_relative      0.000     0.000     0.000       0.0
    B-return_time.period_mod      0.000     0.000     0.000       0.0
                B-month_name      0.000     0.000     0.000       0.0
                B-day_number      0.000     0.000     0.000       0.0
 I-return_date.date_relative      0.000     0.000     0.000       3.0
I-return_date.today_relative      0.000     0.000     0.000       0.0
      B-stoploc.airport_name      0.000     0.000     0.000       0.0
             B-time_relative      0.000     0.000     0.000       0.0
                      I-time      0.000     0.000     0.000       0.0
    I-return_date.day_number      0.000     0.000     0.000       0.0
          I-meal_description      0.000     0.000     0.000       0.0
B-return_date.today_relative      0.000     0.000     0.000       0.0
      B-return_date.day_name      0.000     0.000     0.000       2.0

                   micro avg      0.906     0.907     0.906    3657.0
                   macro avg      0.460     0.440     0.430    3657.0
                weighted avg      0.894     0.907     0.891    3657.0

INFO:tensorflow:Best Slot F1: 0.923, Intent Acc: 0.965
Reading ../data/atis.train.w-intent.iob

/usr/local/lib/python3.6/dist-packages/sklearn/metrics/_classification.py:1272: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.
  _warn_prf(average, modifier, msg_start, len(result))
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/_classification.py:1272: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples. Use `zero_division` parameter to control this behavior.
  _warn_prf(average, modifier, msg_start, len(result))

INFO:tensorflow:Step 2500 | Loss: 1.5369 | Loss_intent: 1.1774 | Loss_slots: 0.3594 | Spent: 5.2 secs | LR: 0.000797
INFO:tensorflow:Step 2550 | Loss: 1.6217 | Loss_intent: 1.1764 | Loss_slots: 0.4454 | Spent: 2.9 secs | LR: 0.000783
INFO:tensorflow:Step 2600 | Loss: 1.6029 | Loss_intent: 1.1899 | Loss_slots: 0.4130 | Spent: 3.0 secs | LR: 0.000769
INFO:tensorflow:Step 2650 | Loss: 1.5024 | Loss_intent: 1.1455 | Loss_slots: 0.3569 | Spent: 3.0 secs | LR: 0.000755
INFO:tensorflow:Step 2700 | Loss: 1.5901 | Loss_intent: 1.1923 | Loss_slots: 0.3977 | Spent: 3.0 secs | LR: 0.000741
INFO:tensorflow:Step 2750 | Loss: 1.6015 | Loss_intent: 1.1858 | Loss_slots: 0.4158 | Spent: 2.9 secs | LR: 0.000727
INFO:tensorflow:Step 2800 | Loss: 1.5524 | Loss_intent: 1.2102 | Loss_slots: 0.3422 | Spent: 2.9 secs | LR: 0.000713
Reading ../data/atis.test.w-intent.iob
INFO:tensorflow:Slot F1: 0.926, Intent Acc: 0.970
INFO:tensorflow:Best Slot F1: 0.926, Intent Acc: 0.970
Reading ../data/atis.train.w-intent.iob
INFO:tensorflow:Step 2850 | Loss: 1.5665 | Loss_intent: 1.1083 | Loss_slots: 0.4581 | Spent: 5.1 secs | LR: 0.000698
INFO:tensorflow:Step 2900 | Loss: 1.4182 | Loss_intent: 1.1135 | Loss_slots: 0.3047 | Spent: 3.1 secs | LR: 0.000684
INFO:tensorflow:Step 2950 | Loss: 1.4993 | Loss_intent: 1.1146 | Loss_slots: 0.3847 | Spent: 3.0 secs | LR: 0.000670
INFO:tensorflow:Step 3000 | Loss: 1.5416 | Loss_intent: 1.1041 | Loss_slots: 0.4375 | Spent: 2.9 secs | LR: 0.000656
INFO:tensorflow:Step 3050 | Loss: 1.4782 | Loss_intent: 1.0994 | Loss_slots: 0.3788 | Spent: 2.9 secs | LR: 0.000642
INFO:tensorflow:Step 3100 | Loss: 1.5915 | Loss_intent: 1.2277 | Loss_slots: 0.3638 | Spent: 2.9 secs | LR: 0.000628
Reading ../data/atis.test.w-intent.iob
INFO:tensorflow:Slot F1: 0.934, Intent Acc: 0.970
INFO:tensorflow:Best Slot F1: 0.934, Intent Acc: 0.970
Reading ../data/atis.train.w-intent.iob
INFO:tensorflow:Step 3150 | Loss: 1.5055 | Loss_intent: 1.2135 | Loss_slots: 0.2920 | Spent: 5.0 secs | LR: 0.000614
INFO:tensorflow:Step 3200 | Loss: 1.4654 | Loss_intent: 1.1660 | Loss_slots: 0.2993 | Spent: 2.9 secs | LR: 0.000600
INFO:tensorflow:Step 3250 | Loss: 1.3893 | Loss_intent: 1.1175 | Loss_slots: 0.2718 | Spent: 2.9 secs | LR: 0.000586
INFO:tensorflow:Step 3300 | Loss: 1.4596 | Loss_intent: 1.1085 | Loss_slots: 0.3511 | Spent: 2.9 secs | LR: 0.000572
INFO:tensorflow:Step 3350 | Loss: 1.4953 | Loss_intent: 1.1441 | Loss_slots: 0.3512 | Spent: 2.9 secs | LR: 0.000558
INFO:tensorflow:Step 3400 | Loss: 1.5368 | Loss_intent: 1.1227 | Loss_slots: 0.4141 | Spent: 2.9 secs | LR: 0.000544
Reading ../data/atis.test.w-intent.iob
INFO:tensorflow:Slot F1: 0.940, Intent Acc: 0.971
INFO:tensorflow:Best Slot F1: 0.940, Intent Acc: 0.971
Reading ../data/atis.train.w-intent.iob
INFO:tensorflow:Step 3450 | Loss: 1.4602 | Loss_intent: 1.0965 | Loss_slots: 0.3637 | Spent: 5.0 secs | LR: 0.000530
INFO:tensorflow:Step 3500 | Loss: 1.3599 | Loss_intent: 1.0850 | Loss_slots: 0.2749 | Spent: 3.0 secs | LR: 0.000516
INFO:tensorflow:Step 3550 | Loss: 1.5684 | Loss_intent: 1.1008 | Loss_slots: 0.4676 | Spent: 2.9 secs | LR: 0.000502
INFO:tensorflow:Step 3600 | Loss: 1.5110 | Loss_intent: 1.1447 | Loss_slots: 0.3663 | Spent: 2.9 secs | LR: 0.000488
INFO:tensorflow:Step 3650 | Loss: 1.4317 | Loss_intent: 1.0913 | Loss_slots: 0.3404 | Spent: 3.1 secs | LR: 0.000473
INFO:tensorflow:Step 3700 | Loss: 1.4745 | Loss_intent: 1.0904 | Loss_slots: 0.3841 | Spent: 2.9 secs | LR: 0.000459
Reading ../data/atis.test.w-intent.iob
INFO:tensorflow:Slot F1: 0.941, Intent Acc: 0.974
INFO:tensorflow:Best Slot F1: 0.941, Intent Acc: 0.974
Reading ../data/atis.train.w-intent.iob
INFO:tensorflow:Step 3750 | Loss: 1.4672 | Loss_intent: 1.0859 | Loss_slots: 0.3813 | Spent: 5.1 secs | LR: 0.000445
INFO:tensorflow:Step 3800 | Loss: 1.4498 | Loss_intent: 1.0874 | Loss_slots: 0.3625 | Spent: 2.9 secs | LR: 0.000431
INFO:tensorflow:Step 3850 | Loss: 1.5024 | Loss_intent: 1.0888 | Loss_slots: 0.4136 | Spent: 2.9 secs | LR: 0.000417
INFO:tensorflow:Step 3900 | Loss: 1.4910 | Loss_intent: 1.1020 | Loss_slots: 0.3890 | Spent: 3.0 secs | LR: 0.000403
INFO:tensorflow:Step 3950 | Loss: 1.4249 | Loss_intent: 1.0914 | Loss_slots: 0.3336 | Spent: 2.9 secs | LR: 0.000389
INFO:tensorflow:Step 4000 | Loss: 1.4214 | Loss_intent: 1.0917 | Loss_slots: 0.3297 | Spent: 2.9 secs | LR: 0.000375
INFO:tensorflow:Step 4050 | Loss: 1.4349 | Loss_intent: 1.1214 | Loss_slots: 0.3135 | Spent: 2.9 secs | LR: 0.000361
Reading ../data/atis.test.w-intent.iob
INFO:tensorflow:Slot F1: 0.946, Intent Acc: 0.976
INFO:tensorflow:Best Slot F1: 0.946, Intent Acc: 0.976
Reading ../data/atis.train.w-intent.iob
INFO:tensorflow:Step 4100 | Loss: 1.5065 | Loss_intent: 1.0861 | Loss_slots: 0.4205 | Spent: 5.1 secs | LR: 0.000347
INFO:tensorflow:Step 4150 | Loss: 1.5227 | Loss_intent: 1.1061 | Loss_slots: 0.4166 | Spent: 2.8 secs | LR: 0.000333
INFO:tensorflow:Step 4200 | Loss: 1.4454 | Loss_intent: 1.0880 | Loss_slots: 0.3573 | Spent: 2.9 secs | LR: 0.000319
INFO:tensorflow:Step 4250 | Loss: 1.4277 | Loss_intent: 1.0846 | Loss_slots: 0.3431 | Spent: 2.9 secs | LR: 0.000305
INFO:tensorflow:Step 4300 | Loss: 1.3385 | Loss_intent: 1.0878 | Loss_slots: 0.2506 | Spent: 2.9 secs | LR: 0.000291
INFO:tensorflow:Step 4350 | Loss: 1.4741 | Loss_intent: 1.1048 | Loss_slots: 0.3693 | Spent: 2.9 secs | LR: 0.000277
Reading ../data/atis.test.w-intent.iob
INFO:tensorflow:Slot F1: 0.944, Intent Acc: 0.974
INFO:tensorflow:Best Slot F1: 0.946, Intent Acc: 0.976
Reading ../data/atis.train.w-intent.iob
INFO:tensorflow:Step 4400 | Loss: 1.4092 | Loss_intent: 1.0937 | Loss_slots: 0.3155 | Spent: 5.1 secs | LR: 0.000263
INFO:tensorflow:Step 4450 | Loss: 1.4929 | Loss_intent: 1.0835 | Loss_slots: 0.4095 | Spent: 2.9 secs | LR: 0.000248
INFO:tensorflow:Step 4500 | Loss: 1.3725 | Loss_intent: 1.0894 | Loss_slots: 0.2831 | Spent: 3.0 secs | LR: 0.000234
INFO:tensorflow:Step 4550 | Loss: 1.4944 | Loss_intent: 1.0849 | Loss_slots: 0.4095 | Spent: 3.0 secs | LR: 0.000220
INFO:tensorflow:Step 4600 | Loss: 1.4266 | Loss_intent: 1.0874 | Loss_slots: 0.3393 | Spent: 3.2 secs | LR: 0.000206
INFO:tensorflow:Step 4650 | Loss: 1.4549 | Loss_intent: 1.0851 | Loss_slots: 0.3697 | Spent: 3.1 secs | LR: 0.000192
Reading ../data/atis.test.w-intent.iob
INFO:tensorflow:Slot F1: 0.946, Intent Acc: 0.974
INFO:tensorflow:Best Slot F1: 0.946, Intent Acc: 0.976
Reading ../data/atis.train.w-intent.iob
INFO:tensorflow:Step 4700 | Loss: 1.2630 | Loss_intent: 1.0901 | Loss_slots: 0.1729 | Spent: 5.7 secs | LR: 0.000178
INFO:tensorflow:Step 4750 | Loss: 1.4411 | Loss_intent: 1.0942 | Loss_slots: 0.3469 | Spent: 2.9 secs | LR: 0.000164
INFO:tensorflow:Step 4800 | Loss: 1.4662 | Loss_intent: 1.0826 | Loss_slots: 0.3836 | Spent: 2.9 secs | LR: 0.000150
INFO:tensorflow:Step 4850 | Loss: 1.4832 | Loss_intent: 1.0921 | Loss_slots: 0.3911 | Spent: 2.9 secs | LR: 0.000136
INFO:tensorflow:Step 4900 | Loss: 1.4259 | Loss_intent: 1.0825 | Loss_slots: 0.3435 | Spent: 2.9 secs | LR: 0.000122
INFO:tensorflow:Step 4950 | Loss: 1.4892 | Loss_intent: 1.0888 | Loss_slots: 0.4005 | Spent: 2.9 secs | LR: 0.000108
Reading ../data/atis.test.w-intent.iob
INFO:tensorflow:Slot F1: 0.947, Intent Acc: 0.973
INFO:tensorflow:
                                          precision    recall  f1-score   support

                             atis_flight      0.986     0.992     0.989       632
                            atis_airfare      0.941     1.000     0.970        48
                     atis_ground_service      0.973     1.000     0.986        36
                            atis_airline      1.000     1.000     1.000        38
                       atis_abbreviation      1.000     0.970     0.985        33
                           atis_aircraft      1.000     0.889     0.941         9
                        atis_flight_time      0.500     1.000     0.667         1
                           atis_quantity      0.429     1.000     0.600         3
                atis_flight#atis_airfare      0.833     0.417     0.556        12
                            atis_airport      0.857     1.000     0.923        18
                           atis_distance      1.000     1.000     1.000        10
                               atis_city      1.000     0.667     0.800         6
                        atis_ground_fare      1.000     0.714     0.833         7
                           atis_capacity      1.000     1.000     1.000        21
                          atis_flight_no      1.000     1.000     1.000         8
                               atis_meal      1.000     0.833     0.909         6
                        atis_restriction      0.000     0.000     0.000         0
             atis_airline#atis_flight_no      0.000     0.000     0.000         0
    atis_ground_service#atis_ground_fare      0.000     0.000     0.000         0
           atis_airfare#atis_flight_time      0.000     0.000     0.000         0
                           atis_cheapest      0.000     0.000     0.000         0
atis_aircraft#atis_flight#atis_flight_no      0.000     0.000     0.000         0

                               micro avg      0.973     0.979     0.976       888
                               macro avg      0.660     0.658     0.644       888
                            weighted avg      0.978     0.979     0.976       888

INFO:tensorflow:
                              precision    recall  f1-score   support

                           O      0.000     0.000     0.000       0.0
           B-toloc.city_name      0.973     0.996     0.984     716.0
         B-fromloc.city_name      0.980     0.994     0.987     704.0
           I-toloc.city_name      0.974     0.985     0.979     265.0
      B-depart_date.day_name      0.990     0.981     0.986     212.0
              B-airline_name      0.981     1.000     0.990     101.0
         I-fromloc.city_name      0.951     0.994     0.972     177.0
 B-depart_time.period_of_day      0.983     0.900     0.940     130.0
              I-airline_name      1.000     1.000     1.000      65.0
    B-depart_date.day_number      0.981     0.964     0.972      55.0
    B-depart_date.month_name      0.982     0.964     0.973      56.0
          B-depart_time.time      0.812     0.982     0.889      57.0
                B-round_trip      1.000     0.986     0.993      73.0
             B-cost_relative      1.000     0.973     0.986      37.0
                I-round_trip      1.000     1.000     1.000      71.0
                B-flight_mod      1.000     0.958     0.979      24.0
 B-depart_time.time_relative      0.969     0.969     0.969      65.0
          I-depart_time.time      0.897     1.000     0.945      52.0
         B-stoploc.city_name      1.000     0.950     0.974      20.0
                 B-city_name      0.844     0.667     0.745      57.0
                B-class_type      0.923     1.000     0.960      24.0
          B-arrive_time.time      0.970     0.941     0.955      34.0
 B-arrive_time.time_relative      0.857     0.968     0.909      31.0
                I-class_type      1.000     1.000     1.000      17.0
               B-flight_stop      1.000     1.000     1.000      21.0
          I-arrive_time.time      1.000     0.971     0.986      35.0
              B-airline_code      1.000     0.853     0.921      34.0
    I-depart_date.day_number      1.000     1.000     1.000      15.0
      I-fromloc.airport_name      0.455     1.000     0.625      15.0
      B-fromloc.airport_name      0.292     0.583     0.389      12.0
      B-arrive_date.day_name      0.688     1.000     0.815      11.0
          B-toloc.state_code      1.000     1.000     1.000      18.0
B-depart_date.today_relative      1.000     1.000     1.000       9.0
             B-flight_number      0.526     0.909     0.667      11.0
 B-depart_date.date_relative      0.895     1.000     0.944      17.0
          B-toloc.state_name      0.885     0.821     0.852      28.0
           B-fare_basis_code      0.917     0.647     0.759      17.0
               B-flight_time      1.000     1.000     1.000       1.0
                        B-or      1.000     1.000     1.000       3.0
 B-arrive_time.period_of_day      0.750     1.000     0.857       6.0
          B-meal_description      1.000     0.900     0.947      10.0
             I-cost_relative      1.000     0.667     0.800       3.0
              I-airport_name      1.000     0.345     0.513      29.0
               B-fare_amount      0.500     0.500     0.500       2.0
               I-fare_amount      1.000     1.000     1.000       2.0
                 I-city_name      0.900     0.600     0.720      30.0
        I-toloc.airport_name      0.600     1.000     0.750       3.0
            B-transport_type      1.000     1.000     1.000      10.0
    B-arrive_date.month_name      0.714     0.833     0.769       6.0
    B-arrive_date.day_number      0.714     0.833     0.769       6.0
         I-stoploc.city_name      1.000     0.800     0.889      10.0
                      B-meal      1.000     1.000     1.000      16.0
        B-fromloc.state_code      1.000     1.000     1.000      23.0
    B-depart_time.period_mod      0.833     1.000     0.909       5.0
                   B-connect      1.000     1.000     1.000       6.0
               B-flight_days      1.000     1.000     1.000      10.0
        B-toloc.airport_name      1.000     0.333     0.500       3.0
        B-fromloc.state_name      0.810     1.000     0.895      17.0
              B-airport_name      0.857     0.286     0.429      21.0
                   B-economy      1.000     1.000     1.000       6.0
               I-flight_time      1.000     1.000     1.000       1.0
             B-aircraft_code      0.967     0.879     0.921      33.0
                       B-mod      0.500     0.500     0.500       2.0
              B-airport_code      0.833     0.556     0.667       9.0
    B-depart_time.start_time      0.500     0.667     0.571       3.0
      B-depart_time.end_time      1.000     0.667     0.800       3.0
          B-depart_date.year      1.000     0.667     0.800       3.0
            I-transport_type      0.000     0.000     0.000       1.0
          B-restriction_code      1.000     1.000     1.000       4.0
    B-arrive_time.start_time      0.889     1.000     0.941       8.0
        B-toloc.airport_code      1.000     0.500     0.667       4.0
      B-arrive_time.end_time      0.800     1.000     0.889       8.0
      I-arrive_time.end_time      0.889     1.000     0.941       8.0
      I-depart_time.end_time      1.000     0.333     0.500       3.0
               I-flight_stop      0.000     0.000     0.000       0.0
      B-fromloc.airport_code      1.000     1.000     1.000       5.0
          I-restriction_code      0.273     1.000     0.429       3.0
    I-depart_time.start_time      1.000     1.000     1.000       1.0
          I-toloc.state_name      0.500     1.000     0.667       1.0
I-depart_date.today_relative      0.000     0.000     0.000       0.0
 B-arrive_date.date_relative      1.000     1.000     1.000       2.0
                I-flight_mod      1.000     0.167     0.286       6.0
                   I-economy      0.000     0.000     0.000       0.0
 B-return_date.date_relative      0.000     0.000     0.000       3.0
        I-fromloc.state_name      0.000     0.000     0.000       1.0
                B-state_code      1.000     1.000     1.000       1.0
    I-arrive_time.start_time      1.000     1.000     1.000       1.0
    I-arrive_date.day_number      0.000     0.000     0.000       0.0
                 B-meal_code      0.000     0.000     0.000       1.0
 I-depart_time.period_of_day      0.000     0.000     0.000       1.0
                  B-day_name      1.000     0.500     0.667       2.0
             B-period_of_day      1.000     0.250     0.400       4.0
        B-stoploc.state_code      0.000     0.000     0.000       0.0
    B-return_date.month_name      0.000     0.000     0.000       0.0
    B-return_date.day_number      0.000     0.000     0.000       0.0
    B-arrive_time.period_mod      0.000     0.000     0.000       0.0
                 I-meal_code      0.000     0.000     0.000       0.0
        B-toloc.country_name      1.000     1.000     1.000       1.0
                 B-days_code      1.000     1.000     1.000       1.0
 I-arrive_time.period_of_day      0.000     0.000     0.000       0.0
            I-today_relative      0.000     0.000     0.000       0.0
 B-return_time.period_of_day      0.000     0.000     0.000       0.0
                      B-time      0.000     0.000     0.000       0.0
           I-fare_basis_code      0.000     0.000     0.000       0.0
 I-arrive_time.time_relative      0.000     0.000     0.000       4.0
 I-depart_time.time_relative      0.000     0.000     0.000       1.0
            B-today_relative      0.000     0.000     0.000       0.0
                B-state_name      0.000     0.000     0.000       9.0
B-arrive_date.today_relative      0.000     0.000     0.000       0.0
    B-return_time.period_mod      0.000     0.000     0.000       0.0
                B-month_name      0.000     0.000     0.000       0.0
                B-day_number      0.000     0.000     0.000       0.0
 I-return_date.date_relative      0.000     0.000     0.000       3.0
I-return_date.today_relative      0.000     0.000     0.000       0.0
      B-stoploc.airport_name      0.000     0.000     0.000       0.0
             B-time_relative      0.000     0.000     0.000       0.0
                      I-time      0.000     0.000     0.000       0.0
    I-return_date.day_number      0.000     0.000     0.000       0.0
          I-meal_description      0.000     0.000     0.000       0.0
B-return_date.today_relative      0.000     0.000     0.000       0.0
      B-return_date.day_name      0.000     0.000     0.000       2.0

                   micro avg      0.946     0.948     0.947    3657.0
                   macro avg      0.624     0.601     0.595    3657.0
                weighted avg      0.951     0.948     0.944    3657.0

INFO:tensorflow:Best Slot F1: 0.947, Intent Acc: 0.973
Reading ../data/atis.train.w-intent.iob
INFO:tensorflow:Step 5000 | Loss: 1.4748 | Loss_intent: 1.0809 | Loss_slots: 0.3939 | Spent: 5.1 secs | LR: 0.000103
INFO:tensorflow:Step 5050 | Loss: 1.4286 | Loss_intent: 1.0806 | Loss_slots: 0.3480 | Spent: 2.8 secs | LR: 0.000110
INFO:tensorflow:Step 5100 | Loss: 1.4031 | Loss_intent: 1.0878 | Loss_slots: 0.3153 | Spent: 2.9 secs | LR: 0.000117
INFO:tensorflow:Step 5150 | Loss: 1.4094 | Loss_intent: 1.0995 | Loss_slots: 0.3099 | Spent: 2.8 secs | LR: 0.000124
INFO:tensorflow:Step 5200 | Loss: 1.4146 | Loss_intent: 1.0853 | Loss_slots: 0.3293 | Spent: 2.9 secs | LR: 0.000131
INFO:tensorflow:Step 5250 | Loss: 1.4568 | Loss_intent: 1.0897 | Loss_slots: 0.3671 | Spent: 2.9 secs | LR: 0.000138
INFO:tensorflow:Step 5300 | Loss: 1.4687 | Loss_intent: 1.0825 | Loss_slots: 0.3862 | Spent: 2.9 secs | LR: 0.000145
Reading ../data/atis.test.w-intent.iob
INFO:tensorflow:Slot F1: 0.946, Intent Acc: 0.972
INFO:tensorflow:Best Slot F1: 0.947, Intent Acc: 0.973
Reading ../data/atis.train.w-intent.iob
INFO:tensorflow:Step 5350 | Loss: 1.4464 | Loss_intent: 1.1050 | Loss_slots: 0.3414 | Spent: 5.0 secs | LR: 0.000152
INFO:tensorflow:Step 5400 | Loss: 1.4590 | Loss_intent: 1.0970 | Loss_slots: 0.3620 | Spent: 2.9 secs | LR: 0.000159
INFO:tensorflow:Step 5450 | Loss: 1.2717 | Loss_intent: 1.0858 | Loss_slots: 0.1859 | Spent: 2.9 secs | LR: 0.000166
INFO:tensorflow:Step 5500 | Loss: 1.3704 | Loss_intent: 1.1096 | Loss_slots: 0.2609 | Spent: 2.9 secs | LR: 0.000173
INFO:tensorflow:Step 5550 | Loss: 1.4065 | Loss_intent: 1.0826 | Loss_slots: 0.3239 | Spent: 2.8 secs | LR: 0.000180
INFO:tensorflow:Step 5600 | Loss: 1.3408 | Loss_intent: 1.0848 | Loss_slots: 0.2560 | Spent: 2.8 secs | LR: 0.000187
Reading ../data/atis.test.w-intent.iob
INFO:tensorflow:Slot F1: 0.950, Intent Acc: 0.970
INFO:tensorflow:Best Slot F1: 0.950, Intent Acc: 0.970
Reading ../data/atis.train.w-intent.iob
INFO:tensorflow:Step 5650 | Loss: 1.4909 | Loss_intent: 1.0854 | Loss_slots: 0.4055 | Spent: 5.1 secs | LR: 0.000194
INFO:tensorflow:Step 5700 | Loss: 1.4639 | Loss_intent: 1.0853 | Loss_slots: 0.3786 | Spent: 3.0 secs | LR: 0.000202
INFO:tensorflow:Step 5750 | Loss: 1.5579 | Loss_intent: 1.0832 | Loss_slots: 0.4747 | Spent: 2.9 secs | LR: 0.000209
INFO:tensorflow:Step 5800 | Loss: 1.4115 | Loss_intent: 1.1002 | Loss_slots: 0.3113 | Spent: 2.9 secs | LR: 0.000216
INFO:tensorflow:Step 5850 | Loss: 1.4163 | Loss_intent: 1.0863 | Loss_slots: 0.3300 | Spent: 2.9 secs | LR: 0.000223
INFO:tensorflow:Step 5900 | Loss: 1.4184 | Loss_intent: 1.0858 | Loss_slots: 0.3326 | Spent: 2.8 secs | LR: 0.000230
Reading ../data/atis.test.w-intent.iob
INFO:tensorflow:Slot F1: 0.950, Intent Acc: 0.974
INFO:tensorflow:Best Slot F1: 0.950, Intent Acc: 0.970
Reading ../data/atis.train.w-intent.iob
INFO:tensorflow:Step 5950 | Loss: 1.4133 | Loss_intent: 1.0941 | Loss_slots: 0.3192 | Spent: 5.0 secs | LR: 0.000237
INFO:tensorflow:Step 6000 | Loss: 1.5657 | Loss_intent: 1.1644 | Loss_slots: 0.4013 | Spent: 2.9 secs | LR: 0.000244
INFO:tensorflow:Step 6050 | Loss: 1.4703 | Loss_intent: 1.0896 | Loss_slots: 0.3807 | Spent: 2.9 secs | LR: 0.000251
INFO:tensorflow:Step 6100 | Loss: 1.3905 | Loss_intent: 1.0877 | Loss_slots: 0.3028 | Spent: 2.9 secs | LR: 0.000258
INFO:tensorflow:Step 6150 | Loss: 1.5131 | Loss_intent: 1.0842 | Loss_slots: 0.4288 | Spent: 2.9 secs | LR: 0.000265
INFO:tensorflow:Step 6200 | Loss: 1.4393 | Loss_intent: 1.0912 | Loss_slots: 0.3481 | Spent: 2.9 secs | LR: 0.000272
Reading ../data/atis.test.w-intent.iob
INFO:tensorflow:Slot F1: 0.946, Intent Acc: 0.975
INFO:tensorflow:Best Slot F1: 0.950, Intent Acc: 0.970
Reading ../data/atis.train.w-intent.iob
INFO:tensorflow:Step 6250 | Loss: 1.3964 | Loss_intent: 1.0898 | Loss_slots: 0.3065 | Spent: 5.0 secs | LR: 0.000279
INFO:tensorflow:Step 6300 | Loss: 1.3707 | Loss_intent: 1.1383 | Loss_slots: 0.2324 | Spent: 2.9 secs | LR: 0.000286
INFO:tensorflow:Step 6350 | Loss: 1.3545 | Loss_intent: 1.0928 | Loss_slots: 0.2617 | Spent: 2.9 secs | LR: 0.000293
INFO:tensorflow:Step 6400 | Loss: 1.4254 | Loss_intent: 1.1385 | Loss_slots: 0.2869 | Spent: 2.9 secs | LR: 0.000300
INFO:tensorflow:Step 6450 | Loss: 1.4012 | Loss_intent: 1.1400 | Loss_slots: 0.2612 | Spent: 2.9 secs | LR: 0.000307
INFO:tensorflow:Step 6500 | Loss: 1.4244 | Loss_intent: 1.0937 | Loss_slots: 0.3307 | Spent: 2.9 secs | LR: 0.000314
INFO:tensorflow:Step 6550 | Loss: 1.4732 | Loss_intent: 1.1151 | Loss_slots: 0.3581 | Spent: 2.8 secs | LR: 0.000321
Reading ../data/atis.test.w-intent.iob
INFO:tensorflow:Slot F1: 0.944, Intent Acc: 0.972
INFO:tensorflow:Best Slot F1: 0.950, Intent Acc: 0.970
Reading ../data/atis.train.w-intent.iob
INFO:tensorflow:Step 6600 | Loss: 1.3969 | Loss_intent: 1.0837 | Loss_slots: 0.3132 | Spent: 5.1 secs | LR: 0.000328
INFO:tensorflow:Step 6650 | Loss: 1.3768 | Loss_intent: 1.0890 | Loss_slots: 0.2878 | Spent: 2.9 secs | LR: 0.000335
INFO:tensorflow:Step 6700 | Loss: 1.5133 | Loss_intent: 1.0880 | Loss_slots: 0.4254 | Spent: 2.9 secs | LR: 0.000342
INFO:tensorflow:Step 6750 | Loss: 1.4116 | Loss_intent: 1.0922 | Loss_slots: 0.3193 | Spent: 2.9 secs | LR: 0.000349
INFO:tensorflow:Step 6800 | Loss: 1.3452 | Loss_intent: 1.1087 | Loss_slots: 0.2364 | Spent: 2.9 secs | LR: 0.000356
INFO:tensorflow:Step 6850 | Loss: 1.4649 | Loss_intent: 1.1480 | Loss_slots: 0.3169 | Spent: 2.9 secs | LR: 0.000363
Reading ../data/atis.test.w-intent.iob
INFO:tensorflow:Slot F1: 0.948, Intent Acc: 0.971
INFO:tensorflow:Best Slot F1: 0.950, Intent Acc: 0.970
Reading ../data/atis.train.w-intent.iob
INFO:tensorflow:Step 6900 | Loss: 1.4388 | Loss_intent: 1.1219 | Loss_slots: 0.3169 | Spent: 5.0 secs | LR: 0.000370
INFO:tensorflow:Step 6950 | Loss: 1.4995 | Loss_intent: 1.0886 | Loss_slots: 0.4109 | Spent: 2.9 secs | LR: 0.000377
INFO:tensorflow:Step 7000 | Loss: 1.4521 | Loss_intent: 1.0893 | Loss_slots: 0.3628 | Spent: 2.9 secs | LR: 0.000384
INFO:tensorflow:Step 7050 | Loss: 1.3369 | Loss_intent: 1.0894 | Loss_slots: 0.2475 | Spent: 2.9 secs | LR: 0.000391
INFO:tensorflow:Step 7100 | Loss: 1.5341 | Loss_intent: 1.0960 | Loss_slots: 0.4381 | Spent: 2.9 secs | LR: 0.000398
INFO:tensorflow:Step 7150 | Loss: 1.5066 | Loss_intent: 1.0917 | Loss_slots: 0.4149 | Spent: 2.8 secs | LR: 0.000405
Reading ../data/atis.test.w-intent.iob
INFO:tensorflow:Slot F1: 0.940, Intent Acc: 0.971
INFO:tensorflow:Best Slot F1: 0.950, Intent Acc: 0.970
Reading ../data/atis.train.w-intent.iob
INFO:tensorflow:Step 7200 | Loss: 1.5160 | Loss_intent: 1.0849 | Loss_slots: 0.4312 | Spent: 5.0 secs | LR: 0.000412
INFO:tensorflow:Step 7250 | Loss: 1.4653 | Loss_intent: 1.1367 | Loss_slots: 0.3286 | Spent: 2.9 secs | LR: 0.000419
INFO:tensorflow:Step 7300 | Loss: 1.4320 | Loss_intent: 1.0818 | Loss_slots: 0.3502 | Spent: 2.9 secs | LR: 0.000427
INFO:tensorflow:Step 7350 | Loss: 1.5053 | Loss_intent: 1.1072 | Loss_slots: 0.3981 | Spent: 2.9 secs | LR: 0.000434
INFO:tensorflow:Step 7400 | Loss: 1.4314 | Loss_intent: 1.1088 | Loss_slots: 0.3226 | Spent: 2.9 secs | LR: 0.000441
INFO:tensorflow:Step 7450 | Loss: 1.4696 | Loss_intent: 1.0857 | Loss_slots: 0.3839 | Spent: 2.9 secs | LR: 0.000448
Reading ../data/atis.test.w-intent.iob
INFO:tensorflow:Slot F1: 0.945, Intent Acc: 0.972
INFO:tensorflow:
                                          precision    recall  f1-score   support

                             atis_flight      0.990     0.984     0.987       632
                            atis_airfare      0.979     0.979     0.979        48
                     atis_ground_service      0.973     1.000     0.986        36
                            atis_airline      0.974     1.000     0.987        38
                       atis_abbreviation      1.000     0.970     0.985        33
                           atis_aircraft      0.692     1.000     0.818         9
                        atis_flight_time      0.500     1.000     0.667         1
                           atis_quantity      0.429     1.000     0.600         3
                atis_flight#atis_airfare      0.889     0.667     0.762        12
                            atis_airport      0.857     1.000     0.923        18
                           atis_distance      1.000     1.000     1.000        10
                               atis_city      1.000     0.667     0.800         6
                        atis_ground_fare      1.000     0.857     0.923         7
                           atis_capacity      1.000     1.000     1.000        21
                          atis_flight_no      1.000     1.000     1.000         8
                               atis_meal      0.833     0.833     0.833         6
                        atis_restriction      0.000     0.000     0.000         0
             atis_airline#atis_flight_no      0.000     0.000     0.000         0
    atis_ground_service#atis_ground_fare      0.000     0.000     0.000         0
           atis_airfare#atis_flight_time      0.000     0.000     0.000         0
                           atis_cheapest      0.000     0.000     0.000         0
atis_aircraft#atis_flight#atis_flight_no      0.000     0.000     0.000         0

                               micro avg      0.972     0.977     0.975       888
                               macro avg      0.642     0.680     0.648       888
                            weighted avg      0.979     0.977     0.977       888

INFO:tensorflow:
                              precision    recall  f1-score   support

                           O      0.000     0.000     0.000       0.0
           B-toloc.city_name      0.979     0.993     0.986     716.0
         B-fromloc.city_name      0.975     0.997     0.986     704.0
           I-toloc.city_name      0.981     0.989     0.985     265.0
      B-depart_date.day_name      0.972     0.986     0.979     212.0
              B-airline_name      0.981     1.000     0.990     101.0
         I-fromloc.city_name      0.962     0.994     0.978     177.0
 B-depart_time.period_of_day      1.000     0.877     0.934     130.0
              I-airline_name      0.985     1.000     0.992      65.0
    B-depart_date.day_number      0.981     0.945     0.963      55.0
    B-depart_date.month_name      0.982     0.964     0.973      56.0
          B-depart_time.time      0.833     0.965     0.894      57.0
                B-round_trip      1.000     0.986     0.993      73.0
             B-cost_relative      1.000     0.973     0.986      37.0
                I-round_trip      1.000     1.000     1.000      71.0
                B-flight_mod      0.828     1.000     0.906      24.0
 B-depart_time.time_relative      0.984     0.969     0.977      65.0
          I-depart_time.time      0.864     0.981     0.919      52.0
         B-stoploc.city_name      1.000     0.950     0.974      20.0
                 B-city_name      0.900     0.632     0.742      57.0
                B-class_type      0.960     1.000     0.980      24.0
          B-arrive_time.time      0.943     0.971     0.957      34.0
 B-arrive_time.time_relative      0.861     1.000     0.925      31.0
                I-class_type      1.000     1.000     1.000      17.0
               B-flight_stop      1.000     1.000     1.000      21.0
          I-arrive_time.time      0.944     0.971     0.958      35.0
              B-airline_code      0.967     0.853     0.906      34.0
    I-depart_date.day_number      1.000     0.867     0.929      15.0
      I-fromloc.airport_name      0.417     1.000     0.588      15.0
      B-fromloc.airport_name      0.458     0.917     0.611      12.0
      B-arrive_date.day_name      0.667     0.727     0.696      11.0
          B-toloc.state_code      1.000     1.000     1.000      18.0
B-depart_date.today_relative      1.000     0.889     0.941       9.0
             B-flight_number      0.440     1.000     0.611      11.0
 B-depart_date.date_relative      0.895     1.000     0.944      17.0
          B-toloc.state_name      0.885     0.821     0.852      28.0
           B-fare_basis_code      0.900     0.529     0.667      17.0
               B-flight_time      0.500     1.000     0.667       1.0
                        B-or      1.000     1.000     1.000       3.0
 B-arrive_time.period_of_day      0.667     1.000     0.800       6.0
          B-meal_description      1.000     0.900     0.947      10.0
             I-cost_relative      1.000     0.667     0.800       3.0
              I-airport_name      0.900     0.310     0.462      29.0
               B-fare_amount      0.500     0.500     0.500       2.0
               I-fare_amount      1.000     1.000     1.000       2.0
                 I-city_name      0.783     0.600     0.679      30.0
        I-toloc.airport_name      1.000     1.000     1.000       3.0
            B-transport_type      1.000     1.000     1.000      10.0
    B-arrive_date.month_name      0.714     0.833     0.769       6.0
    B-arrive_date.day_number      0.625     0.833     0.714       6.0
         I-stoploc.city_name      1.000     0.900     0.947      10.0
                      B-meal      1.000     1.000     1.000      16.0
        B-fromloc.state_code      1.000     1.000     1.000      23.0
    B-depart_time.period_mod      0.833     1.000     0.909       5.0
                   B-connect      1.000     1.000     1.000       6.0
               B-flight_days      1.000     1.000     1.000      10.0
        B-toloc.airport_name      1.000     0.667     0.800       3.0
        B-fromloc.state_name      0.739     1.000     0.850      17.0
              B-airport_name      0.778     0.333     0.467      21.0
                   B-economy      1.000     1.000     1.000       6.0
               I-flight_time      0.500     1.000     0.667       1.0
             B-aircraft_code      1.000     0.788     0.881      33.0
                       B-mod      1.000     0.500     0.667       2.0
              B-airport_code      0.667     0.444     0.533       9.0
    B-depart_time.start_time      0.500     0.667     0.571       3.0
      B-depart_time.end_time      1.000     0.333     0.500       3.0
          B-depart_date.year      1.000     0.667     0.800       3.0
            I-transport_type      0.000     0.000     0.000       1.0
          B-restriction_code      0.667     1.000     0.800       4.0
    B-arrive_time.start_time      0.889     1.000     0.941       8.0
        B-toloc.airport_code      1.000     0.500     0.667       4.0
      B-arrive_time.end_time      0.889     1.000     0.941       8.0
      I-arrive_time.end_time      0.875     0.875     0.875       8.0
      I-depart_time.end_time      1.000     0.333     0.500       3.0
               I-flight_stop      0.000     0.000     0.000       0.0
      B-fromloc.airport_code      1.000     1.000     1.000       5.0
          I-restriction_code      0.273     1.000     0.429       3.0
    I-depart_time.start_time      1.000     1.000     1.000       1.0
          I-toloc.state_name      1.000     1.000     1.000       1.0
I-depart_date.today_relative      0.000     0.000     0.000       0.0
 B-arrive_date.date_relative      1.000     1.000     1.000       2.0
                I-flight_mod      0.000     0.000     0.000       6.0
                   I-economy      0.000     0.000     0.000       0.0
 B-return_date.date_relative      0.500     0.333     0.400       3.0
        I-fromloc.state_name      1.000     1.000     1.000       1.0
                B-state_code      1.000     1.000     1.000       1.0
    I-arrive_time.start_time      1.000     1.000     1.000       1.0
    I-arrive_date.day_number      0.000     0.000     0.000       0.0
                 B-meal_code      0.000     0.000     0.000       1.0
 I-depart_time.period_of_day      1.000     1.000     1.000       1.0
                  B-day_name      0.000     0.000     0.000       2.0
             B-period_of_day      1.000     0.500     0.667       4.0
        B-stoploc.state_code      0.000     0.000     0.000       0.0
    B-return_date.month_name      0.000     0.000     0.000       0.0
    B-return_date.day_number      0.000     0.000     0.000       0.0
    B-arrive_time.period_mod      0.000     0.000     0.000       0.0
                 I-meal_code      0.000     0.000     0.000       0.0
        B-toloc.country_name      0.000     0.000     0.000       1.0
                 B-days_code      1.000     1.000     1.000       1.0
 I-arrive_time.period_of_day      0.000     0.000     0.000       0.0
            I-today_relative      0.000     0.000     0.000       0.0
 B-return_time.period_of_day      0.000     0.000     0.000       0.0
                      B-time      0.000     0.000     0.000       0.0
           I-fare_basis_code      0.000     0.000     0.000       0.0
 I-arrive_time.time_relative      0.000     0.000     0.000       4.0
 I-depart_time.time_relative      0.000     0.000     0.000       1.0
            B-today_relative      0.000     0.000     0.000       0.0
                B-state_name      0.000     0.000     0.000       9.0
B-arrive_date.today_relative      0.000     0.000     0.000       0.0
    B-return_time.period_mod      0.000     0.000     0.000       0.0
                B-month_name      0.000     0.000     0.000       0.0
                B-day_number      0.000     0.000     0.000       0.0
 I-return_date.date_relative      0.000     0.000     0.000       3.0
I-return_date.today_relative      0.000     0.000     0.000       0.0
      B-stoploc.airport_name      0.000     0.000     0.000       0.0
             B-time_relative      0.000     0.000     0.000       0.0
                      I-time      0.000     0.000     0.000       0.0
    I-return_date.day_number      0.000     0.000     0.000       0.0
          I-meal_description      0.000     0.000     0.000       0.0
B-return_date.today_relative      0.000     0.000     0.000       0.0
      B-return_date.day_name      0.000     0.000     0.000       2.0

                   micro avg      0.944     0.945     0.945    3657.0
                   macro avg      0.614     0.605     0.594    3657.0
                weighted avg      0.947     0.945     0.941    3657.0

INFO:tensorflow:Best Slot F1: 0.950, Intent Acc: 0.970
Reading ../data/atis.train.w-intent.iob
INFO:tensorflow:Step 7500 | Loss: 1.4239 | Loss_intent: 1.0859 | Loss_slots: 0.3380 | Spent: 5.1 secs | LR: 0.000445
INFO:tensorflow:Step 7550 | Loss: 1.6293 | Loss_intent: 1.2646 | Loss_slots: 0.3647 | Spent: 3.0 secs | LR: 0.000438
INFO:tensorflow:Step 7600 | Loss: 1.5355 | Loss_intent: 1.0927 | Loss_slots: 0.4428 | Spent: 3.0 secs | LR: 0.000431
INFO:tensorflow:Step 7650 | Loss: 1.5196 | Loss_intent: 1.2707 | Loss_slots: 0.2489 | Spent: 2.9 secs | LR: 0.000424
INFO:tensorflow:Step 7700 | Loss: 1.4385 | Loss_intent: 1.0896 | Loss_slots: 0.3489 | Spent: 2.9 secs | LR: 0.000417
INFO:tensorflow:Step 7750 | Loss: 1.4738 | Loss_intent: 1.0885 | Loss_slots: 0.3854 | Spent: 2.9 secs | LR: 0.000410
Reading ../data/atis.test.w-intent.iob
INFO:tensorflow:Slot F1: 0.940, Intent Acc: 0.944
INFO:tensorflow:Best Slot F1: 0.950, Intent Acc: 0.970
Reading ../data/atis.train.w-intent.iob
INFO:tensorflow:Step 7800 | Loss: 1.4695 | Loss_intent: 1.0911 | Loss_slots: 0.3784 | Spent: 5.0 secs | LR: 0.000403
INFO:tensorflow:Step 7850 | Loss: 1.5171 | Loss_intent: 1.0872 | Loss_slots: 0.4299 | Spent: 2.9 secs | LR: 0.000396
INFO:tensorflow:Step 7900 | Loss: 1.5047 | Loss_intent: 1.1049 | Loss_slots: 0.3998 | Spent: 2.9 secs | LR: 0.000389
INFO:tensorflow:Step 7950 | Loss: 1.4076 | Loss_intent: 1.0900 | Loss_slots: 0.3176 | Spent: 2.9 secs | LR: 0.000382
INFO:tensorflow:Step 8000 | Loss: 1.3545 | Loss_intent: 1.0876 | Loss_slots: 0.2669 | Spent: 2.9 secs | LR: 0.000375
INFO:tensorflow:Step 8050 | Loss: 1.3811 | Loss_intent: 1.0842 | Loss_slots: 0.2969 | Spent: 2.9 secs | LR: 0.000368
INFO:tensorflow:Step 8100 | Loss: 1.4021 | Loss_intent: 1.0878 | Loss_slots: 0.3143 | Spent: 2.9 secs | LR: 0.000361
Reading ../data/atis.test.w-intent.iob
INFO:tensorflow:Slot F1: 0.946, Intent Acc: 0.972
INFO:tensorflow:Best Slot F1: 0.950, Intent Acc: 0.970
Reading ../data/atis.train.w-intent.iob
INFO:tensorflow:Step 8150 | Loss: 1.4526 | Loss_intent: 1.0945 | Loss_slots: 0.3580 | Spent: 5.1 secs | LR: 0.000354
INFO:tensorflow:Step 8200 | Loss: 1.3413 | Loss_intent: 1.0865 | Loss_slots: 0.2548 | Spent: 2.9 secs | LR: 0.000347
INFO:tensorflow:Step 8250 | Loss: 1.3864 | Loss_intent: 1.1126 | Loss_slots: 0.2738 | Spent: 3.0 secs | LR: 0.000340
INFO:tensorflow:Step 8300 | Loss: 1.4922 | Loss_intent: 1.0817 | Loss_slots: 0.4104 | Spent: 2.9 secs | LR: 0.000333
INFO:tensorflow:Step 8350 | Loss: 1.4712 | Loss_intent: 1.1749 | Loss_slots: 0.2963 | Spent: 2.9 secs | LR: 0.000326
INFO:tensorflow:Step 8400 | Loss: 1.4259 | Loss_intent: 1.0823 | Loss_slots: 0.3436 | Spent: 2.9 secs | LR: 0.000319
Reading ../data/atis.test.w-intent.iob
INFO:tensorflow:Slot F1: 0.952, Intent Acc: 0.974
INFO:tensorflow:Best Slot F1: 0.952, Intent Acc: 0.974
Reading ../data/atis.train.w-intent.iob
INFO:tensorflow:Step 8450 | Loss: 1.3611 | Loss_intent: 1.0839 | Loss_slots: 0.2772 | Spent: 5.1 secs | LR: 0.000312
INFO:tensorflow:Step 8500 | Loss: 1.4335 | Loss_intent: 1.0825 | Loss_slots: 0.3510 | Spent: 2.9 secs | LR: 0.000305
INFO:tensorflow:Step 8550 | Loss: 1.4350 | Loss_intent: 1.0879 | Loss_slots: 0.3471 | Spent: 2.9 secs | LR: 0.000298
INFO:tensorflow:Step 8600 | Loss: 1.4848 | Loss_intent: 1.0854 | Loss_slots: 0.3994 | Spent: 2.9 secs | LR: 0.000291
INFO:tensorflow:Step 8650 | Loss: 1.4257 | Loss_intent: 1.0814 | Loss_slots: 0.3443 | Spent: 2.9 secs | LR: 0.000284
INFO:tensorflow:Step 8700 | Loss: 1.4050 | Loss_intent: 1.0840 | Loss_slots: 0.3209 | Spent: 2.9 secs | LR: 0.000277
Reading ../data/atis.test.w-intent.iob
INFO:tensorflow:Slot F1: 0.951, Intent Acc: 0.971
INFO:tensorflow:Best Slot F1: 0.952, Intent Acc: 0.974
Reading ../data/atis.train.w-intent.iob
INFO:tensorflow:Step 8750 | Loss: 1.3803 | Loss_intent: 1.0841 | Loss_slots: 0.2962 | Spent: 5.0 secs | LR: 0.000270
INFO:tensorflow:Step 8800 | Loss: 1.4618 | Loss_intent: 1.0853 | Loss_slots: 0.3764 | Spent: 2.9 secs | LR: 0.000263
INFO:tensorflow:Step 8850 | Loss: 1.4524 | Loss_intent: 1.0827 | Loss_slots: 0.3697 | Spent: 2.9 secs | LR: 0.000256
INFO:tensorflow:Step 8900 | Loss: 1.4433 | Loss_intent: 1.0812 | Loss_slots: 0.3621 | Spent: 2.9 secs | LR: 0.000248
INFO:tensorflow:Step 8950 | Loss: 1.4756 | Loss_intent: 1.0824 | Loss_slots: 0.3932 | Spent: 2.9 secs | LR: 0.000241
INFO:tensorflow:Step 9000 | Loss: 1.4921 | Loss_intent: 1.0862 | Loss_slots: 0.4058 | Spent: 2.9 secs | LR: 0.000234
Reading ../data/atis.test.w-intent.iob
INFO:tensorflow:Slot F1: 0.952, Intent Acc: 0.972
INFO:tensorflow:Best Slot F1: 0.952, Intent Acc: 0.972
Reading ../data/atis.train.w-intent.iob
INFO:tensorflow:Step 9050 | Loss: 1.3400 | Loss_intent: 1.0804 | Loss_slots: 0.2595 | Spent: 5.0 secs | LR: 0.000227
INFO:tensorflow:Step 9100 | Loss: 1.3841 | Loss_intent: 1.0838 | Loss_slots: 0.3003 | Spent: 2.8 secs | LR: 0.000220
INFO:tensorflow:Step 9150 | Loss: 1.5242 | Loss_intent: 1.0926 | Loss_slots: 0.4315 | Spent: 2.9 secs | LR: 0.000213
INFO:tensorflow:Step 9200 | Loss: 1.4498 | Loss_intent: 1.0832 | Loss_slots: 0.3666 | Spent: 2.8 secs | LR: 0.000206
INFO:tensorflow:Step 9250 | Loss: 1.4878 | Loss_intent: 1.0880 | Loss_slots: 0.3997 | Spent: 2.9 secs | LR: 0.000199
INFO:tensorflow:Step 9300 | Loss: 1.4080 | Loss_intent: 1.0812 | Loss_slots: 0.3268 | Spent: 2.8 secs | LR: 0.000192
INFO:tensorflow:Step 9350 | Loss: 1.4222 | Loss_intent: 1.0816 | Loss_slots: 0.3406 | Spent: 2.9 secs | LR: 0.000185
Reading ../data/atis.test.w-intent.iob
INFO:tensorflow:Slot F1: 0.952, Intent Acc: 0.972
INFO:tensorflow:Best Slot F1: 0.952, Intent Acc: 0.972
Reading ../data/atis.train.w-intent.iob
INFO:tensorflow:Step 9400 | Loss: 1.4781 | Loss_intent: 1.0836 | Loss_slots: 0.3946 | Spent: 5.5 secs | LR: 0.000178
INFO:tensorflow:Step 9450 | Loss: 1.4028 | Loss_intent: 1.0811 | Loss_slots: 0.3216 | Spent: 3.1 secs | LR: 0.000171
INFO:tensorflow:Step 9500 | Loss: 1.3783 | Loss_intent: 1.0817 | Loss_slots: 0.2966 | Spent: 3.1 secs | LR: 0.000164
INFO:tensorflow:Step 9550 | Loss: 1.4095 | Loss_intent: 1.0838 | Loss_slots: 0.3256 | Spent: 3.0 secs | LR: 0.000157
INFO:tensorflow:Step 9600 | Loss: 1.3961 | Loss_intent: 1.0813 | Loss_slots: 0.3148 | Spent: 2.9 secs | LR: 0.000150
INFO:tensorflow:Step 9650 | Loss: 1.4813 | Loss_intent: 1.0807 | Loss_slots: 0.4006 | Spent: 2.9 secs | LR: 0.000143
Reading ../data/atis.test.w-intent.iob
INFO:tensorflow:Slot F1: 0.954, Intent Acc: 0.972
INFO:tensorflow:Best Slot F1: 0.954, Intent Acc: 0.972
Reading ../data/atis.train.w-intent.iob
INFO:tensorflow:Step 9700 | Loss: 1.5152 | Loss_intent: 1.0809 | Loss_slots: 0.4343 | Spent: 5.0 secs | LR: 0.000136
INFO:tensorflow:Step 9750 | Loss: 1.4249 | Loss_intent: 1.0862 | Loss_slots: 0.3387 | Spent: 2.8 secs | LR: 0.000129
INFO:tensorflow:Step 9800 | Loss: 1.3693 | Loss_intent: 1.0807 | Loss_slots: 0.2887 | Spent: 2.9 secs | LR: 0.000122
INFO:tensorflow:Step 9850 | Loss: 1.4522 | Loss_intent: 1.0826 | Loss_slots: 0.3695 | Spent: 2.9 secs | LR: 0.000115
INFO:tensorflow:Step 9900 | Loss: 1.2516 | Loss_intent: 1.0817 | Loss_slots: 0.1698 | Spent: 2.9 secs | LR: 0.000108
INFO:tensorflow:Step 9950 | Loss: 1.4493 | Loss_intent: 1.0830 | Loss_slots: 0.3663 | Spent: 2.9 secs | LR: 0.000101
Reading ../data/atis.test.w-intent.iob
INFO:tensorflow:Slot F1: 0.953, Intent Acc: 0.972
INFO:tensorflow:
                                          precision    recall  f1-score   support

                             atis_flight      0.983     0.991     0.987       632
                            atis_airfare      0.979     0.979     0.979        48
                     atis_ground_service      1.000     1.000     1.000        36
                            atis_airline      1.000     1.000     1.000        38
                       atis_abbreviation      0.941     0.970     0.955        33
                           atis_aircraft      1.000     0.889     0.941         9
                        atis_flight_time      0.500     1.000     0.667         1
                           atis_quantity      0.429     1.000     0.600         3
                atis_flight#atis_airfare      0.833     0.417     0.556        12
                            atis_airport      0.900     1.000     0.947        18
                           atis_distance      1.000     1.000     1.000        10
                               atis_city      1.000     0.667     0.800         6
                        atis_ground_fare      1.000     0.857     0.923         7
                           atis_capacity      1.000     1.000     1.000        21
                          atis_flight_no      0.889     1.000     0.941         8
                               atis_meal      1.000     0.833     0.909         6
                        atis_restriction      0.000     0.000     0.000         0
             atis_airline#atis_flight_no      0.000     0.000     0.000         0
    atis_ground_service#atis_ground_fare      0.000     0.000     0.000         0
           atis_airfare#atis_flight_time      0.000     0.000     0.000         0
                           atis_cheapest      0.000     0.000     0.000         0
atis_aircraft#atis_flight#atis_flight_no      0.000     0.000     0.000         0

                               micro avg      0.972     0.977     0.975       888
                               macro avg      0.657     0.664     0.646       888
                            weighted avg      0.977     0.977     0.975       888

INFO:tensorflow:
                              precision    recall  f1-score   support

                           O      0.000     0.000     0.000       0.0
           B-toloc.city_name      0.974     0.996     0.985     716.0
         B-fromloc.city_name      0.989     0.997     0.993     704.0
           I-toloc.city_name      0.985     0.992     0.989     265.0
      B-depart_date.day_name      0.990     0.981     0.986     212.0
              B-airline_name      0.962     1.000     0.981     101.0
         I-fromloc.city_name      0.962     0.994     0.978     177.0
 B-depart_time.period_of_day      0.992     0.923     0.956     130.0
              I-airline_name      1.000     1.000     1.000      65.0
    B-depart_date.day_number      0.981     0.964     0.972      55.0
    B-depart_date.month_name      0.982     0.964     0.973      56.0
          B-depart_time.time      0.848     0.982     0.911      57.0
                B-round_trip      1.000     0.986     0.993      73.0
             B-cost_relative      1.000     0.973     0.986      37.0
                I-round_trip      1.000     1.000     1.000      71.0
                B-flight_mod      1.000     1.000     1.000      24.0
 B-depart_time.time_relative      0.969     0.969     0.969      65.0
          I-depart_time.time      0.981     0.981     0.981      52.0
         B-stoploc.city_name      1.000     0.950     0.974      20.0
                 B-city_name      0.900     0.632     0.742      57.0
                B-class_type      0.960     1.000     0.980      24.0
          B-arrive_time.time      0.971     0.971     0.971      34.0
 B-arrive_time.time_relative      0.857     0.968     0.909      31.0
                I-class_type      1.000     1.000     1.000      17.0
               B-flight_stop      1.000     1.000     1.000      21.0
          I-arrive_time.time      1.000     0.971     0.986      35.0
              B-airline_code      1.000     0.882     0.938      34.0
    I-depart_date.day_number      1.000     0.933     0.966      15.0
      I-fromloc.airport_name      0.405     1.000     0.577      15.0
      B-fromloc.airport_name      0.423     0.917     0.579      12.0
      B-arrive_date.day_name      0.688     1.000     0.815      11.0
          B-toloc.state_code      1.000     1.000     1.000      18.0
B-depart_date.today_relative      1.000     1.000     1.000       9.0
             B-flight_number      0.440     1.000     0.611      11.0
 B-depart_date.date_relative      0.944     1.000     0.971      17.0
          B-toloc.state_name      0.958     0.821     0.885      28.0
           B-fare_basis_code      0.917     0.647     0.759      17.0
               B-flight_time      1.000     1.000     1.000       1.0
                        B-or      1.000     1.000     1.000       3.0
 B-arrive_time.period_of_day      0.667     1.000     0.800       6.0
          B-meal_description      1.000     0.900     0.947      10.0
             I-cost_relative      1.000     0.667     0.800       3.0
              I-airport_name      0.909     0.345     0.500      29.0
               B-fare_amount      1.000     0.500     0.667       2.0
               I-fare_amount      1.000     1.000     1.000       2.0
                 I-city_name      0.947     0.600     0.735      30.0
        I-toloc.airport_name      1.000     1.000     1.000       3.0
            B-transport_type      1.000     1.000     1.000      10.0
    B-arrive_date.month_name      0.714     0.833     0.769       6.0
    B-arrive_date.day_number      0.714     0.833     0.769       6.0
         I-stoploc.city_name      1.000     0.900     0.947      10.0
                      B-meal      1.000     1.000     1.000      16.0
        B-fromloc.state_code      1.000     1.000     1.000      23.0
    B-depart_time.period_mod      0.833     1.000     0.909       5.0
                   B-connect      1.000     1.000     1.000       6.0
               B-flight_days      1.000     1.000     1.000      10.0
        B-toloc.airport_name      1.000     1.000     1.000       3.0
        B-fromloc.state_name      0.739     1.000     0.850      17.0
              B-airport_name      0.778     0.333     0.467      21.0
                   B-economy      1.000     1.000     1.000       6.0
               I-flight_time      1.000     1.000     1.000       1.0
             B-aircraft_code      1.000     0.848     0.918      33.0
                       B-mod      1.000     0.500     0.667       2.0
              B-airport_code      0.833     0.556     0.667       9.0
    B-depart_time.start_time      0.400     0.667     0.500       3.0
      B-depart_time.end_time      0.667     0.667     0.667       3.0
          B-depart_date.year      1.000     0.667     0.800       3.0
            I-transport_type      1.000     1.000     1.000       1.0
          B-restriction_code      1.000     1.000     1.000       4.0
    B-arrive_time.start_time      0.889     1.000     0.941       8.0
        B-toloc.airport_code      1.000     0.500     0.667       4.0
      B-arrive_time.end_time      0.889     1.000     0.941       8.0
      I-arrive_time.end_time      0.889     1.000     0.941       8.0
      I-depart_time.end_time      0.667     0.667     0.667       3.0
               I-flight_stop      0.000     0.000     0.000       0.0
      B-fromloc.airport_code      1.000     1.000     1.000       5.0
          I-restriction_code      0.300     1.000     0.462       3.0
    I-depart_time.start_time      1.000     1.000     1.000       1.0
          I-toloc.state_name      1.000     1.000     1.000       1.0
I-depart_date.today_relative      0.000     0.000     0.000       0.0
 B-arrive_date.date_relative      1.000     1.000     1.000       2.0
                I-flight_mod      1.000     0.167     0.286       6.0
                   I-economy      0.000     0.000     0.000       0.0
 B-return_date.date_relative      0.667     0.667     0.667       3.0
        I-fromloc.state_name      1.000     1.000     1.000       1.0
                B-state_code      1.000     1.000     1.000       1.0
    I-arrive_time.start_time      1.000     1.000     1.000       1.0
    I-arrive_date.day_number      0.000     0.000     0.000       0.0
                 B-meal_code      0.000     0.000     0.000       1.0
 I-depart_time.period_of_day      0.000     0.000     0.000       1.0
                  B-day_name      1.000     0.500     0.667       2.0
             B-period_of_day      1.000     0.500     0.667       4.0
        B-stoploc.state_code      0.000     0.000     0.000       0.0
    B-return_date.month_name      0.000     0.000     0.000       0.0
    B-return_date.day_number      0.000     0.000     0.000       0.0
    B-arrive_time.period_mod      0.000     0.000     0.000       0.0
                 I-meal_code      0.000     0.000     0.000       0.0
        B-toloc.country_name      1.000     1.000     1.000       1.0
                 B-days_code      1.000     1.000     1.000       1.0
 I-arrive_time.period_of_day      0.000     0.000     0.000       0.0
            I-today_relative      0.000     0.000     0.000       0.0
 B-return_time.period_of_day      0.000     0.000     0.000       0.0
                      B-time      0.000     0.000     0.000       0.0
           I-fare_basis_code      0.000     0.000     0.000       0.0
 I-arrive_time.time_relative      0.000     0.000     0.000       4.0
 I-depart_time.time_relative      0.000     0.000     0.000       1.0
            B-today_relative      0.000     0.000     0.000       0.0
                B-state_name      0.000     0.000     0.000       9.0
B-arrive_date.today_relative      0.000     0.000     0.000       0.0
    B-return_time.period_mod      0.000     0.000     0.000       0.0
                B-month_name      0.000     0.000     0.000       0.0
                B-day_number      0.000     0.000     0.000       0.0
 I-return_date.date_relative      1.000     0.333     0.500       3.0
I-return_date.today_relative      0.000     0.000     0.000       0.0
      B-stoploc.airport_name      0.000     0.000     0.000       0.0
             B-time_relative      0.000     0.000     0.000       0.0
                      I-time      0.000     0.000     0.000       0.0
    I-return_date.day_number      0.000     0.000     0.000       0.0
          I-meal_description      0.000     0.000     0.000       0.0
B-return_date.today_relative      0.000     0.000     0.000       0.0
      B-return_date.day_name      0.000     0.000     0.000       2.0

                   micro avg      0.952     0.954     0.953    3657.0
                   macro avg      0.666     0.641     0.637    3657.0
                weighted avg      0.960     0.954     0.951    3657.0

INFO:tensorflow:Best Slot F1: 0.954, Intent Acc: 0.972
Reading ../data/atis.train.w-intent.iob
INFO:tensorflow:Step 10000 | Loss: 1.3375 | Loss_intent: 1.0813 | Loss_slots: 0.2563 | Spent: 5.1 secs | LR: 0.000103
INFO:tensorflow:Step 10050 | Loss: 1.3522 | Loss_intent: 1.0812 | Loss_slots: 0.2711 | Spent: 2.9 secs | LR: 0.000107
INFO:tensorflow:Step 10100 | Loss: 1.2997 | Loss_intent: 1.0809 | Loss_slots: 0.2188 | Spent: 2.9 secs | LR: 0.000110
INFO:tensorflow:Step 10150 | Loss: 1.4426 | Loss_intent: 1.0879 | Loss_slots: 0.3547 | Spent: 2.9 secs | LR: 0.000114
INFO:tensorflow:Step 10200 | Loss: 1.4605 | Loss_intent: 1.0858 | Loss_slots: 0.3746 | Spent: 2.9 secs | LR: 0.000117
INFO:tensorflow:Step 10250 | Loss: 1.3621 | Loss_intent: 1.0820 | Loss_slots: 0.2800 | Spent: 2.9 secs | LR: 0.000121
Reading ../data/atis.test.w-intent.iob
INFO:tensorflow:Slot F1: 0.957, Intent Acc: 0.972
INFO:tensorflow:Best Slot F1: 0.957, Intent Acc: 0.972
Reading ../data/atis.train.w-intent.iob
INFO:tensorflow:Step 10300 | Loss: 1.5363 | Loss_intent: 1.0809 | Loss_slots: 0.4554 | Spent: 5.1 secs | LR: 0.000124
INFO:tensorflow:Step 10350 | Loss: 1.4903 | Loss_intent: 1.0821 | Loss_slots: 0.4082 | Spent: 2.9 secs | LR: 0.000128
INFO:tensorflow:Step 10400 | Loss: 1.2266 | Loss_intent: 1.0837 | Loss_slots: 0.1429 | Spent: 2.8 secs | LR: 0.000131
INFO:tensorflow:Step 10450 | Loss: 1.4292 | Loss_intent: 1.0837 | Loss_slots: 0.3455 | Spent: 2.9 secs | LR: 0.000135
INFO:tensorflow:Step 10500 | Loss: 1.3993 | Loss_intent: 1.0820 | Loss_slots: 0.3173 | Spent: 2.9 secs | LR: 0.000138
INFO:tensorflow:Step 10550 | Loss: 1.4216 | Loss_intent: 1.0841 | Loss_slots: 0.3376 | Spent: 2.9 secs | LR: 0.000142
INFO:tensorflow:Step 10600 | Loss: 1.3884 | Loss_intent: 1.0832 | Loss_slots: 0.3052 | Spent: 2.9 secs | LR: 0.000145
Reading ../data/atis.test.w-intent.iob
INFO:tensorflow:Slot F1: 0.955, Intent Acc: 0.971
INFO:tensorflow:Best Slot F1: 0.957, Intent Acc: 0.972
Reading ../data/atis.train.w-intent.iob
INFO:tensorflow:Step 10650 | Loss: 1.3176 | Loss_intent: 1.0901 | Loss_slots: 0.2275 | Spent: 5.0 secs | LR: 0.000149
INFO:tensorflow:Step 10700 | Loss: 1.4084 | Loss_intent: 1.0821 | Loss_slots: 0.3263 | Spent: 2.9 secs | LR: 0.000152
INFO:tensorflow:Step 10750 | Loss: 1.4860 | Loss_intent: 1.0885 | Loss_slots: 0.3976 | Spent: 3.0 secs | LR: 0.000156
INFO:tensorflow:Step 10800 | Loss: 1.3692 | Loss_intent: 1.0822 | Loss_slots: 0.2870 | Spent: 2.9 secs | LR: 0.000159
INFO:tensorflow:Step 10850 | Loss: 1.4918 | Loss_intent: 1.0818 | Loss_slots: 0.4100 | Spent: 2.8 secs | LR: 0.000163
INFO:tensorflow:Step 10900 | Loss: 1.3464 | Loss_intent: 1.0807 | Loss_slots: 0.2657 | Spent: 2.8 secs | LR: 0.000166
Reading ../data/atis.test.w-intent.iob
INFO:tensorflow:Slot F1: 0.953, Intent Acc: 0.974
INFO:tensorflow:Best Slot F1: 0.957, Intent Acc: 0.972
Reading ../data/atis.train.w-intent.iob
INFO:tensorflow:Step 10950 | Loss: 1.2808 | Loss_intent: 1.0913 | Loss_slots: 0.1894 | Spent: 5.0 secs | LR: 0.000170
INFO:tensorflow:Step 11000 | Loss: 1.4376 | Loss_intent: 1.0842 | Loss_slots: 0.3533 | Spent: 2.9 secs | LR: 0.000173
INFO:tensorflow:Step 11050 | Loss: 1.4070 | Loss_intent: 1.0807 | Loss_slots: 0.3264 | Spent: 2.8 secs | LR: 0.000177
INFO:tensorflow:Step 11100 | Loss: 1.4772 | Loss_intent: 1.0825 | Loss_slots: 0.3947 | Spent: 2.9 secs | LR: 0.000180
INFO:tensorflow:Step 11150 | Loss: 1.3868 | Loss_intent: 1.0844 | Loss_slots: 0.3024 | Spent: 2.8 secs | LR: 0.000184
INFO:tensorflow:Step 11200 | Loss: 1.4005 | Loss_intent: 1.0833 | Loss_slots: 0.3172 | Spent: 2.9 secs | LR: 0.000187
Reading ../data/atis.test.w-intent.iob
INFO:tensorflow:Slot F1: 0.951, Intent Acc: 0.973
INFO:tensorflow:Best Slot F1: 0.957, Intent Acc: 0.972
Reading ../data/atis.train.w-intent.iob
INFO:tensorflow:Step 11250 | Loss: 1.4200 | Loss_intent: 1.1014 | Loss_slots: 0.3186 | Spent: 5.2 secs | LR: 0.000191
INFO:tensorflow:Step 11300 | Loss: 1.4110 | Loss_intent: 1.0811 | Loss_slots: 0.3298 | Spent: 2.9 secs | LR: 0.000194
INFO:tensorflow:Step 11350 | Loss: 1.4429 | Loss_intent: 1.0808 | Loss_slots: 0.3621 | Spent: 2.9 secs | LR: 0.000198
INFO:tensorflow:Step 11400 | Loss: 1.6142 | Loss_intent: 1.2940 | Loss_slots: 0.3202 | Spent: 2.9 secs | LR: 0.000202
INFO:tensorflow:Step 11450 | Loss: 1.3753 | Loss_intent: 1.0887 | Loss_slots: 0.2866 | Spent: 2.9 secs | LR: 0.000205
INFO:tensorflow:Step 11500 | Loss: 1.3691 | Loss_intent: 1.0856 | Loss_slots: 0.2834 | Spent: 2.9 secs | LR: 0.000209
Reading ../data/atis.test.w-intent.iob
INFO:tensorflow:Slot F1: 0.953, Intent Acc: 0.974
INFO:tensorflow:Best Slot F1: 0.957, Intent Acc: 0.972
Reading ../data/atis.train.w-intent.iob
INFO:tensorflow:Step 11550 | Loss: 1.7158 | Loss_intent: 1.3072 | Loss_slots: 0.4087 | Spent: 5.0 secs | LR: 0.000212
INFO:tensorflow:Step 11600 | Loss: 1.3801 | Loss_intent: 1.0837 | Loss_slots: 0.2964 | Spent: 2.9 secs | LR: 0.000216
INFO:tensorflow:Step 11650 | Loss: 1.3268 | Loss_intent: 1.0839 | Loss_slots: 0.2430 | Spent: 2.9 secs | LR: 0.000219
INFO:tensorflow:Step 11700 | Loss: 1.3962 | Loss_intent: 1.0816 | Loss_slots: 0.3146 | Spent: 2.9 secs | LR: 0.000223
INFO:tensorflow:Step 11750 | Loss: 1.4175 | Loss_intent: 1.0901 | Loss_slots: 0.3274 | Spent: 2.9 secs | LR: 0.000226
INFO:tensorflow:Step 11800 | Loss: 1.4049 | Loss_intent: 1.0875 | Loss_slots: 0.3174 | Spent: 2.8 secs | LR: 0.000230
INFO:tensorflow:Step 11850 | Loss: 1.4367 | Loss_intent: 1.0889 | Loss_slots: 0.3478 | Spent: 2.8 secs | LR: 0.000233
Reading ../data/atis.test.w-intent.iob
INFO:tensorflow:Slot F1: 0.952, Intent Acc: 0.974
INFO:tensorflow:Best Slot F1: 0.957, Intent Acc: 0.972
Reading ../data/atis.train.w-intent.iob
INFO:tensorflow:Step 11900 | Loss: 1.4206 | Loss_intent: 1.0827 | Loss_slots: 0.3379 | Spent: 5.0 secs | LR: 0.000237
INFO:tensorflow:Step 11950 | Loss: 1.4495 | Loss_intent: 1.0841 | Loss_slots: 0.3654 | Spent: 2.9 secs | LR: 0.000240
INFO:tensorflow:Step 12000 | Loss: 1.3447 | Loss_intent: 1.0819 | Loss_slots: 0.2628 | Spent: 2.9 secs | LR: 0.000244
INFO:tensorflow:Step 12050 | Loss: 1.4885 | Loss_intent: 1.0812 | Loss_slots: 0.4073 | Spent: 2.9 secs | LR: 0.000247
INFO:tensorflow:Step 12100 | Loss: 1.4853 | Loss_intent: 1.0810 | Loss_slots: 0.4042 | Spent: 2.8 secs | LR: 0.000251
INFO:tensorflow:Step 12150 | Loss: 1.4359 | Loss_intent: 1.0812 | Loss_slots: 0.3547 | Spent: 2.8 secs | LR: 0.000254
Reading ../data/atis.test.w-intent.iob
INFO:tensorflow:Slot F1: 0.952, Intent Acc: 0.976
INFO:tensorflow:Best Slot F1: 0.957, Intent Acc: 0.972
Reading ../data/atis.train.w-intent.iob
INFO:tensorflow:Step 12200 | Loss: 1.4177 | Loss_intent: 1.0840 | Loss_slots: 0.3337 | Spent: 5.1 secs | LR: 0.000258
INFO:tensorflow:Step 12250 | Loss: 1.5390 | Loss_intent: 1.0820 | Loss_slots: 0.4570 | Spent: 2.9 secs | LR: 0.000261
INFO:tensorflow:Step 12300 | Loss: 1.4923 | Loss_intent: 1.0813 | Loss_slots: 0.4110 | Spent: 2.9 secs | LR: 0.000265
INFO:tensorflow:Step 12350 | Loss: 1.4654 | Loss_intent: 1.0833 | Loss_slots: 0.3822 | Spent: 2.9 secs | LR: 0.000268
INFO:tensorflow:Step 12400 | Loss: 1.4417 | Loss_intent: 1.0845 | Loss_slots: 0.3572 | Spent: 2.8 secs | LR: 0.000272
INFO:tensorflow:Step 12450 | Loss: 1.4122 | Loss_intent: 1.0846 | Loss_slots: 0.3276 | Spent: 2.8 secs | LR: 0.000275
Reading ../data/atis.test.w-intent.iob
INFO:tensorflow:Slot F1: 0.951, Intent Acc: 0.974
INFO:tensorflow:
                                          precision    recall  f1-score   support

                             atis_flight      0.983     0.992     0.987       632
                            atis_airfare      0.979     0.979     0.979        48
                     atis_ground_service      0.973     1.000     0.986        36
                            atis_airline      1.000     1.000     1.000        38
                       atis_abbreviation      0.941     0.970     0.955        33
                           atis_aircraft      1.000     1.000     1.000         9
                        atis_flight_time      0.500     1.000     0.667         1
                           atis_quantity      0.429     1.000     0.600         3
                atis_flight#atis_airfare      0.833     0.417     0.556        12
                            atis_airport      1.000     1.000     1.000        18
                           atis_distance      1.000     1.000     1.000        10
                               atis_city      1.000     0.667     0.800         6
                        atis_ground_fare      1.000     0.857     0.923         7
                           atis_capacity      1.000     1.000     1.000        21
                          atis_flight_no      0.889     1.000     0.941         8
                               atis_meal      1.000     0.833     0.909         6
                        atis_restriction      0.000     0.000     0.000         0
             atis_airline#atis_flight_no      0.000     0.000     0.000         0
    atis_ground_service#atis_ground_fare      0.000     0.000     0.000         0
           atis_airfare#atis_flight_time      0.000     0.000     0.000         0
                           atis_cheapest      0.000     0.000     0.000         0
atis_aircraft#atis_flight#atis_flight_no      0.000     0.000     0.000         0

                               micro avg      0.974     0.980     0.977       888
                               macro avg      0.660     0.669     0.650       888
                            weighted avg      0.978     0.980     0.977       888

INFO:tensorflow:
                              precision    recall  f1-score   support

                           O      0.000     0.000     0.000       0.0
           B-toloc.city_name      0.975     0.994     0.985     716.0
         B-fromloc.city_name      0.982     0.997     0.989     704.0
           I-toloc.city_name      0.989     0.981     0.985     265.0
      B-depart_date.day_name      0.991     0.991     0.991     212.0
              B-airline_name      0.981     1.000     0.990     101.0
         I-fromloc.city_name      0.936     0.994     0.964     177.0
 B-depart_time.period_of_day      0.992     0.923     0.956     130.0
              I-airline_name      0.985     1.000     0.992      65.0
    B-depart_date.day_number      0.981     0.964     0.972      55.0
    B-depart_date.month_name      0.982     0.964     0.973      56.0
          B-depart_time.time      0.848     0.982     0.911      57.0
                B-round_trip      1.000     0.986     0.993      73.0
             B-cost_relative      1.000     0.973     0.986      37.0
                I-round_trip      1.000     1.000     1.000      71.0
                B-flight_mod      1.000     0.958     0.979      24.0
 B-depart_time.time_relative      0.984     0.969     0.977      65.0
          I-depart_time.time      0.963     1.000     0.981      52.0
         B-stoploc.city_name      1.000     0.950     0.974      20.0
                 B-city_name      0.805     0.579     0.673      57.0
                B-class_type      0.923     1.000     0.960      24.0
          B-arrive_time.time      0.971     0.971     0.971      34.0
 B-arrive_time.time_relative      0.909     0.968     0.937      31.0
                I-class_type      1.000     1.000     1.000      17.0
               B-flight_stop      1.000     1.000     1.000      21.0
          I-arrive_time.time      0.971     0.971     0.971      35.0
              B-airline_code      1.000     0.941     0.970      34.0
    I-depart_date.day_number      1.000     0.933     0.966      15.0
      I-fromloc.airport_name      0.405     1.000     0.577      15.0
      B-fromloc.airport_name      0.423     0.917     0.579      12.0
      B-arrive_date.day_name      0.786     1.000     0.880      11.0
          B-toloc.state_code      1.000     1.000     1.000      18.0
B-depart_date.today_relative      1.000     0.889     0.941       9.0
             B-flight_number      0.524     1.000     0.688      11.0
 B-depart_date.date_relative      0.895     1.000     0.944      17.0
          B-toloc.state_name      0.885     0.821     0.852      28.0
           B-fare_basis_code      0.917     0.647     0.759      17.0
               B-flight_time      1.000     1.000     1.000       1.0
                        B-or      1.000     1.000     1.000       3.0
 B-arrive_time.period_of_day      0.750     1.000     0.857       6.0
          B-meal_description      1.000     0.900     0.947      10.0
             I-cost_relative      1.000     0.667     0.800       3.0
              I-airport_name      1.000     0.345     0.513      29.0
               B-fare_amount      0.500     0.500     0.500       2.0
               I-fare_amount      1.000     1.000     1.000       2.0
                 I-city_name      0.947     0.600     0.735      30.0
        I-toloc.airport_name      0.500     0.667     0.571       3.0
            B-transport_type      1.000     1.000     1.000      10.0
    B-arrive_date.month_name      0.714     0.833     0.769       6.0
    B-arrive_date.day_number      0.714     0.833     0.769       6.0
         I-stoploc.city_name      0.889     0.800     0.842      10.0
                      B-meal      1.000     1.000     1.000      16.0
        B-fromloc.state_code      1.000     1.000     1.000      23.0
    B-depart_time.period_mod      0.833     1.000     0.909       5.0
                   B-connect      1.000     1.000     1.000       6.0
               B-flight_days      1.000     1.000     1.000      10.0
        B-toloc.airport_name      1.000     1.000     1.000       3.0
        B-fromloc.state_name      0.773     1.000     0.872      17.0
              B-airport_name      0.857     0.286     0.429      21.0
                   B-economy      1.000     1.000     1.000       6.0
               I-flight_time      1.000     1.000     1.000       1.0
             B-aircraft_code      1.000     0.818     0.900      33.0
                       B-mod      1.000     0.500     0.667       2.0
              B-airport_code      0.800     0.444     0.571       9.0
    B-depart_time.start_time      0.286     0.667     0.400       3.0
      B-depart_time.end_time      1.000     0.667     0.800       3.0
          B-depart_date.year      1.000     1.000     1.000       3.0
            I-transport_type      1.000     1.000     1.000       1.0
          B-restriction_code      1.000     1.000     1.000       4.0
    B-arrive_time.start_time      0.889     1.000     0.941       8.0
        B-toloc.airport_code      1.000     0.500     0.667       4.0
      B-arrive_time.end_time      0.889     1.000     0.941       8.0
      I-arrive_time.end_time      0.875     0.875     0.875       8.0
      I-depart_time.end_time      1.000     0.667     0.800       3.0
               I-flight_stop      0.000     0.000     0.000       0.0
      B-fromloc.airport_code      1.000     1.000     1.000       5.0
          I-restriction_code      0.333     1.000     0.500       3.0
    I-depart_time.start_time      1.000     1.000     1.000       1.0
          I-toloc.state_name      1.000     1.000     1.000       1.0
I-depart_date.today_relative      0.000     0.000     0.000       0.0
 B-arrive_date.date_relative      1.000     1.000     1.000       2.0
                I-flight_mod      1.000     0.167     0.286       6.0
                   I-economy      0.000     0.000     0.000       0.0
 B-return_date.date_relative      1.000     0.333     0.500       3.0
        I-fromloc.state_name      1.000     1.000     1.000       1.0
                B-state_code      1.000     1.000     1.000       1.0
    I-arrive_time.start_time      1.000     1.000     1.000       1.0
    I-arrive_date.day_number      0.000     0.000     0.000       0.0
                 B-meal_code      0.000     0.000     0.000       1.0
 I-depart_time.period_of_day      1.000     1.000     1.000       1.0
                  B-day_name      1.000     0.500     0.667       2.0
             B-period_of_day      1.000     0.250     0.400       4.0
        B-stoploc.state_code      0.000     0.000     0.000       0.0
    B-return_date.month_name      0.000     0.000     0.000       0.0
    B-return_date.day_number      0.000     0.000     0.000       0.0
    B-arrive_time.period_mod      0.000     0.000     0.000       0.0
                 I-meal_code      0.000     0.000     0.000       0.0
        B-toloc.country_name      0.000     0.000     0.000       1.0
                 B-days_code      1.000     1.000     1.000       1.0
 I-arrive_time.period_of_day      0.000     0.000     0.000       0.0
            I-today_relative      0.000     0.000     0.000       0.0
 B-return_time.period_of_day      0.000     0.000     0.000       0.0
                      B-time      0.000     0.000     0.000       0.0
           I-fare_basis_code      0.000     0.000     0.000       0.0
 I-arrive_time.time_relative      1.000     0.500     0.667       4.0
 I-depart_time.time_relative      0.000     0.000     0.000       1.0
            B-today_relative      0.000     0.000     0.000       0.0
                B-state_name      0.000     0.000     0.000       9.0
B-arrive_date.today_relative      0.000     0.000     0.000       0.0
    B-return_time.period_mod      0.000     0.000     0.000       0.0
                B-month_name      0.000     0.000     0.000       0.0
                B-day_number      0.000     0.000     0.000       0.0
 I-return_date.date_relative      0.750     1.000     0.857       3.0
I-return_date.today_relative      0.000     0.000     0.000       0.0
      B-stoploc.airport_name      0.000     0.000     0.000       0.0
             B-time_relative      0.000     0.000     0.000       0.0
                      I-time      0.000     0.000     0.000       0.0
    I-return_date.day_number      0.000     0.000     0.000       0.0
          I-meal_description      0.000     0.000     0.000       0.0
B-return_date.today_relative      0.000     0.000     0.000       0.0
      B-return_date.day_name      0.000     0.000     0.000       2.0

                   micro avg      0.950     0.952     0.951    3657.0
                   macro avg      0.672     0.641     0.638    3657.0
                weighted avg      0.958     0.952     0.949    3657.0

INFO:tensorflow:Best Slot F1: 0.957, Intent Acc: 0.972
Reading ../data/atis.train.w-intent.iob
INFO:tensorflow:Step 12500 | Loss: 1.3995 | Loss_intent: 1.0852 | Loss_slots: 0.3144 | Spent: 5.0 secs | LR: 0.000271
INFO:tensorflow:Step 12550 | Loss: 1.4174 | Loss_intent: 1.1130 | Loss_slots: 0.3044 | Spent: 2.8 secs | LR: 0.000268
INFO:tensorflow:Step 12600 | Loss: 1.5131 | Loss_intent: 1.0913 | Loss_slots: 0.4218 | Spent: 2.9 secs | LR: 0.000264
INFO:tensorflow:Step 12650 | Loss: 1.4491 | Loss_intent: 1.0841 | Loss_slots: 0.3649 | Spent: 2.8 secs | LR: 0.000261
INFO:tensorflow:Step 12700 | Loss: 1.4258 | Loss_intent: 1.1025 | Loss_slots: 0.3233 | Spent: 2.8 secs | LR: 0.000257
INFO:tensorflow:Step 12750 | Loss: 1.4282 | Loss_intent: 1.0979 | Loss_slots: 0.3304 | Spent: 2.9 secs | LR: 0.000254
Reading ../data/atis.test.w-intent.iob
INFO:tensorflow:Slot F1: 0.951, Intent Acc: 0.968
INFO:tensorflow:Best Slot F1: 0.957, Intent Acc: 0.972
Reading ../data/atis.train.w-intent.iob
INFO:tensorflow:Step 12800 | Loss: 1.4141 | Loss_intent: 1.0967 | Loss_slots: 0.3173 | Spent: 5.0 secs | LR: 0.000250
INFO:tensorflow:Step 12850 | Loss: 1.4918 | Loss_intent: 1.0932 | Loss_slots: 0.3986 | Spent: 2.8 secs | LR: 0.000247
INFO:tensorflow:Step 12900 | Loss: 1.4135 | Loss_intent: 1.0986 | Loss_slots: 0.3149 | Spent: 3.0 secs | LR: 0.000243
INFO:tensorflow:Step 12950 | Loss: 1.4347 | Loss_intent: 1.1033 | Loss_slots: 0.3314 | Spent: 2.8 secs | LR: 0.000239
INFO:tensorflow:Step 13000 | Loss: 1.3178 | Loss_intent: 1.0819 | Loss_slots: 0.2359 | Spent: 2.8 secs | LR: 0.000236
INFO:tensorflow:Step 13050 | Loss: 1.4022 | Loss_intent: 1.0819 | Loss_slots: 0.3203 | Spent: 2.8 secs | LR: 0.000232
INFO:tensorflow:Step 13100 | Loss: 1.4203 | Loss_intent: 1.0812 | Loss_slots: 0.3391 | Spent: 2.8 secs | LR: 0.000229
Reading ../data/atis.test.w-intent.iob
INFO:tensorflow:Slot F1: 0.952, Intent Acc: 0.974
INFO:tensorflow:Best Slot F1: 0.957, Intent Acc: 0.972
Reading ../data/atis.train.w-intent.iob
INFO:tensorflow:Step 13150 | Loss: 1.3110 | Loss_intent: 1.0804 | Loss_slots: 0.2306 | Spent: 5.1 secs | LR: 0.000225
INFO:tensorflow:Step 13200 | Loss: 1.4557 | Loss_intent: 1.0847 | Loss_slots: 0.3710 | Spent: 2.9 secs | LR: 0.000222
INFO:tensorflow:Step 13250 | Loss: 1.4398 | Loss_intent: 1.0872 | Loss_slots: 0.3526 | Spent: 2.9 secs | LR: 0.000218
INFO:tensorflow:Step 13300 | Loss: 1.4442 | Loss_intent: 1.0871 | Loss_slots: 0.3571 | Spent: 2.9 secs | LR: 0.000215
INFO:tensorflow:Step 13350 | Loss: 1.3710 | Loss_intent: 1.0954 | Loss_slots: 0.2757 | Spent: 2.9 secs | LR: 0.000211
INFO:tensorflow:Step 13400 | Loss: 1.4107 | Loss_intent: 1.0806 | Loss_slots: 0.3301 | Spent: 2.8 secs | LR: 0.000208
Reading ../data/atis.test.w-intent.iob
INFO:tensorflow:Slot F1: 0.953, Intent Acc: 0.972
INFO:tensorflow:Best Slot F1: 0.957, Intent Acc: 0.972
Reading ../data/atis.train.w-intent.iob
INFO:tensorflow:Step 13450 | Loss: 1.4279 | Loss_intent: 1.0806 | Loss_slots: 0.3473 | Spent: 4.9 secs | LR: 0.000204
INFO:tensorflow:Step 13500 | Loss: 1.5142 | Loss_intent: 1.0838 | Loss_slots: 0.4304 | Spent: 2.8 secs | LR: 0.000201
INFO:tensorflow:Step 13550 | Loss: 1.3822 | Loss_intent: 1.0864 | Loss_slots: 0.2958 | Spent: 2.8 secs | LR: 0.000197
INFO:tensorflow:Step 13600 | Loss: 1.4321 | Loss_intent: 1.0810 | Loss_slots: 0.3511 | Spent: 2.8 secs | LR: 0.000194
INFO:tensorflow:Step 13650 | Loss: 1.2726 | Loss_intent: 1.0804 | Loss_slots: 0.1922 | Spent: 2.8 secs | LR: 0.000190
INFO:tensorflow:Step 13700 | Loss: 1.3997 | Loss_intent: 1.0842 | Loss_slots: 0.3155 | Spent: 2.8 secs | LR: 0.000187
Reading ../data/atis.test.w-intent.iob
INFO:tensorflow:Slot F1: 0.954, Intent Acc: 0.973
INFO:tensorflow:Best Slot F1: 0.957, Intent Acc: 0.972
Reading ../data/atis.train.w-intent.iob
INFO:tensorflow:Step 13750 | Loss: 1.4152 | Loss_intent: 1.0828 | Loss_slots: 0.3324 | Spent: 5.0 secs | LR: 0.000183
INFO:tensorflow:Step 13800 | Loss: 1.4260 | Loss_intent: 1.0816 | Loss_slots: 0.3444 | Spent: 2.8 secs | LR: 0.000180
INFO:tensorflow:Step 13850 | Loss: 1.2697 | Loss_intent: 1.0836 | Loss_slots: 0.1861 | Spent: 2.8 secs | LR: 0.000176
INFO:tensorflow:Step 13900 | Loss: 1.3874 | Loss_intent: 1.0835 | Loss_slots: 0.3039 | Spent: 2.9 secs | LR: 0.000173
INFO:tensorflow:Step 13950 | Loss: 1.4173 | Loss_intent: 1.0857 | Loss_slots: 0.3316 | Spent: 2.9 secs | LR: 0.000169
INFO:tensorflow:Step 14000 | Loss: 1.4430 | Loss_intent: 1.1068 | Loss_slots: 0.3362 | Spent: 2.8 secs | LR: 0.000166
Reading ../data/atis.test.w-intent.iob
INFO:tensorflow:Slot F1: 0.950, Intent Acc: 0.974
INFO:tensorflow:Best Slot F1: 0.957, Intent Acc: 0.972
Reading ../data/atis.train.w-intent.iob
INFO:tensorflow:Step 14050 | Loss: 1.4132 | Loss_intent: 1.0870 | Loss_slots: 0.3262 | Spent: 5.1 secs | LR: 0.000162
INFO:tensorflow:Step 14100 | Loss: 1.3509 | Loss_intent: 1.0820 | Loss_slots: 0.2688 | Spent: 2.9 secs | LR: 0.000159
INFO:tensorflow:Step 14150 | Loss: 1.4339 | Loss_intent: 1.0820 | Loss_slots: 0.3519 | Spent: 2.9 secs | LR: 0.000155
INFO:tensorflow:Step 14200 | Loss: 1.4630 | Loss_intent: 1.0807 | Loss_slots: 0.3822 | Spent: 3.3 secs | LR: 0.000152
INFO:tensorflow:Step 14250 | Loss: 1.4361 | Loss_intent: 1.0832 | Loss_slots: 0.3529 | Spent: 3.7 secs | LR: 0.000148
INFO:tensorflow:Step 14300 | Loss: 1.3210 | Loss_intent: 1.0926 | Loss_slots: 0.2284 | Spent: 3.2 secs | LR: 0.000145
INFO:tensorflow:Step 14350 | Loss: 1.4798 | Loss_intent: 1.0860 | Loss_slots: 0.3937 | Spent: 2.9 secs | LR: 0.000141
Reading ../data/atis.test.w-intent.iob
INFO:tensorflow:Slot F1: 0.952, Intent Acc: 0.973
INFO:tensorflow:Best Slot F1: 0.957, Intent Acc: 0.972
Reading ../data/atis.train.w-intent.iob
INFO:tensorflow:Step 14400 | Loss: 1.2812 | Loss_intent: 1.0827 | Loss_slots: 0.1985 | Spent: 5.0 secs | LR: 0.000138
INFO:tensorflow:Step 14450 | Loss: 1.4551 | Loss_intent: 1.0803 | Loss_slots: 0.3748 | Spent: 2.9 secs | LR: 0.000134
INFO:tensorflow:Step 14500 | Loss: 1.3288 | Loss_intent: 1.0846 | Loss_slots: 0.2441 | Spent: 3.0 secs | LR: 0.000131
INFO:tensorflow:Step 14550 | Loss: 1.3957 | Loss_intent: 1.0814 | Loss_slots: 0.3143 | Spent: 3.0 secs | LR: 0.000127
INFO:tensorflow:Step 14600 | Loss: 1.4318 | Loss_intent: 1.0810 | Loss_slots: 0.3509 | Spent: 2.9 secs | LR: 0.000123
INFO:tensorflow:Step 14650 | Loss: 1.4313 | Loss_intent: 1.0904 | Loss_slots: 0.3409 | Spent: 3.0 secs | LR: 0.000120
Reading ../data/atis.test.w-intent.iob
INFO:tensorflow:Slot F1: 0.954, Intent Acc: 0.974
INFO:tensorflow:Best Slot F1: 0.957, Intent Acc: 0.972
Reading ../data/atis.train.w-intent.iob
INFO:tensorflow:Step 14700 | Loss: 1.3615 | Loss_intent: 1.0820 | Loss_slots: 0.2794 | Spent: 5.1 secs | LR: 0.000116
INFO:tensorflow:Step 14750 | Loss: 1.3664 | Loss_intent: 1.0811 | Loss_slots: 0.2852 | Spent: 2.9 secs | LR: 0.000113
INFO:tensorflow:Step 14800 | Loss: 1.3354 | Loss_intent: 1.0801 | Loss_slots: 0.2553 | Spent: 2.8 secs | LR: 0.000109
INFO:tensorflow:Step 14850 | Loss: 1.3684 | Loss_intent: 1.0811 | Loss_slots: 0.2874 | Spent: 2.9 secs | LR: 0.000106
INFO:tensorflow:Step 14900 | Loss: 1.4389 | Loss_intent: 1.0814 | Loss_slots: 0.3575 | Spent: 3.0 secs | LR: 0.000102
INFO:tensorflow:Step 14950 | Loss: 1.5298 | Loss_intent: 1.0806 | Loss_slots: 0.4493 | Spent: 2.9 secs | LR: 0.000101
Reading ../data/atis.test.w-intent.iob
INFO:tensorflow:Slot F1: 0.955, Intent Acc: 0.973
INFO:tensorflow:
                                          precision    recall  f1-score   support

                             atis_flight      0.983     0.989     0.986       632
                            atis_airfare      0.979     0.979     0.979        48
                     atis_ground_service      1.000     1.000     1.000        36
                            atis_airline      0.974     1.000     0.987        38
                       atis_abbreviation      0.941     0.970     0.955        33
                           atis_aircraft      0.900     1.000     0.947         9
                        atis_flight_time      1.000     1.000     1.000         1
                           atis_quantity      0.429     1.000     0.600         3
                atis_flight#atis_airfare      0.833     0.417     0.556        12
                            atis_airport      0.947     1.000     0.973        18
                           atis_distance      1.000     1.000     1.000        10
                               atis_city      1.000     0.667     0.800         6
                        atis_ground_fare      1.000     1.000     1.000         7
                           atis_capacity      1.000     1.000     1.000        21
                          atis_flight_no      1.000     1.000     1.000         8
                               atis_meal      1.000     0.833     0.909         6
                        atis_restriction      0.000     0.000     0.000         0
             atis_airline#atis_flight_no      0.000     0.000     0.000         0
    atis_ground_service#atis_ground_fare      0.000     0.000     0.000         0
           atis_airfare#atis_flight_time      0.000     0.000     0.000         0
                           atis_cheapest      0.000     0.000     0.000         0
atis_aircraft#atis_flight#atis_flight_no      0.000     0.000     0.000         0

                               micro avg      0.973     0.979     0.976       888
                               macro avg      0.681     0.675     0.668       888
                            weighted avg      0.977     0.979     0.976       888

INFO:tensorflow:
                              precision    recall  f1-score   support

                           O      0.000     0.000     0.000       0.0
           B-toloc.city_name      0.977     0.994     0.985     716.0
         B-fromloc.city_name      0.987     0.997     0.992     704.0
           I-toloc.city_name      0.974     0.992     0.983     265.0
      B-depart_date.day_name      0.991     0.986     0.988     212.0
              B-airline_name      0.981     1.000     0.990     101.0
         I-fromloc.city_name      0.962     0.994     0.978     177.0
 B-depart_time.period_of_day      1.000     0.923     0.960     130.0
              I-airline_name      1.000     1.000     1.000      65.0
    B-depart_date.day_number      0.981     0.964     0.972      55.0
    B-depart_date.month_name      0.982     0.964     0.973      56.0
          B-depart_time.time      0.848     0.982     0.911      57.0
                B-round_trip      1.000     0.986     0.993      73.0
             B-cost_relative      1.000     0.973     0.986      37.0
                I-round_trip      1.000     1.000     1.000      71.0
                B-flight_mod      1.000     1.000     1.000      24.0
 B-depart_time.time_relative      0.984     0.969     0.977      65.0
          I-depart_time.time      0.963     1.000     0.981      52.0
         B-stoploc.city_name      1.000     0.950     0.974      20.0
                 B-city_name      0.881     0.649     0.747      57.0
                B-class_type      0.960     1.000     0.980      24.0
          B-arrive_time.time      0.971     0.971     0.971      34.0
 B-arrive_time.time_relative      0.833     0.968     0.896      31.0
                I-class_type      1.000     1.000     1.000      17.0
               B-flight_stop      1.000     1.000     1.000      21.0
          I-arrive_time.time      1.000     0.971     0.986      35.0
              B-airline_code      1.000     0.941     0.970      34.0
    I-depart_date.day_number      1.000     0.933     0.966      15.0
      I-fromloc.airport_name      0.417     1.000     0.588      15.0
      B-fromloc.airport_name      0.423     0.917     0.579      12.0
      B-arrive_date.day_name      0.733     1.000     0.846      11.0
          B-toloc.state_code      1.000     1.000     1.000      18.0
B-depart_date.today_relative      1.000     1.000     1.000       9.0
             B-flight_number      0.440     1.000     0.611      11.0
 B-depart_date.date_relative      0.944     1.000     0.971      17.0
          B-toloc.state_name      0.920     0.821     0.868      28.0
           B-fare_basis_code      0.944     1.000     0.971      17.0
               B-flight_time      1.000     1.000     1.000       1.0
                        B-or      1.000     1.000     1.000       3.0
 B-arrive_time.period_of_day      0.750     1.000     0.857       6.0
          B-meal_description      1.000     0.900     0.947      10.0
             I-cost_relative      1.000     0.667     0.800       3.0
              I-airport_name      0.917     0.379     0.537      29.0
               B-fare_amount      0.500     0.500     0.500       2.0
               I-fare_amount      1.000     1.000     1.000       2.0
                 I-city_name      1.000     0.500     0.667      30.0
        I-toloc.airport_name      1.000     1.000     1.000       3.0
            B-transport_type      1.000     1.000     1.000      10.0
    B-arrive_date.month_name      0.714     0.833     0.769       6.0
    B-arrive_date.day_number      0.714     0.833     0.769       6.0
         I-stoploc.city_name      1.000     0.900     0.947      10.0
                      B-meal      1.000     1.000     1.000      16.0
        B-fromloc.state_code      1.000     1.000     1.000      23.0
    B-depart_time.period_mod      0.833     1.000     0.909       5.0
                   B-connect      1.000     1.000     1.000       6.0
               B-flight_days      1.000     1.000     1.000      10.0
        B-toloc.airport_name      1.000     1.000     1.000       3.0
        B-fromloc.state_name      0.810     1.000     0.895      17.0
              B-airport_name      0.692     0.429     0.529      21.0
                   B-economy      1.000     1.000     1.000       6.0
               I-flight_time      1.000     1.000     1.000       1.0
             B-aircraft_code      1.000     0.788     0.881      33.0
                       B-mod      1.000     0.500     0.667       2.0
              B-airport_code      0.833     0.556     0.667       9.0
    B-depart_time.start_time      0.400     0.667     0.500       3.0
      B-depart_time.end_time      1.000     0.667     0.800       3.0
          B-depart_date.year      1.000     0.667     0.800       3.0
            I-transport_type      0.000     0.000     0.000       1.0
          B-restriction_code      1.000     1.000     1.000       4.0
    B-arrive_time.start_time      0.889     1.000     0.941       8.0
        B-toloc.airport_code      1.000     0.500     0.667       4.0
      B-arrive_time.end_time      0.889     1.000     0.941       8.0
      I-arrive_time.end_time      0.889     1.000     0.941       8.0
      I-depart_time.end_time      1.000     0.667     0.800       3.0
               I-flight_stop      0.000     0.000     0.000       0.0
      B-fromloc.airport_code      1.000     1.000     1.000       5.0
          I-restriction_code      1.000     1.000     1.000       3.0
    I-depart_time.start_time      1.000     1.000     1.000       1.0
          I-toloc.state_name      1.000     1.000     1.000       1.0
I-depart_date.today_relative      0.000     0.000     0.000       0.0
 B-arrive_date.date_relative      1.000     1.000     1.000       2.0
                I-flight_mod      1.000     0.167     0.286       6.0
                   I-economy      0.000     0.000     0.000       0.0
 B-return_date.date_relative      1.000     0.333     0.500       3.0
        I-fromloc.state_name      1.000     1.000     1.000       1.0
                B-state_code      1.000     1.000     1.000       1.0
    I-arrive_time.start_time      1.000     1.000     1.000       1.0
    I-arrive_date.day_number      0.000     0.000     0.000       0.0
                 B-meal_code      0.000     0.000     0.000       1.0
 I-depart_time.period_of_day      1.000     1.000     1.000       1.0
                  B-day_name      1.000     0.500     0.667       2.0
             B-period_of_day      1.000     0.250     0.400       4.0
        B-stoploc.state_code      0.000     0.000     0.000       0.0
    B-return_date.month_name      0.000     0.000     0.000       0.0
    B-return_date.day_number      0.000     0.000     0.000       0.0
    B-arrive_time.period_mod      0.000     0.000     0.000       0.0
                 I-meal_code      0.000     0.000     0.000       0.0
        B-toloc.country_name      1.000     1.000     1.000       1.0
                 B-days_code      1.000     1.000     1.000       1.0
 I-arrive_time.period_of_day      0.000     0.000     0.000       0.0
            I-today_relative      0.000     0.000     0.000       0.0
 B-return_time.period_of_day      0.000     0.000     0.000       0.0
                      B-time      0.000     0.000     0.000       0.0
           I-fare_basis_code      0.000     0.000     0.000       0.0
 I-arrive_time.time_relative      0.000     0.000     0.000       4.0
 I-depart_time.time_relative      0.000     0.000     0.000       1.0
            B-today_relative      0.000     0.000     0.000       0.0
                B-state_name      0.000     0.000     0.000       9.0
B-arrive_date.today_relative      0.000     0.000     0.000       0.0
    B-return_time.period_mod      0.000     0.000     0.000       0.0
                B-month_name      0.000     0.000     0.000       0.0
                B-day_number      0.000     0.000     0.000       0.0
 I-return_date.date_relative      0.667     0.667     0.667       3.0
I-return_date.today_relative      0.000     0.000     0.000       0.0
      B-stoploc.airport_name      0.000     0.000     0.000       0.0
             B-time_relative      0.000     0.000     0.000       0.0
                      I-time      0.000     0.000     0.000       0.0
    I-return_date.day_number      0.000     0.000     0.000       0.0
          I-meal_description      0.000     0.000     0.000       0.0
B-return_date.today_relative      0.000     0.000     0.000       0.0
      B-return_date.day_name      0.000     0.000     0.000       2.0

                   micro avg      0.954     0.955     0.955    3657.0
                   macro avg      0.674     0.642     0.643    3657.0
                weighted avg      0.961     0.955     0.953    3657.0

INFO:tensorflow:Best Slot F1: 0.957, Intent Acc: 0.972
Reading ../data/atis.train.w-intent.iob
INFO:tensorflow:Step 15000 | Loss: 1.3960 | Loss_intent: 1.0820 | Loss_slots: 0.3140 | Spent: 5.3 secs | LR: 0.000102
INFO:tensorflow:Step 15050 | Loss: 1.3315 | Loss_intent: 1.0815 | Loss_slots: 0.2501 | Spent: 3.1 secs | LR: 0.000104
INFO:tensorflow:Step 15100 | Loss: 1.3765 | Loss_intent: 1.0803 | Loss_slots: 0.2962 | Spent: 3.1 secs | LR: 0.000106
INFO:tensorflow:Step 15150 | Loss: 1.5083 | Loss_intent: 1.0824 | Loss_slots: 0.4258 | Spent: 3.0 secs | LR: 0.000108
INFO:tensorflow:Step 15200 | Loss: 1.4895 | Loss_intent: 1.0905 | Loss_slots: 0.3990 | Spent: 3.0 secs | LR: 0.000109
INFO:tensorflow:Step 15250 | Loss: 1.3936 | Loss_intent: 1.0813 | Loss_slots: 0.3123 | Spent: 3.0 secs | LR: 0.000111
Reading ../data/atis.test.w-intent.iob
INFO:tensorflow:Slot F1: 0.954, Intent Acc: 0.972
INFO:tensorflow:Best Slot F1: 0.957, Intent Acc: 0.972
Reading ../data/atis.train.w-intent.iob
INFO:tensorflow:Step 15300 | Loss: 1.3964 | Loss_intent: 1.0817 | Loss_slots: 0.3147 | Spent: 5.2 secs | LR: 0.000113
INFO:tensorflow:Step 15350 | Loss: 1.4042 | Loss_intent: 1.0801 | Loss_slots: 0.3241 | Spent: 2.9 secs | LR: 0.000115
INFO:tensorflow:Step 15400 | Loss: 1.5063 | Loss_intent: 1.0813 | Loss_slots: 0.4250 | Spent: 3.1 secs | LR: 0.000116
INFO:tensorflow:Step 15450 | Loss: 1.5378 | Loss_intent: 1.0825 | Loss_slots: 0.4553 | Spent: 3.1 secs | LR: 0.000118
INFO:tensorflow:Step 15500 | Loss: 1.3422 | Loss_intent: 1.0819 | Loss_slots: 0.2603 | Spent: 3.0 secs | LR: 0.000120
INFO:tensorflow:Step 15550 | Loss: 1.4014 | Loss_intent: 1.0857 | Loss_slots: 0.3157 | Spent: 3.1 secs | LR: 0.000122
Reading ../data/atis.test.w-intent.iob
INFO:tensorflow:Slot F1: 0.952, Intent Acc: 0.972
INFO:tensorflow:Best Slot F1: 0.957, Intent Acc: 0.972
Reading ../data/atis.train.w-intent.iob
INFO:tensorflow:Step 15600 | Loss: 1.3861 | Loss_intent: 1.0810 | Loss_slots: 0.3051 | Spent: 5.4 secs | LR: 0.000123
INFO:tensorflow:Step 15650 | Loss: 1.3418 | Loss_intent: 1.0820 | Loss_slots: 0.2598 | Spent: 3.1 secs | LR: 0.000125
INFO:tensorflow:Step 15700 | Loss: 1.3778 | Loss_intent: 1.0820 | Loss_slots: 0.2958 | Spent: 3.1 secs | LR: 0.000127
INFO:tensorflow:Step 15750 | Loss: 1.4430 | Loss_intent: 1.0848 | Loss_slots: 0.3582 | Spent: 3.1 secs | LR: 0.000129
INFO:tensorflow:Step 15800 | Loss: 1.3862 | Loss_intent: 1.0835 | Loss_slots: 0.3027 | Spent: 3.1 secs | LR: 0.000130
INFO:tensorflow:Step 15850 | Loss: 1.4865 | Loss_intent: 1.0815 | Loss_slots: 0.4050 | Spent: 3.1 secs | LR: 0.000132
INFO:tensorflow:Step 15900 | Loss: 1.3127 | Loss_intent: 1.0812 | Loss_slots: 0.2315 | Spent: 3.1 secs | LR: 0.000134
Reading ../data/atis.test.w-intent.iob
INFO:tensorflow:Slot F1: 0.955, Intent Acc: 0.973
INFO:tensorflow:Best Slot F1: 0.957, Intent Acc: 0.972
Reading ../data/atis.train.w-intent.iob
INFO:tensorflow:Step 15950 | Loss: 1.3937 | Loss_intent: 1.0810 | Loss_slots: 0.3128 | Spent: 5.4 secs | LR: 0.000136
INFO:tensorflow:Step 16000 | Loss: 1.2612 | Loss_intent: 1.0823 | Loss_slots: 0.1789 | Spent: 3.1 secs | LR: 0.000137
INFO:tensorflow:Step 16050 | Loss: 1.4411 | Loss_intent: 1.0843 | Loss_slots: 0.3568 | Spent: 3.1 secs | LR: 0.000139
INFO:tensorflow:Step 16100 | Loss: 1.5215 | Loss_intent: 1.0811 | Loss_slots: 0.4403 | Spent: 3.0 secs | LR: 0.000141
INFO:tensorflow:Step 16150 | Loss: 1.3259 | Loss_intent: 1.0814 | Loss_slots: 0.2444 | Spent: 3.0 secs | LR: 0.000143
INFO:tensorflow:Step 16200 | Loss: 1.3885 | Loss_intent: 1.0875 | Loss_slots: 0.3011 | Spent: 3.1 secs | LR: 0.000145
Reading ../data/atis.test.w-intent.iob
INFO:tensorflow:Slot F1: 0.955, Intent Acc: 0.974
INFO:tensorflow:Best Slot F1: 0.957, Intent Acc: 0.972
Reading ../data/atis.train.w-intent.iob
INFO:tensorflow:Step 16250 | Loss: 1.4110 | Loss_intent: 1.0810 | Loss_slots: 0.3301 | Spent: 5.4 secs | LR: 0.000146
INFO:tensorflow:Step 16300 | Loss: 1.3730 | Loss_intent: 1.0811 | Loss_slots: 0.2919 | Spent: 3.1 secs | LR: 0.000148
INFO:tensorflow:Step 16350 | Loss: 1.3763 | Loss_intent: 1.0819 | Loss_slots: 0.2944 | Spent: 3.0 secs | LR: 0.000150
INFO:tensorflow:Step 16400 | Loss: 1.5598 | Loss_intent: 1.0800 | Loss_slots: 0.4798 | Spent: 3.0 secs | LR: 0.000152
INFO:tensorflow:Step 16450 | Loss: 1.4740 | Loss_intent: 1.0804 | Loss_slots: 0.3936 | Spent: 3.0 secs | LR: 0.000153
INFO:tensorflow:Step 16500 | Loss: 1.3704 | Loss_intent: 1.0813 | Loss_slots: 0.2891 | Spent: 3.0 secs | LR: 0.000155
Reading ../data/atis.test.w-intent.iob
INFO:tensorflow:Slot F1: 0.951, Intent Acc: 0.972
INFO:tensorflow:Best Slot F1: 0.957, Intent Acc: 0.972
Reading ../data/atis.train.w-intent.iob
INFO:tensorflow:Step 16550 | Loss: 1.5418 | Loss_intent: 1.0814 | Loss_slots: 0.4604 | Spent: 5.1 secs | LR: 0.000157
INFO:tensorflow:Step 16600 | Loss: 1.3904 | Loss_intent: 1.0837 | Loss_slots: 0.3067 | Spent: 3.0 secs | LR: 0.000159
INFO:tensorflow:Step 16650 | Loss: 1.3729 | Loss_intent: 1.0818 | Loss_slots: 0.2911 | Spent: 3.0 secs | LR: 0.000160
INFO:tensorflow:Step 16700 | Loss: 1.4192 | Loss_intent: 1.0807 | Loss_slots: 0.3385 | Spent: 3.0 secs | LR: 0.000162
INFO:tensorflow:Step 16750 | Loss: 1.5211 | Loss_intent: 1.0817 | Loss_slots: 0.4394 | Spent: 3.0 secs | LR: 0.000164
INFO:tensorflow:Step 16800 | Loss: 1.3772 | Loss_intent: 1.0808 | Loss_slots: 0.2964 | Spent: 3.0 secs | LR: 0.000166
Reading ../data/atis.test.w-intent.iob
INFO:tensorflow:Slot F1: 0.955, Intent Acc: 0.973
INFO:tensorflow:Best Slot F1: 0.957, Intent Acc: 0.972
Reading ../data/atis.train.w-intent.iob
INFO:tensorflow:Step 16850 | Loss: 1.4449 | Loss_intent: 1.0817 | Loss_slots: 0.3632 | Spent: 5.3 secs | LR: 0.000167
INFO:tensorflow:Step 16900 | Loss: 1.4095 | Loss_intent: 1.0826 | Loss_slots: 0.3269 | Spent: 3.2 secs | LR: 0.000169
INFO:tensorflow:Step 16950 | Loss: 1.4247 | Loss_intent: 1.0862 | Loss_slots: 0.3385 | Spent: 3.1 secs | LR: 0.000171
INFO:tensorflow:Step 17000 | Loss: 1.3547 | Loss_intent: 1.0817 | Loss_slots: 0.2730 | Spent: 3.0 secs | LR: 0.000173
INFO:tensorflow:Step 17050 | Loss: 1.3856 | Loss_intent: 1.0808 | Loss_slots: 0.3047 | Spent: 3.0 secs | LR: 0.000174
INFO:tensorflow:Step 17100 | Loss: 1.4106 | Loss_intent: 1.0819 | Loss_slots: 0.3287 | Spent: 3.0 secs | LR: 0.000176
INFO:tensorflow:Step 17150 | Loss: 1.4348 | Loss_intent: 1.0873 | Loss_slots: 0.3475 | Spent: 3.0 secs | LR: 0.000178
Reading ../data/atis.test.w-intent.iob
INFO:tensorflow:Slot F1: 0.954, Intent Acc: 0.973
INFO:tensorflow:Best Slot F1: 0.957, Intent Acc: 0.972
Reading ../data/atis.train.w-intent.iob
INFO:tensorflow:Step 17200 | Loss: 1.4798 | Loss_intent: 1.0812 | Loss_slots: 0.3987 | Spent: 5.3 secs | LR: 0.000180
INFO:tensorflow:Step 17250 | Loss: 1.4390 | Loss_intent: 1.0813 | Loss_slots: 0.3577 | Spent: 3.1 secs | LR: 0.000181
INFO:tensorflow:Step 17300 | Loss: 1.5239 | Loss_intent: 1.0851 | Loss_slots: 0.4388 | Spent: 3.1 secs | LR: 0.000183
INFO:tensorflow:Step 17350 | Loss: 1.4873 | Loss_intent: 1.0889 | Loss_slots: 0.3984 | Spent: 3.0 secs | LR: 0.000185
INFO:tensorflow:Step 17400 | Loss: 1.4362 | Loss_intent: 1.0819 | Loss_slots: 0.3543 | Spent: 3.0 secs | LR: 0.000187
INFO:tensorflow:Step 17450 | Loss: 1.3747 | Loss_intent: 1.1895 | Loss_slots: 0.1852 | Spent: 3.1 secs | LR: 0.000187
Reading ../data/atis.test.w-intent.iob
INFO:tensorflow:Slot F1: 0.958, Intent Acc: 0.975
INFO:tensorflow:
                                          precision    recall  f1-score   support

                             atis_flight      0.986     0.992     0.989       632
                            atis_airfare      0.979     0.979     0.979        48
                     atis_ground_service      0.973     1.000     0.986        36
                            atis_airline      1.000     1.000     1.000        38
                       atis_abbreviation      0.941     0.970     0.955        33
                           atis_aircraft      1.000     0.889     0.941         9
                        atis_flight_time      1.000     1.000     1.000         1
                           atis_quantity      0.429     1.000     0.600         3
                atis_flight#atis_airfare      0.778     0.583     0.667        12
                            atis_airport      1.000     1.000     1.000        18
                           atis_distance      1.000     1.000     1.000        10
                               atis_city      1.000     0.667     0.800         6
                        atis_ground_fare      1.000     0.857     0.923         7
                           atis_capacity      0.955     1.000     0.977        21
                          atis_flight_no      0.889     1.000     0.941         8
                               atis_meal      1.000     0.833     0.909         6
                        atis_restriction      0.000     0.000     0.000         0
             atis_airline#atis_flight_no      0.000     0.000     0.000         0
    atis_ground_service#atis_ground_fare      0.000     0.000     0.000         0
           atis_airfare#atis_flight_time      0.000     0.000     0.000         0
                           atis_cheapest      0.000     0.000     0.000         0
atis_aircraft#atis_flight#atis_flight_no      0.000     0.000     0.000         0

                               micro avg      0.975     0.981     0.978       888
                               macro avg      0.679     0.671     0.667       888
                            weighted avg      0.979     0.981     0.979       888

INFO:tensorflow:
                              precision    recall  f1-score   support

                           O      0.000     0.000     0.000       0.0
           B-toloc.city_name      0.974     0.996     0.985     716.0
         B-fromloc.city_name      0.992     0.997     0.994     704.0
           I-toloc.city_name      0.978     0.992     0.985     265.0
      B-depart_date.day_name      0.991     0.986     0.988     212.0
              B-airline_name      0.981     1.000     0.990     101.0
         I-fromloc.city_name      0.978     0.994     0.986     177.0
 B-depart_time.period_of_day      0.984     0.931     0.957     130.0
              I-airline_name      1.000     1.000     1.000      65.0
    B-depart_date.day_number      0.981     0.964     0.972      55.0
    B-depart_date.month_name      0.982     0.964     0.973      56.0
          B-depart_time.time      0.836     0.982     0.903      57.0
                B-round_trip      1.000     0.986     0.993      73.0
             B-cost_relative      1.000     0.973     0.986      37.0
                I-round_trip      1.000     1.000     1.000      71.0
                B-flight_mod      1.000     1.000     1.000      24.0
 B-depart_time.time_relative      0.984     0.969     0.977      65.0
          I-depart_time.time      0.981     1.000     0.990      52.0
         B-stoploc.city_name      0.952     1.000     0.976      20.0
                 B-city_name      0.857     0.632     0.727      57.0
                B-class_type      0.960     1.000     0.980      24.0
          B-arrive_time.time      0.971     0.971     0.971      34.0
 B-arrive_time.time_relative      0.857     0.968     0.909      31.0
                I-class_type      1.000     1.000     1.000      17.0
               B-flight_stop      1.000     1.000     1.000      21.0
          I-arrive_time.time      0.971     0.971     0.971      35.0
              B-airline_code      1.000     0.853     0.921      34.0
    I-depart_date.day_number      1.000     0.933     0.966      15.0
      I-fromloc.airport_name      0.469     1.000     0.638      15.0
      B-fromloc.airport_name      0.440     0.917     0.595      12.0
      B-arrive_date.day_name      0.733     1.000     0.846      11.0
          B-toloc.state_code      1.000     1.000     1.000      18.0
B-depart_date.today_relative      1.000     1.000     1.000       9.0
             B-flight_number      0.500     1.000     0.667      11.0
 B-depart_date.date_relative      0.944     1.000     0.971      17.0
          B-toloc.state_name      0.962     0.893     0.926      28.0
           B-fare_basis_code      0.944     1.000     0.971      17.0
               B-flight_time      1.000     1.000     1.000       1.0
                        B-or      1.000     1.000     1.000       3.0
 B-arrive_time.period_of_day      0.750     1.000     0.857       6.0
          B-meal_description      1.000     0.900     0.947      10.0
             I-cost_relative      1.000     0.667     0.800       3.0
              I-airport_name      0.938     0.517     0.667      29.0
               B-fare_amount      1.000     0.500     0.667       2.0
               I-fare_amount      1.000     1.000     1.000       2.0
                 I-city_name      1.000     0.600     0.750      30.0
        I-toloc.airport_name      1.000     1.000     1.000       3.0
            B-transport_type      1.000     1.000     1.000      10.0
    B-arrive_date.month_name      0.714     0.833     0.769       6.0
    B-arrive_date.day_number      0.714     0.833     0.769       6.0
         I-stoploc.city_name      1.000     1.000     1.000      10.0
                      B-meal      1.000     1.000     1.000      16.0
        B-fromloc.state_code      1.000     1.000     1.000      23.0
    B-depart_time.period_mod      0.833     1.000     0.909       5.0
                   B-connect      1.000     1.000     1.000       6.0
               B-flight_days      1.000     1.000     1.000      10.0
        B-toloc.airport_name      1.000     1.000     1.000       3.0
        B-fromloc.state_name      0.850     1.000     0.919      17.0
              B-airport_name      0.692     0.429     0.529      21.0
                   B-economy      1.000     1.000     1.000       6.0
               I-flight_time      1.000     1.000     1.000       1.0
             B-aircraft_code      0.967     0.879     0.921      33.0
                       B-mod      1.000     0.500     0.667       2.0
              B-airport_code      0.833     0.556     0.667       9.0
    B-depart_time.start_time      0.400     0.667     0.500       3.0
      B-depart_time.end_time      0.667     0.667     0.667       3.0
          B-depart_date.year      1.000     0.667     0.800       3.0
            I-transport_type      0.000     0.000     0.000       1.0
          B-restriction_code      1.000     1.000     1.000       4.0
    B-arrive_time.start_time      0.800     1.000     0.889       8.0
        B-toloc.airport_code      1.000     0.500     0.667       4.0
      B-arrive_time.end_time      0.889     1.000     0.941       8.0
      I-arrive_time.end_time      0.875     0.875     0.875       8.0
      I-depart_time.end_time      1.000     0.667     0.800       3.0
               I-flight_stop      0.000     0.000     0.000       0.0
      B-fromloc.airport_code      1.000     1.000     1.000       5.0
          I-restriction_code      0.750     1.000     0.857       3.0
    I-depart_time.start_time      1.000     1.000     1.000       1.0
          I-toloc.state_name      1.000     1.000     1.000       1.0
I-depart_date.today_relative      0.000     0.000     0.000       0.0
 B-arrive_date.date_relative      1.000     1.000     1.000       2.0
                I-flight_mod      1.000     0.167     0.286       6.0
                   I-economy      0.000     0.000     0.000       0.0
 B-return_date.date_relative      1.000     0.667     0.800       3.0
        I-fromloc.state_name      1.000     1.000     1.000       1.0
                B-state_code      1.000     1.000     1.000       1.0
    I-arrive_time.start_time      1.000     1.000     1.000       1.0
    I-arrive_date.day_number      0.000     0.000     0.000       0.0
                 B-meal_code      0.000     0.000     0.000       1.0
 I-depart_time.period_of_day      0.000     0.000     0.000       1.0
                  B-day_name      1.000     0.500     0.667       2.0
             B-period_of_day      1.000     0.250     0.400       4.0
        B-stoploc.state_code      0.000     0.000     0.000       0.0
    B-return_date.month_name      0.000     0.000     0.000       0.0
    B-return_date.day_number      0.000     0.000     0.000       0.0
    B-arrive_time.period_mod      0.000     0.000     0.000       0.0
                 I-meal_code      0.000     0.000     0.000       0.0
        B-toloc.country_name      1.000     1.000     1.000       1.0
                 B-days_code      1.000     1.000     1.000       1.0
 I-arrive_time.period_of_day      0.000     0.000     0.000       0.0
            I-today_relative      0.000     0.000     0.000       0.0
 B-return_time.period_of_day      0.000     0.000     0.000       0.0
                      B-time      0.000     0.000     0.000       0.0
           I-fare_basis_code      0.000     0.000     0.000       0.0
 I-arrive_time.time_relative      0.000     0.000     0.000       4.0
 I-depart_time.time_relative      0.000     0.000     0.000       1.0
            B-today_relative      0.000     0.000     0.000       0.0
                B-state_name      0.000     0.000     0.000       9.0
B-arrive_date.today_relative      0.000     0.000     0.000       0.0
    B-return_time.period_mod      0.000     0.000     0.000       0.0
                B-month_name      0.000     0.000     0.000       0.0
                B-day_number      0.000     0.000     0.000       0.0
 I-return_date.date_relative      1.000     1.000     1.000       3.0
I-return_date.today_relative      0.000     0.000     0.000       0.0
      B-stoploc.airport_name      0.000     0.000     0.000       0.0
             B-time_relative      0.000     0.000     0.000       0.0
                      I-time      0.000     0.000     0.000       0.0
    I-return_date.day_number      0.000     0.000     0.000       0.0
          I-meal_description      0.000     0.000     0.000       0.0
B-return_date.today_relative      0.000     0.000     0.000       0.0
      B-return_date.day_name      0.000     0.000     0.000       2.0

                   micro avg      0.957     0.959     0.958    3657.0
                   macro avg      0.668     0.642     0.642    3657.0
                weighted avg      0.961     0.959     0.956    3657.0

INFO:tensorflow:Best Slot F1: 0.958, Intent Acc: 0.975
Reading ../data/atis.train.w-intent.iob
INFO:tensorflow:Step 17500 | Loss: 1.4350 | Loss_intent: 1.0824 | Loss_slots: 0.3526 | Spent: 5.4 secs | LR: 0.000185
INFO:tensorflow:Step 17550 | Loss: 1.3775 | Loss_intent: 1.0818 | Loss_slots: 0.2957 | Spent: 3.0 secs | LR: 0.000183
INFO:tensorflow:Step 17600 | Loss: 1.5026 | Loss_intent: 1.0805 | Loss_slots: 0.4220 | Spent: 3.1 secs | LR: 0.000181
INFO:tensorflow:Step 17650 | Loss: 1.4078 | Loss_intent: 1.0861 | Loss_slots: 0.3217 | Spent: 3.1 secs | LR: 0.000180
INFO:tensorflow:Step 17700 | Loss: 1.3914 | Loss_intent: 1.0809 | Loss_slots: 0.3105 | Spent: 3.0 secs | LR: 0.000178
INFO:tensorflow:Step 17750 | Loss: 1.3963 | Loss_intent: 1.0820 | Loss_slots: 0.3143 | Spent: 3.0 secs | LR: 0.000176
Reading ../data/atis.test.w-intent.iob
INFO:tensorflow:Slot F1: 0.955, Intent Acc: 0.973
INFO:tensorflow:Best Slot F1: 0.958, Intent Acc: 0.975
Reading ../data/atis.train.w-intent.iob
INFO:tensorflow:Step 17800 | Loss: 1.3722 | Loss_intent: 1.0816 | Loss_slots: 0.2906 | Spent: 5.3 secs | LR: 0.000174
INFO:tensorflow:Step 17850 | Loss: 1.4085 | Loss_intent: 1.0814 | Loss_slots: 0.3270 | Spent: 2.9 secs | LR: 0.000172
INFO:tensorflow:Step 17900 | Loss: 1.4077 | Loss_intent: 1.0829 | Loss_slots: 0.3248 | Spent: 3.0 secs | LR: 0.000171
INFO:tensorflow:Step 17950 | Loss: 1.3696 | Loss_intent: 1.0819 | Loss_slots: 0.2877 | Spent: 3.0 secs | LR: 0.000169
INFO:tensorflow:Step 18000 | Loss: 1.4390 | Loss_intent: 1.0813 | Loss_slots: 0.3578 | Spent: 3.1 secs | LR: 0.000167
INFO:tensorflow:Step 18050 | Loss: 1.4288 | Loss_intent: 1.0802 | Loss_slots: 0.3486 | Spent: 3.1 secs | LR: 0.000165
Reading ../data/atis.test.w-intent.iob
INFO:tensorflow:Slot F1: 0.950, Intent Acc: 0.978
INFO:tensorflow:Best Slot F1: 0.958, Intent Acc: 0.975
Reading ../data/atis.train.w-intent.iob
INFO:tensorflow:Step 18100 | Loss: 1.4677 | Loss_intent: 1.0828 | Loss_slots: 0.3849 | Spent: 5.4 secs | LR: 0.000164
INFO:tensorflow:Step 18150 | Loss: 1.4148 | Loss_intent: 1.0818 | Loss_slots: 0.3330 | Spent: 3.1 secs | LR: 0.000162
INFO:tensorflow:Step 18200 | Loss: 1.4552 | Loss_intent: 1.0815 | Loss_slots: 0.3737 | Spent: 3.1 secs | LR: 0.000160
INFO:tensorflow:Step 18250 | Loss: 1.2705 | Loss_intent: 1.0815 | Loss_slots: 0.1889 | Spent: 3.1 secs | LR: 0.000158
INFO:tensorflow:Step 18300 | Loss: 1.3954 | Loss_intent: 1.0865 | Loss_slots: 0.3089 | Spent: 3.2 secs | LR: 0.000157
INFO:tensorflow:Step 18350 | Loss: 1.4291 | Loss_intent: 1.0848 | Loss_slots: 0.3444 | Spent: 3.1 secs | LR: 0.000155
INFO:tensorflow:Step 18400 | Loss: 1.3951 | Loss_intent: 1.0799 | Loss_slots: 0.3151 | Spent: 3.1 secs | LR: 0.000153
Reading ../data/atis.test.w-intent.iob
INFO:tensorflow:Slot F1: 0.952, Intent Acc: 0.978
INFO:tensorflow:Best Slot F1: 0.958, Intent Acc: 0.975
Reading ../data/atis.train.w-intent.iob
INFO:tensorflow:Step 18450 | Loss: 1.4428 | Loss_intent: 1.0810 | Loss_slots: 0.3617 | Spent: 5.4 secs | LR: 0.000151
INFO:tensorflow:Step 18500 | Loss: 1.4484 | Loss_intent: 1.0810 | Loss_slots: 0.3673 | Spent: 3.1 secs | LR: 0.000150
INFO:tensorflow:Step 18550 | Loss: 1.4481 | Loss_intent: 1.0811 | Loss_slots: 0.3670 | Spent: 3.1 secs | LR: 0.000148
INFO:tensorflow:Step 18600 | Loss: 1.4120 | Loss_intent: 1.0815 | Loss_slots: 0.3306 | Spent: 3.1 secs | LR: 0.000146
INFO:tensorflow:Step 18650 | Loss: 1.4423 | Loss_intent: 1.0805 | Loss_slots: 0.3618 | Spent: 3.0 secs | LR: 0.000144
INFO:tensorflow:Step 18700 | Loss: 1.4753 | Loss_intent: 1.0805 | Loss_slots: 0.3948 | Spent: 3.1 secs | LR: 0.000143
Reading ../data/atis.test.w-intent.iob
INFO:tensorflow:Slot F1: 0.957, Intent Acc: 0.974
INFO:tensorflow:Best Slot F1: 0.958, Intent Acc: 0.975
Reading ../data/atis.train.w-intent.iob
INFO:tensorflow:Step 18750 | Loss: 1.3914 | Loss_intent: 1.0819 | Loss_slots: 0.3095 | Spent: 5.5 secs | LR: 0.000141
INFO:tensorflow:Step 18800 | Loss: 1.4237 | Loss_intent: 1.0833 | Loss_slots: 0.3404 | Spent: 3.3 secs | LR: 0.000139
INFO:tensorflow:Step 18850 | Loss: 1.5260 | Loss_intent: 1.0800 | Loss_slots: 0.4461 | Spent: 3.3 secs | LR: 0.000137
INFO:tensorflow:Step 18900 | Loss: 1.2655 | Loss_intent: 1.0814 | Loss_slots: 0.1841 | Spent: 3.1 secs | LR: 0.000136
INFO:tensorflow:Step 18950 | Loss: 1.4255 | Loss_intent: 1.0814 | Loss_slots: 0.3441 | Spent: 3.0 secs | LR: 0.000134
INFO:tensorflow:Step 19000 | Loss: 1.3790 | Loss_intent: 1.0831 | Loss_slots: 0.2959 | Spent: 3.0 secs | LR: 0.000132
Reading ../data/atis.test.w-intent.iob
INFO:tensorflow:Slot F1: 0.952, Intent Acc: 0.976
INFO:tensorflow:Best Slot F1: 0.958, Intent Acc: 0.975
Reading ../data/atis.train.w-intent.iob
INFO:tensorflow:Step 19050 | Loss: 1.4697 | Loss_intent: 1.0828 | Loss_slots: 0.3869 | Spent: 5.3 secs | LR: 0.000130
INFO:tensorflow:Step 19100 | Loss: 1.3835 | Loss_intent: 1.0805 | Loss_slots: 0.3030 | Spent: 3.0 secs | LR: 0.000129
INFO:tensorflow:Step 19150 | Loss: 1.3273 | Loss_intent: 1.0822 | Loss_slots: 0.2451 | Spent: 3.1 secs | LR: 0.000127
INFO:tensorflow:Step 19200 | Loss: 1.4785 | Loss_intent: 1.0808 | Loss_slots: 0.3977 | Spent: 3.1 secs | LR: 0.000125
INFO:tensorflow:Step 19250 | Loss: 1.3789 | Loss_intent: 1.0819 | Loss_slots: 0.2970 | Spent: 3.1 secs | LR: 0.000123
INFO:tensorflow:Step 19300 | Loss: 1.4633 | Loss_intent: 1.0820 | Loss_slots: 0.3813 | Spent: 3.1 secs | LR: 0.000122
Reading ../data/atis.test.w-intent.iob
INFO:tensorflow:Slot F1: 0.954, Intent Acc: 0.974
INFO:tensorflow:Best Slot F1: 0.958, Intent Acc: 0.975
Reading ../data/atis.train.w-intent.iob
INFO:tensorflow:Step 19350 | Loss: 1.3944 | Loss_intent: 1.0824 | Loss_slots: 0.3120 | Spent: 5.4 secs | LR: 0.000120
INFO:tensorflow:Step 19400 | Loss: 1.3656 | Loss_intent: 1.0802 | Loss_slots: 0.2854 | Spent: 3.1 secs | LR: 0.000118
INFO:tensorflow:Step 19450 | Loss: 1.3292 | Loss_intent: 1.0809 | Loss_slots: 0.2483 | Spent: 3.0 secs | LR: 0.000116
INFO:tensorflow:Step 19500 | Loss: 1.4138 | Loss_intent: 1.0807 | Loss_slots: 0.3332 | Spent: 3.0 secs | LR: 0.000114
INFO:tensorflow:Step 19550 | Loss: 1.3227 | Loss_intent: 1.0810 | Loss_slots: 0.2417 | Spent: 3.0 secs | LR: 0.000113
INFO:tensorflow:Step 19600 | Loss: 1.3403 | Loss_intent: 1.0826 | Loss_slots: 0.2577 | Spent: 3.0 secs | LR: 0.000111
INFO:tensorflow:Step 19650 | Loss: 1.2463 | Loss_intent: 1.0808 | Loss_slots: 0.1655 | Spent: 3.0 secs | LR: 0.000109
Reading ../data/atis.test.w-intent.iob
INFO:tensorflow:Slot F1: 0.954, Intent Acc: 0.974
INFO:tensorflow:Best Slot F1: 0.958, Intent Acc: 0.975
Reading ../data/atis.train.w-intent.iob
INFO:tensorflow:Step 19700 | Loss: 1.4686 | Loss_intent: 1.0803 | Loss_slots: 0.3883 | Spent: 5.3 secs | LR: 0.000107
INFO:tensorflow:Step 19750 | Loss: 1.4562 | Loss_intent: 1.0807 | Loss_slots: 0.3756 | Spent: 3.1 secs | LR: 0.000106
INFO:tensorflow:Step 19800 | Loss: 1.4102 | Loss_intent: 1.0823 | Loss_slots: 0.3279 | Spent: 3.0 secs | LR: 0.000104
INFO:tensorflow:Step 19850 | Loss: 1.4452 | Loss_intent: 1.0843 | Loss_slots: 0.3609 | Spent: 3.0 secs | LR: 0.000102
INFO:tensorflow:Step 19900 | Loss: 1.2984 | Loss_intent: 1.0824 | Loss_slots: 0.2160 | Spent: 3.0 secs | LR: 0.000100
INFO:tensorflow:Step 19950 | Loss: 1.3162 | Loss_intent: 1.0848 | Loss_slots: 0.2314 | Spent: 3.0 secs | LR: 0.000101
Reading ../data/atis.test.w-intent.iob
INFO:tensorflow:Slot F1: 0.957, Intent Acc: 0.973
INFO:tensorflow:
                                          precision    recall  f1-score   support

                             atis_flight      0.983     0.992     0.987       632
                            atis_airfare      0.979     0.979     0.979        48
                     atis_ground_service      0.973     1.000     0.986        36
                            atis_airline      0.974     1.000     0.987        38
                       atis_abbreviation      0.941     0.970     0.955        33
                           atis_aircraft      1.000     0.889     0.941         9
                        atis_flight_time      1.000     1.000     1.000         1
                           atis_quantity      0.375     1.000     0.545         3
                atis_flight#atis_airfare      0.833     0.417     0.556        12
                            atis_airport      1.000     1.000     1.000        18
                           atis_distance      1.000     1.000     1.000        10
                               atis_city      1.000     0.667     0.800         6
                        atis_ground_fare      1.000     0.857     0.923         7
                           atis_capacity      1.000     1.000     1.000        21
                          atis_flight_no      0.889     1.000     0.941         8
                               atis_meal      1.000     0.833     0.909         6
                        atis_restriction      0.000     0.000     0.000         0
             atis_airline#atis_flight_no      0.000     0.000     0.000         0
    atis_ground_service#atis_ground_fare      0.000     0.000     0.000         0
           atis_airfare#atis_flight_time      0.000     0.000     0.000         0
                           atis_cheapest      0.000     0.000     0.000         0
atis_aircraft#atis_flight#atis_flight_no      0.000     0.000     0.000         0

                               micro avg      0.973     0.979     0.976       888
                               macro avg      0.679     0.664     0.660       888
                            weighted avg      0.977     0.979     0.976       888

INFO:tensorflow:
                              precision    recall  f1-score   support

                           O      0.000     0.000     0.000       0.0
           B-toloc.city_name      0.977     0.996     0.986     716.0
         B-fromloc.city_name      0.990     0.997     0.994     704.0
           I-toloc.city_name      0.978     0.992     0.985     265.0
      B-depart_date.day_name      0.991     0.991     0.991     212.0
              B-airline_name      0.981     1.000     0.990     101.0
         I-fromloc.city_name      0.983     0.994     0.989     177.0
 B-depart_time.period_of_day      1.000     0.923     0.960     130.0
              I-airline_name      1.000     1.000     1.000      65.0
    B-depart_date.day_number      0.981     0.964     0.972      55.0
    B-depart_date.month_name      0.982     0.964     0.973      56.0
          B-depart_time.time      0.836     0.982     0.903      57.0
                B-round_trip      1.000     0.986     0.993      73.0
             B-cost_relative      1.000     0.973     0.986      37.0
                I-round_trip      1.000     1.000     1.000      71.0
                B-flight_mod      1.000     1.000     1.000      24.0
 B-depart_time.time_relative      0.984     0.969     0.977      65.0
          I-depart_time.time      0.945     1.000     0.972      52.0
         B-stoploc.city_name      1.000     1.000     1.000      20.0
                 B-city_name      0.881     0.649     0.747      57.0
                B-class_type      0.960     1.000     0.980      24.0
          B-arrive_time.time      0.971     0.971     0.971      34.0
 B-arrive_time.time_relative      0.909     0.968     0.937      31.0
                I-class_type      1.000     1.000     1.000      17.0
               B-flight_stop      1.000     1.000     1.000      21.0
          I-arrive_time.time      0.971     0.971     0.971      35.0
              B-airline_code      1.000     0.882     0.938      34.0
    I-depart_date.day_number      1.000     0.933     0.966      15.0
      I-fromloc.airport_name      0.405     1.000     0.577      15.0
      B-fromloc.airport_name      0.423     0.917     0.579      12.0
      B-arrive_date.day_name      0.786     1.000     0.880      11.0
          B-toloc.state_code      1.000     1.000     1.000      18.0
B-depart_date.today_relative      1.000     1.000     1.000       9.0
             B-flight_number      0.500     1.000     0.667      11.0
 B-depart_date.date_relative      0.944     1.000     0.971      17.0
          B-toloc.state_name      0.923     0.857     0.889      28.0
           B-fare_basis_code      0.944     1.000     0.971      17.0
               B-flight_time      1.000     1.000     1.000       1.0
                        B-or      1.000     1.000     1.000       3.0
 B-arrive_time.period_of_day      0.750     1.000     0.857       6.0
          B-meal_description      1.000     0.900     0.947      10.0
             I-cost_relative      1.000     0.667     0.800       3.0
              I-airport_name      0.917     0.379     0.537      29.0
               B-fare_amount      1.000     0.500     0.667       2.0
               I-fare_amount      1.000     1.000     1.000       2.0
                 I-city_name      0.947     0.600     0.735      30.0
        I-toloc.airport_name      1.000     1.000     1.000       3.0
            B-transport_type      1.000     1.000     1.000      10.0
    B-arrive_date.month_name      0.714     0.833     0.769       6.0
    B-arrive_date.day_number      0.714     0.833     0.769       6.0
         I-stoploc.city_name      1.000     1.000     1.000      10.0
                      B-meal      1.000     1.000     1.000      16.0
        B-fromloc.state_code      1.000     1.000     1.000      23.0
    B-depart_time.period_mod      0.833     1.000     0.909       5.0
                   B-connect      1.000     1.000     1.000       6.0
               B-flight_days      1.000     1.000     1.000      10.0
        B-toloc.airport_name      1.000     1.000     1.000       3.0
        B-fromloc.state_name      0.850     1.000     0.919      17.0
              B-airport_name      0.692     0.429     0.529      21.0
                   B-economy      1.000     1.000     1.000       6.0
               I-flight_time      1.000     1.000     1.000       1.0
             B-aircraft_code      1.000     0.818     0.900      33.0
                       B-mod      1.000     0.500     0.667       2.0
              B-airport_code      0.800     0.444     0.571       9.0
    B-depart_time.start_time      0.500     0.667     0.571       3.0
      B-depart_time.end_time      0.667     0.667     0.667       3.0
          B-depart_date.year      1.000     0.667     0.800       3.0
            I-transport_type      0.000     0.000     0.000       1.0
          B-restriction_code      1.000     1.000     1.000       4.0
    B-arrive_time.start_time      0.800     1.000     0.889       8.0
        B-toloc.airport_code      1.000     0.500     0.667       4.0
      B-arrive_time.end_time      0.889     1.000     0.941       8.0
      I-arrive_time.end_time      0.875     0.875     0.875       8.0
      I-depart_time.end_time      1.000     0.667     0.800       3.0
               I-flight_stop      0.000     0.000     0.000       0.0
      B-fromloc.airport_code      1.000     1.000     1.000       5.0
          I-restriction_code      0.750     1.000     0.857       3.0
    I-depart_time.start_time      1.000     1.000     1.000       1.0
          I-toloc.state_name      1.000     1.000     1.000       1.0
I-depart_date.today_relative      0.000     0.000     0.000       0.0
 B-arrive_date.date_relative      1.000     1.000     1.000       2.0
                I-flight_mod      1.000     0.167     0.286       6.0
                   I-economy      0.000     0.000     0.000       0.0
 B-return_date.date_relative      1.000     0.333     0.500       3.0
        I-fromloc.state_name      1.000     1.000     1.000       1.0
                B-state_code      1.000     1.000     1.000       1.0
    I-arrive_time.start_time      1.000     1.000     1.000       1.0
    I-arrive_date.day_number      0.000     0.000     0.000       0.0
                 B-meal_code      0.000     0.000     0.000       1.0
 I-depart_time.period_of_day      1.000     1.000     1.000       1.0
                  B-day_name      1.000     0.500     0.667       2.0
             B-period_of_day      1.000     0.500     0.667       4.0
        B-stoploc.state_code      0.000     0.000     0.000       0.0
    B-return_date.month_name      0.000     0.000     0.000       0.0
    B-return_date.day_number      0.000     0.000     0.000       0.0
    B-arrive_time.period_mod      0.000     0.000     0.000       0.0
                 I-meal_code      0.000     0.000     0.000       0.0
        B-toloc.country_name      1.000     1.000     1.000       1.0
                 B-days_code      1.000     1.000     1.000       1.0
 I-arrive_time.period_of_day      0.000     0.000     0.000       0.0
            I-today_relative      0.000     0.000     0.000       0.0
 B-return_time.period_of_day      0.000     0.000     0.000       0.0
                      B-time      0.000     0.000     0.000       0.0
           I-fare_basis_code      0.000     0.000     0.000       0.0
 I-arrive_time.time_relative      1.000     0.500     0.667       4.0
 I-depart_time.time_relative      0.000     0.000     0.000       1.0
            B-today_relative      0.000     0.000     0.000       0.0
                B-state_name      0.000     0.000     0.000       9.0
B-arrive_date.today_relative      0.000     0.000     0.000       0.0
    B-return_time.period_mod      0.000     0.000     0.000       0.0
                B-month_name      0.000     0.000     0.000       0.0
                B-day_number      0.000     0.000     0.000       0.0
 I-return_date.date_relative      0.750     1.000     0.857       3.0
I-return_date.today_relative      0.000     0.000     0.000       0.0
      B-stoploc.airport_name      0.000     0.000     0.000       0.0
             B-time_relative      0.000     0.000     0.000       0.0
                      I-time      0.000     0.000     0.000       0.0
    I-return_date.day_number      0.000     0.000     0.000       0.0
          I-meal_description      0.000     0.000     0.000       0.0
B-return_date.today_relative      0.000     0.000     0.000       0.0
      B-return_date.day_name      0.000     0.000     0.000       2.0

                   micro avg      0.956     0.958     0.957    3657.0
                   macro avg      0.683     0.651     0.653    3657.0
                weighted avg      0.963     0.958     0.956    3657.0

INFO:tensorflow:Best Slot F1: 0.958, Intent Acc: 0.975