"""
We use following lines because we are running on Google Colab
If you are running notebook on a local computer, you don't need this cell
"""
from google.colab import drive
drive.mount('/content/gdrive')
import os
os.chdir('/content/gdrive/My Drive/finch/tensorflow1/spoken_language_understanding/atis/main')
Drive already mounted at /content/gdrive; to attempt to forcibly remount, call drive.mount("/content/gdrive", force_remount=True).
import tensorflow as tf
import tensorflow_hub as hub
import pprint
import logging
import time
import numpy as np
from sklearn.metrics import classification_report, f1_score
from pathlib import Path
print("TensorFlow Version", tf.__version__)
print('GPU Enabled:', tf.test.is_gpu_available())
TensorFlow Version 1.14.0 GPU Enabled: True
def get_vocab(vocab_path):
word2idx = {}
with open(vocab_path) as f:
for i, line in enumerate(f):
line = line.rstrip()
word2idx[line] = i
return word2idx
def data_generator(f_path, params):
print('Reading', f_path)
with open(f_path) as f:
for line in f:
line = line.rstrip()
text, slot_intent = line.split('\t')
words = text.split()[1:-1]
slot_intent = slot_intent.split()
slots, intent = slot_intent[1:-1], slot_intent[-1]
assert len(words) == len(slots)
yield (words, (intent, slots))
def dataset(is_training, params):
_shapes = ([None], ((), [None]))
_types = (tf.string, (tf.string, tf.string))
_pads = ('<pad>', ('_', 'O'))
if is_training:
ds = tf.data.Dataset.from_generator(
lambda: data_generator(params['train_path'], params),
output_shapes = _shapes,
output_types = _types,)
ds = ds.shuffle(params['num_samples'])
ds = ds.padded_batch(params['batch_size'], _shapes, _pads)
ds = ds.prefetch(tf.data.experimental.AUTOTUNE)
else:
ds = tf.data.Dataset.from_generator(
lambda: data_generator(params['test_path'], params),
output_shapes = _shapes,
output_types = _types,)
ds = ds.padded_batch(1, _shapes, _pads)
ds = ds.prefetch(tf.data.experimental.AUTOTUNE)
return ds
def model_fn(features, labels, mode, params):
is_training = (mode == tf.estimator.ModeKeys.TRAIN)
vocab = tf.contrib.lookup.index_table_from_file(
params['word_path'], num_oov_buckets=1)
words = vocab.lookup(features)
seq_len = tf.count_nonzero(words, 1, dtype=tf.int32)
embedding = np.load(params['vocab_path'])
embedding = tf.Variable(embedding, name='embedding', dtype=tf.float32)
x = tf.nn.embedding_lookup(embedding, words)
elmo = hub.Module("https://tfhub.dev/google/elmo/2", trainable=is_training)
e = elmo(inputs={'tokens':features, 'sequence_len':seq_len}, signature='tokens', as_dict=True)['elmo']
x = tf.concat((x, e), -1)
x = tf.layers.dropout(x, params['dropout_rate'], training=is_training)
x = tf.layers.dense(x, params['rnn_units'], tf.nn.elu)
x = tf.layers.dropout(x, params['dropout_rate'], training=is_training)
cell_fw = tf.nn.rnn_cell.GRUCell(params['rnn_units'])
cell_bw = tf.nn.rnn_cell.GRUCell(params['rnn_units'])
o, _ = tf.nn.bidirectional_dynamic_rnn(cell_fw, cell_bw, x, seq_len, dtype=tf.float32)
x = tf.concat(o, -1)
y_intent = tf.layers.dense(tf.reduce_max(x, 1), params['intent_size'])
y_slots = tf.layers.dense(x, params['slot_size'])
if labels is not None:
intent, slots = labels
vocab = tf.contrib.lookup.index_table_from_file(
params['intent_path'], num_oov_buckets=1)
intent = vocab.lookup(intent)
vocab = tf.contrib.lookup.index_table_from_file(
params['slot_path'], num_oov_buckets=1)
slots = vocab.lookup(slots)
loss_intent = tf.nn.sparse_softmax_cross_entropy_with_logits(
labels=intent, logits=y_intent)
loss_intent = tf.reduce_mean(loss_intent)
loss_slots = tf.nn.sparse_softmax_cross_entropy_with_logits(labels=slots, logits=y_slots)
weights = tf.cast(tf.sign(slots), tf.float32)
padding = tf.fill(tf.shape(weights), 1e-2)
weights = tf.where(tf.equal(weights, 0.), padding, weights)
loss_slots = tf.reduce_mean(loss_slots * weights)
loss_op = loss_intent + loss_slots
if mode == tf.estimator.ModeKeys.TRAIN:
variables = tf.trainable_variables()
tf.logging.info('\n'+pprint.pformat(variables))
grads = tf.gradients(loss_op, variables)
grads, _ = tf.clip_by_global_norm(grads, params['clip_norm'])
global_step=tf.train.get_or_create_global_step()
decay_lr = tf.train.exponential_decay(
params['lr'], global_step, 1000, .9)
hook = tf.train.LoggingTensorHook({'lr': decay_lr}, every_n_iter=100)
optim = tf.train.AdamOptimizer(decay_lr)
train_op = optim.apply_gradients(
zip(grads, variables), global_step=global_step)
return tf.estimator.EstimatorSpec(mode=mode,
loss=loss_op,
train_op=train_op,
training_hooks=[hook],)
if mode == tf.estimator.ModeKeys.EVAL:
return tf.estimator.EstimatorSpec(mode=mode,
loss=loss_op)
if mode == tf.estimator.ModeKeys.PREDICT:
return tf.estimator.EstimatorSpec(mode,
predictions={'intent': tf.argmax(y_intent, -1),
'slots': tf.argmax(y_slots, -1)})
params = {
'model_dir': '../model/elmo_bigru',
'log_path': '../log/elmo_bigru.txt',
'train_path': '../data/atis.train.w-intent.iob',
'test_path': '../data/atis.test.w-intent.iob',
'word_path': '../vocab/word.txt',
'vocab_path': '../vocab/word.npy',
'intent_path': '../vocab/intent.txt',
'slot_path': '../vocab/slot.txt',
'batch_size': 16,
'num_samples': 4978,
'rnn_units': 300,
'dropout_rate': 0.2,
'clip_norm': 5.0,
'lr': 3e-4,
'num_patience': 3,
}
params['word2idx'] = get_vocab(params['word_path'])
params['intent2idx'] = get_vocab(params['intent_path'])
params['slot2idx'] = get_vocab(params['slot_path'])
params['word_size'] = len(params['word2idx']) + 1
params['intent_size'] = len(params['intent2idx']) + 1
params['slot_size'] = len(params['slot2idx']) + 1
def is_descending(history: list):
history = history[-(params['num_patience']+1):]
for i in range(1, len(history)):
if history[i-1] <= history[i]:
return False
return True
# Create directory if not exist
Path(os.path.dirname(params['log_path'])).mkdir(exist_ok=True)
Path(params['model_dir']).mkdir(exist_ok=True, parents=True)
# Logging
logger = logging.getLogger('tensorflow')
logger.setLevel(logging.INFO)
fh = logging.FileHandler(params['log_path'])
logger.addHandler(fh)
# Create an estimator
_eval_steps = params['num_samples']//params['batch_size'] + 1
config = tf.estimator.RunConfig(
save_checkpoints_steps=_eval_steps,)
estimator = tf.estimator.Estimator(
model_fn=model_fn,
model_dir=params['model_dir'],
config=config,
params=params)
# Train on training data and Evaluate on testing data
train_spec = tf.estimator.TrainSpec(
input_fn=lambda: dataset(is_training=True, params=params),)
eval_spec = tf.estimator.EvalSpec(
input_fn=lambda: dataset(is_training=False, params=params),
steps=None,
throttle_secs=10,)
best_f1 = .0
history_f1 = []
tf.enable_eager_execution()
while True:
tf.estimator.train_and_evaluate(estimator, train_spec, eval_spec)
intent = []
slots = []
for w, (i, s) in dataset(is_training=False, params=params):
intent.append(i.numpy())
slots.append(s.numpy())
intent = [i for batch in intent for i in batch]
intent = [params['intent2idx'].get(str(t, 'utf-8'), len(params['intent2idx'])) for t in intent]
slots = [j for batch in slots for i in batch for j in i]
slots = [params['slot2idx'].get(str(s, 'utf-8'), len(params['slot2idx'])) for s in slots]
predicted = list(estimator.predict(input_fn=lambda: dataset(is_training=False, params=params)))
y_slots = [j for i in predicted for j in i['slots']]
y_intent = [i['intent'] for i in predicted]
logger.info('\n'+classification_report(y_true = intent,
y_pred = y_intent,
labels = list(params['intent2idx'].values()),
target_names = list(params['intent2idx'].keys()),
digits=3))
logger.info('\n'+classification_report(y_true = slots,
y_pred = y_slots,
labels = list(params['slot2idx'].values()),
target_names = list(params['slot2idx'].keys()),
sample_weight = np.sign(slots),
digits=3))
f1_slots = f1_score(y_true = slots,
y_pred = y_slots,
labels = list(params['slot2idx'].values()),
sample_weight = np.sign(slots),
average='micro',)
history_f1.append(f1_slots)
if f1_slots > best_f1:
best_f1 = f1_slots
logger.info("Best Slot F1: {:.3f}".format(best_f1))
if len(history_f1) > params['num_patience'] and is_descending(history_f1):
logger.info("Testing Slot F1 not improved over {} epochs, Early Stop".format(params['num_patience']))
break
WARNING: Logging before flag parsing goes to stderr.
I0716 02:58:30.483514 139795883440000 estimator.py:209] Using config: {'_model_dir': '../model/elmo_bigru', '_tf_random_seed': None, '_save_summary_steps': 100, '_save_checkpoints_steps': 312, '_save_checkpoints_secs': None, '_session_config': allow_soft_placement: true
graph_options {
rewrite_options {
meta_optimizer_iterations: ONE
}
}
, '_keep_checkpoint_max': 5, '_keep_checkpoint_every_n_hours': 10000, '_log_step_count_steps': 100, '_train_distribute': None, '_device_fn': None, '_protocol': None, '_eval_distribute': None, '_experimental_distribute': None, '_experimental_max_worker_delay_secs': None, '_service': None, '_cluster_spec': <tensorflow.python.training.server_lib.ClusterSpec object at 0x7f2460d49240>, '_task_type': 'worker', '_task_id': 0, '_global_id_in_cluster': 0, '_master': '', '_evaluation_master': '', '_is_chief': True, '_num_ps_replicas': 0, '_num_worker_replicas': 1}
I0716 02:58:30.489848 139795883440000 estimator_training.py:186] Not using Distribute Coordinator.
I0716 02:58:30.493601 139795883440000 training.py:612] Running training and evaluation locally (non-distributed).
I0716 02:58:30.498357 139795883440000 training.py:700] Start train and evaluate loop. The evaluate will happen after every checkpoint. Checkpoint frequency is determined based on RunConfig arguments: save_checkpoints_steps 312 or save_checkpoints_secs None.
W0716 02:58:30.509913 139795883440000 deprecation.py:323] From /usr/local/lib/python3.6/dist-packages/tensorflow/python/training/training_util.py:236: Variable.initialized_value (from tensorflow.python.ops.variables) is deprecated and will be removed in a future version.
Instructions for updating:
Use Variable.read_value. Variables in 2.X are initialized automatically both in eager and graph (inside tf.defun) contexts.
W0716 02:58:30.533959 139795883440000 deprecation.py:323] From /usr/local/lib/python3.6/dist-packages/tensorflow/python/data/ops/dataset_ops.py:494: py_func (from tensorflow.python.ops.script_ops) is deprecated and will be removed in a future version.
Instructions for updating:
tf.py_func is deprecated in TF V2. Instead, there are two
options available in V2.
- tf.py_function takes a python function which manipulates tf eager
tensors instead of numpy arrays. It's easy to convert a tf eager tensor to
an ndarray (just call tensor.numpy()) but having access to eager tensors
means `tf.py_function`s can use accelerators such as GPUs as well as
being differentiable using a gradient tape.
- tf.numpy_function maintains the semantics of the deprecated tf.py_func
(it is not differentiable, and manipulates numpy arrays). It drops the
stateful argument making all functions stateful.
I0716 02:58:30.582866 139795883440000 estimator.py:1145] Calling model_fn.
W0716 02:58:31.239827 139795883440000 lazy_loader.py:50]
The TensorFlow contrib module will not be included in TensorFlow 2.0.
For more information, please see:
* https://github.com/tensorflow/community/blob/master/rfcs/20180907-contrib-sunset.md
* https://github.com/tensorflow/addons
* https://github.com/tensorflow/io (for I/O related ops)
If you depend on functionality not listed there, please file an issue.
W0716 02:58:31.253417 139795883440000 deprecation.py:323] From /usr/local/lib/python3.6/dist-packages/tensorflow/python/ops/lookup_ops.py:978: add_dispatch_support.<locals>.wrapper (from tensorflow.python.ops.array_ops) is deprecated and will be removed in a future version.
Instructions for updating:
Use tf.where in 2.0, which has the same broadcast rule as np.where
W0716 02:58:31.257234 139795883440000 deprecation.py:506] From /usr/local/lib/python3.6/dist-packages/tensorflow/python/util/deprecation.py:507: calling count_nonzero (from tensorflow.python.ops.math_ops) with axis is deprecated and will be removed in a future version.
Instructions for updating:
reduction_indices is deprecated, use axis instead
I0716 02:58:32.649416 139795883440000 saver.py:1499] Saver not created because there are no variables in the graph to restore
W0716 02:58:32.737046 139795883440000 deprecation.py:323] From <ipython-input-6-d6747b40360b>:17: dropout (from tensorflow.python.layers.core) is deprecated and will be removed in a future version.
Instructions for updating:
Use keras.layers.dropout instead.
W0716 02:58:32.812015 139795883440000 deprecation.py:323] From <ipython-input-6-d6747b40360b>:18: dense (from tensorflow.python.layers.core) is deprecated and will be removed in a future version.
Instructions for updating:
Use keras.layers.dense instead.
W0716 02:58:32.817591 139795883440000 deprecation.py:506] From /usr/local/lib/python3.6/dist-packages/tensorflow/python/ops/init_ops.py:1251: calling VarianceScaling.__init__ (from tensorflow.python.ops.init_ops) with dtype is deprecated and will be removed in a future version.
Instructions for updating:
Call initializer instance with the dtype argument instead of passing it to the constructor
W0716 02:58:33.183183 139795883440000 deprecation.py:323] From <ipython-input-6-d6747b40360b>:21: GRUCell.__init__ (from tensorflow.python.ops.rnn_cell_impl) is deprecated and will be removed in a future version.
Instructions for updating:
This class is equivalent as tf.keras.layers.GRUCell, and will be replaced by that in Tensorflow 2.0.
W0716 02:58:33.185412 139795883440000 deprecation.py:323] From <ipython-input-6-d6747b40360b>:23: bidirectional_dynamic_rnn (from tensorflow.python.ops.rnn) is deprecated and will be removed in a future version.
Instructions for updating:
Please use `keras.layers.Bidirectional(keras.layers.RNN(cell))`, which is equivalent to this API
W0716 02:58:33.187494 139795883440000 deprecation.py:323] From /usr/local/lib/python3.6/dist-packages/tensorflow/python/ops/rnn.py:464: dynamic_rnn (from tensorflow.python.ops.rnn) is deprecated and will be removed in a future version.
Instructions for updating:
Please use `keras.layers.RNN(cell)`, which is equivalent to this API
W0716 02:58:33.282541 139795883440000 deprecation.py:506] From /usr/local/lib/python3.6/dist-packages/tensorflow/python/ops/rnn_cell_impl.py:564: calling Constant.__init__ (from tensorflow.python.ops.init_ops) with dtype is deprecated and will be removed in a future version.
Instructions for updating:
Call initializer instance with the dtype argument instead of passing it to the constructor
W0716 02:58:33.300598 139795883440000 deprecation.py:506] From /usr/local/lib/python3.6/dist-packages/tensorflow/python/ops/rnn_cell_impl.py:574: calling Zeros.__init__ (from tensorflow.python.ops.init_ops) with dtype is deprecated and will be removed in a future version.
Instructions for updating:
Call initializer instance with the dtype argument instead of passing it to the constructor
I0716 02:58:33.824702 139795883440000 <ipython-input-6-d6747b40360b>:55]
[<tf.Variable 'embedding:0' shape=(750, 300) dtype=float32_ref>,
<tf.Variable 'module/aggregation/weights:0' shape=(3,) dtype=float32>,
<tf.Variable 'module/aggregation/scaling:0' shape=() dtype=float32>,
<tf.Variable 'dense/kernel:0' shape=(1324, 300) dtype=float32_ref>,
<tf.Variable 'dense/bias:0' shape=(300,) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/fw/gru_cell/gates/kernel:0' shape=(600, 600) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/fw/gru_cell/gates/bias:0' shape=(600,) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/fw/gru_cell/candidate/kernel:0' shape=(600, 300) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/fw/gru_cell/candidate/bias:0' shape=(300,) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/bw/gru_cell/gates/kernel:0' shape=(600, 600) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/bw/gru_cell/gates/bias:0' shape=(600,) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/bw/gru_cell/candidate/kernel:0' shape=(600, 300) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/bw/gru_cell/candidate/bias:0' shape=(300,) dtype=float32_ref>,
<tf.Variable 'dense_1/kernel:0' shape=(600, 23) dtype=float32_ref>,
<tf.Variable 'dense_1/bias:0' shape=(23,) dtype=float32_ref>,
<tf.Variable 'dense_2/kernel:0' shape=(600, 122) dtype=float32_ref>,
<tf.Variable 'dense_2/bias:0' shape=(122,) dtype=float32_ref>]
I0716 02:58:34.888393 139795883440000 estimator.py:1147] Done calling model_fn.
I0716 02:58:34.892489 139795883440000 basic_session_run_hooks.py:541] Create CheckpointSaverHook.
I0716 02:58:36.013862 139795883440000 monitored_session.py:240] Graph was finalized.
I0716 02:58:39.056770 139795883440000 session_manager.py:500] Running local_init_op.
I0716 02:58:39.108992 139795883440000 session_manager.py:502] Done running local_init_op.
I0716 02:58:41.689479 139795883440000 basic_session_run_hooks.py:606] Saving checkpoints for 0 into ../model/elmo_bigru/model.ckpt.
Reading ../data/atis.train.w-intent.iob
I0716 02:58:47.781142 139795883440000 basic_session_run_hooks.py:262] loss = 4.089126, step = 0 I0716 02:58:47.785583 139795883440000 basic_session_run_hooks.py:262] lr = 0.0003 I0716 02:59:17.091764 139795883440000 basic_session_run_hooks.py:692] global_step/sec: 3.41163 I0716 02:59:17.094635 139795883440000 basic_session_run_hooks.py:260] loss = 0.57814324, step = 100 (29.314 sec) I0716 02:59:17.100646 139795883440000 basic_session_run_hooks.py:260] lr = 0.00029685578 (29.315 sec) I0716 02:59:43.655862 139795883440000 basic_session_run_hooks.py:692] global_step/sec: 3.76447 I0716 02:59:43.659226 139795883440000 basic_session_run_hooks.py:260] loss = 0.25135267, step = 200 (26.565 sec) I0716 02:59:43.661857 139795883440000 basic_session_run_hooks.py:260] lr = 0.00029374452 (26.561 sec) I0716 03:00:09.468027 139795883440000 basic_session_run_hooks.py:692] global_step/sec: 3.87415 I0716 03:00:09.473734 139795883440000 basic_session_run_hooks.py:260] loss = 0.28173244, step = 300 (25.815 sec) I0716 03:00:09.475708 139795883440000 basic_session_run_hooks.py:260] lr = 0.00029066586 (25.814 sec) I0716 03:00:12.225202 139795883440000 basic_session_run_hooks.py:606] Saving checkpoints for 312 into ../model/elmo_bigru/model.ckpt. I0716 03:00:14.384853 139795883440000 estimator.py:1145] Calling model_fn. I0716 03:00:15.675234 139795883440000 saver.py:1499] Saver not created because there are no variables in the graph to restore I0716 03:00:16.476772 139795883440000 estimator.py:1147] Done calling model_fn. I0716 03:00:16.506482 139795883440000 evaluation.py:255] Starting evaluation at 2019-07-16T03:00:16Z I0716 03:00:16.733509 139795883440000 monitored_session.py:240] Graph was finalized. W0716 03:00:16.740108 139795883440000 deprecation.py:323] From /usr/local/lib/python3.6/dist-packages/tensorflow/python/training/saver.py:1276: checkpoint_exists (from tensorflow.python.training.checkpoint_management) is deprecated and will be removed in a future version. Instructions for updating: Use standard file APIs to check for files with this prefix. I0716 03:00:16.754307 139795883440000 saver.py:1280] Restoring parameters from ../model/elmo_bigru/model.ckpt-312 I0716 03:00:17.626637 139795883440000 session_manager.py:500] Running local_init_op. I0716 03:00:17.679666 139795883440000 session_manager.py:502] Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
I0716 03:01:17.274649 139795883440000 evaluation.py:275] Finished evaluation at 2019-07-16-03:01:17 I0716 03:01:17.276630 139795883440000 estimator.py:2039] Saving dict for global step 312: global_step = 312, loss = 0.6538741 I0716 03:01:17.900297 139795883440000 estimator.py:2099] Saving 'checkpoint_path' summary for global step 312: ../model/elmo_bigru/model.ckpt-312 I0716 03:01:17.977829 139795883440000 estimator.py:368] Loss for final step: 0.9662949.
Reading ../data/atis.test.w-intent.iob
I0716 03:01:18.593478 139795883440000 estimator.py:1145] Calling model_fn. I0716 03:01:20.062015 139795883440000 saver.py:1499] Saver not created because there are no variables in the graph to restore I0716 03:01:20.566077 139795883440000 estimator.py:1147] Done calling model_fn. I0716 03:01:20.785360 139795883440000 monitored_session.py:240] Graph was finalized. I0716 03:01:20.803046 139795883440000 saver.py:1280] Restoring parameters from ../model/elmo_bigru/model.ckpt-312 I0716 03:01:21.685840 139795883440000 session_manager.py:500] Running local_init_op. I0716 03:01:21.728985 139795883440000 session_manager.py:502] Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.
'precision', 'predicted', average, warn_for)
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1439: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples.
'recall', 'true', average, warn_for)
I0716 03:02:19.941967 139795883440000 interactiveshell.py:2882]
precision recall f1-score support
atis_flight 0.948 0.991 0.969 632
atis_airfare 0.887 0.979 0.931 48
atis_ground_service 0.923 1.000 0.960 36
atis_airline 0.925 0.974 0.949 38
atis_abbreviation 0.943 1.000 0.971 33
atis_aircraft 1.000 1.000 1.000 9
atis_flight_time 0.333 1.000 0.500 1
atis_quantity 0.429 1.000 0.600 3
atis_flight#atis_airfare 0.000 0.000 0.000 12
atis_airport 1.000 0.722 0.839 18
atis_distance 1.000 0.900 0.947 10
atis_city 0.000 0.000 0.000 6
atis_ground_fare 1.000 0.714 0.833 7
atis_capacity 1.000 0.952 0.976 21
atis_flight_no 0.000 0.000 0.000 8
atis_meal 0.000 0.000 0.000 6
atis_restriction 0.000 0.000 0.000 0
atis_airline#atis_flight_no 0.000 0.000 0.000 0
atis_ground_service#atis_ground_fare 0.000 0.000 0.000 0
atis_airfare#atis_flight_time 0.000 0.000 0.000 0
atis_cheapest 0.000 0.000 0.000 0
atis_aircraft#atis_flight#atis_flight_no 0.000 0.000 0.000 0
micro avg 0.940 0.945 0.942 888
macro avg 0.472 0.511 0.476 888
weighted avg 0.910 0.945 0.926 888
I0716 03:02:19.991601 139795883440000 interactiveshell.py:2882]
precision recall f1-score support
O 0.000 0.000 0.000 0.0
B-toloc.city_name 0.927 0.979 0.952 716.0
B-fromloc.city_name 0.923 0.994 0.958 704.0
I-toloc.city_name 0.847 0.958 0.899 265.0
B-depart_date.day_name 0.876 1.000 0.934 212.0
B-airline_name 0.952 0.980 0.966 101.0
I-fromloc.city_name 0.856 0.944 0.898 177.0
B-depart_time.period_of_day 0.804 0.885 0.842 130.0
I-airline_name 0.875 0.969 0.920 65.0
B-depart_date.day_number 0.720 0.982 0.831 55.0
B-depart_date.month_name 0.889 1.000 0.941 56.0
B-depart_time.time 0.496 0.982 0.659 57.0
B-round_trip 0.986 0.973 0.979 73.0
B-cost_relative 0.925 1.000 0.961 37.0
I-round_trip 0.887 1.000 0.940 71.0
B-flight_mod 0.700 0.875 0.778 24.0
B-depart_time.time_relative 0.735 0.938 0.824 65.0
I-depart_time.time 0.486 1.000 0.654 52.0
B-stoploc.city_name 0.586 0.850 0.694 20.0
B-city_name 0.471 0.579 0.520 57.0
B-class_type 1.000 0.708 0.829 24.0
B-arrive_time.time 0.714 0.294 0.417 34.0
B-arrive_time.time_relative 0.682 0.484 0.566 31.0
I-class_type 1.000 1.000 1.000 17.0
B-flight_stop 1.000 1.000 1.000 21.0
I-arrive_time.time 1.000 0.029 0.056 35.0
B-airline_code 0.533 0.706 0.608 34.0
I-depart_date.day_number 0.000 0.000 0.000 15.0
I-fromloc.airport_name 0.244 0.667 0.357 15.0
B-fromloc.airport_name 0.000 0.000 0.000 12.0
B-arrive_date.day_name 0.000 0.000 0.000 11.0
B-toloc.state_code 0.882 0.833 0.857 18.0
B-depart_date.today_relative 0.000 0.000 0.000 9.0
B-flight_number 0.538 0.636 0.583 11.0
B-depart_date.date_relative 0.857 0.706 0.774 17.0
B-toloc.state_name 0.000 0.000 0.000 28.0
B-fare_basis_code 0.500 1.000 0.667 17.0
B-flight_time 0.500 1.000 0.667 1.0
B-or 0.000 0.000 0.000 3.0
B-arrive_time.period_of_day 0.000 0.000 0.000 6.0
B-meal_description 0.538 0.700 0.609 10.0
I-cost_relative 0.000 0.000 0.000 3.0
I-airport_name 0.000 0.000 0.000 29.0
B-fare_amount 0.222 1.000 0.364 2.0
I-fare_amount 0.667 1.000 0.800 2.0
I-city_name 0.000 0.000 0.000 30.0
I-toloc.airport_name 0.000 0.000 0.000 3.0
B-transport_type 1.000 0.900 0.947 10.0
B-arrive_date.month_name 0.000 0.000 0.000 6.0
B-arrive_date.day_number 0.000 0.000 0.000 6.0
I-stoploc.city_name 0.000 0.000 0.000 10.0
B-meal 0.000 0.000 0.000 16.0
B-fromloc.state_code 1.000 0.087 0.160 23.0
B-depart_time.period_mod 0.000 0.000 0.000 5.0
B-connect 0.000 0.000 0.000 6.0
B-flight_days 1.000 0.100 0.182 10.0
B-toloc.airport_name 0.000 0.000 0.000 3.0
B-fromloc.state_name 0.000 0.000 0.000 17.0
B-airport_name 0.000 0.000 0.000 21.0
B-economy 0.000 0.000 0.000 6.0
I-flight_time 0.000 0.000 0.000 1.0
B-aircraft_code 0.000 0.000 0.000 33.0
B-mod 0.000 0.000 0.000 2.0
B-airport_code 0.000 0.000 0.000 9.0
B-depart_time.start_time 0.000 0.000 0.000 3.0
B-depart_time.end_time 0.000 0.000 0.000 3.0
B-depart_date.year 0.000 0.000 0.000 3.0
I-transport_type 0.000 0.000 0.000 1.0
B-restriction_code 0.000 0.000 0.000 4.0
B-arrive_time.start_time 0.000 0.000 0.000 8.0
B-toloc.airport_code 0.000 0.000 0.000 4.0
B-arrive_time.end_time 0.000 0.000 0.000 8.0
I-arrive_time.end_time 0.000 0.000 0.000 8.0
I-depart_time.end_time 0.000 0.000 0.000 3.0
I-flight_stop 0.000 0.000 0.000 0.0
B-fromloc.airport_code 0.000 0.000 0.000 5.0
I-restriction_code 0.000 0.000 0.000 3.0
I-depart_time.start_time 0.000 0.000 0.000 1.0
I-toloc.state_name 0.000 0.000 0.000 1.0
I-depart_date.today_relative 0.000 0.000 0.000 0.0
B-arrive_date.date_relative 0.000 0.000 0.000 2.0
I-flight_mod 0.000 0.000 0.000 6.0
I-economy 0.000 0.000 0.000 0.0
B-return_date.date_relative 0.000 0.000 0.000 3.0
I-fromloc.state_name 0.000 0.000 0.000 1.0
B-state_code 0.000 0.000 0.000 1.0
I-arrive_time.start_time 0.000 0.000 0.000 1.0
I-arrive_date.day_number 0.000 0.000 0.000 0.0
B-meal_code 0.000 0.000 0.000 1.0
I-depart_time.period_of_day 0.000 0.000 0.000 1.0
B-day_name 0.000 0.000 0.000 2.0
B-period_of_day 0.000 0.000 0.000 4.0
B-stoploc.state_code 0.000 0.000 0.000 0.0
B-return_date.month_name 0.000 0.000 0.000 0.0
B-return_date.day_number 0.000 0.000 0.000 0.0
B-arrive_time.period_mod 0.000 0.000 0.000 0.0
I-meal_code 0.000 0.000 0.000 0.0
B-toloc.country_name 0.000 0.000 0.000 1.0
B-days_code 0.000 0.000 0.000 1.0
I-arrive_time.period_of_day 0.000 0.000 0.000 0.0
I-today_relative 0.000 0.000 0.000 0.0
B-return_time.period_of_day 0.000 0.000 0.000 0.0
B-time 0.000 0.000 0.000 0.0
I-fare_basis_code 0.000 0.000 0.000 0.0
I-arrive_time.time_relative 0.000 0.000 0.000 4.0
I-depart_time.time_relative 0.000 0.000 0.000 1.0
B-today_relative 0.000 0.000 0.000 0.0
B-state_name 0.000 0.000 0.000 9.0
B-arrive_date.today_relative 0.000 0.000 0.000 0.0
B-return_time.period_mod 0.000 0.000 0.000 0.0
B-month_name 0.000 0.000 0.000 0.0
B-day_number 0.000 0.000 0.000 0.0
I-return_date.date_relative 0.000 0.000 0.000 3.0
I-return_date.today_relative 0.000 0.000 0.000 0.0
B-stoploc.airport_name 0.000 0.000 0.000 0.0
B-time_relative 0.000 0.000 0.000 0.0
I-time 0.000 0.000 0.000 0.0
I-return_date.day_number 0.000 0.000 0.000 0.0
I-meal_description 0.000 0.000 0.000 0.0
B-return_date.today_relative 0.000 0.000 0.000 0.0
B-return_date.day_name 0.000 0.000 0.000 2.0
micro avg 0.827 0.829 0.828 3657.0
macro avg 0.238 0.254 0.228 3657.0
weighted avg 0.768 0.829 0.784 3657.0
I0716 03:02:20.012179 139795883440000 interactiveshell.py:2882] Best Slot F1: 0.828
I0716 03:02:20.013774 139795883440000 estimator_training.py:186] Not using Distribute Coordinator.
I0716 03:02:20.015699 139795883440000 training.py:612] Running training and evaluation locally (non-distributed).
I0716 03:02:20.017799 139795883440000 training.py:700] Start train and evaluate loop. The evaluate will happen after every checkpoint. Checkpoint frequency is determined based on RunConfig arguments: save_checkpoints_steps 312 or save_checkpoints_secs None.
I0716 03:02:20.084496 139795883440000 estimator.py:1145] Calling model_fn.
I0716 03:02:21.517312 139795883440000 saver.py:1499] Saver not created because there are no variables in the graph to restore
I0716 03:02:22.273025 139795883440000 <ipython-input-6-d6747b40360b>:55]
[<tf.Variable 'embedding:0' shape=(750, 300) dtype=float32_ref>,
<tf.Variable 'module/aggregation/weights:0' shape=(3,) dtype=float32>,
<tf.Variable 'module/aggregation/scaling:0' shape=() dtype=float32>,
<tf.Variable 'dense/kernel:0' shape=(1324, 300) dtype=float32_ref>,
<tf.Variable 'dense/bias:0' shape=(300,) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/fw/gru_cell/gates/kernel:0' shape=(600, 600) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/fw/gru_cell/gates/bias:0' shape=(600,) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/fw/gru_cell/candidate/kernel:0' shape=(600, 300) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/fw/gru_cell/candidate/bias:0' shape=(300,) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/bw/gru_cell/gates/kernel:0' shape=(600, 600) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/bw/gru_cell/gates/bias:0' shape=(600,) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/bw/gru_cell/candidate/kernel:0' shape=(600, 300) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/bw/gru_cell/candidate/bias:0' shape=(300,) dtype=float32_ref>,
<tf.Variable 'dense_1/kernel:0' shape=(600, 23) dtype=float32_ref>,
<tf.Variable 'dense_1/bias:0' shape=(23,) dtype=float32_ref>,
<tf.Variable 'dense_2/kernel:0' shape=(600, 122) dtype=float32_ref>,
<tf.Variable 'dense_2/bias:0' shape=(122,) dtype=float32_ref>]
I0716 03:02:23.352547 139795883440000 estimator.py:1147] Done calling model_fn.
I0716 03:02:23.356692 139795883440000 basic_session_run_hooks.py:541] Create CheckpointSaverHook.
I0716 03:02:23.677159 139795883440000 monitored_session.py:240] Graph was finalized.
I0716 03:02:23.693130 139795883440000 saver.py:1280] Restoring parameters from ../model/elmo_bigru/model.ckpt-312
W0716 03:02:24.570057 139795883440000 deprecation.py:323] From /usr/local/lib/python3.6/dist-packages/tensorflow/python/training/saver.py:1066: get_checkpoint_mtimes (from tensorflow.python.training.checkpoint_management) is deprecated and will be removed in a future version.
Instructions for updating:
Use standard file utilities to get mtimes.
I0716 03:02:24.701941 139795883440000 session_manager.py:500] Running local_init_op.
I0716 03:02:24.764321 139795883440000 session_manager.py:502] Done running local_init_op.
I0716 03:02:27.727120 139795883440000 basic_session_run_hooks.py:606] Saving checkpoints for 312 into ../model/elmo_bigru/model.ckpt.
Reading ../data/atis.train.w-intent.iob
I0716 03:02:32.666477 139795883440000 basic_session_run_hooks.py:262] loss = 0.19418588, step = 312 I0716 03:02:32.668385 139795883440000 basic_session_run_hooks.py:262] lr = 0.0002902986 I0716 03:03:00.168136 139795883440000 basic_session_run_hooks.py:692] global_step/sec: 3.63607 I0716 03:03:00.173908 139795883440000 basic_session_run_hooks.py:260] loss = 0.44005728, step = 412 (27.507 sec) I0716 03:03:00.177161 139795883440000 basic_session_run_hooks.py:260] lr = 0.00028725603 (27.509 sec) I0716 03:03:26.463520 139795883440000 basic_session_run_hooks.py:692] global_step/sec: 3.80294 I0716 03:03:26.471512 139795883440000 basic_session_run_hooks.py:260] loss = 0.43844122, step = 512 (26.298 sec) I0716 03:03:26.473572 139795883440000 basic_session_run_hooks.py:260] lr = 0.00028424538 (26.296 sec) I0716 03:03:53.220065 139795883440000 basic_session_run_hooks.py:692] global_step/sec: 3.73741 I0716 03:03:53.226559 139795883440000 basic_session_run_hooks.py:260] loss = 0.13767746, step = 612 (26.755 sec) I0716 03:03:53.228701 139795883440000 basic_session_run_hooks.py:260] lr = 0.00028126626 (26.755 sec) I0716 03:03:56.350152 139795883440000 basic_session_run_hooks.py:606] Saving checkpoints for 624 into ../model/elmo_bigru/model.ckpt. I0716 03:03:58.483096 139795883440000 estimator.py:1145] Calling model_fn. I0716 03:03:59.825540 139795883440000 saver.py:1499] Saver not created because there are no variables in the graph to restore I0716 03:04:00.421510 139795883440000 estimator.py:1147] Done calling model_fn. I0716 03:04:00.455102 139795883440000 evaluation.py:255] Starting evaluation at 2019-07-16T03:04:00Z I0716 03:04:00.696269 139795883440000 monitored_session.py:240] Graph was finalized. I0716 03:04:00.711647 139795883440000 saver.py:1280] Restoring parameters from ../model/elmo_bigru/model.ckpt-624 I0716 03:04:01.609956 139795883440000 session_manager.py:500] Running local_init_op. I0716 03:04:01.664305 139795883440000 session_manager.py:502] Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
I0716 03:05:00.591241 139795883440000 evaluation.py:275] Finished evaluation at 2019-07-16-03:05:00 I0716 03:05:00.593144 139795883440000 estimator.py:2039] Saving dict for global step 624: global_step = 624, loss = 0.39170766 I0716 03:05:00.599527 139795883440000 estimator.py:2099] Saving 'checkpoint_path' summary for global step 624: ../model/elmo_bigru/model.ckpt-624 I0716 03:05:00.683492 139795883440000 estimator.py:368] Loss for final step: 0.19239207.
Reading ../data/atis.test.w-intent.iob
I0716 03:05:01.279512 139795883440000 estimator.py:1145] Calling model_fn. I0716 03:05:02.865846 139795883440000 saver.py:1499] Saver not created because there are no variables in the graph to restore I0716 03:05:03.398653 139795883440000 estimator.py:1147] Done calling model_fn. I0716 03:05:03.780389 139795883440000 monitored_session.py:240] Graph was finalized. I0716 03:05:03.796601 139795883440000 saver.py:1280] Restoring parameters from ../model/elmo_bigru/model.ckpt-624 I0716 03:05:04.675439 139795883440000 session_manager.py:500] Running local_init_op. I0716 03:05:04.724084 139795883440000 session_manager.py:502] Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.
'precision', 'predicted', average, warn_for)
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1439: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples.
'recall', 'true', average, warn_for)
I0716 03:06:02.881382 139795883440000 interactiveshell.py:2882]
precision recall f1-score support
atis_flight 0.981 0.987 0.984 632
atis_airfare 0.889 1.000 0.941 48
atis_ground_service 0.857 1.000 0.923 36
atis_airline 0.950 1.000 0.974 38
atis_abbreviation 0.917 1.000 0.957 33
atis_aircraft 1.000 1.000 1.000 9
atis_flight_time 0.333 1.000 0.500 1
atis_quantity 0.429 1.000 0.600 3
atis_flight#atis_airfare 0.833 0.417 0.556 12
atis_airport 1.000 0.944 0.971 18
atis_distance 1.000 0.600 0.750 10
atis_city 1.000 0.500 0.667 6
atis_ground_fare 1.000 0.857 0.923 7
atis_capacity 1.000 0.952 0.976 21
atis_flight_no 1.000 1.000 1.000 8
atis_meal 0.000 0.000 0.000 6
atis_restriction 0.000 0.000 0.000 0
atis_airline#atis_flight_no 0.000 0.000 0.000 0
atis_ground_service#atis_ground_fare 0.000 0.000 0.000 0
atis_airfare#atis_flight_time 0.000 0.000 0.000 0
atis_cheapest 0.000 0.000 0.000 0
atis_aircraft#atis_flight#atis_flight_no 0.000 0.000 0.000 0
micro avg 0.960 0.965 0.962 888
macro avg 0.600 0.603 0.578 888
weighted avg 0.958 0.965 0.958 888
I0716 03:06:02.919304 139795883440000 interactiveshell.py:2882]
precision recall f1-score support
O 0.000 0.000 0.000 0.0
B-toloc.city_name 0.958 0.989 0.973 716.0
B-fromloc.city_name 0.955 0.997 0.976 704.0
I-toloc.city_name 0.886 0.996 0.938 265.0
B-depart_date.day_name 0.922 1.000 0.959 212.0
B-airline_name 0.971 0.990 0.980 101.0
I-fromloc.city_name 0.935 0.972 0.953 177.0
B-depart_time.period_of_day 0.949 0.862 0.903 130.0
I-airline_name 1.000 0.969 0.984 65.0
B-depart_date.day_number 0.902 1.000 0.948 55.0
B-depart_date.month_name 0.902 0.982 0.940 56.0
B-depart_time.time 0.750 0.895 0.816 57.0
B-round_trip 0.986 0.973 0.979 73.0
B-cost_relative 0.974 1.000 0.987 37.0
I-round_trip 0.973 1.000 0.986 71.0
B-flight_mod 0.786 0.917 0.846 24.0
B-depart_time.time_relative 0.964 0.831 0.893 65.0
I-depart_time.time 0.885 0.885 0.885 52.0
B-stoploc.city_name 0.760 0.950 0.844 20.0
B-city_name 0.810 0.596 0.687 57.0
B-class_type 0.960 1.000 0.980 24.0
B-arrive_time.time 0.582 0.941 0.719 34.0
B-arrive_time.time_relative 0.667 0.968 0.789 31.0
I-class_type 1.000 1.000 1.000 17.0
B-flight_stop 1.000 1.000 1.000 21.0
I-arrive_time.time 0.611 0.943 0.742 35.0
B-airline_code 0.659 0.794 0.720 34.0
I-depart_date.day_number 0.833 1.000 0.909 15.0
I-fromloc.airport_name 0.387 0.800 0.522 15.0
B-fromloc.airport_name 0.158 0.250 0.194 12.0
B-arrive_date.day_name 0.500 0.091 0.154 11.0
B-toloc.state_code 0.900 1.000 0.947 18.0
B-depart_date.today_relative 1.000 0.556 0.714 9.0
B-flight_number 0.833 0.909 0.870 11.0
B-depart_date.date_relative 0.810 1.000 0.895 17.0
B-toloc.state_name 0.545 0.214 0.308 28.0
B-fare_basis_code 0.762 0.941 0.842 17.0
B-flight_time 1.000 1.000 1.000 1.0
B-or 1.000 1.000 1.000 3.0
B-arrive_time.period_of_day 0.429 0.500 0.462 6.0
B-meal_description 1.000 0.800 0.889 10.0
I-cost_relative 1.000 0.667 0.800 3.0
I-airport_name 0.818 0.310 0.450 29.0
B-fare_amount 1.000 1.000 1.000 2.0
I-fare_amount 1.000 1.000 1.000 2.0
I-city_name 0.760 0.633 0.691 30.0
I-toloc.airport_name 1.000 0.333 0.500 3.0
B-transport_type 0.909 1.000 0.952 10.0
B-arrive_date.month_name 0.000 0.000 0.000 6.0
B-arrive_date.day_number 0.000 0.000 0.000 6.0
I-stoploc.city_name 0.000 0.000 0.000 10.0
B-meal 0.941 1.000 0.970 16.0
B-fromloc.state_code 1.000 0.913 0.955 23.0
B-depart_time.period_mod 0.714 1.000 0.833 5.0
B-connect 1.000 0.667 0.800 6.0
B-flight_days 1.000 1.000 1.000 10.0
B-toloc.airport_name 1.000 0.333 0.500 3.0
B-fromloc.state_name 0.000 0.000 0.000 17.0
B-airport_name 0.500 0.286 0.364 21.0
B-economy 1.000 0.167 0.286 6.0
I-flight_time 1.000 1.000 1.000 1.0
B-aircraft_code 1.000 0.758 0.862 33.0
B-mod 0.000 0.000 0.000 2.0
B-airport_code 1.000 0.222 0.364 9.0
B-depart_time.start_time 0.667 0.667 0.667 3.0
B-depart_time.end_time 0.000 0.000 0.000 3.0
B-depart_date.year 0.000 0.000 0.000 3.0
I-transport_type 0.000 0.000 0.000 1.0
B-restriction_code 0.500 0.500 0.500 4.0
B-arrive_time.start_time 0.000 0.000 0.000 8.0
B-toloc.airport_code 0.000 0.000 0.000 4.0
B-arrive_time.end_time 0.000 0.000 0.000 8.0
I-arrive_time.end_time 0.000 0.000 0.000 8.0
I-depart_time.end_time 0.000 0.000 0.000 3.0
I-flight_stop 0.000 0.000 0.000 0.0
B-fromloc.airport_code 0.000 0.000 0.000 5.0
I-restriction_code 0.000 0.000 0.000 3.0
I-depart_time.start_time 0.000 0.000 0.000 1.0
I-toloc.state_name 0.000 0.000 0.000 1.0
I-depart_date.today_relative 0.000 0.000 0.000 0.0
B-arrive_date.date_relative 0.000 0.000 0.000 2.0
I-flight_mod 0.000 0.000 0.000 6.0
I-economy 0.000 0.000 0.000 0.0
B-return_date.date_relative 0.000 0.000 0.000 3.0
I-fromloc.state_name 0.000 0.000 0.000 1.0
B-state_code 0.000 0.000 0.000 1.0
I-arrive_time.start_time 0.000 0.000 0.000 1.0
I-arrive_date.day_number 0.000 0.000 0.000 0.0
B-meal_code 0.000 0.000 0.000 1.0
I-depart_time.period_of_day 0.000 0.000 0.000 1.0
B-day_name 0.000 0.000 0.000 2.0
B-period_of_day 0.000 0.000 0.000 4.0
B-stoploc.state_code 0.000 0.000 0.000 0.0
B-return_date.month_name 0.000 0.000 0.000 0.0
B-return_date.day_number 0.000 0.000 0.000 0.0
B-arrive_time.period_mod 0.000 0.000 0.000 0.0
I-meal_code 0.000 0.000 0.000 0.0
B-toloc.country_name 0.000 0.000 0.000 1.0
B-days_code 0.000 0.000 0.000 1.0
I-arrive_time.period_of_day 0.000 0.000 0.000 0.0
I-today_relative 0.000 0.000 0.000 0.0
B-return_time.period_of_day 0.000 0.000 0.000 0.0
B-time 0.000 0.000 0.000 0.0
I-fare_basis_code 0.000 0.000 0.000 0.0
I-arrive_time.time_relative 0.000 0.000 0.000 4.0
I-depart_time.time_relative 0.000 0.000 0.000 1.0
B-today_relative 0.000 0.000 0.000 0.0
B-state_name 0.000 0.000 0.000 9.0
B-arrive_date.today_relative 0.000 0.000 0.000 0.0
B-return_time.period_mod 0.000 0.000 0.000 0.0
B-month_name 0.000 0.000 0.000 0.0
B-day_number 0.000 0.000 0.000 0.0
I-return_date.date_relative 0.000 0.000 0.000 3.0
I-return_date.today_relative 0.000 0.000 0.000 0.0
B-stoploc.airport_name 0.000 0.000 0.000 0.0
B-time_relative 0.000 0.000 0.000 0.0
I-time 0.000 0.000 0.000 0.0
I-return_date.day_number 0.000 0.000 0.000 0.0
I-meal_description 0.000 0.000 0.000 0.0
B-return_date.today_relative 0.000 0.000 0.000 0.0
B-return_date.day_name 0.000 0.000 0.000 2.0
micro avg 0.898 0.900 0.899 3657.0
macro avg 0.419 0.396 0.394 3657.0
weighted avg 0.876 0.900 0.881 3657.0
I0716 03:06:02.940419 139795883440000 interactiveshell.py:2882] Best Slot F1: 0.899
I0716 03:06:02.942004 139795883440000 estimator_training.py:186] Not using Distribute Coordinator.
I0716 03:06:02.943432 139795883440000 training.py:612] Running training and evaluation locally (non-distributed).
I0716 03:06:02.944901 139795883440000 training.py:700] Start train and evaluate loop. The evaluate will happen after every checkpoint. Checkpoint frequency is determined based on RunConfig arguments: save_checkpoints_steps 312 or save_checkpoints_secs None.
I0716 03:06:03.011088 139795883440000 estimator.py:1145] Calling model_fn.
I0716 03:06:04.299739 139795883440000 saver.py:1499] Saver not created because there are no variables in the graph to restore
I0716 03:06:05.053959 139795883440000 <ipython-input-6-d6747b40360b>:55]
[<tf.Variable 'embedding:0' shape=(750, 300) dtype=float32_ref>,
<tf.Variable 'module/aggregation/weights:0' shape=(3,) dtype=float32>,
<tf.Variable 'module/aggregation/scaling:0' shape=() dtype=float32>,
<tf.Variable 'dense/kernel:0' shape=(1324, 300) dtype=float32_ref>,
<tf.Variable 'dense/bias:0' shape=(300,) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/fw/gru_cell/gates/kernel:0' shape=(600, 600) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/fw/gru_cell/gates/bias:0' shape=(600,) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/fw/gru_cell/candidate/kernel:0' shape=(600, 300) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/fw/gru_cell/candidate/bias:0' shape=(300,) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/bw/gru_cell/gates/kernel:0' shape=(600, 600) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/bw/gru_cell/gates/bias:0' shape=(600,) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/bw/gru_cell/candidate/kernel:0' shape=(600, 300) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/bw/gru_cell/candidate/bias:0' shape=(300,) dtype=float32_ref>,
<tf.Variable 'dense_1/kernel:0' shape=(600, 23) dtype=float32_ref>,
<tf.Variable 'dense_1/bias:0' shape=(23,) dtype=float32_ref>,
<tf.Variable 'dense_2/kernel:0' shape=(600, 122) dtype=float32_ref>,
<tf.Variable 'dense_2/bias:0' shape=(122,) dtype=float32_ref>]
I0716 03:06:06.150071 139795883440000 estimator.py:1147] Done calling model_fn.
I0716 03:06:06.154184 139795883440000 basic_session_run_hooks.py:541] Create CheckpointSaverHook.
I0716 03:06:06.486130 139795883440000 monitored_session.py:240] Graph was finalized.
I0716 03:06:06.497994 139795883440000 saver.py:1280] Restoring parameters from ../model/elmo_bigru/model.ckpt-624
I0716 03:06:07.499193 139795883440000 session_manager.py:500] Running local_init_op.
I0716 03:06:07.571006 139795883440000 session_manager.py:502] Done running local_init_op.
I0716 03:06:10.528747 139795883440000 basic_session_run_hooks.py:606] Saving checkpoints for 624 into ../model/elmo_bigru/model.ckpt.
Reading ../data/atis.train.w-intent.iob
I0716 03:06:15.286043 139795883440000 basic_session_run_hooks.py:262] loss = 0.10650402, step = 624 I0716 03:06:15.287670 139795883440000 basic_session_run_hooks.py:262] lr = 0.00028091087 I0716 03:06:42.979780 139795883440000 basic_session_run_hooks.py:692] global_step/sec: 3.61086 I0716 03:06:42.986958 139795883440000 basic_session_run_hooks.py:260] loss = 0.10334292, step = 724 (27.701 sec) I0716 03:06:42.989081 139795883440000 basic_session_run_hooks.py:260] lr = 0.00027796673 (27.701 sec) I0716 03:07:09.221698 139795883440000 basic_session_run_hooks.py:692] global_step/sec: 3.81069 I0716 03:07:09.226385 139795883440000 basic_session_run_hooks.py:260] loss = 0.07500512, step = 824 (26.239 sec) I0716 03:07:09.230600 139795883440000 basic_session_run_hooks.py:260] lr = 0.00027505343 (26.242 sec) I0716 03:07:35.807775 139795883440000 basic_session_run_hooks.py:692] global_step/sec: 3.76137 I0716 03:07:35.813550 139795883440000 basic_session_run_hooks.py:260] loss = 0.91219175, step = 924 (26.587 sec) I0716 03:07:35.817075 139795883440000 basic_session_run_hooks.py:260] lr = 0.00027217067 (26.586 sec) I0716 03:07:38.272199 139795883440000 basic_session_run_hooks.py:606] Saving checkpoints for 936 into ../model/elmo_bigru/model.ckpt. I0716 03:07:40.166394 139795883440000 estimator.py:1145] Calling model_fn. I0716 03:07:41.555429 139795883440000 saver.py:1499] Saver not created because there are no variables in the graph to restore I0716 03:07:42.150777 139795883440000 estimator.py:1147] Done calling model_fn. I0716 03:07:42.180353 139795883440000 evaluation.py:255] Starting evaluation at 2019-07-16T03:07:42Z I0716 03:07:42.412451 139795883440000 monitored_session.py:240] Graph was finalized. I0716 03:07:42.427135 139795883440000 saver.py:1280] Restoring parameters from ../model/elmo_bigru/model.ckpt-936 I0716 03:07:43.321085 139795883440000 session_manager.py:500] Running local_init_op. I0716 03:07:43.375441 139795883440000 session_manager.py:502] Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
I0716 03:08:42.309874 139795883440000 evaluation.py:275] Finished evaluation at 2019-07-16-03:08:42 I0716 03:08:42.311658 139795883440000 estimator.py:2039] Saving dict for global step 936: global_step = 936, loss = 0.3205106 I0716 03:08:42.318030 139795883440000 estimator.py:2099] Saving 'checkpoint_path' summary for global step 936: ../model/elmo_bigru/model.ckpt-936 I0716 03:08:42.402586 139795883440000 estimator.py:368] Loss for final step: 0.17703222.
Reading ../data/atis.test.w-intent.iob
I0716 03:08:43.001334 139795883440000 estimator.py:1145] Calling model_fn. I0716 03:08:44.594808 139795883440000 saver.py:1499] Saver not created because there are no variables in the graph to restore I0716 03:08:45.103219 139795883440000 estimator.py:1147] Done calling model_fn. I0716 03:08:45.491380 139795883440000 monitored_session.py:240] Graph was finalized. I0716 03:08:45.505645 139795883440000 saver.py:1280] Restoring parameters from ../model/elmo_bigru/model.ckpt-936 I0716 03:08:46.373692 139795883440000 session_manager.py:500] Running local_init_op. I0716 03:08:46.417907 139795883440000 session_manager.py:502] Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.
'precision', 'predicted', average, warn_for)
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1439: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples.
'recall', 'true', average, warn_for)
I0716 03:09:44.563587 139795883440000 interactiveshell.py:2882]
precision recall f1-score support
atis_flight 0.984 0.991 0.987 632
atis_airfare 0.960 1.000 0.980 48
atis_ground_service 0.947 1.000 0.973 36
atis_airline 1.000 1.000 1.000 38
atis_abbreviation 0.943 1.000 0.971 33
atis_aircraft 0.900 1.000 0.947 9
atis_flight_time 0.143 1.000 0.250 1
atis_quantity 0.429 1.000 0.600 3
atis_flight#atis_airfare 0.800 0.333 0.471 12
atis_airport 1.000 1.000 1.000 18
atis_distance 1.000 0.900 0.947 10
atis_city 1.000 0.500 0.667 6
atis_ground_fare 1.000 0.857 0.923 7
atis_capacity 1.000 0.952 0.976 21
atis_flight_no 0.889 1.000 0.941 8
atis_meal 1.000 0.333 0.500 6
atis_restriction 0.000 0.000 0.000 0
atis_airline#atis_flight_no 0.000 0.000 0.000 0
atis_ground_service#atis_ground_fare 0.000 0.000 0.000 0
atis_airfare#atis_flight_time 0.000 0.000 0.000 0
atis_cheapest 0.000 0.000 0.000 0
atis_aircraft#atis_flight#atis_flight_no 0.000 0.000 0.000 0
micro avg 0.968 0.973 0.970 888
macro avg 0.636 0.630 0.597 888
weighted avg 0.975 0.973 0.970 888
I0716 03:09:44.601177 139795883440000 interactiveshell.py:2882]
precision recall f1-score support
O 0.000 0.000 0.000 0.0
B-toloc.city_name 0.970 0.993 0.981 716.0
B-fromloc.city_name 0.966 0.999 0.982 704.0
I-toloc.city_name 0.940 0.996 0.967 265.0
B-depart_date.day_name 0.946 0.991 0.968 212.0
B-airline_name 0.962 0.990 0.976 101.0
I-fromloc.city_name 0.967 0.989 0.978 177.0
B-depart_time.period_of_day 0.942 0.869 0.904 130.0
I-airline_name 1.000 0.954 0.976 65.0
B-depart_date.day_number 0.964 0.982 0.973 55.0
B-depart_date.month_name 0.965 0.982 0.973 56.0
B-depart_time.time 0.778 0.982 0.868 57.0
B-round_trip 1.000 0.986 0.993 73.0
B-cost_relative 0.973 0.973 0.973 37.0
I-round_trip 0.986 1.000 0.993 71.0
B-flight_mod 0.786 0.917 0.846 24.0
B-depart_time.time_relative 0.915 1.000 0.956 65.0
I-depart_time.time 0.812 1.000 0.897 52.0
B-stoploc.city_name 0.900 0.900 0.900 20.0
B-city_name 0.872 0.596 0.708 57.0
B-class_type 0.960 1.000 0.980 24.0
B-arrive_time.time 0.865 0.941 0.901 34.0
B-arrive_time.time_relative 0.879 0.935 0.906 31.0
I-class_type 1.000 1.000 1.000 17.0
B-flight_stop 1.000 1.000 1.000 21.0
I-arrive_time.time 0.800 0.914 0.853 35.0
B-airline_code 0.757 0.824 0.789 34.0
I-depart_date.day_number 1.000 1.000 1.000 15.0
I-fromloc.airport_name 0.394 0.867 0.542 15.0
B-fromloc.airport_name 0.194 0.500 0.279 12.0
B-arrive_date.day_name 0.750 0.545 0.632 11.0
B-toloc.state_code 0.900 1.000 0.947 18.0
B-depart_date.today_relative 1.000 0.778 0.875 9.0
B-flight_number 0.846 1.000 0.917 11.0
B-depart_date.date_relative 0.850 1.000 0.919 17.0
B-toloc.state_name 0.735 0.893 0.806 28.0
B-fare_basis_code 0.882 0.882 0.882 17.0
B-flight_time 1.000 1.000 1.000 1.0
B-or 1.000 1.000 1.000 3.0
B-arrive_time.period_of_day 0.800 0.667 0.727 6.0
B-meal_description 0.900 0.900 0.900 10.0
I-cost_relative 1.000 0.667 0.800 3.0
I-airport_name 0.900 0.310 0.462 29.0
B-fare_amount 1.000 1.000 1.000 2.0
I-fare_amount 1.000 1.000 1.000 2.0
I-city_name 0.800 0.533 0.640 30.0
I-toloc.airport_name 1.000 0.333 0.500 3.0
B-transport_type 0.909 1.000 0.952 10.0
B-arrive_date.month_name 0.667 0.667 0.667 6.0
B-arrive_date.day_number 0.800 0.667 0.727 6.0
I-stoploc.city_name 0.875 0.700 0.778 10.0
B-meal 1.000 0.938 0.968 16.0
B-fromloc.state_code 1.000 0.913 0.955 23.0
B-depart_time.period_mod 0.714 1.000 0.833 5.0
B-connect 1.000 1.000 1.000 6.0
B-flight_days 1.000 1.000 1.000 10.0
B-toloc.airport_name 1.000 0.333 0.500 3.0
B-fromloc.state_name 0.909 0.588 0.714 17.0
B-airport_name 0.714 0.238 0.357 21.0
B-economy 1.000 0.333 0.500 6.0
I-flight_time 1.000 1.000 1.000 1.0
B-aircraft_code 1.000 0.788 0.881 33.0
B-mod 0.000 0.000 0.000 2.0
B-airport_code 0.600 0.333 0.429 9.0
B-depart_time.start_time 0.222 0.667 0.333 3.0
B-depart_time.end_time 0.286 0.667 0.400 3.0
B-depart_date.year 1.000 1.000 1.000 3.0
I-transport_type 0.000 0.000 0.000 1.0
B-restriction_code 0.667 0.500 0.571 4.0
B-arrive_time.start_time 0.000 0.000 0.000 8.0
B-toloc.airport_code 0.000 0.000 0.000 4.0
B-arrive_time.end_time 0.000 0.000 0.000 8.0
I-arrive_time.end_time 0.000 0.000 0.000 8.0
I-depart_time.end_time 0.000 0.000 0.000 3.0
I-flight_stop 0.000 0.000 0.000 0.0
B-fromloc.airport_code 0.000 0.000 0.000 5.0
I-restriction_code 1.000 1.000 1.000 3.0
I-depart_time.start_time 0.500 1.000 0.667 1.0
I-toloc.state_name 0.000 0.000 0.000 1.0
I-depart_date.today_relative 0.000 0.000 0.000 0.0
B-arrive_date.date_relative 0.000 0.000 0.000 2.0
I-flight_mod 0.000 0.000 0.000 6.0
I-economy 0.000 0.000 0.000 0.0
B-return_date.date_relative 0.000 0.000 0.000 3.0
I-fromloc.state_name 0.000 0.000 0.000 1.0
B-state_code 1.000 1.000 1.000 1.0
I-arrive_time.start_time 0.000 0.000 0.000 1.0
I-arrive_date.day_number 0.000 0.000 0.000 0.0
B-meal_code 0.000 0.000 0.000 1.0
I-depart_time.period_of_day 0.000 0.000 0.000 1.0
B-day_name 0.000 0.000 0.000 2.0
B-period_of_day 0.000 0.000 0.000 4.0
B-stoploc.state_code 0.000 0.000 0.000 0.0
B-return_date.month_name 0.000 0.000 0.000 0.0
B-return_date.day_number 0.000 0.000 0.000 0.0
B-arrive_time.period_mod 0.000 0.000 0.000 0.0
I-meal_code 0.000 0.000 0.000 0.0
B-toloc.country_name 0.000 0.000 0.000 1.0
B-days_code 0.000 0.000 0.000 1.0
I-arrive_time.period_of_day 0.000 0.000 0.000 0.0
I-today_relative 0.000 0.000 0.000 0.0
B-return_time.period_of_day 0.000 0.000 0.000 0.0
B-time 0.000 0.000 0.000 0.0
I-fare_basis_code 0.000 0.000 0.000 0.0
I-arrive_time.time_relative 0.000 0.000 0.000 4.0
I-depart_time.time_relative 0.000 0.000 0.000 1.0
B-today_relative 0.000 0.000 0.000 0.0
B-state_name 0.000 0.000 0.000 9.0
B-arrive_date.today_relative 0.000 0.000 0.000 0.0
B-return_time.period_mod 0.000 0.000 0.000 0.0
B-month_name 0.000 0.000 0.000 0.0
B-day_number 0.000 0.000 0.000 0.0
I-return_date.date_relative 0.000 0.000 0.000 3.0
I-return_date.today_relative 0.000 0.000 0.000 0.0
B-stoploc.airport_name 0.000 0.000 0.000 0.0
B-time_relative 0.000 0.000 0.000 0.0
I-time 0.000 0.000 0.000 0.0
I-return_date.day_number 0.000 0.000 0.000 0.0
I-meal_description 0.000 0.000 0.000 0.0
B-return_date.today_relative 0.000 0.000 0.000 0.0
B-return_date.day_name 0.000 0.000 0.000 2.0
micro avg 0.923 0.925 0.924 3657.0
macro avg 0.494 0.479 0.474 3657.0
weighted avg 0.913 0.925 0.914 3657.0
I0716 03:09:44.621426 139795883440000 interactiveshell.py:2882] Best Slot F1: 0.924
I0716 03:09:44.623023 139795883440000 estimator_training.py:186] Not using Distribute Coordinator.
I0716 03:09:44.624618 139795883440000 training.py:612] Running training and evaluation locally (non-distributed).
I0716 03:09:44.627876 139795883440000 training.py:700] Start train and evaluate loop. The evaluate will happen after every checkpoint. Checkpoint frequency is determined based on RunConfig arguments: save_checkpoints_steps 312 or save_checkpoints_secs None.
I0716 03:09:44.691657 139795883440000 estimator.py:1145] Calling model_fn.
I0716 03:09:45.981238 139795883440000 saver.py:1499] Saver not created because there are no variables in the graph to restore
I0716 03:09:46.733167 139795883440000 <ipython-input-6-d6747b40360b>:55]
[<tf.Variable 'embedding:0' shape=(750, 300) dtype=float32_ref>,
<tf.Variable 'module/aggregation/weights:0' shape=(3,) dtype=float32>,
<tf.Variable 'module/aggregation/scaling:0' shape=() dtype=float32>,
<tf.Variable 'dense/kernel:0' shape=(1324, 300) dtype=float32_ref>,
<tf.Variable 'dense/bias:0' shape=(300,) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/fw/gru_cell/gates/kernel:0' shape=(600, 600) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/fw/gru_cell/gates/bias:0' shape=(600,) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/fw/gru_cell/candidate/kernel:0' shape=(600, 300) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/fw/gru_cell/candidate/bias:0' shape=(300,) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/bw/gru_cell/gates/kernel:0' shape=(600, 600) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/bw/gru_cell/gates/bias:0' shape=(600,) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/bw/gru_cell/candidate/kernel:0' shape=(600, 300) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/bw/gru_cell/candidate/bias:0' shape=(300,) dtype=float32_ref>,
<tf.Variable 'dense_1/kernel:0' shape=(600, 23) dtype=float32_ref>,
<tf.Variable 'dense_1/bias:0' shape=(23,) dtype=float32_ref>,
<tf.Variable 'dense_2/kernel:0' shape=(600, 122) dtype=float32_ref>,
<tf.Variable 'dense_2/bias:0' shape=(122,) dtype=float32_ref>]
I0716 03:09:47.810201 139795883440000 estimator.py:1147] Done calling model_fn.
I0716 03:09:47.814795 139795883440000 basic_session_run_hooks.py:541] Create CheckpointSaverHook.
I0716 03:09:48.135759 139795883440000 monitored_session.py:240] Graph was finalized.
I0716 03:09:48.153249 139795883440000 saver.py:1280] Restoring parameters from ../model/elmo_bigru/model.ckpt-936
I0716 03:09:49.177926 139795883440000 session_manager.py:500] Running local_init_op.
I0716 03:09:49.244561 139795883440000 session_manager.py:502] Done running local_init_op.
I0716 03:09:52.202533 139795883440000 basic_session_run_hooks.py:606] Saving checkpoints for 936 into ../model/elmo_bigru/model.ckpt.
Reading ../data/atis.train.w-intent.iob
I0716 03:09:57.132408 139795883440000 basic_session_run_hooks.py:262] loss = 0.09598224, step = 936 I0716 03:09:57.134508 139795883440000 basic_session_run_hooks.py:262] lr = 0.00027182678 I0716 03:10:25.028632 139795883440000 basic_session_run_hooks.py:692] global_step/sec: 3.58465 I0716 03:10:25.032232 139795883440000 basic_session_run_hooks.py:260] loss = 0.11424209, step = 1036 (27.900 sec) I0716 03:10:25.036499 139795883440000 basic_session_run_hooks.py:260] lr = 0.00026897783 (27.902 sec) I0716 03:10:51.357235 139795883440000 basic_session_run_hooks.py:692] global_step/sec: 3.79814 I0716 03:10:51.364395 139795883440000 basic_session_run_hooks.py:260] loss = 0.070536286, step = 1136 (26.332 sec) I0716 03:10:51.366651 139795883440000 basic_session_run_hooks.py:260] lr = 0.00026615875 (26.330 sec) I0716 03:11:17.504055 139795883440000 basic_session_run_hooks.py:692] global_step/sec: 3.82456 I0716 03:11:17.510384 139795883440000 basic_session_run_hooks.py:260] loss = 0.25277418, step = 1236 (26.146 sec) I0716 03:11:17.513742 139795883440000 basic_session_run_hooks.py:260] lr = 0.0002633692 (26.147 sec) I0716 03:11:20.337857 139795883440000 basic_session_run_hooks.py:606] Saving checkpoints for 1248 into ../model/elmo_bigru/model.ckpt. I0716 03:11:22.240674 139795883440000 estimator.py:1145] Calling model_fn. I0716 03:11:23.577543 139795883440000 saver.py:1499] Saver not created because there are no variables in the graph to restore I0716 03:11:24.163589 139795883440000 estimator.py:1147] Done calling model_fn. I0716 03:11:24.192977 139795883440000 evaluation.py:255] Starting evaluation at 2019-07-16T03:11:24Z I0716 03:11:24.418920 139795883440000 monitored_session.py:240] Graph was finalized. I0716 03:11:24.435197 139795883440000 saver.py:1280] Restoring parameters from ../model/elmo_bigru/model.ckpt-1248 I0716 03:11:25.325025 139795883440000 session_manager.py:500] Running local_init_op. I0716 03:11:25.378224 139795883440000 session_manager.py:502] Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
I0716 03:12:24.341384 139795883440000 evaluation.py:275] Finished evaluation at 2019-07-16-03:12:24 I0716 03:12:24.343268 139795883440000 estimator.py:2039] Saving dict for global step 1248: global_step = 1248, loss = 0.29128006 I0716 03:12:24.351259 139795883440000 estimator.py:2099] Saving 'checkpoint_path' summary for global step 1248: ../model/elmo_bigru/model.ckpt-1248 I0716 03:12:24.436446 139795883440000 estimator.py:368] Loss for final step: 0.08681949.
Reading ../data/atis.test.w-intent.iob
I0716 03:12:25.030044 139795883440000 estimator.py:1145] Calling model_fn. I0716 03:12:26.625313 139795883440000 saver.py:1499] Saver not created because there are no variables in the graph to restore I0716 03:12:27.137457 139795883440000 estimator.py:1147] Done calling model_fn. I0716 03:12:27.531526 139795883440000 monitored_session.py:240] Graph was finalized. I0716 03:12:27.549257 139795883440000 saver.py:1280] Restoring parameters from ../model/elmo_bigru/model.ckpt-1248 I0716 03:12:28.412033 139795883440000 session_manager.py:500] Running local_init_op. I0716 03:12:28.456053 139795883440000 session_manager.py:502] Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.
'precision', 'predicted', average, warn_for)
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1439: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples.
'recall', 'true', average, warn_for)
I0716 03:13:26.352146 139795883440000 interactiveshell.py:2882]
precision recall f1-score support
atis_flight 0.987 0.989 0.988 632
atis_airfare 0.906 1.000 0.950 48
atis_ground_service 0.878 1.000 0.935 36
atis_airline 1.000 1.000 1.000 38
atis_abbreviation 0.943 1.000 0.971 33
atis_aircraft 1.000 1.000 1.000 9
atis_flight_time 0.333 1.000 0.500 1
atis_quantity 0.429 1.000 0.600 3
atis_flight#atis_airfare 0.857 0.500 0.632 12
atis_airport 1.000 0.944 0.971 18
atis_distance 1.000 1.000 1.000 10
atis_city 1.000 0.167 0.286 6
atis_ground_fare 1.000 0.857 0.923 7
atis_capacity 1.000 0.952 0.976 21
atis_flight_no 0.889 1.000 0.941 8
atis_meal 1.000 0.667 0.800 6
atis_restriction 0.000 0.000 0.000 0
atis_airline#atis_flight_no 0.000 0.000 0.000 0
atis_ground_service#atis_ground_fare 0.000 0.000 0.000 0
atis_airfare#atis_flight_time 0.000 0.000 0.000 0
atis_cheapest 0.000 0.000 0.000 0
atis_aircraft#atis_flight#atis_flight_no 0.000 0.000 0.000 0
micro avg 0.969 0.974 0.971 888
macro avg 0.646 0.640 0.612 888
weighted avg 0.973 0.974 0.970 888
I0716 03:13:26.388364 139795883440000 interactiveshell.py:2882]
precision recall f1-score support
O 0.000 0.000 0.000 0.0
B-toloc.city_name 0.974 0.994 0.984 716.0
B-fromloc.city_name 0.989 0.999 0.994 704.0
I-toloc.city_name 0.936 0.996 0.965 265.0
B-depart_date.day_name 0.977 0.995 0.986 212.0
B-airline_name 0.962 0.990 0.976 101.0
I-fromloc.city_name 0.989 0.972 0.980 177.0
B-depart_time.period_of_day 0.975 0.885 0.927 130.0
I-airline_name 1.000 0.954 0.976 65.0
B-depart_date.day_number 0.982 0.982 0.982 55.0
B-depart_date.month_name 0.982 0.964 0.973 56.0
B-depart_time.time 0.826 1.000 0.905 57.0
B-round_trip 1.000 0.986 0.993 73.0
B-cost_relative 0.973 0.973 0.973 37.0
I-round_trip 0.986 1.000 0.993 71.0
B-flight_mod 0.786 0.917 0.846 24.0
B-depart_time.time_relative 0.942 1.000 0.970 65.0
I-depart_time.time 0.852 1.000 0.920 52.0
B-stoploc.city_name 0.864 0.950 0.905 20.0
B-city_name 0.857 0.632 0.727 57.0
B-class_type 0.960 1.000 0.980 24.0
B-arrive_time.time 1.000 0.971 0.985 34.0
B-arrive_time.time_relative 0.882 0.968 0.923 31.0
I-class_type 1.000 1.000 1.000 17.0
B-flight_stop 1.000 1.000 1.000 21.0
I-arrive_time.time 0.810 0.971 0.883 35.0
B-airline_code 0.795 0.912 0.849 34.0
I-depart_date.day_number 1.000 1.000 1.000 15.0
I-fromloc.airport_name 0.500 1.000 0.667 15.0
B-fromloc.airport_name 0.355 0.917 0.512 12.0
B-arrive_date.day_name 0.786 1.000 0.880 11.0
B-toloc.state_code 0.900 1.000 0.947 18.0
B-depart_date.today_relative 1.000 0.778 0.875 9.0
B-flight_number 0.846 1.000 0.917 11.0
B-depart_date.date_relative 0.850 1.000 0.919 17.0
B-toloc.state_name 0.806 0.893 0.847 28.0
B-fare_basis_code 0.895 1.000 0.944 17.0
B-flight_time 1.000 1.000 1.000 1.0
B-or 1.000 1.000 1.000 3.0
B-arrive_time.period_of_day 0.667 1.000 0.800 6.0
B-meal_description 1.000 0.900 0.947 10.0
I-cost_relative 1.000 0.667 0.800 3.0
I-airport_name 0.800 0.414 0.545 29.0
B-fare_amount 1.000 1.000 1.000 2.0
I-fare_amount 1.000 1.000 1.000 2.0
I-city_name 0.810 0.567 0.667 30.0
I-toloc.airport_name 1.000 1.000 1.000 3.0
B-transport_type 0.909 1.000 0.952 10.0
B-arrive_date.month_name 0.714 0.833 0.769 6.0
B-arrive_date.day_number 0.714 0.833 0.769 6.0
I-stoploc.city_name 0.889 0.800 0.842 10.0
B-meal 1.000 1.000 1.000 16.0
B-fromloc.state_code 1.000 0.913 0.955 23.0
B-depart_time.period_mod 1.000 1.000 1.000 5.0
B-connect 1.000 1.000 1.000 6.0
B-flight_days 1.000 1.000 1.000 10.0
B-toloc.airport_name 1.000 0.333 0.500 3.0
B-fromloc.state_name 0.941 0.941 0.941 17.0
B-airport_name 0.556 0.238 0.333 21.0
B-economy 1.000 0.333 0.500 6.0
I-flight_time 1.000 1.000 1.000 1.0
B-aircraft_code 1.000 0.848 0.918 33.0
B-mod 1.000 0.500 0.667 2.0
B-airport_code 0.750 0.333 0.462 9.0
B-depart_time.start_time 1.000 0.333 0.500 3.0
B-depart_time.end_time 1.000 0.333 0.500 3.0
B-depart_date.year 1.000 1.000 1.000 3.0
I-transport_type 0.000 0.000 0.000 1.0
B-restriction_code 0.667 0.500 0.571 4.0
B-arrive_time.start_time 0.889 1.000 0.941 8.0
B-toloc.airport_code 1.000 0.750 0.857 4.0
B-arrive_time.end_time 0.889 1.000 0.941 8.0
I-arrive_time.end_time 0.000 0.000 0.000 8.0
I-depart_time.end_time 0.000 0.000 0.000 3.0
I-flight_stop 0.000 0.000 0.000 0.0
B-fromloc.airport_code 0.800 0.800 0.800 5.0
I-restriction_code 1.000 1.000 1.000 3.0
I-depart_time.start_time 1.000 1.000 1.000 1.0
I-toloc.state_name 1.000 1.000 1.000 1.0
I-depart_date.today_relative 0.000 0.000 0.000 0.0
B-arrive_date.date_relative 1.000 0.500 0.667 2.0
I-flight_mod 0.000 0.000 0.000 6.0
I-economy 0.000 0.000 0.000 0.0
B-return_date.date_relative 0.000 0.000 0.000 3.0
I-fromloc.state_name 0.000 0.000 0.000 1.0
B-state_code 1.000 1.000 1.000 1.0
I-arrive_time.start_time 1.000 1.000 1.000 1.0
I-arrive_date.day_number 0.000 0.000 0.000 0.0
B-meal_code 0.000 0.000 0.000 1.0
I-depart_time.period_of_day 0.000 0.000 0.000 1.0
B-day_name 0.000 0.000 0.000 2.0
B-period_of_day 0.000 0.000 0.000 4.0
B-stoploc.state_code 0.000 0.000 0.000 0.0
B-return_date.month_name 0.000 0.000 0.000 0.0
B-return_date.day_number 0.000 0.000 0.000 0.0
B-arrive_time.period_mod 0.000 0.000 0.000 0.0
I-meal_code 0.000 0.000 0.000 0.0
B-toloc.country_name 0.000 0.000 0.000 1.0
B-days_code 0.000 0.000 0.000 1.0
I-arrive_time.period_of_day 0.000 0.000 0.000 0.0
I-today_relative 0.000 0.000 0.000 0.0
B-return_time.period_of_day 0.000 0.000 0.000 0.0
B-time 0.000 0.000 0.000 0.0
I-fare_basis_code 0.000 0.000 0.000 0.0
I-arrive_time.time_relative 0.000 0.000 0.000 4.0
I-depart_time.time_relative 0.000 0.000 0.000 1.0
B-today_relative 0.000 0.000 0.000 0.0
B-state_name 0.000 0.000 0.000 9.0
B-arrive_date.today_relative 0.000 0.000 0.000 0.0
B-return_time.period_mod 0.000 0.000 0.000 0.0
B-month_name 0.000 0.000 0.000 0.0
B-day_number 0.000 0.000 0.000 0.0
I-return_date.date_relative 0.000 0.000 0.000 3.0
I-return_date.today_relative 0.000 0.000 0.000 0.0
B-stoploc.airport_name 0.000 0.000 0.000 0.0
B-time_relative 0.000 0.000 0.000 0.0
I-time 0.000 0.000 0.000 0.0
I-return_date.day_number 0.000 0.000 0.000 0.0
I-meal_description 0.000 0.000 0.000 0.0
B-return_date.today_relative 0.000 0.000 0.000 0.0
B-return_date.day_name 0.000 0.000 0.000 2.0
micro avg 0.942 0.944 0.943 3657.0
macro avg 0.580 0.556 0.556 3657.0
weighted avg 0.936 0.944 0.936 3657.0
I0716 03:13:26.409072 139795883440000 interactiveshell.py:2882] Best Slot F1: 0.943
I0716 03:13:26.410695 139795883440000 estimator_training.py:186] Not using Distribute Coordinator.
I0716 03:13:26.413439 139795883440000 training.py:612] Running training and evaluation locally (non-distributed).
I0716 03:13:26.417328 139795883440000 training.py:700] Start train and evaluate loop. The evaluate will happen after every checkpoint. Checkpoint frequency is determined based on RunConfig arguments: save_checkpoints_steps 312 or save_checkpoints_secs None.
I0716 03:13:26.484791 139795883440000 estimator.py:1145] Calling model_fn.
I0716 03:13:27.782305 139795883440000 saver.py:1499] Saver not created because there are no variables in the graph to restore
I0716 03:13:28.523794 139795883440000 <ipython-input-6-d6747b40360b>:55]
[<tf.Variable 'embedding:0' shape=(750, 300) dtype=float32_ref>,
<tf.Variable 'module/aggregation/weights:0' shape=(3,) dtype=float32>,
<tf.Variable 'module/aggregation/scaling:0' shape=() dtype=float32>,
<tf.Variable 'dense/kernel:0' shape=(1324, 300) dtype=float32_ref>,
<tf.Variable 'dense/bias:0' shape=(300,) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/fw/gru_cell/gates/kernel:0' shape=(600, 600) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/fw/gru_cell/gates/bias:0' shape=(600,) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/fw/gru_cell/candidate/kernel:0' shape=(600, 300) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/fw/gru_cell/candidate/bias:0' shape=(300,) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/bw/gru_cell/gates/kernel:0' shape=(600, 600) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/bw/gru_cell/gates/bias:0' shape=(600,) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/bw/gru_cell/candidate/kernel:0' shape=(600, 300) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/bw/gru_cell/candidate/bias:0' shape=(300,) dtype=float32_ref>,
<tf.Variable 'dense_1/kernel:0' shape=(600, 23) dtype=float32_ref>,
<tf.Variable 'dense_1/bias:0' shape=(23,) dtype=float32_ref>,
<tf.Variable 'dense_2/kernel:0' shape=(600, 122) dtype=float32_ref>,
<tf.Variable 'dense_2/bias:0' shape=(122,) dtype=float32_ref>]
I0716 03:13:29.597792 139795883440000 estimator.py:1147] Done calling model_fn.
I0716 03:13:29.601427 139795883440000 basic_session_run_hooks.py:541] Create CheckpointSaverHook.
I0716 03:13:29.932820 139795883440000 monitored_session.py:240] Graph was finalized.
I0716 03:13:29.950454 139795883440000 saver.py:1280] Restoring parameters from ../model/elmo_bigru/model.ckpt-1248
I0716 03:13:30.966139 139795883440000 session_manager.py:500] Running local_init_op.
I0716 03:13:31.033563 139795883440000 session_manager.py:502] Done running local_init_op.
I0716 03:13:34.017776 139795883440000 basic_session_run_hooks.py:606] Saving checkpoints for 1248 into ../model/elmo_bigru/model.ckpt.
Reading ../data/atis.train.w-intent.iob
I0716 03:13:38.845421 139795883440000 basic_session_run_hooks.py:262] loss = 0.056573045, step = 1248 I0716 03:13:38.847369 139795883440000 basic_session_run_hooks.py:262] lr = 0.00026303643 I0716 03:14:06.612472 139795883440000 basic_session_run_hooks.py:692] global_step/sec: 3.60131 I0716 03:14:06.617980 139795883440000 basic_session_run_hooks.py:260] loss = 0.044386838, step = 1348 (27.773 sec) I0716 03:14:06.620636 139795883440000 basic_session_run_hooks.py:260] lr = 0.00026027963 (27.773 sec) I0716 03:14:33.374242 139795883440000 basic_session_run_hooks.py:692] global_step/sec: 3.73667 I0716 03:14:33.379942 139795883440000 basic_session_run_hooks.py:260] loss = 0.04956132, step = 1448 (26.762 sec) I0716 03:14:33.382454 139795883440000 basic_session_run_hooks.py:260] lr = 0.00025755167 (26.762 sec) I0716 03:15:00.112639 139795883440000 basic_session_run_hooks.py:692] global_step/sec: 3.73994 I0716 03:15:00.118970 139795883440000 basic_session_run_hooks.py:260] loss = 0.038445197, step = 1548 (26.739 sec) I0716 03:15:00.121211 139795883440000 basic_session_run_hooks.py:260] lr = 0.00025485238 (26.739 sec) I0716 03:15:02.877131 139795883440000 basic_session_run_hooks.py:606] Saving checkpoints for 1560 into ../model/elmo_bigru/model.ckpt. W0716 03:15:04.569809 139795883440000 deprecation.py:323] From /usr/local/lib/python3.6/dist-packages/tensorflow/python/training/saver.py:960: remove_checkpoint (from tensorflow.python.training.checkpoint_management) is deprecated and will be removed in a future version. Instructions for updating: Use standard file APIs to delete files with this prefix. I0716 03:15:05.077725 139795883440000 estimator.py:1145] Calling model_fn. I0716 03:15:06.442743 139795883440000 saver.py:1499] Saver not created because there are no variables in the graph to restore I0716 03:15:07.043466 139795883440000 estimator.py:1147] Done calling model_fn. I0716 03:15:07.072723 139795883440000 evaluation.py:255] Starting evaluation at 2019-07-16T03:15:07Z I0716 03:15:07.301808 139795883440000 monitored_session.py:240] Graph was finalized. I0716 03:15:07.318938 139795883440000 saver.py:1280] Restoring parameters from ../model/elmo_bigru/model.ckpt-1560 I0716 03:15:08.226789 139795883440000 session_manager.py:500] Running local_init_op. I0716 03:15:08.283047 139795883440000 session_manager.py:502] Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
I0716 03:16:07.308760 139795883440000 evaluation.py:275] Finished evaluation at 2019-07-16-03:16:07 I0716 03:16:07.310705 139795883440000 estimator.py:2039] Saving dict for global step 1560: global_step = 1560, loss = 0.28729033 I0716 03:16:07.316600 139795883440000 estimator.py:2099] Saving 'checkpoint_path' summary for global step 1560: ../model/elmo_bigru/model.ckpt-1560 I0716 03:16:07.401273 139795883440000 estimator.py:368] Loss for final step: 0.008345632.
Reading ../data/atis.test.w-intent.iob
I0716 03:16:07.986275 139795883440000 estimator.py:1145] Calling model_fn. I0716 03:16:09.618869 139795883440000 saver.py:1499] Saver not created because there are no variables in the graph to restore I0716 03:16:10.127372 139795883440000 estimator.py:1147] Done calling model_fn. I0716 03:16:10.338745 139795883440000 monitored_session.py:240] Graph was finalized. I0716 03:16:10.351769 139795883440000 saver.py:1280] Restoring parameters from ../model/elmo_bigru/model.ckpt-1560 I0716 03:16:11.246006 139795883440000 session_manager.py:500] Running local_init_op. I0716 03:16:11.290265 139795883440000 session_manager.py:502] Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.
'precision', 'predicted', average, warn_for)
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1439: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples.
'recall', 'true', average, warn_for)
I0716 03:17:09.232142 139795883440000 interactiveshell.py:2882]
precision recall f1-score support
atis_flight 0.984 0.992 0.988 632
atis_airfare 0.960 1.000 0.980 48
atis_ground_service 0.973 1.000 0.986 36
atis_airline 1.000 1.000 1.000 38
atis_abbreviation 0.943 1.000 0.971 33
atis_aircraft 1.000 0.889 0.941 9
atis_flight_time 0.333 1.000 0.500 1
atis_quantity 0.375 1.000 0.545 3
atis_flight#atis_airfare 0.833 0.417 0.556 12
atis_airport 1.000 0.944 0.971 18
atis_distance 1.000 1.000 1.000 10
atis_city 0.600 0.500 0.545 6
atis_ground_fare 1.000 0.857 0.923 7
atis_capacity 1.000 0.952 0.976 21
atis_flight_no 0.889 1.000 0.941 8
atis_meal 1.000 0.667 0.800 6
atis_restriction 0.000 0.000 0.000 0
atis_airline#atis_flight_no 0.000 0.000 0.000 0
atis_ground_service#atis_ground_fare 0.000 0.000 0.000 0
atis_airfare#atis_flight_time 0.000 0.000 0.000 0
atis_cheapest 0.000 0.000 0.000 0
atis_aircraft#atis_flight#atis_flight_no 0.000 0.000 0.000 0
micro avg 0.971 0.976 0.974 888
macro avg 0.631 0.646 0.619 888
weighted avg 0.975 0.976 0.973 888
I0716 03:17:09.269594 139795883440000 interactiveshell.py:2882]
precision recall f1-score support
O 0.000 0.000 0.000 0.0
B-toloc.city_name 0.970 0.997 0.983 716.0
B-fromloc.city_name 0.992 0.997 0.994 704.0
I-toloc.city_name 0.953 1.000 0.976 265.0
B-depart_date.day_name 0.986 0.991 0.988 212.0
B-airline_name 0.980 0.990 0.985 101.0
I-fromloc.city_name 0.994 0.977 0.986 177.0
B-depart_time.period_of_day 0.983 0.877 0.927 130.0
I-airline_name 1.000 0.969 0.984 65.0
B-depart_date.day_number 0.981 0.964 0.972 55.0
B-depart_date.month_name 0.982 0.964 0.973 56.0
B-depart_time.time 0.836 0.982 0.903 57.0
B-round_trip 1.000 0.986 0.993 73.0
B-cost_relative 0.973 0.973 0.973 37.0
I-round_trip 0.986 1.000 0.993 71.0
B-flight_mod 0.800 1.000 0.889 24.0
B-depart_time.time_relative 0.968 0.938 0.953 65.0
I-depart_time.time 0.870 0.904 0.887 52.0
B-stoploc.city_name 0.950 0.950 0.950 20.0
B-city_name 0.850 0.596 0.701 57.0
B-class_type 0.960 1.000 0.980 24.0
B-arrive_time.time 0.971 0.971 0.971 34.0
B-arrive_time.time_relative 0.833 0.968 0.896 31.0
I-class_type 1.000 1.000 1.000 17.0
B-flight_stop 1.000 1.000 1.000 21.0
I-arrive_time.time 0.723 0.971 0.829 35.0
B-airline_code 0.773 1.000 0.872 34.0
I-depart_date.day_number 1.000 0.933 0.966 15.0
I-fromloc.airport_name 0.484 1.000 0.652 15.0
B-fromloc.airport_name 0.429 1.000 0.600 12.0
B-arrive_date.day_name 0.786 1.000 0.880 11.0
B-toloc.state_code 0.900 1.000 0.947 18.0
B-depart_date.today_relative 1.000 0.778 0.875 9.0
B-flight_number 0.786 1.000 0.880 11.0
B-depart_date.date_relative 0.850 1.000 0.919 17.0
B-toloc.state_name 0.889 0.857 0.873 28.0
B-fare_basis_code 0.850 1.000 0.919 17.0
B-flight_time 1.000 1.000 1.000 1.0
B-or 1.000 1.000 1.000 3.0
B-arrive_time.period_of_day 0.600 1.000 0.750 6.0
B-meal_description 1.000 0.900 0.947 10.0
I-cost_relative 1.000 0.667 0.800 3.0
I-airport_name 0.765 0.448 0.565 29.0
B-fare_amount 1.000 1.000 1.000 2.0
I-fare_amount 1.000 1.000 1.000 2.0
I-city_name 0.792 0.633 0.704 30.0
I-toloc.airport_name 1.000 1.000 1.000 3.0
B-transport_type 1.000 0.700 0.824 10.0
B-arrive_date.month_name 0.714 0.833 0.769 6.0
B-arrive_date.day_number 0.625 0.833 0.714 6.0
I-stoploc.city_name 0.909 1.000 0.952 10.0
B-meal 1.000 1.000 1.000 16.0
B-fromloc.state_code 1.000 0.913 0.955 23.0
B-depart_time.period_mod 0.833 1.000 0.909 5.0
B-connect 1.000 1.000 1.000 6.0
B-flight_days 1.000 1.000 1.000 10.0
B-toloc.airport_name 1.000 0.333 0.500 3.0
B-fromloc.state_name 0.941 0.941 0.941 17.0
B-airport_name 0.615 0.381 0.471 21.0
B-economy 1.000 0.333 0.500 6.0
I-flight_time 1.000 1.000 1.000 1.0
B-aircraft_code 1.000 0.848 0.918 33.0
B-mod 1.000 0.500 0.667 2.0
B-airport_code 1.000 0.111 0.200 9.0
B-depart_time.start_time 1.000 0.667 0.800 3.0
B-depart_time.end_time 1.000 0.333 0.500 3.0
B-depart_date.year 1.000 1.000 1.000 3.0
I-transport_type 0.200 1.000 0.333 1.0
B-restriction_code 0.667 0.500 0.571 4.0
B-arrive_time.start_time 0.889 1.000 0.941 8.0
B-toloc.airport_code 1.000 0.750 0.857 4.0
B-arrive_time.end_time 0.889 1.000 0.941 8.0
I-arrive_time.end_time 1.000 0.250 0.400 8.0
I-depart_time.end_time 1.000 0.667 0.800 3.0
I-flight_stop 0.000 0.000 0.000 0.0
B-fromloc.airport_code 0.800 0.800 0.800 5.0
I-restriction_code 1.000 1.000 1.000 3.0
I-depart_time.start_time 1.000 1.000 1.000 1.0
I-toloc.state_name 1.000 1.000 1.000 1.0
I-depart_date.today_relative 0.000 0.000 0.000 0.0
B-arrive_date.date_relative 1.000 0.500 0.667 2.0
I-flight_mod 0.000 0.000 0.000 6.0
I-economy 0.000 0.000 0.000 0.0
B-return_date.date_relative 0.667 0.667 0.667 3.0
I-fromloc.state_name 0.000 0.000 0.000 1.0
B-state_code 1.000 1.000 1.000 1.0
I-arrive_time.start_time 0.000 0.000 0.000 1.0
I-arrive_date.day_number 0.000 0.000 0.000 0.0
B-meal_code 0.000 0.000 0.000 1.0
I-depart_time.period_of_day 0.000 0.000 0.000 1.0
B-day_name 0.500 0.500 0.500 2.0
B-period_of_day 0.000 0.000 0.000 4.0
B-stoploc.state_code 0.000 0.000 0.000 0.0
B-return_date.month_name 0.000 0.000 0.000 0.0
B-return_date.day_number 0.000 0.000 0.000 0.0
B-arrive_time.period_mod 0.000 0.000 0.000 0.0
I-meal_code 0.000 0.000 0.000 0.0
B-toloc.country_name 0.000 0.000 0.000 1.0
B-days_code 0.000 0.000 0.000 1.0
I-arrive_time.period_of_day 0.000 0.000 0.000 0.0
I-today_relative 0.000 0.000 0.000 0.0
B-return_time.period_of_day 0.000 0.000 0.000 0.0
B-time 0.000 0.000 0.000 0.0
I-fare_basis_code 0.000 0.000 0.000 0.0
I-arrive_time.time_relative 0.000 0.000 0.000 4.0
I-depart_time.time_relative 0.000 0.000 0.000 1.0
B-today_relative 0.000 0.000 0.000 0.0
B-state_name 0.000 0.000 0.000 9.0
B-arrive_date.today_relative 0.000 0.000 0.000 0.0
B-return_time.period_mod 0.000 0.000 0.000 0.0
B-month_name 0.000 0.000 0.000 0.0
B-day_number 0.000 0.000 0.000 0.0
I-return_date.date_relative 0.000 0.000 0.000 3.0
I-return_date.today_relative 0.000 0.000 0.000 0.0
B-stoploc.airport_name 0.000 0.000 0.000 0.0
B-time_relative 0.000 0.000 0.000 0.0
I-time 0.000 0.000 0.000 0.0
I-return_date.day_number 0.000 0.000 0.000 0.0
I-meal_description 0.000 0.000 0.000 0.0
B-return_date.today_relative 0.000 0.000 0.000 0.0
B-return_date.day_name 0.000 0.000 0.000 2.0
micro avg 0.943 0.945 0.944 3657.0
macro avg 0.601 0.575 0.570 3657.0
weighted avg 0.942 0.945 0.939 3657.0
I0716 03:17:09.291633 139795883440000 interactiveshell.py:2882] Best Slot F1: 0.944
I0716 03:17:09.293515 139795883440000 estimator_training.py:186] Not using Distribute Coordinator.
I0716 03:17:09.296332 139795883440000 training.py:612] Running training and evaluation locally (non-distributed).
I0716 03:17:09.299224 139795883440000 training.py:700] Start train and evaluate loop. The evaluate will happen after every checkpoint. Checkpoint frequency is determined based on RunConfig arguments: save_checkpoints_steps 312 or save_checkpoints_secs None.
I0716 03:17:09.362334 139795883440000 estimator.py:1145] Calling model_fn.
I0716 03:17:10.946285 139795883440000 saver.py:1499] Saver not created because there are no variables in the graph to restore
I0716 03:17:11.521585 139795883440000 <ipython-input-6-d6747b40360b>:55]
[<tf.Variable 'embedding:0' shape=(750, 300) dtype=float32_ref>,
<tf.Variable 'module/aggregation/weights:0' shape=(3,) dtype=float32>,
<tf.Variable 'module/aggregation/scaling:0' shape=() dtype=float32>,
<tf.Variable 'dense/kernel:0' shape=(1324, 300) dtype=float32_ref>,
<tf.Variable 'dense/bias:0' shape=(300,) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/fw/gru_cell/gates/kernel:0' shape=(600, 600) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/fw/gru_cell/gates/bias:0' shape=(600,) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/fw/gru_cell/candidate/kernel:0' shape=(600, 300) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/fw/gru_cell/candidate/bias:0' shape=(300,) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/bw/gru_cell/gates/kernel:0' shape=(600, 600) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/bw/gru_cell/gates/bias:0' shape=(600,) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/bw/gru_cell/candidate/kernel:0' shape=(600, 300) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/bw/gru_cell/candidate/bias:0' shape=(300,) dtype=float32_ref>,
<tf.Variable 'dense_1/kernel:0' shape=(600, 23) dtype=float32_ref>,
<tf.Variable 'dense_1/bias:0' shape=(23,) dtype=float32_ref>,
<tf.Variable 'dense_2/kernel:0' shape=(600, 122) dtype=float32_ref>,
<tf.Variable 'dense_2/bias:0' shape=(122,) dtype=float32_ref>]
I0716 03:17:12.603037 139795883440000 estimator.py:1147] Done calling model_fn.
I0716 03:17:12.607095 139795883440000 basic_session_run_hooks.py:541] Create CheckpointSaverHook.
I0716 03:17:13.122910 139795883440000 monitored_session.py:240] Graph was finalized.
I0716 03:17:13.142551 139795883440000 saver.py:1280] Restoring parameters from ../model/elmo_bigru/model.ckpt-1560
I0716 03:17:14.162578 139795883440000 session_manager.py:500] Running local_init_op.
I0716 03:17:14.228501 139795883440000 session_manager.py:502] Done running local_init_op.
I0716 03:17:16.980484 139795883440000 basic_session_run_hooks.py:606] Saving checkpoints for 1560 into ../model/elmo_bigru/model.ckpt.
Reading ../data/atis.train.w-intent.iob
I0716 03:17:21.701616 139795883440000 basic_session_run_hooks.py:262] loss = 0.05464157, step = 1560 I0716 03:17:21.703541 139795883440000 basic_session_run_hooks.py:262] lr = 0.00025453034 I0716 03:17:49.344164 139795883440000 basic_session_run_hooks.py:692] global_step/sec: 3.61755 I0716 03:17:49.347166 139795883440000 basic_session_run_hooks.py:260] loss = 0.08211179, step = 1660 (27.646 sec) I0716 03:17:49.349646 139795883440000 basic_session_run_hooks.py:260] lr = 0.0002518627 (27.646 sec) I0716 03:18:15.868099 139795883440000 basic_session_run_hooks.py:692] global_step/sec: 3.77017 I0716 03:18:15.871267 139795883440000 basic_session_run_hooks.py:260] loss = 0.029694248, step = 1760 (26.524 sec) I0716 03:18:15.873329 139795883440000 basic_session_run_hooks.py:260] lr = 0.00024922297 (26.524 sec) I0716 03:18:41.795644 139795883440000 basic_session_run_hooks.py:692] global_step/sec: 3.8569 I0716 03:18:41.803750 139795883440000 basic_session_run_hooks.py:260] loss = 0.057554252, step = 1860 (25.932 sec) I0716 03:18:41.805616 139795883440000 basic_session_run_hooks.py:260] lr = 0.00024661093 (25.932 sec) I0716 03:18:44.854101 139795883440000 basic_session_run_hooks.py:606] Saving checkpoints for 1872 into ../model/elmo_bigru/model.ckpt. I0716 03:18:46.823345 139795883440000 estimator.py:1145] Calling model_fn. I0716 03:18:48.140162 139795883440000 saver.py:1499] Saver not created because there are no variables in the graph to restore I0716 03:18:48.964244 139795883440000 estimator.py:1147] Done calling model_fn. I0716 03:18:48.994191 139795883440000 evaluation.py:255] Starting evaluation at 2019-07-16T03:18:48Z I0716 03:18:49.214614 139795883440000 monitored_session.py:240] Graph was finalized. I0716 03:18:49.229626 139795883440000 saver.py:1280] Restoring parameters from ../model/elmo_bigru/model.ckpt-1872 I0716 03:18:50.094576 139795883440000 session_manager.py:500] Running local_init_op. I0716 03:18:50.148860 139795883440000 session_manager.py:502] Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
I0716 03:19:49.197244 139795883440000 evaluation.py:275] Finished evaluation at 2019-07-16-03:19:49 I0716 03:19:49.198988 139795883440000 estimator.py:2039] Saving dict for global step 1872: global_step = 1872, loss = 0.27500156 I0716 03:19:49.205196 139795883440000 estimator.py:2099] Saving 'checkpoint_path' summary for global step 1872: ../model/elmo_bigru/model.ckpt-1872 I0716 03:19:49.288769 139795883440000 estimator.py:368] Loss for final step: 0.023545232.
Reading ../data/atis.test.w-intent.iob
I0716 03:19:49.877800 139795883440000 estimator.py:1145] Calling model_fn. I0716 03:19:50.902939 139795883440000 saver.py:1499] Saver not created because there are no variables in the graph to restore I0716 03:19:51.873017 139795883440000 estimator.py:1147] Done calling model_fn. I0716 03:19:52.085095 139795883440000 monitored_session.py:240] Graph was finalized. I0716 03:19:52.101049 139795883440000 saver.py:1280] Restoring parameters from ../model/elmo_bigru/model.ckpt-1872 I0716 03:19:52.988541 139795883440000 session_manager.py:500] Running local_init_op. I0716 03:19:53.032260 139795883440000 session_manager.py:502] Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.
'precision', 'predicted', average, warn_for)
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1439: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples.
'recall', 'true', average, warn_for)
I0716 03:20:51.023446 139795883440000 interactiveshell.py:2882]
precision recall f1-score support
atis_flight 0.987 0.991 0.989 632
atis_airfare 0.906 1.000 0.950 48
atis_ground_service 0.923 1.000 0.960 36
atis_airline 1.000 1.000 1.000 38
atis_abbreviation 0.943 1.000 0.971 33
atis_aircraft 1.000 0.889 0.941 9
atis_flight_time 0.500 1.000 0.667 1
atis_quantity 0.429 1.000 0.600 3
atis_flight#atis_airfare 0.833 0.417 0.556 12
atis_airport 0.944 0.944 0.944 18
atis_distance 1.000 1.000 1.000 10
atis_city 1.000 0.500 0.667 6
atis_ground_fare 1.000 0.857 0.923 7
atis_capacity 0.952 0.952 0.952 21
atis_flight_no 0.889 1.000 0.941 8
atis_meal 1.000 0.667 0.800 6
atis_restriction 0.000 0.000 0.000 0
atis_airline#atis_flight_no 0.000 0.000 0.000 0
atis_ground_service#atis_ground_fare 0.000 0.000 0.000 0
atis_airfare#atis_flight_time 0.000 0.000 0.000 0
atis_cheapest 0.000 0.000 0.000 0
atis_aircraft#atis_flight#atis_flight_no 0.000 0.000 0.000 0
micro avg 0.970 0.975 0.972 888
macro avg 0.650 0.646 0.630 888
weighted avg 0.973 0.975 0.971 888
I0716 03:20:51.059612 139795883440000 interactiveshell.py:2882]
precision recall f1-score support
O 0.000 0.000 0.000 0.0
B-toloc.city_name 0.969 0.994 0.981 716.0
B-fromloc.city_name 0.990 0.999 0.994 704.0
I-toloc.city_name 0.960 1.000 0.980 265.0
B-depart_date.day_name 0.986 0.991 0.988 212.0
B-airline_name 0.981 1.000 0.990 101.0
I-fromloc.city_name 0.989 0.994 0.992 177.0
B-depart_time.period_of_day 0.983 0.885 0.931 130.0
I-airline_name 1.000 0.969 0.984 65.0
B-depart_date.day_number 0.981 0.945 0.963 55.0
B-depart_date.month_name 0.982 0.964 0.973 56.0
B-depart_time.time 0.792 1.000 0.884 57.0
B-round_trip 1.000 0.986 0.993 73.0
B-cost_relative 0.973 0.973 0.973 37.0
I-round_trip 0.986 1.000 0.993 71.0
B-flight_mod 0.800 1.000 0.889 24.0
B-depart_time.time_relative 0.969 0.954 0.961 65.0
I-depart_time.time 0.879 0.981 0.927 52.0
B-stoploc.city_name 0.905 0.950 0.927 20.0
B-city_name 0.889 0.561 0.688 57.0
B-class_type 0.960 1.000 0.980 24.0
B-arrive_time.time 1.000 0.971 0.985 34.0
B-arrive_time.time_relative 0.857 0.968 0.909 31.0
I-class_type 1.000 1.000 1.000 17.0
B-flight_stop 1.000 1.000 1.000 21.0
I-arrive_time.time 0.944 0.971 0.958 35.0
B-airline_code 0.825 0.971 0.892 34.0
I-depart_date.day_number 1.000 0.933 0.966 15.0
I-fromloc.airport_name 0.500 1.000 0.667 15.0
B-fromloc.airport_name 0.400 1.000 0.571 12.0
B-arrive_date.day_name 0.786 1.000 0.880 11.0
B-toloc.state_code 1.000 1.000 1.000 18.0
B-depart_date.today_relative 1.000 0.778 0.875 9.0
B-flight_number 0.769 0.909 0.833 11.0
B-depart_date.date_relative 0.944 1.000 0.971 17.0
B-toloc.state_name 0.828 0.857 0.842 28.0
B-fare_basis_code 0.850 1.000 0.919 17.0
B-flight_time 1.000 1.000 1.000 1.0
B-or 1.000 1.000 1.000 3.0
B-arrive_time.period_of_day 0.750 1.000 0.857 6.0
B-meal_description 1.000 0.900 0.947 10.0
I-cost_relative 1.000 0.667 0.800 3.0
I-airport_name 0.867 0.448 0.591 29.0
B-fare_amount 1.000 1.000 1.000 2.0
I-fare_amount 1.000 1.000 1.000 2.0
I-city_name 1.000 0.600 0.750 30.0
I-toloc.airport_name 1.000 1.000 1.000 3.0
B-transport_type 1.000 0.900 0.947 10.0
B-arrive_date.month_name 0.714 0.833 0.769 6.0
B-arrive_date.day_number 0.556 0.833 0.667 6.0
I-stoploc.city_name 0.909 1.000 0.952 10.0
B-meal 1.000 1.000 1.000 16.0
B-fromloc.state_code 1.000 1.000 1.000 23.0
B-depart_time.period_mod 0.833 1.000 0.909 5.0
B-connect 1.000 1.000 1.000 6.0
B-flight_days 1.000 1.000 1.000 10.0
B-toloc.airport_name 1.000 0.333 0.500 3.0
B-fromloc.state_name 0.941 0.941 0.941 17.0
B-airport_name 0.615 0.381 0.471 21.0
B-economy 1.000 0.333 0.500 6.0
I-flight_time 1.000 1.000 1.000 1.0
B-aircraft_code 1.000 0.788 0.881 33.0
B-mod 1.000 0.500 0.667 2.0
B-airport_code 1.000 0.333 0.500 9.0
B-depart_time.start_time 1.000 1.000 1.000 3.0
B-depart_time.end_time 1.000 0.667 0.800 3.0
B-depart_date.year 1.000 1.000 1.000 3.0
I-transport_type 0.167 1.000 0.286 1.0
B-restriction_code 1.000 0.500 0.667 4.0
B-arrive_time.start_time 1.000 1.000 1.000 8.0
B-toloc.airport_code 1.000 0.750 0.857 4.0
B-arrive_time.end_time 1.000 1.000 1.000 8.0
I-arrive_time.end_time 1.000 0.875 0.933 8.0
I-depart_time.end_time 1.000 0.667 0.800 3.0
I-flight_stop 0.000 0.000 0.000 0.0
B-fromloc.airport_code 0.833 1.000 0.909 5.0
I-restriction_code 1.000 1.000 1.000 3.0
I-depart_time.start_time 1.000 1.000 1.000 1.0
I-toloc.state_name 1.000 1.000 1.000 1.0
I-depart_date.today_relative 0.000 0.000 0.000 0.0
B-arrive_date.date_relative 1.000 1.000 1.000 2.0
I-flight_mod 0.000 0.000 0.000 6.0
I-economy 0.000 0.000 0.000 0.0
B-return_date.date_relative 0.500 0.667 0.571 3.0
I-fromloc.state_name 0.000 0.000 0.000 1.0
B-state_code 1.000 1.000 1.000 1.0
I-arrive_time.start_time 1.000 1.000 1.000 1.0
I-arrive_date.day_number 0.000 0.000 0.000 0.0
B-meal_code 0.000 0.000 0.000 1.0
I-depart_time.period_of_day 0.000 0.000 0.000 1.0
B-day_name 1.000 0.500 0.667 2.0
B-period_of_day 1.000 0.500 0.667 4.0
B-stoploc.state_code 0.000 0.000 0.000 0.0
B-return_date.month_name 0.000 0.000 0.000 0.0
B-return_date.day_number 0.000 0.000 0.000 0.0
B-arrive_time.period_mod 0.000 0.000 0.000 0.0
I-meal_code 0.000 0.000 0.000 0.0
B-toloc.country_name 0.000 0.000 0.000 1.0
B-days_code 0.000 0.000 0.000 1.0
I-arrive_time.period_of_day 0.000 0.000 0.000 0.0
I-today_relative 0.000 0.000 0.000 0.0
B-return_time.period_of_day 0.000 0.000 0.000 0.0
B-time 0.000 0.000 0.000 0.0
I-fare_basis_code 0.000 0.000 0.000 0.0
I-arrive_time.time_relative 0.000 0.000 0.000 4.0
I-depart_time.time_relative 0.000 0.000 0.000 1.0
B-today_relative 0.000 0.000 0.000 0.0
B-state_name 0.000 0.000 0.000 9.0
B-arrive_date.today_relative 0.000 0.000 0.000 0.0
B-return_time.period_mod 0.000 0.000 0.000 0.0
B-month_name 0.000 0.000 0.000 0.0
B-day_number 0.000 0.000 0.000 0.0
I-return_date.date_relative 0.000 0.000 0.000 3.0
I-return_date.today_relative 0.000 0.000 0.000 0.0
B-stoploc.airport_name 0.000 0.000 0.000 0.0
B-time_relative 0.000 0.000 0.000 0.0
I-time 0.000 0.000 0.000 0.0
I-return_date.day_number 0.000 0.000 0.000 0.0
I-meal_description 0.000 0.000 0.000 0.0
B-return_date.today_relative 0.000 0.000 0.000 0.0
B-return_date.day_name 0.000 0.000 0.000 2.0
micro avg 0.949 0.951 0.950 3657.0
macro avg 0.631 0.607 0.604 3657.0
weighted avg 0.950 0.951 0.946 3657.0
I0716 03:20:51.080179 139795883440000 interactiveshell.py:2882] Best Slot F1: 0.950
I0716 03:20:51.081658 139795883440000 estimator_training.py:186] Not using Distribute Coordinator.
I0716 03:20:51.085367 139795883440000 training.py:612] Running training and evaluation locally (non-distributed).
I0716 03:20:51.088035 139795883440000 training.py:700] Start train and evaluate loop. The evaluate will happen after every checkpoint. Checkpoint frequency is determined based on RunConfig arguments: save_checkpoints_steps 312 or save_checkpoints_secs None.
I0716 03:20:51.152100 139795883440000 estimator.py:1145] Calling model_fn.
I0716 03:20:52.592101 139795883440000 saver.py:1499] Saver not created because there are no variables in the graph to restore
I0716 03:20:53.185715 139795883440000 <ipython-input-6-d6747b40360b>:55]
[<tf.Variable 'embedding:0' shape=(750, 300) dtype=float32_ref>,
<tf.Variable 'module/aggregation/weights:0' shape=(3,) dtype=float32>,
<tf.Variable 'module/aggregation/scaling:0' shape=() dtype=float32>,
<tf.Variable 'dense/kernel:0' shape=(1324, 300) dtype=float32_ref>,
<tf.Variable 'dense/bias:0' shape=(300,) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/fw/gru_cell/gates/kernel:0' shape=(600, 600) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/fw/gru_cell/gates/bias:0' shape=(600,) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/fw/gru_cell/candidate/kernel:0' shape=(600, 300) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/fw/gru_cell/candidate/bias:0' shape=(300,) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/bw/gru_cell/gates/kernel:0' shape=(600, 600) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/bw/gru_cell/gates/bias:0' shape=(600,) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/bw/gru_cell/candidate/kernel:0' shape=(600, 300) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/bw/gru_cell/candidate/bias:0' shape=(300,) dtype=float32_ref>,
<tf.Variable 'dense_1/kernel:0' shape=(600, 23) dtype=float32_ref>,
<tf.Variable 'dense_1/bias:0' shape=(23,) dtype=float32_ref>,
<tf.Variable 'dense_2/kernel:0' shape=(600, 122) dtype=float32_ref>,
<tf.Variable 'dense_2/bias:0' shape=(122,) dtype=float32_ref>]
I0716 03:20:54.254271 139795883440000 estimator.py:1147] Done calling model_fn.
I0716 03:20:54.257858 139795883440000 basic_session_run_hooks.py:541] Create CheckpointSaverHook.
I0716 03:20:54.765297 139795883440000 monitored_session.py:240] Graph was finalized.
I0716 03:20:54.785214 139795883440000 saver.py:1280] Restoring parameters from ../model/elmo_bigru/model.ckpt-1872
I0716 03:20:55.794133 139795883440000 session_manager.py:500] Running local_init_op.
I0716 03:20:55.866584 139795883440000 session_manager.py:502] Done running local_init_op.
I0716 03:20:58.639599 139795883440000 basic_session_run_hooks.py:606] Saving checkpoints for 1872 into ../model/elmo_bigru/model.ckpt.
Reading ../data/atis.train.w-intent.iob
I0716 03:21:03.386334 139795883440000 basic_session_run_hooks.py:262] loss = 0.05346859, step = 1872 I0716 03:21:03.388282 139795883440000 basic_session_run_hooks.py:262] lr = 0.0002462993 I0716 03:21:30.505856 139795883440000 basic_session_run_hooks.py:692] global_step/sec: 3.68729 I0716 03:21:30.513989 139795883440000 basic_session_run_hooks.py:260] loss = 0.04573856, step = 1972 (27.128 sec) I0716 03:21:30.516146 139795883440000 basic_session_run_hooks.py:260] lr = 0.00024371794 (27.128 sec) I0716 03:21:57.266182 139795883440000 basic_session_run_hooks.py:692] global_step/sec: 3.73689 I0716 03:21:57.273312 139795883440000 basic_session_run_hooks.py:260] loss = 0.043409802, step = 2072 (26.759 sec) I0716 03:21:57.276177 139795883440000 basic_session_run_hooks.py:260] lr = 0.00024116358 (26.760 sec) I0716 03:22:24.066598 139795883440000 basic_session_run_hooks.py:692] global_step/sec: 3.73127 I0716 03:22:24.074424 139795883440000 basic_session_run_hooks.py:260] loss = 0.040823795, step = 2172 (26.801 sec) I0716 03:22:24.075997 139795883440000 basic_session_run_hooks.py:260] lr = 0.00023863598 (26.800 sec) I0716 03:22:26.920837 139795883440000 basic_session_run_hooks.py:606] Saving checkpoints for 2184 into ../model/elmo_bigru/model.ckpt. I0716 03:22:29.112518 139795883440000 estimator.py:1145] Calling model_fn. I0716 03:22:30.449255 139795883440000 saver.py:1499] Saver not created because there are no variables in the graph to restore I0716 03:22:31.278379 139795883440000 estimator.py:1147] Done calling model_fn. I0716 03:22:31.310838 139795883440000 evaluation.py:255] Starting evaluation at 2019-07-16T03:22:31Z I0716 03:22:31.531050 139795883440000 monitored_session.py:240] Graph was finalized. I0716 03:22:31.549881 139795883440000 saver.py:1280] Restoring parameters from ../model/elmo_bigru/model.ckpt-2184 I0716 03:22:32.448242 139795883440000 session_manager.py:500] Running local_init_op. I0716 03:22:32.499600 139795883440000 session_manager.py:502] Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
I0716 03:23:31.168509 139795883440000 evaluation.py:275] Finished evaluation at 2019-07-16-03:23:31 I0716 03:23:31.170503 139795883440000 estimator.py:2039] Saving dict for global step 2184: global_step = 2184, loss = 0.28009278 I0716 03:23:31.177702 139795883440000 estimator.py:2099] Saving 'checkpoint_path' summary for global step 2184: ../model/elmo_bigru/model.ckpt-2184 I0716 03:23:31.260535 139795883440000 estimator.py:368] Loss for final step: 0.01047217.
Reading ../data/atis.test.w-intent.iob
I0716 03:23:31.841433 139795883440000 estimator.py:1145] Calling model_fn. I0716 03:23:32.880039 139795883440000 saver.py:1499] Saver not created because there are no variables in the graph to restore I0716 03:23:33.836756 139795883440000 estimator.py:1147] Done calling model_fn. I0716 03:23:34.046449 139795883440000 monitored_session.py:240] Graph was finalized. I0716 03:23:34.061011 139795883440000 saver.py:1280] Restoring parameters from ../model/elmo_bigru/model.ckpt-2184 I0716 03:23:34.944166 139795883440000 session_manager.py:500] Running local_init_op. I0716 03:23:34.986050 139795883440000 session_manager.py:502] Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.
'precision', 'predicted', average, warn_for)
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1439: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples.
'recall', 'true', average, warn_for)
I0716 03:24:32.635773 139795883440000 interactiveshell.py:2882]
precision recall f1-score support
atis_flight 0.986 0.992 0.989 632
atis_airfare 0.941 1.000 0.970 48
atis_ground_service 1.000 1.000 1.000 36
atis_airline 1.000 1.000 1.000 38
atis_abbreviation 0.943 1.000 0.971 33
atis_aircraft 1.000 1.000 1.000 9
atis_flight_time 0.333 1.000 0.500 1
atis_quantity 0.429 1.000 0.600 3
atis_flight#atis_airfare 0.833 0.417 0.556 12
atis_airport 1.000 1.000 1.000 18
atis_distance 1.000 1.000 1.000 10
atis_city 0.750 0.500 0.600 6
atis_ground_fare 1.000 0.857 0.923 7
atis_capacity 1.000 0.952 0.976 21
atis_flight_no 0.889 1.000 0.941 8
atis_meal 1.000 0.833 0.909 6
atis_restriction 0.000 0.000 0.000 0
atis_airline#atis_flight_no 0.000 0.000 0.000 0
atis_ground_service#atis_ground_fare 0.000 0.000 0.000 0
atis_airfare#atis_flight_time 0.000 0.000 0.000 0
atis_cheapest 0.000 0.000 0.000 0
atis_aircraft#atis_flight#atis_flight_no 0.000 0.000 0.000 0
micro avg 0.974 0.980 0.977 888
macro avg 0.641 0.661 0.633 888
weighted avg 0.977 0.980 0.976 888
I0716 03:24:32.672422 139795883440000 interactiveshell.py:2882]
precision recall f1-score support
O 0.000 0.000 0.000 0.0
B-toloc.city_name 0.974 0.992 0.983 716.0
B-fromloc.city_name 0.989 0.999 0.994 704.0
I-toloc.city_name 0.967 0.996 0.981 265.0
B-depart_date.day_name 0.986 0.991 0.988 212.0
B-airline_name 0.981 1.000 0.990 101.0
I-fromloc.city_name 0.972 0.994 0.983 177.0
B-depart_time.period_of_day 0.983 0.915 0.948 130.0
I-airline_name 1.000 0.969 0.984 65.0
B-depart_date.day_number 0.964 0.964 0.964 55.0
B-depart_date.month_name 0.982 0.964 0.973 56.0
B-depart_time.time 0.826 1.000 0.905 57.0
B-round_trip 1.000 0.986 0.993 73.0
B-cost_relative 0.973 0.973 0.973 37.0
I-round_trip 0.986 1.000 0.993 71.0
B-flight_mod 0.800 1.000 0.889 24.0
B-depart_time.time_relative 0.970 1.000 0.985 65.0
I-depart_time.time 0.929 1.000 0.963 52.0
B-stoploc.city_name 0.870 1.000 0.930 20.0
B-city_name 0.895 0.596 0.716 57.0
B-class_type 0.960 1.000 0.980 24.0
B-arrive_time.time 1.000 0.971 0.985 34.0
B-arrive_time.time_relative 0.882 0.968 0.923 31.0
I-class_type 1.000 1.000 1.000 17.0
B-flight_stop 1.000 1.000 1.000 21.0
I-arrive_time.time 1.000 0.971 0.986 35.0
B-airline_code 0.810 1.000 0.895 34.0
I-depart_date.day_number 1.000 0.933 0.966 15.0
I-fromloc.airport_name 0.484 1.000 0.652 15.0
B-fromloc.airport_name 0.400 1.000 0.571 12.0
B-arrive_date.day_name 0.786 1.000 0.880 11.0
B-toloc.state_code 1.000 1.000 1.000 18.0
B-depart_date.today_relative 1.000 0.778 0.875 9.0
B-flight_number 0.786 1.000 0.880 11.0
B-depart_date.date_relative 0.895 1.000 0.944 17.0
B-toloc.state_name 0.833 0.893 0.862 28.0
B-fare_basis_code 0.895 1.000 0.944 17.0
B-flight_time 1.000 1.000 1.000 1.0
B-or 1.000 1.000 1.000 3.0
B-arrive_time.period_of_day 0.750 1.000 0.857 6.0
B-meal_description 1.000 0.900 0.947 10.0
I-cost_relative 1.000 0.667 0.800 3.0
I-airport_name 0.765 0.448 0.565 29.0
B-fare_amount 1.000 1.000 1.000 2.0
I-fare_amount 1.000 1.000 1.000 2.0
I-city_name 1.000 0.533 0.696 30.0
I-toloc.airport_name 1.000 1.000 1.000 3.0
B-transport_type 0.909 1.000 0.952 10.0
B-arrive_date.month_name 0.714 0.833 0.769 6.0
B-arrive_date.day_number 0.625 0.833 0.714 6.0
I-stoploc.city_name 0.909 1.000 0.952 10.0
B-meal 1.000 1.000 1.000 16.0
B-fromloc.state_code 1.000 1.000 1.000 23.0
B-depart_time.period_mod 1.000 1.000 1.000 5.0
B-connect 1.000 1.000 1.000 6.0
B-flight_days 1.000 1.000 1.000 10.0
B-toloc.airport_name 1.000 1.000 1.000 3.0
B-fromloc.state_name 0.941 0.941 0.941 17.0
B-airport_name 0.636 0.333 0.437 21.0
B-economy 1.000 0.333 0.500 6.0
I-flight_time 1.000 1.000 1.000 1.0
B-aircraft_code 1.000 0.758 0.862 33.0
B-mod 1.000 0.500 0.667 2.0
B-airport_code 1.000 0.444 0.615 9.0
B-depart_time.start_time 1.000 0.667 0.800 3.0
B-depart_time.end_time 1.000 0.333 0.500 3.0
B-depart_date.year 1.000 1.000 1.000 3.0
I-transport_type 0.000 0.000 0.000 1.0
B-restriction_code 1.000 0.500 0.667 4.0
B-arrive_time.start_time 0.889 1.000 0.941 8.0
B-toloc.airport_code 1.000 0.750 0.857 4.0
B-arrive_time.end_time 0.889 1.000 0.941 8.0
I-arrive_time.end_time 1.000 1.000 1.000 8.0
I-depart_time.end_time 1.000 0.667 0.800 3.0
I-flight_stop 0.000 0.000 0.000 0.0
B-fromloc.airport_code 0.714 1.000 0.833 5.0
I-restriction_code 1.000 1.000 1.000 3.0
I-depart_time.start_time 1.000 1.000 1.000 1.0
I-toloc.state_name 1.000 1.000 1.000 1.0
I-depart_date.today_relative 0.000 0.000 0.000 0.0
B-arrive_date.date_relative 1.000 1.000 1.000 2.0
I-flight_mod 0.000 0.000 0.000 6.0
I-economy 0.000 0.000 0.000 0.0
B-return_date.date_relative 0.667 0.667 0.667 3.0
I-fromloc.state_name 0.000 0.000 0.000 1.0
B-state_code 1.000 1.000 1.000 1.0
I-arrive_time.start_time 1.000 1.000 1.000 1.0
I-arrive_date.day_number 0.000 0.000 0.000 0.0
B-meal_code 0.000 0.000 0.000 1.0
I-depart_time.period_of_day 0.000 0.000 0.000 1.0
B-day_name 1.000 0.500 0.667 2.0
B-period_of_day 1.000 0.500 0.667 4.0
B-stoploc.state_code 0.000 0.000 0.000 0.0
B-return_date.month_name 0.000 0.000 0.000 0.0
B-return_date.day_number 0.000 0.000 0.000 0.0
B-arrive_time.period_mod 0.000 0.000 0.000 0.0
I-meal_code 0.000 0.000 0.000 0.0
B-toloc.country_name 0.000 0.000 0.000 1.0
B-days_code 0.000 0.000 0.000 1.0
I-arrive_time.period_of_day 0.000 0.000 0.000 0.0
I-today_relative 0.000 0.000 0.000 0.0
B-return_time.period_of_day 0.000 0.000 0.000 0.0
B-time 0.000 0.000 0.000 0.0
I-fare_basis_code 0.000 0.000 0.000 0.0
I-arrive_time.time_relative 0.000 0.000 0.000 4.0
I-depart_time.time_relative 0.000 0.000 0.000 1.0
B-today_relative 0.000 0.000 0.000 0.0
B-state_name 0.000 0.000 0.000 9.0
B-arrive_date.today_relative 0.000 0.000 0.000 0.0
B-return_time.period_mod 0.000 0.000 0.000 0.0
B-month_name 0.000 0.000 0.000 0.0
B-day_number 0.000 0.000 0.000 0.0
I-return_date.date_relative 0.000 0.000 0.000 3.0
I-return_date.today_relative 0.000 0.000 0.000 0.0
B-stoploc.airport_name 0.000 0.000 0.000 0.0
B-time_relative 0.000 0.000 0.000 0.0
I-time 0.000 0.000 0.000 0.0
I-return_date.day_number 0.000 0.000 0.000 0.0
I-meal_description 0.000 0.000 0.000 0.0
B-return_date.today_relative 0.000 0.000 0.000 0.0
B-return_date.day_name 0.000 0.000 0.000 2.0
micro avg 0.952 0.953 0.952 3657.0
macro avg 0.630 0.603 0.604 3657.0
weighted avg 0.951 0.953 0.948 3657.0
I0716 03:24:32.695803 139795883440000 interactiveshell.py:2882] Best Slot F1: 0.952
I0716 03:24:32.697161 139795883440000 estimator_training.py:186] Not using Distribute Coordinator.
I0716 03:24:32.700752 139795883440000 training.py:612] Running training and evaluation locally (non-distributed).
I0716 03:24:32.704385 139795883440000 training.py:700] Start train and evaluate loop. The evaluate will happen after every checkpoint. Checkpoint frequency is determined based on RunConfig arguments: save_checkpoints_steps 312 or save_checkpoints_secs None.
I0716 03:24:32.768478 139795883440000 estimator.py:1145] Calling model_fn.
I0716 03:24:34.197245 139795883440000 saver.py:1499] Saver not created because there are no variables in the graph to restore
I0716 03:24:34.793243 139795883440000 <ipython-input-6-d6747b40360b>:55]
[<tf.Variable 'embedding:0' shape=(750, 300) dtype=float32_ref>,
<tf.Variable 'module/aggregation/weights:0' shape=(3,) dtype=float32>,
<tf.Variable 'module/aggregation/scaling:0' shape=() dtype=float32>,
<tf.Variable 'dense/kernel:0' shape=(1324, 300) dtype=float32_ref>,
<tf.Variable 'dense/bias:0' shape=(300,) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/fw/gru_cell/gates/kernel:0' shape=(600, 600) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/fw/gru_cell/gates/bias:0' shape=(600,) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/fw/gru_cell/candidate/kernel:0' shape=(600, 300) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/fw/gru_cell/candidate/bias:0' shape=(300,) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/bw/gru_cell/gates/kernel:0' shape=(600, 600) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/bw/gru_cell/gates/bias:0' shape=(600,) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/bw/gru_cell/candidate/kernel:0' shape=(600, 300) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/bw/gru_cell/candidate/bias:0' shape=(300,) dtype=float32_ref>,
<tf.Variable 'dense_1/kernel:0' shape=(600, 23) dtype=float32_ref>,
<tf.Variable 'dense_1/bias:0' shape=(23,) dtype=float32_ref>,
<tf.Variable 'dense_2/kernel:0' shape=(600, 122) dtype=float32_ref>,
<tf.Variable 'dense_2/bias:0' shape=(122,) dtype=float32_ref>]
I0716 03:24:35.869684 139795883440000 estimator.py:1147] Done calling model_fn.
I0716 03:24:35.874225 139795883440000 basic_session_run_hooks.py:541] Create CheckpointSaverHook.
I0716 03:24:36.372488 139795883440000 monitored_session.py:240] Graph was finalized.
I0716 03:24:36.389989 139795883440000 saver.py:1280] Restoring parameters from ../model/elmo_bigru/model.ckpt-2184
I0716 03:24:37.422055 139795883440000 session_manager.py:500] Running local_init_op.
I0716 03:24:37.489724 139795883440000 session_manager.py:502] Done running local_init_op.
I0716 03:24:40.400031 139795883440000 basic_session_run_hooks.py:606] Saving checkpoints for 2184 into ../model/elmo_bigru/model.ckpt.
Reading ../data/atis.train.w-intent.iob
I0716 03:24:45.578757 139795883440000 basic_session_run_hooks.py:262] loss = 0.063943386, step = 2184 I0716 03:24:45.580540 139795883440000 basic_session_run_hooks.py:262] lr = 0.00023833448 I0716 03:25:12.153598 139795883440000 basic_session_run_hooks.py:692] global_step/sec: 3.76288 I0716 03:25:12.158720 139795883440000 basic_session_run_hooks.py:260] loss = 0.03020731, step = 2284 (26.580 sec) I0716 03:25:12.161730 139795883440000 basic_session_run_hooks.py:260] lr = 0.00023583656 (26.581 sec) I0716 03:25:39.568844 139795883440000 basic_session_run_hooks.py:692] global_step/sec: 3.6476 I0716 03:25:39.576084 139795883440000 basic_session_run_hooks.py:260] loss = 0.03212006, step = 2384 (27.417 sec) I0716 03:25:39.578353 139795883440000 basic_session_run_hooks.py:260] lr = 0.00023336482 (27.417 sec) I0716 03:26:05.588598 139795883440000 basic_session_run_hooks.py:692] global_step/sec: 3.84323 I0716 03:26:05.595238 139795883440000 basic_session_run_hooks.py:260] loss = 0.14932747, step = 2484 (26.019 sec) I0716 03:26:05.597378 139795883440000 basic_session_run_hooks.py:260] lr = 0.00023091899 (26.019 sec) I0716 03:26:08.265371 139795883440000 basic_session_run_hooks.py:606] Saving checkpoints for 2496 into ../model/elmo_bigru/model.ckpt. I0716 03:26:10.106337 139795883440000 estimator.py:1145] Calling model_fn. I0716 03:26:11.422580 139795883440000 saver.py:1499] Saver not created because there are no variables in the graph to restore I0716 03:26:12.238691 139795883440000 estimator.py:1147] Done calling model_fn. I0716 03:26:12.267982 139795883440000 evaluation.py:255] Starting evaluation at 2019-07-16T03:26:12Z I0716 03:26:12.498234 139795883440000 monitored_session.py:240] Graph was finalized. I0716 03:26:12.514398 139795883440000 saver.py:1280] Restoring parameters from ../model/elmo_bigru/model.ckpt-2496 I0716 03:26:13.408679 139795883440000 session_manager.py:500] Running local_init_op. I0716 03:26:13.464711 139795883440000 session_manager.py:502] Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
I0716 03:27:11.979592 139795883440000 evaluation.py:275] Finished evaluation at 2019-07-16-03:27:11 I0716 03:27:11.981393 139795883440000 estimator.py:2039] Saving dict for global step 2496: global_step = 2496, loss = 0.280597 I0716 03:27:11.989431 139795883440000 estimator.py:2099] Saving 'checkpoint_path' summary for global step 2496: ../model/elmo_bigru/model.ckpt-2496 I0716 03:27:12.073182 139795883440000 estimator.py:368] Loss for final step: 0.07099749.
Reading ../data/atis.test.w-intent.iob
I0716 03:27:12.656421 139795883440000 estimator.py:1145] Calling model_fn. I0716 03:27:13.679173 139795883440000 saver.py:1499] Saver not created because there are no variables in the graph to restore I0716 03:27:14.652549 139795883440000 estimator.py:1147] Done calling model_fn. I0716 03:27:14.870477 139795883440000 monitored_session.py:240] Graph was finalized. I0716 03:27:14.887186 139795883440000 saver.py:1280] Restoring parameters from ../model/elmo_bigru/model.ckpt-2496 I0716 03:27:15.759628 139795883440000 session_manager.py:500] Running local_init_op. I0716 03:27:15.808227 139795883440000 session_manager.py:502] Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.
'precision', 'predicted', average, warn_for)
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1439: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples.
'recall', 'true', average, warn_for)
I0716 03:28:13.653455 139795883440000 interactiveshell.py:2882]
precision recall f1-score support
atis_flight 0.987 0.992 0.990 632
atis_airfare 0.941 1.000 0.970 48
atis_ground_service 0.973 1.000 0.986 36
atis_airline 1.000 1.000 1.000 38
atis_abbreviation 0.943 1.000 0.971 33
atis_aircraft 1.000 0.889 0.941 9
atis_flight_time 0.333 1.000 0.500 1
atis_quantity 0.429 1.000 0.600 3
atis_flight#atis_airfare 0.857 0.500 0.632 12
atis_airport 1.000 0.944 0.971 18
atis_distance 1.000 1.000 1.000 10
atis_city 0.600 0.500 0.545 6
atis_ground_fare 1.000 0.857 0.923 7
atis_capacity 0.952 0.952 0.952 21
atis_flight_no 0.889 1.000 0.941 8
atis_meal 1.000 0.667 0.800 6
atis_restriction 0.000 0.000 0.000 0
atis_airline#atis_flight_no 0.000 0.000 0.000 0
atis_ground_service#atis_ground_fare 0.000 0.000 0.000 0
atis_airfare#atis_flight_time 0.000 0.000 0.000 0
atis_cheapest 0.000 0.000 0.000 0
atis_aircraft#atis_flight#atis_flight_no 0.000 0.000 0.000 0
micro avg 0.972 0.977 0.975 888
macro avg 0.632 0.650 0.624 888
weighted avg 0.975 0.977 0.975 888
I0716 03:28:13.689820 139795883440000 interactiveshell.py:2882]
precision recall f1-score support
O 0.000 0.000 0.000 0.0
B-toloc.city_name 0.974 0.993 0.983 716.0
B-fromloc.city_name 0.989 0.999 0.994 704.0
I-toloc.city_name 0.964 1.000 0.981 265.0
B-depart_date.day_name 0.990 0.972 0.981 212.0
B-airline_name 0.990 1.000 0.995 101.0
I-fromloc.city_name 0.994 0.994 0.994 177.0
B-depart_time.period_of_day 0.983 0.915 0.948 130.0
I-airline_name 1.000 0.985 0.992 65.0
B-depart_date.day_number 0.981 0.945 0.963 55.0
B-depart_date.month_name 0.981 0.946 0.964 56.0
B-depart_time.time 0.851 1.000 0.919 57.0
B-round_trip 1.000 1.000 1.000 73.0
B-cost_relative 0.973 0.973 0.973 37.0
I-round_trip 1.000 1.000 1.000 71.0
B-flight_mod 0.800 1.000 0.889 24.0
B-depart_time.time_relative 0.969 0.954 0.961 65.0
I-depart_time.time 0.945 1.000 0.972 52.0
B-stoploc.city_name 0.909 1.000 0.952 20.0
B-city_name 0.900 0.632 0.742 57.0
B-class_type 0.960 1.000 0.980 24.0
B-arrive_time.time 0.971 0.971 0.971 34.0
B-arrive_time.time_relative 0.833 0.968 0.896 31.0
I-class_type 1.000 1.000 1.000 17.0
B-flight_stop 1.000 1.000 1.000 21.0
I-arrive_time.time 1.000 0.971 0.986 35.0
B-airline_code 0.850 1.000 0.919 34.0
I-depart_date.day_number 1.000 0.933 0.966 15.0
I-fromloc.airport_name 0.405 1.000 0.577 15.0
B-fromloc.airport_name 0.414 1.000 0.585 12.0
B-arrive_date.day_name 0.611 1.000 0.759 11.0
B-toloc.state_code 1.000 1.000 1.000 18.0
B-depart_date.today_relative 1.000 0.889 0.941 9.0
B-flight_number 0.786 1.000 0.880 11.0
B-depart_date.date_relative 0.944 1.000 0.971 17.0
B-toloc.state_name 0.833 0.893 0.862 28.0
B-fare_basis_code 0.850 1.000 0.919 17.0
B-flight_time 1.000 1.000 1.000 1.0
B-or 1.000 1.000 1.000 3.0
B-arrive_time.period_of_day 0.750 1.000 0.857 6.0
B-meal_description 1.000 0.900 0.947 10.0
I-cost_relative 1.000 0.667 0.800 3.0
I-airport_name 0.833 0.345 0.488 29.0
B-fare_amount 1.000 1.000 1.000 2.0
I-fare_amount 1.000 1.000 1.000 2.0
I-city_name 1.000 0.567 0.723 30.0
I-toloc.airport_name 1.000 1.000 1.000 3.0
B-transport_type 1.000 1.000 1.000 10.0
B-arrive_date.month_name 0.625 0.833 0.714 6.0
B-arrive_date.day_number 0.556 0.833 0.667 6.0
I-stoploc.city_name 0.909 1.000 0.952 10.0
B-meal 1.000 1.000 1.000 16.0
B-fromloc.state_code 1.000 1.000 1.000 23.0
B-depart_time.period_mod 0.833 1.000 0.909 5.0
B-connect 1.000 1.000 1.000 6.0
B-flight_days 1.000 1.000 1.000 10.0
B-toloc.airport_name 1.000 1.000 1.000 3.0
B-fromloc.state_name 0.941 0.941 0.941 17.0
B-airport_name 0.636 0.333 0.437 21.0
B-economy 1.000 0.333 0.500 6.0
I-flight_time 1.000 1.000 1.000 1.0
B-aircraft_code 1.000 0.848 0.918 33.0
B-mod 1.000 0.500 0.667 2.0
B-airport_code 1.000 0.444 0.615 9.0
B-depart_time.start_time 1.000 0.667 0.800 3.0
B-depart_time.end_time 1.000 0.333 0.500 3.0
B-depart_date.year 1.000 1.000 1.000 3.0
I-transport_type 1.000 1.000 1.000 1.0
B-restriction_code 1.000 0.500 0.667 4.0
B-arrive_time.start_time 0.889 1.000 0.941 8.0
B-toloc.airport_code 1.000 0.750 0.857 4.0
B-arrive_time.end_time 0.889 1.000 0.941 8.0
I-arrive_time.end_time 0.889 1.000 0.941 8.0
I-depart_time.end_time 1.000 0.333 0.500 3.0
I-flight_stop 0.000 0.000 0.000 0.0
B-fromloc.airport_code 0.625 1.000 0.769 5.0
I-restriction_code 1.000 1.000 1.000 3.0
I-depart_time.start_time 1.000 1.000 1.000 1.0
I-toloc.state_name 1.000 1.000 1.000 1.0
I-depart_date.today_relative 0.000 0.000 0.000 0.0
B-arrive_date.date_relative 1.000 1.000 1.000 2.0
I-flight_mod 0.000 0.000 0.000 6.0
I-economy 0.000 0.000 0.000 0.0
B-return_date.date_relative 0.500 0.667 0.571 3.0
I-fromloc.state_name 0.000 0.000 0.000 1.0
B-state_code 1.000 1.000 1.000 1.0
I-arrive_time.start_time 1.000 1.000 1.000 1.0
I-arrive_date.day_number 0.000 0.000 0.000 0.0
B-meal_code 0.000 0.000 0.000 1.0
I-depart_time.period_of_day 0.000 0.000 0.000 1.0
B-day_name 1.000 0.500 0.667 2.0
B-period_of_day 1.000 0.750 0.857 4.0
B-stoploc.state_code 0.000 0.000 0.000 0.0
B-return_date.month_name 0.000 0.000 0.000 0.0
B-return_date.day_number 0.000 0.000 0.000 0.0
B-arrive_time.period_mod 0.000 0.000 0.000 0.0
I-meal_code 0.000 0.000 0.000 0.0
B-toloc.country_name 1.000 1.000 1.000 1.0
B-days_code 0.000 0.000 0.000 1.0
I-arrive_time.period_of_day 0.000 0.000 0.000 0.0
I-today_relative 0.000 0.000 0.000 0.0
B-return_time.period_of_day 0.000 0.000 0.000 0.0
B-time 0.000 0.000 0.000 0.0
I-fare_basis_code 0.000 0.000 0.000 0.0
I-arrive_time.time_relative 0.000 0.000 0.000 4.0
I-depart_time.time_relative 0.000 0.000 0.000 1.0
B-today_relative 0.000 0.000 0.000 0.0
B-state_name 0.000 0.000 0.000 9.0
B-arrive_date.today_relative 0.000 0.000 0.000 0.0
B-return_time.period_mod 0.000 0.000 0.000 0.0
B-month_name 0.000 0.000 0.000 0.0
B-day_number 0.000 0.000 0.000 0.0
I-return_date.date_relative 0.000 0.000 0.000 3.0
I-return_date.today_relative 0.000 0.000 0.000 0.0
B-stoploc.airport_name 0.000 0.000 0.000 0.0
B-time_relative 0.000 0.000 0.000 0.0
I-time 0.000 0.000 0.000 0.0
I-return_date.day_number 0.000 0.000 0.000 0.0
I-meal_description 0.000 0.000 0.000 0.0
B-return_date.today_relative 0.000 0.000 0.000 0.0
B-return_date.day_name 0.000 0.000 0.000 2.0
micro avg 0.952 0.954 0.953 3657.0
macro avg 0.641 0.620 0.616 3657.0
weighted avg 0.953 0.954 0.949 3657.0
I0716 03:28:13.711204 139795883440000 interactiveshell.py:2882] Best Slot F1: 0.953
I0716 03:28:13.712770 139795883440000 estimator_training.py:186] Not using Distribute Coordinator.
I0716 03:28:13.714238 139795883440000 training.py:612] Running training and evaluation locally (non-distributed).
I0716 03:28:13.717311 139795883440000 training.py:700] Start train and evaluate loop. The evaluate will happen after every checkpoint. Checkpoint frequency is determined based on RunConfig arguments: save_checkpoints_steps 312 or save_checkpoints_secs None.
I0716 03:28:13.781014 139795883440000 estimator.py:1145] Calling model_fn.
I0716 03:28:15.236994 139795883440000 saver.py:1499] Saver not created because there are no variables in the graph to restore
I0716 03:28:15.821968 139795883440000 <ipython-input-6-d6747b40360b>:55]
[<tf.Variable 'embedding:0' shape=(750, 300) dtype=float32_ref>,
<tf.Variable 'module/aggregation/weights:0' shape=(3,) dtype=float32>,
<tf.Variable 'module/aggregation/scaling:0' shape=() dtype=float32>,
<tf.Variable 'dense/kernel:0' shape=(1324, 300) dtype=float32_ref>,
<tf.Variable 'dense/bias:0' shape=(300,) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/fw/gru_cell/gates/kernel:0' shape=(600, 600) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/fw/gru_cell/gates/bias:0' shape=(600,) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/fw/gru_cell/candidate/kernel:0' shape=(600, 300) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/fw/gru_cell/candidate/bias:0' shape=(300,) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/bw/gru_cell/gates/kernel:0' shape=(600, 600) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/bw/gru_cell/gates/bias:0' shape=(600,) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/bw/gru_cell/candidate/kernel:0' shape=(600, 300) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/bw/gru_cell/candidate/bias:0' shape=(300,) dtype=float32_ref>,
<tf.Variable 'dense_1/kernel:0' shape=(600, 23) dtype=float32_ref>,
<tf.Variable 'dense_1/bias:0' shape=(23,) dtype=float32_ref>,
<tf.Variable 'dense_2/kernel:0' shape=(600, 122) dtype=float32_ref>,
<tf.Variable 'dense_2/bias:0' shape=(122,) dtype=float32_ref>]
I0716 03:28:17.068133 139795883440000 estimator.py:1147] Done calling model_fn.
I0716 03:28:17.072277 139795883440000 basic_session_run_hooks.py:541] Create CheckpointSaverHook.
I0716 03:28:17.395323 139795883440000 monitored_session.py:240] Graph was finalized.
I0716 03:28:17.413097 139795883440000 saver.py:1280] Restoring parameters from ../model/elmo_bigru/model.ckpt-2496
I0716 03:28:18.424999 139795883440000 session_manager.py:500] Running local_init_op.
I0716 03:28:18.496368 139795883440000 session_manager.py:502] Done running local_init_op.
I0716 03:28:21.246337 139795883440000 basic_session_run_hooks.py:606] Saving checkpoints for 2496 into ../model/elmo_bigru/model.ckpt.
Reading ../data/atis.train.w-intent.iob
I0716 03:28:25.928198 139795883440000 basic_session_run_hooks.py:262] loss = 0.040984906, step = 2496 I0716 03:28:25.930085 139795883440000 basic_session_run_hooks.py:262] lr = 0.0002306272 I0716 03:28:52.934669 139795883440000 basic_session_run_hooks.py:692] global_step/sec: 3.70273 I0716 03:28:52.941412 139795883440000 basic_session_run_hooks.py:260] loss = 0.06381842, step = 2596 (27.013 sec) I0716 03:28:52.943911 139795883440000 basic_session_run_hooks.py:260] lr = 0.00022821005 (27.014 sec) I0716 03:29:19.840715 139795883440000 basic_session_run_hooks.py:692] global_step/sec: 3.71663 I0716 03:29:19.844252 139795883440000 basic_session_run_hooks.py:260] loss = 0.023345564, step = 2696 (26.903 sec) I0716 03:29:19.846118 139795883440000 basic_session_run_hooks.py:260] lr = 0.00022581825 (26.902 sec) I0716 03:29:46.032870 139795883440000 basic_session_run_hooks.py:692] global_step/sec: 3.81795 I0716 03:29:46.036398 139795883440000 basic_session_run_hooks.py:260] loss = 0.038965724, step = 2796 (26.192 sec) I0716 03:29:46.041105 139795883440000 basic_session_run_hooks.py:260] lr = 0.00022345151 (26.195 sec) I0716 03:29:48.910070 139795883440000 basic_session_run_hooks.py:606] Saving checkpoints for 2808 into ../model/elmo_bigru/model.ckpt. I0716 03:29:51.028875 139795883440000 estimator.py:1145] Calling model_fn. I0716 03:29:52.357199 139795883440000 saver.py:1499] Saver not created because there are no variables in the graph to restore I0716 03:29:53.193941 139795883440000 estimator.py:1147] Done calling model_fn. I0716 03:29:53.224582 139795883440000 evaluation.py:255] Starting evaluation at 2019-07-16T03:29:53Z I0716 03:29:53.445019 139795883440000 monitored_session.py:240] Graph was finalized. I0716 03:29:53.458856 139795883440000 saver.py:1280] Restoring parameters from ../model/elmo_bigru/model.ckpt-2808 I0716 03:29:54.366625 139795883440000 session_manager.py:500] Running local_init_op. I0716 03:29:54.422200 139795883440000 session_manager.py:502] Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
I0716 03:30:53.035401 139795883440000 evaluation.py:275] Finished evaluation at 2019-07-16-03:30:53 I0716 03:30:53.037463 139795883440000 estimator.py:2039] Saving dict for global step 2808: global_step = 2808, loss = 0.2947929 I0716 03:30:53.044780 139795883440000 estimator.py:2099] Saving 'checkpoint_path' summary for global step 2808: ../model/elmo_bigru/model.ckpt-2808 I0716 03:30:53.122354 139795883440000 estimator.py:368] Loss for final step: 0.00338955.
Reading ../data/atis.test.w-intent.iob
I0716 03:30:53.703723 139795883440000 estimator.py:1145] Calling model_fn. I0716 03:30:54.741008 139795883440000 saver.py:1499] Saver not created because there are no variables in the graph to restore I0716 03:30:55.715564 139795883440000 estimator.py:1147] Done calling model_fn. I0716 03:30:55.930930 139795883440000 monitored_session.py:240] Graph was finalized. I0716 03:30:55.946676 139795883440000 saver.py:1280] Restoring parameters from ../model/elmo_bigru/model.ckpt-2808 I0716 03:30:56.829934 139795883440000 session_manager.py:500] Running local_init_op. I0716 03:30:56.878928 139795883440000 session_manager.py:502] Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.
'precision', 'predicted', average, warn_for)
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1439: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples.
'recall', 'true', average, warn_for)
I0716 03:31:54.581436 139795883440000 interactiveshell.py:2882]
precision recall f1-score support
atis_flight 0.984 0.992 0.988 632
atis_airfare 0.941 1.000 0.970 48
atis_ground_service 0.973 1.000 0.986 36
atis_airline 1.000 1.000 1.000 38
atis_abbreviation 0.943 1.000 0.971 33
atis_aircraft 1.000 0.889 0.941 9
atis_flight_time 0.333 1.000 0.500 1
atis_quantity 0.429 1.000 0.600 3
atis_flight#atis_airfare 0.833 0.417 0.556 12
atis_airport 1.000 0.944 0.971 18
atis_distance 1.000 1.000 1.000 10
atis_city 1.000 0.500 0.667 6
atis_ground_fare 1.000 0.857 0.923 7
atis_capacity 0.952 0.952 0.952 21
atis_flight_no 0.889 1.000 0.941 8
atis_meal 1.000 0.833 0.909 6
atis_restriction 0.000 0.000 0.000 0
atis_airline#atis_flight_no 0.000 0.000 0.000 0
atis_ground_service#atis_ground_fare 0.000 0.000 0.000 0
atis_airfare#atis_flight_time 0.000 0.000 0.000 0
atis_cheapest 0.000 0.000 0.000 0
atis_aircraft#atis_flight#atis_flight_no 0.000 0.000 0.000 0
micro avg 0.972 0.977 0.975 888
macro avg 0.649 0.654 0.631 888
weighted avg 0.975 0.977 0.974 888
I0716 03:31:54.617678 139795883440000 interactiveshell.py:2882]
precision recall f1-score support
O 0.000 0.000 0.000 0.0
B-toloc.city_name 0.974 0.993 0.983 716.0
B-fromloc.city_name 0.992 1.000 0.996 704.0
I-toloc.city_name 0.967 0.992 0.980 265.0
B-depart_date.day_name 0.991 0.991 0.991 212.0
B-airline_name 1.000 1.000 1.000 101.0
I-fromloc.city_name 0.967 0.994 0.981 177.0
B-depart_time.period_of_day 0.983 0.908 0.944 130.0
I-airline_name 0.985 1.000 0.992 65.0
B-depart_date.day_number 0.981 0.964 0.972 55.0
B-depart_date.month_name 0.981 0.946 0.964 56.0
B-depart_time.time 0.851 1.000 0.919 57.0
B-round_trip 1.000 1.000 1.000 73.0
B-cost_relative 0.973 0.973 0.973 37.0
I-round_trip 1.000 1.000 1.000 71.0
B-flight_mod 0.828 1.000 0.906 24.0
B-depart_time.time_relative 0.970 1.000 0.985 65.0
I-depart_time.time 0.912 1.000 0.954 52.0
B-stoploc.city_name 0.870 1.000 0.930 20.0
B-city_name 0.917 0.579 0.710 57.0
B-class_type 0.960 1.000 0.980 24.0
B-arrive_time.time 1.000 0.971 0.985 34.0
B-arrive_time.time_relative 0.882 0.968 0.923 31.0
I-class_type 1.000 1.000 1.000 17.0
B-flight_stop 1.000 1.000 1.000 21.0
I-arrive_time.time 1.000 0.971 0.986 35.0
B-airline_code 0.872 1.000 0.932 34.0
I-depart_date.day_number 1.000 0.933 0.966 15.0
I-fromloc.airport_name 0.441 1.000 0.612 15.0
B-fromloc.airport_name 0.414 1.000 0.585 12.0
B-arrive_date.day_name 0.786 1.000 0.880 11.0
B-toloc.state_code 1.000 1.000 1.000 18.0
B-depart_date.today_relative 1.000 0.889 0.941 9.0
B-flight_number 0.786 1.000 0.880 11.0
B-depart_date.date_relative 0.944 1.000 0.971 17.0
B-toloc.state_name 0.833 0.893 0.862 28.0
B-fare_basis_code 0.895 1.000 0.944 17.0
B-flight_time 1.000 1.000 1.000 1.0
B-or 1.000 1.000 1.000 3.0
B-arrive_time.period_of_day 0.750 1.000 0.857 6.0
B-meal_description 1.000 0.900 0.947 10.0
I-cost_relative 1.000 0.667 0.800 3.0
I-airport_name 0.846 0.379 0.524 29.0
B-fare_amount 1.000 1.000 1.000 2.0
I-fare_amount 1.000 1.000 1.000 2.0
I-city_name 1.000 0.567 0.723 30.0
I-toloc.airport_name 1.000 1.000 1.000 3.0
B-transport_type 1.000 1.000 1.000 10.0
B-arrive_date.month_name 0.625 0.833 0.714 6.0
B-arrive_date.day_number 0.625 0.833 0.714 6.0
I-stoploc.city_name 0.909 1.000 0.952 10.0
B-meal 1.000 1.000 1.000 16.0
B-fromloc.state_code 1.000 1.000 1.000 23.0
B-depart_time.period_mod 1.000 1.000 1.000 5.0
B-connect 1.000 1.000 1.000 6.0
B-flight_days 1.000 1.000 1.000 10.0
B-toloc.airport_name 1.000 1.000 1.000 3.0
B-fromloc.state_name 0.895 1.000 0.944 17.0
B-airport_name 0.636 0.333 0.437 21.0
B-economy 1.000 0.333 0.500 6.0
I-flight_time 1.000 1.000 1.000 1.0
B-aircraft_code 1.000 0.848 0.918 33.0
B-mod 1.000 0.500 0.667 2.0
B-airport_code 1.000 0.444 0.615 9.0
B-depart_time.start_time 1.000 1.000 1.000 3.0
B-depart_time.end_time 1.000 0.667 0.800 3.0
B-depart_date.year 1.000 1.000 1.000 3.0
I-transport_type 0.000 0.000 0.000 1.0
B-restriction_code 0.667 0.500 0.571 4.0
B-arrive_time.start_time 1.000 1.000 1.000 8.0
B-toloc.airport_code 1.000 0.750 0.857 4.0
B-arrive_time.end_time 1.000 1.000 1.000 8.0
I-arrive_time.end_time 1.000 1.000 1.000 8.0
I-depart_time.end_time 1.000 0.667 0.800 3.0
I-flight_stop 0.000 0.000 0.000 0.0
B-fromloc.airport_code 0.625 1.000 0.769 5.0
I-restriction_code 1.000 1.000 1.000 3.0
I-depart_time.start_time 1.000 1.000 1.000 1.0
I-toloc.state_name 1.000 1.000 1.000 1.0
I-depart_date.today_relative 0.000 0.000 0.000 0.0
B-arrive_date.date_relative 1.000 1.000 1.000 2.0
I-flight_mod 1.000 0.167 0.286 6.0
I-economy 0.000 0.000 0.000 0.0
B-return_date.date_relative 0.667 0.667 0.667 3.0
I-fromloc.state_name 0.000 0.000 0.000 1.0
B-state_code 1.000 1.000 1.000 1.0
I-arrive_time.start_time 1.000 1.000 1.000 1.0
I-arrive_date.day_number 0.000 0.000 0.000 0.0
B-meal_code 0.000 0.000 0.000 1.0
I-depart_time.period_of_day 0.000 0.000 0.000 1.0
B-day_name 1.000 0.500 0.667 2.0
B-period_of_day 1.000 0.750 0.857 4.0
B-stoploc.state_code 0.000 0.000 0.000 0.0
B-return_date.month_name 0.000 0.000 0.000 0.0
B-return_date.day_number 0.000 0.000 0.000 0.0
B-arrive_time.period_mod 0.000 0.000 0.000 0.0
I-meal_code 0.000 0.000 0.000 0.0
B-toloc.country_name 0.000 0.000 0.000 1.0
B-days_code 0.000 0.000 0.000 1.0
I-arrive_time.period_of_day 0.000 0.000 0.000 0.0
I-today_relative 0.000 0.000 0.000 0.0
B-return_time.period_of_day 0.000 0.000 0.000 0.0
B-time 0.000 0.000 0.000 0.0
I-fare_basis_code 0.000 0.000 0.000 0.0
I-arrive_time.time_relative 0.000 0.000 0.000 4.0
I-depart_time.time_relative 0.000 0.000 0.000 1.0
B-today_relative 0.000 0.000 0.000 0.0
B-state_name 0.000 0.000 0.000 9.0
B-arrive_date.today_relative 0.000 0.000 0.000 0.0
B-return_time.period_mod 0.000 0.000 0.000 0.0
B-month_name 0.000 0.000 0.000 0.0
B-day_number 0.000 0.000 0.000 0.0
I-return_date.date_relative 1.000 0.333 0.500 3.0
I-return_date.today_relative 0.000 0.000 0.000 0.0
B-stoploc.airport_name 0.000 0.000 0.000 0.0
B-time_relative 0.000 0.000 0.000 0.0
I-time 0.000 0.000 0.000 0.0
I-return_date.day_number 0.000 0.000 0.000 0.0
I-meal_description 0.000 0.000 0.000 0.0
B-return_date.today_relative 0.000 0.000 0.000 0.0
B-return_date.day_name 0.000 0.000 0.000 2.0
micro avg 0.954 0.956 0.955 3657.0
macro avg 0.646 0.617 0.617 3657.0
weighted avg 0.957 0.956 0.951 3657.0
I0716 03:31:54.637969 139795883440000 interactiveshell.py:2882] Best Slot F1: 0.955
I0716 03:31:54.639525 139795883440000 estimator_training.py:186] Not using Distribute Coordinator.
I0716 03:31:54.640842 139795883440000 training.py:612] Running training and evaluation locally (non-distributed).
I0716 03:31:54.644369 139795883440000 training.py:700] Start train and evaluate loop. The evaluate will happen after every checkpoint. Checkpoint frequency is determined based on RunConfig arguments: save_checkpoints_steps 312 or save_checkpoints_secs None.
I0716 03:31:54.707897 139795883440000 estimator.py:1145] Calling model_fn.
I0716 03:31:56.134528 139795883440000 saver.py:1499] Saver not created because there are no variables in the graph to restore
I0716 03:31:56.710400 139795883440000 <ipython-input-6-d6747b40360b>:55]
[<tf.Variable 'embedding:0' shape=(750, 300) dtype=float32_ref>,
<tf.Variable 'module/aggregation/weights:0' shape=(3,) dtype=float32>,
<tf.Variable 'module/aggregation/scaling:0' shape=() dtype=float32>,
<tf.Variable 'dense/kernel:0' shape=(1324, 300) dtype=float32_ref>,
<tf.Variable 'dense/bias:0' shape=(300,) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/fw/gru_cell/gates/kernel:0' shape=(600, 600) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/fw/gru_cell/gates/bias:0' shape=(600,) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/fw/gru_cell/candidate/kernel:0' shape=(600, 300) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/fw/gru_cell/candidate/bias:0' shape=(300,) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/bw/gru_cell/gates/kernel:0' shape=(600, 600) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/bw/gru_cell/gates/bias:0' shape=(600,) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/bw/gru_cell/candidate/kernel:0' shape=(600, 300) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/bw/gru_cell/candidate/bias:0' shape=(300,) dtype=float32_ref>,
<tf.Variable 'dense_1/kernel:0' shape=(600, 23) dtype=float32_ref>,
<tf.Variable 'dense_1/bias:0' shape=(23,) dtype=float32_ref>,
<tf.Variable 'dense_2/kernel:0' shape=(600, 122) dtype=float32_ref>,
<tf.Variable 'dense_2/bias:0' shape=(122,) dtype=float32_ref>]
I0716 03:31:57.955630 139795883440000 estimator.py:1147] Done calling model_fn.
I0716 03:31:57.959849 139795883440000 basic_session_run_hooks.py:541] Create CheckpointSaverHook.
I0716 03:31:58.293367 139795883440000 monitored_session.py:240] Graph was finalized.
I0716 03:31:58.313203 139795883440000 saver.py:1280] Restoring parameters from ../model/elmo_bigru/model.ckpt-2808
I0716 03:31:59.349800 139795883440000 session_manager.py:500] Running local_init_op.
I0716 03:31:59.419791 139795883440000 session_manager.py:502] Done running local_init_op.
I0716 03:32:02.178851 139795883440000 basic_session_run_hooks.py:606] Saving checkpoints for 2808 into ../model/elmo_bigru/model.ckpt.
Reading ../data/atis.train.w-intent.iob
I0716 03:32:07.011035 139795883440000 basic_session_run_hooks.py:262] loss = 0.0189911, step = 2808 I0716 03:32:07.013246 139795883440000 basic_session_run_hooks.py:262] lr = 0.00022316918 I0716 03:32:34.563869 139795883440000 basic_session_run_hooks.py:692] global_step/sec: 3.62933 I0716 03:32:34.570090 139795883440000 basic_session_run_hooks.py:260] loss = 0.03583881, step = 2908 (27.559 sec) I0716 03:32:34.573180 139795883440000 basic_session_run_hooks.py:260] lr = 0.00022083019 (27.560 sec) I0716 03:33:01.101560 139795883440000 basic_session_run_hooks.py:692] global_step/sec: 3.76822 I0716 03:33:01.107414 139795883440000 basic_session_run_hooks.py:260] loss = 0.032352217, step = 3008 (26.537 sec) I0716 03:33:01.111151 139795883440000 basic_session_run_hooks.py:260] lr = 0.00021851574 (26.538 sec) I0716 03:33:27.693025 139795883440000 basic_session_run_hooks.py:692] global_step/sec: 3.76061 I0716 03:33:27.695716 139795883440000 basic_session_run_hooks.py:260] loss = 0.026988506, step = 3108 (26.588 sec) I0716 03:33:27.700540 139795883440000 basic_session_run_hooks.py:260] lr = 0.00021622553 (26.589 sec) I0716 03:33:30.370072 139795883440000 basic_session_run_hooks.py:606] Saving checkpoints for 3120 into ../model/elmo_bigru/model.ckpt. I0716 03:33:32.249582 139795883440000 estimator.py:1145] Calling model_fn. I0716 03:33:33.591853 139795883440000 saver.py:1499] Saver not created because there are no variables in the graph to restore I0716 03:33:34.398364 139795883440000 estimator.py:1147] Done calling model_fn. I0716 03:33:34.436570 139795883440000 evaluation.py:255] Starting evaluation at 2019-07-16T03:33:34Z I0716 03:33:34.662846 139795883440000 monitored_session.py:240] Graph was finalized. I0716 03:33:34.680534 139795883440000 saver.py:1280] Restoring parameters from ../model/elmo_bigru/model.ckpt-3120 I0716 03:33:35.565032 139795883440000 session_manager.py:500] Running local_init_op. I0716 03:33:35.620205 139795883440000 session_manager.py:502] Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
I0716 03:34:34.403399 139795883440000 evaluation.py:275] Finished evaluation at 2019-07-16-03:34:34 I0716 03:34:34.405219 139795883440000 estimator.py:2039] Saving dict for global step 3120: global_step = 3120, loss = 0.29238617 I0716 03:34:34.413099 139795883440000 estimator.py:2099] Saving 'checkpoint_path' summary for global step 3120: ../model/elmo_bigru/model.ckpt-3120 I0716 03:34:34.496270 139795883440000 estimator.py:368] Loss for final step: 0.018303238.
Reading ../data/atis.test.w-intent.iob
I0716 03:34:35.101234 139795883440000 estimator.py:1145] Calling model_fn. I0716 03:34:36.580038 139795883440000 saver.py:1499] Saver not created because there are no variables in the graph to restore I0716 03:34:37.104504 139795883440000 estimator.py:1147] Done calling model_fn. I0716 03:34:37.313228 139795883440000 monitored_session.py:240] Graph was finalized. I0716 03:34:37.329845 139795883440000 saver.py:1280] Restoring parameters from ../model/elmo_bigru/model.ckpt-3120 I0716 03:34:38.225905 139795883440000 session_manager.py:500] Running local_init_op. I0716 03:34:38.282472 139795883440000 session_manager.py:502] Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.
'precision', 'predicted', average, warn_for)
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1439: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples.
'recall', 'true', average, warn_for)
I0716 03:35:36.634260 139795883440000 interactiveshell.py:2882]
precision recall f1-score support
atis_flight 0.984 0.992 0.988 632
atis_airfare 0.941 1.000 0.970 48
atis_ground_service 0.973 1.000 0.986 36
atis_airline 1.000 1.000 1.000 38
atis_abbreviation 0.943 1.000 0.971 33
atis_aircraft 1.000 0.889 0.941 9
atis_flight_time 0.333 1.000 0.500 1
atis_quantity 0.375 1.000 0.545 3
atis_flight#atis_airfare 0.833 0.417 0.556 12
atis_airport 1.000 0.944 0.971 18
atis_distance 1.000 1.000 1.000 10
atis_city 1.000 0.500 0.667 6
atis_ground_fare 1.000 0.857 0.923 7
atis_capacity 1.000 0.952 0.976 21
atis_flight_no 0.889 1.000 0.941 8
atis_meal 1.000 0.833 0.909 6
atis_restriction 0.000 0.000 0.000 0
atis_airline#atis_flight_no 0.000 0.000 0.000 0
atis_ground_service#atis_ground_fare 0.000 0.000 0.000 0
atis_airfare#atis_flight_time 0.000 0.000 0.000 0
atis_cheapest 0.000 0.000 0.000 0
atis_aircraft#atis_flight#atis_flight_no 0.000 0.000 0.000 0
micro avg 0.972 0.977 0.975 888
macro avg 0.649 0.654 0.629 888
weighted avg 0.976 0.977 0.974 888
I0716 03:35:36.670582 139795883440000 interactiveshell.py:2882]
precision recall f1-score support
O 0.000 0.000 0.000 0.0
B-toloc.city_name 0.974 0.993 0.983 716.0
B-fromloc.city_name 0.992 0.999 0.995 704.0
I-toloc.city_name 0.974 0.989 0.981 265.0
B-depart_date.day_name 0.991 0.991 0.991 212.0
B-airline_name 1.000 1.000 1.000 101.0
I-fromloc.city_name 0.978 0.994 0.986 177.0
B-depart_time.period_of_day 0.984 0.923 0.952 130.0
I-airline_name 1.000 1.000 1.000 65.0
B-depart_date.day_number 0.981 0.964 0.972 55.0
B-depart_date.month_name 0.981 0.946 0.964 56.0
B-depart_time.time 0.864 1.000 0.927 57.0
B-round_trip 1.000 1.000 1.000 73.0
B-cost_relative 0.973 0.973 0.973 37.0
I-round_trip 1.000 1.000 1.000 71.0
B-flight_mod 0.828 1.000 0.906 24.0
B-depart_time.time_relative 0.970 0.985 0.977 65.0
I-depart_time.time 0.945 1.000 0.972 52.0
B-stoploc.city_name 0.870 1.000 0.930 20.0
B-city_name 0.900 0.632 0.742 57.0
B-class_type 0.960 1.000 0.980 24.0
B-arrive_time.time 0.971 0.971 0.971 34.0
B-arrive_time.time_relative 0.857 0.968 0.909 31.0
I-class_type 1.000 1.000 1.000 17.0
B-flight_stop 1.000 1.000 1.000 21.0
I-arrive_time.time 1.000 0.971 0.986 35.0
B-airline_code 0.895 1.000 0.944 34.0
I-depart_date.day_number 1.000 0.933 0.966 15.0
I-fromloc.airport_name 0.429 1.000 0.600 15.0
B-fromloc.airport_name 0.400 1.000 0.571 12.0
B-arrive_date.day_name 0.786 1.000 0.880 11.0
B-toloc.state_code 1.000 1.000 1.000 18.0
B-depart_date.today_relative 1.000 0.889 0.941 9.0
B-flight_number 0.786 1.000 0.880 11.0
B-depart_date.date_relative 0.944 1.000 0.971 17.0
B-toloc.state_name 0.833 0.893 0.862 28.0
B-fare_basis_code 0.895 1.000 0.944 17.0
B-flight_time 1.000 1.000 1.000 1.0
B-or 1.000 1.000 1.000 3.0
B-arrive_time.period_of_day 0.750 1.000 0.857 6.0
B-meal_description 1.000 0.900 0.947 10.0
I-cost_relative 1.000 0.667 0.800 3.0
I-airport_name 0.846 0.379 0.524 29.0
B-fare_amount 1.000 1.000 1.000 2.0
I-fare_amount 1.000 1.000 1.000 2.0
I-city_name 1.000 0.633 0.776 30.0
I-toloc.airport_name 1.000 1.000 1.000 3.0
B-transport_type 1.000 1.000 1.000 10.0
B-arrive_date.month_name 0.625 0.833 0.714 6.0
B-arrive_date.day_number 0.625 0.833 0.714 6.0
I-stoploc.city_name 0.909 1.000 0.952 10.0
B-meal 1.000 1.000 1.000 16.0
B-fromloc.state_code 1.000 1.000 1.000 23.0
B-depart_time.period_mod 1.000 1.000 1.000 5.0
B-connect 1.000 1.000 1.000 6.0
B-flight_days 1.000 1.000 1.000 10.0
B-toloc.airport_name 1.000 1.000 1.000 3.0
B-fromloc.state_name 1.000 1.000 1.000 17.0
B-airport_name 0.636 0.333 0.437 21.0
B-economy 1.000 0.667 0.800 6.0
I-flight_time 1.000 1.000 1.000 1.0
B-aircraft_code 1.000 0.848 0.918 33.0
B-mod 1.000 0.500 0.667 2.0
B-airport_code 1.000 0.444 0.615 9.0
B-depart_time.start_time 1.000 0.667 0.800 3.0
B-depart_time.end_time 1.000 0.333 0.500 3.0
B-depart_date.year 1.000 1.000 1.000 3.0
I-transport_type 1.000 1.000 1.000 1.0
B-restriction_code 0.750 0.750 0.750 4.0
B-arrive_time.start_time 0.889 1.000 0.941 8.0
B-toloc.airport_code 1.000 0.750 0.857 4.0
B-arrive_time.end_time 0.889 1.000 0.941 8.0
I-arrive_time.end_time 0.889 1.000 0.941 8.0
I-depart_time.end_time 1.000 0.333 0.500 3.0
I-flight_stop 0.000 0.000 0.000 0.0
B-fromloc.airport_code 0.625 1.000 0.769 5.0
I-restriction_code 1.000 1.000 1.000 3.0
I-depart_time.start_time 1.000 1.000 1.000 1.0
I-toloc.state_name 1.000 1.000 1.000 1.0
I-depart_date.today_relative 0.000 0.000 0.000 0.0
B-arrive_date.date_relative 1.000 1.000 1.000 2.0
I-flight_mod 1.000 0.167 0.286 6.0
I-economy 0.000 0.000 0.000 0.0
B-return_date.date_relative 0.500 0.333 0.400 3.0
I-fromloc.state_name 1.000 1.000 1.000 1.0
B-state_code 1.000 1.000 1.000 1.0
I-arrive_time.start_time 1.000 1.000 1.000 1.0
I-arrive_date.day_number 0.000 0.000 0.000 0.0
B-meal_code 0.000 0.000 0.000 1.0
I-depart_time.period_of_day 0.000 0.000 0.000 1.0
B-day_name 1.000 0.500 0.667 2.0
B-period_of_day 1.000 0.750 0.857 4.0
B-stoploc.state_code 0.000 0.000 0.000 0.0
B-return_date.month_name 0.000 0.000 0.000 0.0
B-return_date.day_number 0.000 0.000 0.000 0.0
B-arrive_time.period_mod 0.000 0.000 0.000 0.0
I-meal_code 0.000 0.000 0.000 0.0
B-toloc.country_name 1.000 1.000 1.000 1.0
B-days_code 0.000 0.000 0.000 1.0
I-arrive_time.period_of_day 0.000 0.000 0.000 0.0
I-today_relative 0.000 0.000 0.000 0.0
B-return_time.period_of_day 0.000 0.000 0.000 0.0
B-time 0.000 0.000 0.000 0.0
I-fare_basis_code 0.000 0.000 0.000 0.0
I-arrive_time.time_relative 0.000 0.000 0.000 4.0
I-depart_time.time_relative 0.000 0.000 0.000 1.0
B-today_relative 0.000 0.000 0.000 0.0
B-state_name 0.000 0.000 0.000 9.0
B-arrive_date.today_relative 0.000 0.000 0.000 0.0
B-return_time.period_mod 0.000 0.000 0.000 0.0
B-month_name 0.000 0.000 0.000 0.0
B-day_number 0.000 0.000 0.000 0.0
I-return_date.date_relative 1.000 0.333 0.500 3.0
I-return_date.today_relative 0.000 0.000 0.000 0.0
B-stoploc.airport_name 0.000 0.000 0.000 0.0
B-time_relative 0.000 0.000 0.000 0.0
I-time 0.000 0.000 0.000 0.0
I-return_date.day_number 0.000 0.000 0.000 0.0
I-meal_description 0.000 0.000 0.000 0.0
B-return_date.today_relative 0.000 0.000 0.000 0.0
B-return_date.day_name 0.000 0.000 0.000 2.0
micro avg 0.956 0.958 0.957 3657.0
macro avg 0.669 0.636 0.637 3657.0
weighted avg 0.959 0.958 0.954 3657.0
I0716 03:35:36.690505 139795883440000 interactiveshell.py:2882] Best Slot F1: 0.957
I0716 03:35:36.692061 139795883440000 estimator_training.py:186] Not using Distribute Coordinator.
I0716 03:35:36.693465 139795883440000 training.py:612] Running training and evaluation locally (non-distributed).
I0716 03:35:36.696232 139795883440000 training.py:700] Start train and evaluate loop. The evaluate will happen after every checkpoint. Checkpoint frequency is determined based on RunConfig arguments: save_checkpoints_steps 312 or save_checkpoints_secs None.
I0716 03:35:36.762451 139795883440000 estimator.py:1145] Calling model_fn.
I0716 03:35:38.223496 139795883440000 saver.py:1499] Saver not created because there are no variables in the graph to restore
I0716 03:35:38.814246 139795883440000 <ipython-input-6-d6747b40360b>:55]
[<tf.Variable 'embedding:0' shape=(750, 300) dtype=float32_ref>,
<tf.Variable 'module/aggregation/weights:0' shape=(3,) dtype=float32>,
<tf.Variable 'module/aggregation/scaling:0' shape=() dtype=float32>,
<tf.Variable 'dense/kernel:0' shape=(1324, 300) dtype=float32_ref>,
<tf.Variable 'dense/bias:0' shape=(300,) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/fw/gru_cell/gates/kernel:0' shape=(600, 600) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/fw/gru_cell/gates/bias:0' shape=(600,) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/fw/gru_cell/candidate/kernel:0' shape=(600, 300) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/fw/gru_cell/candidate/bias:0' shape=(300,) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/bw/gru_cell/gates/kernel:0' shape=(600, 600) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/bw/gru_cell/gates/bias:0' shape=(600,) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/bw/gru_cell/candidate/kernel:0' shape=(600, 300) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/bw/gru_cell/candidate/bias:0' shape=(300,) dtype=float32_ref>,
<tf.Variable 'dense_1/kernel:0' shape=(600, 23) dtype=float32_ref>,
<tf.Variable 'dense_1/bias:0' shape=(23,) dtype=float32_ref>,
<tf.Variable 'dense_2/kernel:0' shape=(600, 122) dtype=float32_ref>,
<tf.Variable 'dense_2/bias:0' shape=(122,) dtype=float32_ref>]
I0716 03:35:40.079383 139795883440000 estimator.py:1147] Done calling model_fn.
I0716 03:35:40.083529 139795883440000 basic_session_run_hooks.py:541] Create CheckpointSaverHook.
I0716 03:35:40.399303 139795883440000 monitored_session.py:240] Graph was finalized.
I0716 03:35:40.417211 139795883440000 saver.py:1280] Restoring parameters from ../model/elmo_bigru/model.ckpt-3120
I0716 03:35:41.444351 139795883440000 session_manager.py:500] Running local_init_op.
I0716 03:35:41.522269 139795883440000 session_manager.py:502] Done running local_init_op.
I0716 03:35:44.345134 139795883440000 basic_session_run_hooks.py:606] Saving checkpoints for 3120 into ../model/elmo_bigru/model.ckpt.
Reading ../data/atis.train.w-intent.iob
I0716 03:35:49.160026 139795883440000 basic_session_run_hooks.py:262] loss = 0.024522787, step = 3120 I0716 03:35:49.161925 139795883440000 basic_session_run_hooks.py:262] lr = 0.00021595233 I0716 03:36:17.451791 139795883440000 basic_session_run_hooks.py:692] global_step/sec: 3.53452 I0716 03:36:17.455758 139795883440000 basic_session_run_hooks.py:260] loss = 0.018387439, step = 3220 (28.296 sec) I0716 03:36:17.459416 139795883440000 basic_session_run_hooks.py:260] lr = 0.00021368897 (28.297 sec) I0716 03:36:43.690851 139795883440000 basic_session_run_hooks.py:692] global_step/sec: 3.81111 I0716 03:36:43.696416 139795883440000 basic_session_run_hooks.py:260] loss = 0.028793802, step = 3320 (26.241 sec) I0716 03:36:43.700000 139795883440000 basic_session_run_hooks.py:260] lr = 0.00021144934 (26.241 sec) I0716 03:37:10.121157 139795883440000 basic_session_run_hooks.py:692] global_step/sec: 3.78353 I0716 03:37:10.128551 139795883440000 basic_session_run_hooks.py:260] loss = 0.023584345, step = 3420 (26.432 sec) I0716 03:37:10.131360 139795883440000 basic_session_run_hooks.py:260] lr = 0.0002092332 (26.431 sec) I0716 03:37:13.255368 139795883440000 basic_session_run_hooks.py:606] Saving checkpoints for 3432 into ../model/elmo_bigru/model.ckpt. I0716 03:37:15.275616 139795883440000 estimator.py:1145] Calling model_fn. I0716 03:37:16.615622 139795883440000 saver.py:1499] Saver not created because there are no variables in the graph to restore I0716 03:37:17.416712 139795883440000 estimator.py:1147] Done calling model_fn. I0716 03:37:17.444679 139795883440000 evaluation.py:255] Starting evaluation at 2019-07-16T03:37:17Z I0716 03:37:17.674989 139795883440000 monitored_session.py:240] Graph was finalized. I0716 03:37:17.688753 139795883440000 saver.py:1280] Restoring parameters from ../model/elmo_bigru/model.ckpt-3432 I0716 03:37:18.592427 139795883440000 session_manager.py:500] Running local_init_op. I0716 03:37:18.653311 139795883440000 session_manager.py:502] Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
I0716 03:38:17.490103 139795883440000 evaluation.py:275] Finished evaluation at 2019-07-16-03:38:17 I0716 03:38:17.492123 139795883440000 estimator.py:2039] Saving dict for global step 3432: global_step = 3432, loss = 0.28893816 I0716 03:38:17.498728 139795883440000 estimator.py:2099] Saving 'checkpoint_path' summary for global step 3432: ../model/elmo_bigru/model.ckpt-3432 I0716 03:38:17.585021 139795883440000 estimator.py:368] Loss for final step: 0.008152063.
Reading ../data/atis.test.w-intent.iob
I0716 03:38:18.181552 139795883440000 estimator.py:1145] Calling model_fn. I0716 03:38:19.680203 139795883440000 saver.py:1499] Saver not created because there are no variables in the graph to restore I0716 03:38:20.192303 139795883440000 estimator.py:1147] Done calling model_fn. I0716 03:38:20.402776 139795883440000 monitored_session.py:240] Graph was finalized. I0716 03:38:20.419113 139795883440000 saver.py:1280] Restoring parameters from ../model/elmo_bigru/model.ckpt-3432 I0716 03:38:21.312413 139795883440000 session_manager.py:500] Running local_init_op. I0716 03:38:21.359920 139795883440000 session_manager.py:502] Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.
'precision', 'predicted', average, warn_for)
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1439: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples.
'recall', 'true', average, warn_for)
I0716 03:39:19.167973 139795883440000 interactiveshell.py:2882]
precision recall f1-score support
atis_flight 0.986 0.992 0.989 632
atis_airfare 0.941 1.000 0.970 48
atis_ground_service 0.973 1.000 0.986 36
atis_airline 1.000 1.000 1.000 38
atis_abbreviation 0.943 1.000 0.971 33
atis_aircraft 1.000 0.889 0.941 9
atis_flight_time 0.333 1.000 0.500 1
atis_quantity 0.429 1.000 0.600 3
atis_flight#atis_airfare 0.833 0.417 0.556 12
atis_airport 1.000 0.944 0.971 18
atis_distance 1.000 1.000 1.000 10
atis_city 0.750 0.500 0.600 6
atis_ground_fare 1.000 0.857 0.923 7
atis_capacity 0.952 0.952 0.952 21
atis_flight_no 0.889 1.000 0.941 8
atis_meal 1.000 0.833 0.909 6
atis_restriction 0.000 0.000 0.000 0
atis_airline#atis_flight_no 0.000 0.000 0.000 0
atis_ground_service#atis_ground_fare 0.000 0.000 0.000 0
atis_airfare#atis_flight_time 0.000 0.000 0.000 0
atis_cheapest 0.000 0.000 0.000 0
atis_aircraft#atis_flight#atis_flight_no 0.000 0.000 0.000 0
micro avg 0.972 0.977 0.975 888
macro avg 0.638 0.654 0.628 888
weighted avg 0.975 0.977 0.974 888
I0716 03:39:19.204613 139795883440000 interactiveshell.py:2882]
precision recall f1-score support
O 0.000 0.000 0.000 0.0
B-toloc.city_name 0.977 0.993 0.985 716.0
B-fromloc.city_name 0.993 0.999 0.996 704.0
I-toloc.city_name 0.985 0.989 0.987 265.0
B-depart_date.day_name 0.991 0.991 0.991 212.0
B-airline_name 0.990 1.000 0.995 101.0
I-fromloc.city_name 0.967 0.994 0.981 177.0
B-depart_time.period_of_day 0.984 0.923 0.952 130.0
I-airline_name 1.000 0.985 0.992 65.0
B-depart_date.day_number 0.964 0.964 0.964 55.0
B-depart_date.month_name 0.982 0.964 0.973 56.0
B-depart_time.time 0.864 1.000 0.927 57.0
B-round_trip 1.000 1.000 1.000 73.0
B-cost_relative 0.973 0.973 0.973 37.0
I-round_trip 1.000 1.000 1.000 71.0
B-flight_mod 0.828 1.000 0.906 24.0
B-depart_time.time_relative 0.970 0.985 0.977 65.0
I-depart_time.time 0.945 1.000 0.972 52.0
B-stoploc.city_name 0.870 1.000 0.930 20.0
B-city_name 0.886 0.684 0.772 57.0
B-class_type 0.960 1.000 0.980 24.0
B-arrive_time.time 0.971 0.971 0.971 34.0
B-arrive_time.time_relative 0.857 0.968 0.909 31.0
I-class_type 1.000 1.000 1.000 17.0
B-flight_stop 1.000 1.000 1.000 21.0
I-arrive_time.time 1.000 0.971 0.986 35.0
B-airline_code 0.850 1.000 0.919 34.0
I-depart_date.day_number 1.000 1.000 1.000 15.0
I-fromloc.airport_name 0.469 1.000 0.638 15.0
B-fromloc.airport_name 0.429 1.000 0.600 12.0
B-arrive_date.day_name 0.786 1.000 0.880 11.0
B-toloc.state_code 0.947 1.000 0.973 18.0
B-depart_date.today_relative 1.000 0.889 0.941 9.0
B-flight_number 0.846 1.000 0.917 11.0
B-depart_date.date_relative 0.944 1.000 0.971 17.0
B-toloc.state_name 0.774 0.857 0.814 28.0
B-fare_basis_code 0.850 1.000 0.919 17.0
B-flight_time 1.000 1.000 1.000 1.0
B-or 1.000 1.000 1.000 3.0
B-arrive_time.period_of_day 0.750 1.000 0.857 6.0
B-meal_description 1.000 0.900 0.947 10.0
I-cost_relative 1.000 0.667 0.800 3.0
I-airport_name 0.867 0.448 0.591 29.0
B-fare_amount 1.000 1.000 1.000 2.0
I-fare_amount 1.000 1.000 1.000 2.0
I-city_name 1.000 0.667 0.800 30.0
I-toloc.airport_name 1.000 1.000 1.000 3.0
B-transport_type 1.000 1.000 1.000 10.0
B-arrive_date.month_name 0.714 0.833 0.769 6.0
B-arrive_date.day_number 0.714 0.833 0.769 6.0
I-stoploc.city_name 0.909 1.000 0.952 10.0
B-meal 1.000 1.000 1.000 16.0
B-fromloc.state_code 1.000 0.957 0.978 23.0
B-depart_time.period_mod 1.000 1.000 1.000 5.0
B-connect 1.000 1.000 1.000 6.0
B-flight_days 1.000 1.000 1.000 10.0
B-toloc.airport_name 1.000 1.000 1.000 3.0
B-fromloc.state_name 1.000 1.000 1.000 17.0
B-airport_name 0.800 0.381 0.516 21.0
B-economy 1.000 1.000 1.000 6.0
I-flight_time 1.000 1.000 1.000 1.0
B-aircraft_code 1.000 0.848 0.918 33.0
B-mod 1.000 0.500 0.667 2.0
B-airport_code 1.000 0.444 0.615 9.0
B-depart_time.start_time 1.000 1.000 1.000 3.0
B-depart_time.end_time 1.000 0.667 0.800 3.0
B-depart_date.year 1.000 1.000 1.000 3.0
I-transport_type 1.000 1.000 1.000 1.0
B-restriction_code 1.000 0.500 0.667 4.0
B-arrive_time.start_time 1.000 1.000 1.000 8.0
B-toloc.airport_code 1.000 0.750 0.857 4.0
B-arrive_time.end_time 1.000 1.000 1.000 8.0
I-arrive_time.end_time 0.889 1.000 0.941 8.0
I-depart_time.end_time 1.000 0.333 0.500 3.0
I-flight_stop 0.000 0.000 0.000 0.0
B-fromloc.airport_code 0.625 1.000 0.769 5.0
I-restriction_code 1.000 1.000 1.000 3.0
I-depart_time.start_time 1.000 1.000 1.000 1.0
I-toloc.state_name 1.000 1.000 1.000 1.0
I-depart_date.today_relative 0.000 0.000 0.000 0.0
B-arrive_date.date_relative 1.000 1.000 1.000 2.0
I-flight_mod 1.000 0.167 0.286 6.0
I-economy 0.000 0.000 0.000 0.0
B-return_date.date_relative 0.500 0.333 0.400 3.0
I-fromloc.state_name 1.000 1.000 1.000 1.0
B-state_code 1.000 1.000 1.000 1.0
I-arrive_time.start_time 1.000 1.000 1.000 1.0
I-arrive_date.day_number 0.000 0.000 0.000 0.0
B-meal_code 0.000 0.000 0.000 1.0
I-depart_time.period_of_day 0.000 0.000 0.000 1.0
B-day_name 1.000 0.500 0.667 2.0
B-period_of_day 1.000 0.750 0.857 4.0
B-stoploc.state_code 0.000 0.000 0.000 0.0
B-return_date.month_name 0.000 0.000 0.000 0.0
B-return_date.day_number 0.000 0.000 0.000 0.0
B-arrive_time.period_mod 0.000 0.000 0.000 0.0
I-meal_code 0.000 0.000 0.000 0.0
B-toloc.country_name 1.000 1.000 1.000 1.0
B-days_code 0.000 0.000 0.000 1.0
I-arrive_time.period_of_day 0.000 0.000 0.000 0.0
I-today_relative 0.000 0.000 0.000 0.0
B-return_time.period_of_day 0.000 0.000 0.000 0.0
B-time 0.000 0.000 0.000 0.0
I-fare_basis_code 0.000 0.000 0.000 0.0
I-arrive_time.time_relative 0.000 0.000 0.000 4.0
I-depart_time.time_relative 0.000 0.000 0.000 1.0
B-today_relative 0.000 0.000 0.000 0.0
B-state_name 0.000 0.000 0.000 9.0
B-arrive_date.today_relative 0.000 0.000 0.000 0.0
B-return_time.period_mod 0.000 0.000 0.000 0.0
B-month_name 0.000 0.000 0.000 0.0
B-day_number 0.000 0.000 0.000 0.0
I-return_date.date_relative 1.000 0.333 0.500 3.0
I-return_date.today_relative 0.000 0.000 0.000 0.0
B-stoploc.airport_name 0.000 0.000 0.000 0.0
B-time_relative 0.000 0.000 0.000 0.0
I-time 0.000 0.000 0.000 0.0
I-return_date.day_number 0.000 0.000 0.000 0.0
I-meal_description 0.000 0.000 0.000 0.0
B-return_date.today_relative 0.000 0.000 0.000 0.0
B-return_date.day_name 0.000 0.000 0.000 2.0
micro avg 0.959 0.960 0.959 3657.0
macro avg 0.675 0.644 0.646 3657.0
weighted avg 0.960 0.960 0.956 3657.0
I0716 03:39:19.230432 139795883440000 interactiveshell.py:2882] Best Slot F1: 0.959
I0716 03:39:19.231839 139795883440000 estimator_training.py:186] Not using Distribute Coordinator.
I0716 03:39:19.233781 139795883440000 training.py:612] Running training and evaluation locally (non-distributed).
I0716 03:39:19.237845 139795883440000 training.py:700] Start train and evaluate loop. The evaluate will happen after every checkpoint. Checkpoint frequency is determined based on RunConfig arguments: save_checkpoints_steps 312 or save_checkpoints_secs None.
I0716 03:39:19.309082 139795883440000 estimator.py:1145] Calling model_fn.
I0716 03:39:20.755266 139795883440000 saver.py:1499] Saver not created because there are no variables in the graph to restore
I0716 03:39:21.346002 139795883440000 <ipython-input-6-d6747b40360b>:55]
[<tf.Variable 'embedding:0' shape=(750, 300) dtype=float32_ref>,
<tf.Variable 'module/aggregation/weights:0' shape=(3,) dtype=float32>,
<tf.Variable 'module/aggregation/scaling:0' shape=() dtype=float32>,
<tf.Variable 'dense/kernel:0' shape=(1324, 300) dtype=float32_ref>,
<tf.Variable 'dense/bias:0' shape=(300,) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/fw/gru_cell/gates/kernel:0' shape=(600, 600) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/fw/gru_cell/gates/bias:0' shape=(600,) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/fw/gru_cell/candidate/kernel:0' shape=(600, 300) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/fw/gru_cell/candidate/bias:0' shape=(300,) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/bw/gru_cell/gates/kernel:0' shape=(600, 600) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/bw/gru_cell/gates/bias:0' shape=(600,) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/bw/gru_cell/candidate/kernel:0' shape=(600, 300) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/bw/gru_cell/candidate/bias:0' shape=(300,) dtype=float32_ref>,
<tf.Variable 'dense_1/kernel:0' shape=(600, 23) dtype=float32_ref>,
<tf.Variable 'dense_1/bias:0' shape=(23,) dtype=float32_ref>,
<tf.Variable 'dense_2/kernel:0' shape=(600, 122) dtype=float32_ref>,
<tf.Variable 'dense_2/bias:0' shape=(122,) dtype=float32_ref>]
I0716 03:39:22.593348 139795883440000 estimator.py:1147] Done calling model_fn.
I0716 03:39:22.597057 139795883440000 basic_session_run_hooks.py:541] Create CheckpointSaverHook.
I0716 03:39:22.914428 139795883440000 monitored_session.py:240] Graph was finalized.
I0716 03:39:22.932276 139795883440000 saver.py:1280] Restoring parameters from ../model/elmo_bigru/model.ckpt-3432
I0716 03:39:23.970994 139795883440000 session_manager.py:500] Running local_init_op.
I0716 03:39:24.039549 139795883440000 session_manager.py:502] Done running local_init_op.
I0716 03:39:26.760696 139795883440000 basic_session_run_hooks.py:606] Saving checkpoints for 3432 into ../model/elmo_bigru/model.ckpt.
Reading ../data/atis.train.w-intent.iob
I0716 03:39:31.528347 139795883440000 basic_session_run_hooks.py:262] loss = 0.02635604, step = 3432 I0716 03:39:31.530147 139795883440000 basic_session_run_hooks.py:262] lr = 0.00020896882 I0716 03:39:58.348940 139795883440000 basic_session_run_hooks.py:692] global_step/sec: 3.7284 I0716 03:39:58.355597 139795883440000 basic_session_run_hooks.py:260] loss = 0.016621402, step = 3532 (26.827 sec) I0716 03:39:58.357238 139795883440000 basic_session_run_hooks.py:260] lr = 0.00020677868 (26.827 sec) I0716 03:40:25.918095 139795883440000 basic_session_run_hooks.py:692] global_step/sec: 3.62724 I0716 03:40:25.924374 139795883440000 basic_session_run_hooks.py:260] loss = 0.022599978, step = 3632 (27.569 sec) I0716 03:40:25.927362 139795883440000 basic_session_run_hooks.py:260] lr = 0.00020461148 (27.570 sec) I0716 03:40:52.109565 139795883440000 basic_session_run_hooks.py:692] global_step/sec: 3.81804 I0716 03:40:52.116390 139795883440000 basic_session_run_hooks.py:260] loss = 0.032441977, step = 3732 (26.192 sec) I0716 03:40:52.118647 139795883440000 basic_session_run_hooks.py:260] lr = 0.00020246702 (26.191 sec) I0716 03:40:54.840354 139795883440000 basic_session_run_hooks.py:606] Saving checkpoints for 3744 into ../model/elmo_bigru/model.ckpt. I0716 03:40:56.758404 139795883440000 estimator.py:1145] Calling model_fn. I0716 03:40:58.077687 139795883440000 saver.py:1499] Saver not created because there are no variables in the graph to restore I0716 03:40:58.906549 139795883440000 estimator.py:1147] Done calling model_fn. I0716 03:40:58.940602 139795883440000 evaluation.py:255] Starting evaluation at 2019-07-16T03:40:58Z I0716 03:40:59.164282 139795883440000 monitored_session.py:240] Graph was finalized. I0716 03:40:59.180455 139795883440000 saver.py:1280] Restoring parameters from ../model/elmo_bigru/model.ckpt-3744 I0716 03:41:00.080443 139795883440000 session_manager.py:500] Running local_init_op. I0716 03:41:00.139611 139795883440000 session_manager.py:502] Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
I0716 03:41:58.817108 139795883440000 evaluation.py:275] Finished evaluation at 2019-07-16-03:41:58 I0716 03:41:58.819010 139795883440000 estimator.py:2039] Saving dict for global step 3744: global_step = 3744, loss = 0.29848006 I0716 03:41:58.826589 139795883440000 estimator.py:2099] Saving 'checkpoint_path' summary for global step 3744: ../model/elmo_bigru/model.ckpt-3744 I0716 03:41:58.910535 139795883440000 estimator.py:368] Loss for final step: 0.009342727.
Reading ../data/atis.test.w-intent.iob
I0716 03:41:59.536495 139795883440000 estimator.py:1145] Calling model_fn. I0716 03:42:01.035902 139795883440000 saver.py:1499] Saver not created because there are no variables in the graph to restore I0716 03:42:01.564124 139795883440000 estimator.py:1147] Done calling model_fn. I0716 03:42:01.776388 139795883440000 monitored_session.py:240] Graph was finalized. I0716 03:42:01.793310 139795883440000 saver.py:1280] Restoring parameters from ../model/elmo_bigru/model.ckpt-3744 I0716 03:42:02.692504 139795883440000 session_manager.py:500] Running local_init_op. I0716 03:42:02.742432 139795883440000 session_manager.py:502] Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.
'precision', 'predicted', average, warn_for)
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1439: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples.
'recall', 'true', average, warn_for)
I0716 03:43:00.717076 139795883440000 interactiveshell.py:2882]
precision recall f1-score support
atis_flight 0.984 0.991 0.987 632
atis_airfare 0.941 1.000 0.970 48
atis_ground_service 0.973 1.000 0.986 36
atis_airline 1.000 1.000 1.000 38
atis_abbreviation 0.943 1.000 0.971 33
atis_aircraft 1.000 0.889 0.941 9
atis_flight_time 0.500 1.000 0.667 1
atis_quantity 0.375 1.000 0.545 3
atis_flight#atis_airfare 0.833 0.417 0.556 12
atis_airport 0.944 0.944 0.944 18
atis_distance 1.000 1.000 1.000 10
atis_city 0.750 0.500 0.600 6
atis_ground_fare 1.000 0.857 0.923 7
atis_capacity 1.000 0.952 0.976 21
atis_flight_no 0.889 1.000 0.941 8
atis_meal 1.000 0.833 0.909 6
atis_restriction 0.000 0.000 0.000 0
atis_airline#atis_flight_no 0.000 0.000 0.000 0
atis_ground_service#atis_ground_fare 0.000 0.000 0.000 0
atis_airfare#atis_flight_time 0.000 0.000 0.000 0
atis_cheapest 0.000 0.000 0.000 0
atis_aircraft#atis_flight#atis_flight_no 0.000 0.000 0.000 0
micro avg 0.971 0.976 0.974 888
macro avg 0.642 0.654 0.633 888
weighted avg 0.974 0.976 0.973 888
I0716 03:43:00.753411 139795883440000 interactiveshell.py:2882]
precision recall f1-score support
O 0.000 0.000 0.000 0.0
B-toloc.city_name 0.975 0.993 0.984 716.0
B-fromloc.city_name 0.990 1.000 0.995 704.0
I-toloc.city_name 0.981 0.985 0.983 265.0
B-depart_date.day_name 0.991 0.991 0.991 212.0
B-airline_name 0.990 1.000 0.995 101.0
I-fromloc.city_name 0.957 1.000 0.978 177.0
B-depart_time.period_of_day 0.984 0.931 0.957 130.0
I-airline_name 1.000 0.985 0.992 65.0
B-depart_date.day_number 0.981 0.964 0.972 55.0
B-depart_date.month_name 0.982 0.964 0.973 56.0
B-depart_time.time 0.851 1.000 0.919 57.0
B-round_trip 1.000 1.000 1.000 73.0
B-cost_relative 0.973 0.973 0.973 37.0
I-round_trip 1.000 1.000 1.000 71.0
B-flight_mod 0.828 1.000 0.906 24.0
B-depart_time.time_relative 0.955 0.985 0.970 65.0
I-depart_time.time 0.945 1.000 0.972 52.0
B-stoploc.city_name 0.870 1.000 0.930 20.0
B-city_name 0.902 0.649 0.755 57.0
B-class_type 0.960 1.000 0.980 24.0
B-arrive_time.time 1.000 0.971 0.985 34.0
B-arrive_time.time_relative 1.000 0.935 0.967 31.0
I-class_type 1.000 1.000 1.000 17.0
B-flight_stop 1.000 1.000 1.000 21.0
I-arrive_time.time 1.000 0.971 0.986 35.0
B-airline_code 0.872 1.000 0.932 34.0
I-depart_date.day_number 1.000 1.000 1.000 15.0
I-fromloc.airport_name 0.455 1.000 0.625 15.0
B-fromloc.airport_name 0.444 1.000 0.615 12.0
B-arrive_date.day_name 0.786 1.000 0.880 11.0
B-toloc.state_code 1.000 1.000 1.000 18.0
B-depart_date.today_relative 1.000 0.889 0.941 9.0
B-flight_number 0.733 1.000 0.846 11.0
B-depart_date.date_relative 0.944 1.000 0.971 17.0
B-toloc.state_name 0.781 0.893 0.833 28.0
B-fare_basis_code 0.850 1.000 0.919 17.0
B-flight_time 1.000 1.000 1.000 1.0
B-or 1.000 1.000 1.000 3.0
B-arrive_time.period_of_day 0.857 1.000 0.923 6.0
B-meal_description 1.000 0.900 0.947 10.0
I-cost_relative 1.000 0.667 0.800 3.0
I-airport_name 0.833 0.345 0.488 29.0
B-fare_amount 1.000 1.000 1.000 2.0
I-fare_amount 1.000 1.000 1.000 2.0
I-city_name 0.944 0.567 0.708 30.0
I-toloc.airport_name 1.000 1.000 1.000 3.0
B-transport_type 1.000 1.000 1.000 10.0
B-arrive_date.month_name 0.714 0.833 0.769 6.0
B-arrive_date.day_number 0.714 0.833 0.769 6.0
I-stoploc.city_name 0.909 1.000 0.952 10.0
B-meal 1.000 1.000 1.000 16.0
B-fromloc.state_code 1.000 1.000 1.000 23.0
B-depart_time.period_mod 0.833 1.000 0.909 5.0
B-connect 1.000 1.000 1.000 6.0
B-flight_days 1.000 1.000 1.000 10.0
B-toloc.airport_name 1.000 1.000 1.000 3.0
B-fromloc.state_name 0.944 1.000 0.971 17.0
B-airport_name 0.750 0.286 0.414 21.0
B-economy 1.000 1.000 1.000 6.0
I-flight_time 1.000 1.000 1.000 1.0
B-aircraft_code 1.000 0.818 0.900 33.0
B-mod 1.000 0.500 0.667 2.0
B-airport_code 1.000 0.556 0.714 9.0
B-depart_time.start_time 0.750 1.000 0.857 3.0
B-depart_time.end_time 0.667 0.667 0.667 3.0
B-depart_date.year 1.000 1.000 1.000 3.0
I-transport_type 1.000 1.000 1.000 1.0
B-restriction_code 0.667 0.500 0.571 4.0
B-arrive_time.start_time 1.000 0.875 0.933 8.0
B-toloc.airport_code 1.000 0.750 0.857 4.0
B-arrive_time.end_time 1.000 0.875 0.933 8.0
I-arrive_time.end_time 1.000 0.875 0.933 8.0
I-depart_time.end_time 0.667 0.667 0.667 3.0
I-flight_stop 0.000 0.000 0.000 0.0
B-fromloc.airport_code 0.714 1.000 0.833 5.0
I-restriction_code 1.000 1.000 1.000 3.0
I-depart_time.start_time 1.000 1.000 1.000 1.0
I-toloc.state_name 1.000 1.000 1.000 1.0
I-depart_date.today_relative 0.000 0.000 0.000 0.0
B-arrive_date.date_relative 1.000 1.000 1.000 2.0
I-flight_mod 1.000 0.167 0.286 6.0
I-economy 0.000 0.000 0.000 0.0
B-return_date.date_relative 1.000 0.667 0.800 3.0
I-fromloc.state_name 1.000 1.000 1.000 1.0
B-state_code 1.000 1.000 1.000 1.0
I-arrive_time.start_time 1.000 1.000 1.000 1.0
I-arrive_date.day_number 0.000 0.000 0.000 0.0
B-meal_code 0.000 0.000 0.000 1.0
I-depart_time.period_of_day 0.000 0.000 0.000 1.0
B-day_name 1.000 0.500 0.667 2.0
B-period_of_day 1.000 0.750 0.857 4.0
B-stoploc.state_code 0.000 0.000 0.000 0.0
B-return_date.month_name 0.000 0.000 0.000 0.0
B-return_date.day_number 0.000 0.000 0.000 0.0
B-arrive_time.period_mod 0.000 0.000 0.000 0.0
I-meal_code 0.000 0.000 0.000 0.0
B-toloc.country_name 1.000 1.000 1.000 1.0
B-days_code 0.000 0.000 0.000 1.0
I-arrive_time.period_of_day 0.000 0.000 0.000 0.0
I-today_relative 0.000 0.000 0.000 0.0
B-return_time.period_of_day 0.000 0.000 0.000 0.0
B-time 0.000 0.000 0.000 0.0
I-fare_basis_code 0.000 0.000 0.000 0.0
I-arrive_time.time_relative 1.000 1.000 1.000 4.0
I-depart_time.time_relative 0.000 0.000 0.000 1.0
B-today_relative 0.000 0.000 0.000 0.0
B-state_name 0.000 0.000 0.000 9.0
B-arrive_date.today_relative 0.000 0.000 0.000 0.0
B-return_time.period_mod 0.000 0.000 0.000 0.0
B-month_name 0.000 0.000 0.000 0.0
B-day_number 0.000 0.000 0.000 0.0
I-return_date.date_relative 1.000 0.667 0.800 3.0
I-return_date.today_relative 0.000 0.000 0.000 0.0
B-stoploc.airport_name 0.000 0.000 0.000 0.0
B-time_relative 0.000 0.000 0.000 0.0
I-time 0.000 0.000 0.000 0.0
I-return_date.day_number 0.000 0.000 0.000 0.0
I-meal_description 0.000 0.000 0.000 0.0
B-return_date.today_relative 0.000 0.000 0.000 0.0
B-return_date.day_name 0.000 0.000 0.000 2.0
micro avg 0.958 0.959 0.958 3657.0
macro avg 0.677 0.656 0.656 3657.0
weighted avg 0.960 0.959 0.955 3657.0
I0716 03:43:00.773989 139795883440000 interactiveshell.py:2882] Best Slot F1: 0.959
I0716 03:43:00.775548 139795883440000 estimator_training.py:186] Not using Distribute Coordinator.
I0716 03:43:00.779031 139795883440000 training.py:612] Running training and evaluation locally (non-distributed).
I0716 03:43:00.781644 139795883440000 training.py:700] Start train and evaluate loop. The evaluate will happen after every checkpoint. Checkpoint frequency is determined based on RunConfig arguments: save_checkpoints_steps 312 or save_checkpoints_secs None.
I0716 03:43:00.846108 139795883440000 estimator.py:1145] Calling model_fn.
I0716 03:43:02.279944 139795883440000 saver.py:1499] Saver not created because there are no variables in the graph to restore
I0716 03:43:02.867092 139795883440000 <ipython-input-6-d6747b40360b>:55]
[<tf.Variable 'embedding:0' shape=(750, 300) dtype=float32_ref>,
<tf.Variable 'module/aggregation/weights:0' shape=(3,) dtype=float32>,
<tf.Variable 'module/aggregation/scaling:0' shape=() dtype=float32>,
<tf.Variable 'dense/kernel:0' shape=(1324, 300) dtype=float32_ref>,
<tf.Variable 'dense/bias:0' shape=(300,) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/fw/gru_cell/gates/kernel:0' shape=(600, 600) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/fw/gru_cell/gates/bias:0' shape=(600,) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/fw/gru_cell/candidate/kernel:0' shape=(600, 300) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/fw/gru_cell/candidate/bias:0' shape=(300,) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/bw/gru_cell/gates/kernel:0' shape=(600, 600) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/bw/gru_cell/gates/bias:0' shape=(600,) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/bw/gru_cell/candidate/kernel:0' shape=(600, 300) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/bw/gru_cell/candidate/bias:0' shape=(300,) dtype=float32_ref>,
<tf.Variable 'dense_1/kernel:0' shape=(600, 23) dtype=float32_ref>,
<tf.Variable 'dense_1/bias:0' shape=(23,) dtype=float32_ref>,
<tf.Variable 'dense_2/kernel:0' shape=(600, 122) dtype=float32_ref>,
<tf.Variable 'dense_2/bias:0' shape=(122,) dtype=float32_ref>]
I0716 03:43:04.119152 139795883440000 estimator.py:1147] Done calling model_fn.
I0716 03:43:04.123580 139795883440000 basic_session_run_hooks.py:541] Create CheckpointSaverHook.
I0716 03:43:04.439007 139795883440000 monitored_session.py:240] Graph was finalized.
I0716 03:43:04.455222 139795883440000 saver.py:1280] Restoring parameters from ../model/elmo_bigru/model.ckpt-3744
I0716 03:43:05.489947 139795883440000 session_manager.py:500] Running local_init_op.
I0716 03:43:05.554122 139795883440000 session_manager.py:502] Done running local_init_op.
I0716 03:43:08.292708 139795883440000 basic_session_run_hooks.py:606] Saving checkpoints for 3744 into ../model/elmo_bigru/model.ckpt.
Reading ../data/atis.train.w-intent.iob
I0716 03:43:12.975174 139795883440000 basic_session_run_hooks.py:262] loss = 0.021001698, step = 3744 I0716 03:43:12.977192 139795883440000 basic_session_run_hooks.py:262] lr = 0.00020221119 I0716 03:43:39.822550 139795883440000 basic_session_run_hooks.py:692] global_step/sec: 3.72469 I0716 03:43:39.827977 139795883440000 basic_session_run_hooks.py:260] loss = 0.02767645, step = 3844 (26.853 sec) I0716 03:43:39.830882 139795883440000 basic_session_run_hooks.py:260] lr = 0.00020009186 (26.854 sec) I0716 03:44:06.863698 139795883440000 basic_session_run_hooks.py:692] global_step/sec: 3.69806 I0716 03:44:06.869125 139795883440000 basic_session_run_hooks.py:260] loss = 0.018864132, step = 3944 (27.041 sec) I0716 03:44:06.872111 139795883440000 basic_session_run_hooks.py:260] lr = 0.00019799476 (27.041 sec) I0716 03:44:33.213929 139795883440000 basic_session_run_hooks.py:692] global_step/sec: 3.79504 I0716 03:44:33.216701 139795883440000 basic_session_run_hooks.py:260] loss = 0.027750138, step = 4044 (26.348 sec) I0716 03:44:33.222939 139795883440000 basic_session_run_hooks.py:260] lr = 0.00019591961 (26.351 sec) I0716 03:44:36.275014 139795883440000 basic_session_run_hooks.py:606] Saving checkpoints for 4056 into ../model/elmo_bigru/model.ckpt. I0716 03:44:38.252676 139795883440000 estimator.py:1145] Calling model_fn. I0716 03:44:39.814150 139795883440000 saver.py:1499] Saver not created because there are no variables in the graph to restore I0716 03:44:40.388385 139795883440000 estimator.py:1147] Done calling model_fn. I0716 03:44:40.418080 139795883440000 evaluation.py:255] Starting evaluation at 2019-07-16T03:44:40Z I0716 03:44:40.637779 139795883440000 monitored_session.py:240] Graph was finalized. I0716 03:44:40.655734 139795883440000 saver.py:1280] Restoring parameters from ../model/elmo_bigru/model.ckpt-4056 I0716 03:44:41.561204 139795883440000 session_manager.py:500] Running local_init_op. I0716 03:44:41.618479 139795883440000 session_manager.py:502] Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
I0716 03:45:40.799308 139795883440000 evaluation.py:275] Finished evaluation at 2019-07-16-03:45:40 I0716 03:45:40.801134 139795883440000 estimator.py:2039] Saving dict for global step 4056: global_step = 4056, loss = 0.313336 I0716 03:45:40.808373 139795883440000 estimator.py:2099] Saving 'checkpoint_path' summary for global step 4056: ../model/elmo_bigru/model.ckpt-4056 I0716 03:45:40.894330 139795883440000 estimator.py:368] Loss for final step: 0.010575725.
Reading ../data/atis.test.w-intent.iob
I0716 03:45:41.482816 139795883440000 estimator.py:1145] Calling model_fn. I0716 03:45:42.989626 139795883440000 saver.py:1499] Saver not created because there are no variables in the graph to restore I0716 03:45:43.509806 139795883440000 estimator.py:1147] Done calling model_fn. I0716 03:45:43.724382 139795883440000 monitored_session.py:240] Graph was finalized. I0716 03:45:43.739015 139795883440000 saver.py:1280] Restoring parameters from ../model/elmo_bigru/model.ckpt-4056 I0716 03:45:44.637495 139795883440000 session_manager.py:500] Running local_init_op. I0716 03:45:44.689667 139795883440000 session_manager.py:502] Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.
'precision', 'predicted', average, warn_for)
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1439: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples.
'recall', 'true', average, warn_for)
I0716 03:46:42.461421 139795883440000 interactiveshell.py:2882]
precision recall f1-score support
atis_flight 0.986 0.991 0.988 632
atis_airfare 0.941 1.000 0.970 48
atis_ground_service 1.000 1.000 1.000 36
atis_airline 1.000 1.000 1.000 38
atis_abbreviation 0.943 1.000 0.971 33
atis_aircraft 1.000 0.889 0.941 9
atis_flight_time 0.333 1.000 0.500 1
atis_quantity 0.375 1.000 0.545 3
atis_flight#atis_airfare 0.833 0.417 0.556 12
atis_airport 0.947 1.000 0.973 18
atis_distance 1.000 1.000 1.000 10
atis_city 0.750 0.500 0.600 6
atis_ground_fare 1.000 0.857 0.923 7
atis_capacity 1.000 0.952 0.976 21
atis_flight_no 0.889 1.000 0.941 8
atis_meal 1.000 0.833 0.909 6
atis_restriction 0.000 0.000 0.000 0
atis_airline#atis_flight_no 0.000 0.000 0.000 0
atis_ground_service#atis_ground_fare 0.000 0.000 0.000 0
atis_airfare#atis_flight_time 0.000 0.000 0.000 0
atis_cheapest 0.000 0.000 0.000 0
atis_aircraft#atis_flight#atis_flight_no 0.000 0.000 0.000 0
micro avg 0.972 0.977 0.975 888
macro avg 0.636 0.656 0.627 888
weighted avg 0.976 0.977 0.975 888
I0716 03:46:42.499803 139795883440000 interactiveshell.py:2882]
precision recall f1-score support
O 0.000 0.000 0.000 0.0
B-toloc.city_name 0.974 0.993 0.983 716.0
B-fromloc.city_name 0.989 0.999 0.994 704.0
I-toloc.city_name 0.978 0.992 0.985 265.0
B-depart_date.day_name 0.991 0.991 0.991 212.0
B-airline_name 1.000 1.000 1.000 101.0
I-fromloc.city_name 0.978 0.994 0.986 177.0
B-depart_time.period_of_day 0.983 0.915 0.948 130.0
I-airline_name 1.000 1.000 1.000 65.0
B-depart_date.day_number 0.981 0.964 0.972 55.0
B-depart_date.month_name 0.981 0.946 0.964 56.0
B-depart_time.time 0.864 1.000 0.927 57.0
B-round_trip 1.000 1.000 1.000 73.0
B-cost_relative 0.973 0.973 0.973 37.0
I-round_trip 1.000 1.000 1.000 71.0
B-flight_mod 0.828 1.000 0.906 24.0
B-depart_time.time_relative 0.970 0.985 0.977 65.0
I-depart_time.time 0.929 1.000 0.963 52.0
B-stoploc.city_name 0.952 1.000 0.976 20.0
B-city_name 0.923 0.632 0.750 57.0
B-class_type 0.960 1.000 0.980 24.0
B-arrive_time.time 0.971 0.971 0.971 34.0
B-arrive_time.time_relative 0.857 0.968 0.909 31.0
I-class_type 1.000 1.000 1.000 17.0
B-flight_stop 1.000 1.000 1.000 21.0
I-arrive_time.time 1.000 0.971 0.986 35.0
B-airline_code 0.872 1.000 0.932 34.0
I-depart_date.day_number 1.000 0.933 0.966 15.0
I-fromloc.airport_name 0.417 1.000 0.588 15.0
B-fromloc.airport_name 0.400 1.000 0.571 12.0
B-arrive_date.day_name 0.786 1.000 0.880 11.0
B-toloc.state_code 1.000 1.000 1.000 18.0
B-depart_date.today_relative 1.000 0.889 0.941 9.0
B-flight_number 0.733 1.000 0.846 11.0
B-depart_date.date_relative 0.944 1.000 0.971 17.0
B-toloc.state_name 0.781 0.893 0.833 28.0
B-fare_basis_code 0.850 1.000 0.919 17.0
B-flight_time 1.000 1.000 1.000 1.0
B-or 1.000 1.000 1.000 3.0
B-arrive_time.period_of_day 0.750 1.000 0.857 6.0
B-meal_description 1.000 0.900 0.947 10.0
I-cost_relative 1.000 0.667 0.800 3.0
I-airport_name 0.833 0.345 0.488 29.0
B-fare_amount 1.000 1.000 1.000 2.0
I-fare_amount 1.000 1.000 1.000 2.0
I-city_name 1.000 0.567 0.723 30.0
I-toloc.airport_name 1.000 1.000 1.000 3.0
B-transport_type 1.000 1.000 1.000 10.0
B-arrive_date.month_name 0.625 0.833 0.714 6.0
B-arrive_date.day_number 0.625 0.833 0.714 6.0
I-stoploc.city_name 0.909 1.000 0.952 10.0
B-meal 1.000 1.000 1.000 16.0
B-fromloc.state_code 1.000 1.000 1.000 23.0
B-depart_time.period_mod 1.000 1.000 1.000 5.0
B-connect 1.000 1.000 1.000 6.0
B-flight_days 1.000 1.000 1.000 10.0
B-toloc.airport_name 1.000 1.000 1.000 3.0
B-fromloc.state_name 0.944 1.000 0.971 17.0
B-airport_name 0.778 0.333 0.467 21.0
B-economy 1.000 1.000 1.000 6.0
I-flight_time 1.000 1.000 1.000 1.0
B-aircraft_code 1.000 0.818 0.900 33.0
B-mod 1.000 0.500 0.667 2.0
B-airport_code 1.000 0.444 0.615 9.0
B-depart_time.start_time 1.000 0.667 0.800 3.0
B-depart_time.end_time 1.000 0.333 0.500 3.0
B-depart_date.year 1.000 1.000 1.000 3.0
I-transport_type 1.000 1.000 1.000 1.0
B-restriction_code 0.667 0.500 0.571 4.0
B-arrive_time.start_time 0.889 1.000 0.941 8.0
B-toloc.airport_code 1.000 0.750 0.857 4.0
B-arrive_time.end_time 0.889 1.000 0.941 8.0
I-arrive_time.end_time 0.889 1.000 0.941 8.0
I-depart_time.end_time 1.000 0.333 0.500 3.0
I-flight_stop 0.000 0.000 0.000 0.0
B-fromloc.airport_code 0.625 1.000 0.769 5.0
I-restriction_code 1.000 1.000 1.000 3.0
I-depart_time.start_time 1.000 1.000 1.000 1.0
I-toloc.state_name 1.000 1.000 1.000 1.0
I-depart_date.today_relative 0.000 0.000 0.000 0.0
B-arrive_date.date_relative 1.000 1.000 1.000 2.0
I-flight_mod 1.000 0.167 0.286 6.0
I-economy 0.000 0.000 0.000 0.0
B-return_date.date_relative 1.000 0.667 0.800 3.0
I-fromloc.state_name 1.000 1.000 1.000 1.0
B-state_code 1.000 1.000 1.000 1.0
I-arrive_time.start_time 1.000 1.000 1.000 1.0
I-arrive_date.day_number 0.000 0.000 0.000 0.0
B-meal_code 0.000 0.000 0.000 1.0
I-depart_time.period_of_day 0.000 0.000 0.000 1.0
B-day_name 1.000 0.500 0.667 2.0
B-period_of_day 1.000 0.750 0.857 4.0
B-stoploc.state_code 0.000 0.000 0.000 0.0
B-return_date.month_name 0.000 0.000 0.000 0.0
B-return_date.day_number 0.000 0.000 0.000 0.0
B-arrive_time.period_mod 0.000 0.000 0.000 0.0
I-meal_code 0.000 0.000 0.000 0.0
B-toloc.country_name 1.000 1.000 1.000 1.0
B-days_code 0.000 0.000 0.000 1.0
I-arrive_time.period_of_day 0.000 0.000 0.000 0.0
I-today_relative 0.000 0.000 0.000 0.0
B-return_time.period_of_day 0.000 0.000 0.000 0.0
B-time 0.000 0.000 0.000 0.0
I-fare_basis_code 0.000 0.000 0.000 0.0
I-arrive_time.time_relative 0.000 0.000 0.000 4.0
I-depart_time.time_relative 0.000 0.000 0.000 1.0
B-today_relative 0.000 0.000 0.000 0.0
B-state_name 0.000 0.000 0.000 9.0
B-arrive_date.today_relative 0.000 0.000 0.000 0.0
B-return_time.period_mod 0.000 0.000 0.000 0.0
B-month_name 0.000 0.000 0.000 0.0
B-day_number 0.000 0.000 0.000 0.0
I-return_date.date_relative 1.000 0.667 0.800 3.0
I-return_date.today_relative 0.000 0.000 0.000 0.0
B-stoploc.airport_name 0.000 0.000 0.000 0.0
B-time_relative 0.000 0.000 0.000 0.0
I-time 0.000 0.000 0.000 0.0
I-return_date.day_number 0.000 0.000 0.000 0.0
I-meal_description 0.000 0.000 0.000 0.0
B-return_date.today_relative 0.000 0.000 0.000 0.0
B-return_date.day_name 0.000 0.000 0.000 2.0
micro avg 0.956 0.957 0.957 3657.0
macro avg 0.672 0.641 0.642 3657.0
weighted avg 0.959 0.957 0.953 3657.0
I0716 03:46:42.520813 139795883440000 interactiveshell.py:2882] Best Slot F1: 0.959
I0716 03:46:42.522331 139795883440000 estimator_training.py:186] Not using Distribute Coordinator.
I0716 03:46:42.524306 139795883440000 training.py:612] Running training and evaluation locally (non-distributed).
I0716 03:46:42.525812 139795883440000 training.py:700] Start train and evaluate loop. The evaluate will happen after every checkpoint. Checkpoint frequency is determined based on RunConfig arguments: save_checkpoints_steps 312 or save_checkpoints_secs None.
I0716 03:46:42.591641 139795883440000 estimator.py:1145] Calling model_fn.
I0716 03:46:44.033873 139795883440000 saver.py:1499] Saver not created because there are no variables in the graph to restore
I0716 03:46:44.784013 139795883440000 <ipython-input-6-d6747b40360b>:55]
[<tf.Variable 'embedding:0' shape=(750, 300) dtype=float32_ref>,
<tf.Variable 'module/aggregation/weights:0' shape=(3,) dtype=float32>,
<tf.Variable 'module/aggregation/scaling:0' shape=() dtype=float32>,
<tf.Variable 'dense/kernel:0' shape=(1324, 300) dtype=float32_ref>,
<tf.Variable 'dense/bias:0' shape=(300,) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/fw/gru_cell/gates/kernel:0' shape=(600, 600) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/fw/gru_cell/gates/bias:0' shape=(600,) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/fw/gru_cell/candidate/kernel:0' shape=(600, 300) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/fw/gru_cell/candidate/bias:0' shape=(300,) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/bw/gru_cell/gates/kernel:0' shape=(600, 600) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/bw/gru_cell/gates/bias:0' shape=(600,) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/bw/gru_cell/candidate/kernel:0' shape=(600, 300) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/bw/gru_cell/candidate/bias:0' shape=(300,) dtype=float32_ref>,
<tf.Variable 'dense_1/kernel:0' shape=(600, 23) dtype=float32_ref>,
<tf.Variable 'dense_1/bias:0' shape=(23,) dtype=float32_ref>,
<tf.Variable 'dense_2/kernel:0' shape=(600, 122) dtype=float32_ref>,
<tf.Variable 'dense_2/bias:0' shape=(122,) dtype=float32_ref>]
I0716 03:46:45.850750 139795883440000 estimator.py:1147] Done calling model_fn.
I0716 03:46:45.854489 139795883440000 basic_session_run_hooks.py:541] Create CheckpointSaverHook.
I0716 03:46:46.181741 139795883440000 monitored_session.py:240] Graph was finalized.
I0716 03:46:46.198333 139795883440000 saver.py:1280] Restoring parameters from ../model/elmo_bigru/model.ckpt-4056
I0716 03:46:47.237629 139795883440000 session_manager.py:500] Running local_init_op.
I0716 03:46:47.309618 139795883440000 session_manager.py:502] Done running local_init_op.
I0716 03:46:50.114432 139795883440000 basic_session_run_hooks.py:606] Saving checkpoints for 4056 into ../model/elmo_bigru/model.ckpt.
Reading ../data/atis.train.w-intent.iob
I0716 03:46:54.964871 139795883440000 basic_session_run_hooks.py:262] loss = 0.015439781, step = 4056 I0716 03:46:54.966714 139795883440000 basic_session_run_hooks.py:262] lr = 0.00019567205 I0716 03:47:22.732358 139795883440000 basic_session_run_hooks.py:692] global_step/sec: 3.60126 I0716 03:47:22.735428 139795883440000 basic_session_run_hooks.py:260] loss = 0.022223681, step = 4156 (27.771 sec) I0716 03:47:22.740602 139795883440000 basic_session_run_hooks.py:260] lr = 0.0001936213 (27.774 sec) I0716 03:47:49.557179 139795883440000 basic_session_run_hooks.py:692] global_step/sec: 3.72788 I0716 03:47:49.562354 139795883440000 basic_session_run_hooks.py:260] loss = 0.024592336, step = 4256 (26.827 sec) I0716 03:47:49.565015 139795883440000 basic_session_run_hooks.py:260] lr = 0.00019159199 (26.824 sec) I0716 03:48:15.762630 139795883440000 basic_session_run_hooks.py:692] global_step/sec: 3.816 I0716 03:48:15.770267 139795883440000 basic_session_run_hooks.py:260] loss = 0.019611156, step = 4356 (26.208 sec) I0716 03:48:15.772212 139795883440000 basic_session_run_hooks.py:260] lr = 0.00018958394 (26.207 sec) I0716 03:48:18.542946 139795883440000 basic_session_run_hooks.py:606] Saving checkpoints for 4368 into ../model/elmo_bigru/model.ckpt. I0716 03:48:20.516006 139795883440000 estimator.py:1145] Calling model_fn. I0716 03:48:21.880723 139795883440000 saver.py:1499] Saver not created because there are no variables in the graph to restore I0716 03:48:22.436344 139795883440000 estimator.py:1147] Done calling model_fn. I0716 03:48:22.465405 139795883440000 evaluation.py:255] Starting evaluation at 2019-07-16T03:48:22Z I0716 03:48:22.684628 139795883440000 monitored_session.py:240] Graph was finalized. I0716 03:48:22.697028 139795883440000 saver.py:1280] Restoring parameters from ../model/elmo_bigru/model.ckpt-4368 I0716 03:48:23.581039 139795883440000 session_manager.py:500] Running local_init_op. I0716 03:48:23.639722 139795883440000 session_manager.py:502] Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
I0716 03:49:22.386981 139795883440000 evaluation.py:275] Finished evaluation at 2019-07-16-03:49:22 I0716 03:49:22.388689 139795883440000 estimator.py:2039] Saving dict for global step 4368: global_step = 4368, loss = 0.3263626 I0716 03:49:22.396470 139795883440000 estimator.py:2099] Saving 'checkpoint_path' summary for global step 4368: ../model/elmo_bigru/model.ckpt-4368 I0716 03:49:22.478740 139795883440000 estimator.py:368] Loss for final step: 0.012535343.
Reading ../data/atis.test.w-intent.iob
I0716 03:49:23.071287 139795883440000 estimator.py:1145] Calling model_fn. I0716 03:49:24.734490 139795883440000 saver.py:1499] Saver not created because there are no variables in the graph to restore I0716 03:49:25.242751 139795883440000 estimator.py:1147] Done calling model_fn. I0716 03:49:25.466029 139795883440000 monitored_session.py:240] Graph was finalized. I0716 03:49:25.482319 139795883440000 saver.py:1280] Restoring parameters from ../model/elmo_bigru/model.ckpt-4368 I0716 03:49:26.366806 139795883440000 session_manager.py:500] Running local_init_op. I0716 03:49:26.414819 139795883440000 session_manager.py:502] Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.
'precision', 'predicted', average, warn_for)
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1439: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples.
'recall', 'true', average, warn_for)
I0716 03:50:24.146742 139795883440000 interactiveshell.py:2882]
precision recall f1-score support
atis_flight 0.978 0.992 0.985 632
atis_airfare 0.941 1.000 0.970 48
atis_ground_service 1.000 1.000 1.000 36
atis_airline 1.000 1.000 1.000 38
atis_abbreviation 0.943 1.000 0.971 33
atis_aircraft 1.000 0.889 0.941 9
atis_flight_time 0.500 1.000 0.667 1
atis_quantity 0.375 1.000 0.545 3
atis_flight#atis_airfare 0.833 0.417 0.556 12
atis_airport 1.000 0.944 0.971 18
atis_distance 1.000 1.000 1.000 10
atis_city 1.000 0.167 0.286 6
atis_ground_fare 1.000 0.857 0.923 7
atis_capacity 1.000 0.952 0.976 21
atis_flight_no 0.889 1.000 0.941 8
atis_meal 1.000 0.833 0.909 6
atis_restriction 0.000 0.000 0.000 0
atis_airline#atis_flight_no 0.000 0.000 0.000 0
atis_ground_service#atis_ground_fare 0.000 0.000 0.000 0
atis_airfare#atis_flight_time 0.000 0.000 0.000 0
atis_cheapest 0.000 0.000 0.000 0
atis_aircraft#atis_flight#atis_flight_no 0.000 0.000 0.000 0
micro avg 0.970 0.975 0.972 888
macro avg 0.657 0.639 0.620 888
weighted avg 0.973 0.975 0.970 888
I0716 03:50:24.183134 139795883440000 interactiveshell.py:2882]
precision recall f1-score support
O 0.000 0.000 0.000 0.0
B-toloc.city_name 0.973 0.994 0.983 716.0
B-fromloc.city_name 0.989 1.000 0.994 704.0
I-toloc.city_name 0.978 0.989 0.983 265.0
B-depart_date.day_name 0.991 0.991 0.991 212.0
B-airline_name 1.000 1.000 1.000 101.0
I-fromloc.city_name 0.967 1.000 0.983 177.0
B-depart_time.period_of_day 0.984 0.923 0.952 130.0
I-airline_name 1.000 1.000 1.000 65.0
B-depart_date.day_number 0.981 0.964 0.972 55.0
B-depart_date.month_name 0.982 0.964 0.973 56.0
B-depart_time.time 0.864 1.000 0.927 57.0
B-round_trip 1.000 1.000 1.000 73.0
B-cost_relative 0.973 0.973 0.973 37.0
I-round_trip 1.000 1.000 1.000 71.0
B-flight_mod 0.828 1.000 0.906 24.0
B-depart_time.time_relative 0.955 0.985 0.970 65.0
I-depart_time.time 0.945 1.000 0.972 52.0
B-stoploc.city_name 0.952 1.000 0.976 20.0
B-city_name 0.897 0.614 0.729 57.0
B-class_type 0.960 1.000 0.980 24.0
B-arrive_time.time 0.971 0.971 0.971 34.0
B-arrive_time.time_relative 0.853 0.935 0.892 31.0
I-class_type 1.000 1.000 1.000 17.0
B-flight_stop 1.000 1.000 1.000 21.0
I-arrive_time.time 1.000 0.971 0.986 35.0
B-airline_code 0.810 1.000 0.895 34.0
I-depart_date.day_number 1.000 1.000 1.000 15.0
I-fromloc.airport_name 0.455 1.000 0.625 15.0
B-fromloc.airport_name 0.429 1.000 0.600 12.0
B-arrive_date.day_name 0.786 1.000 0.880 11.0
B-toloc.state_code 1.000 1.000 1.000 18.0
B-depart_date.today_relative 1.000 0.889 0.941 9.0
B-flight_number 0.786 1.000 0.880 11.0
B-depart_date.date_relative 0.944 1.000 0.971 17.0
B-toloc.state_name 0.781 0.893 0.833 28.0
B-fare_basis_code 0.850 1.000 0.919 17.0
B-flight_time 1.000 1.000 1.000 1.0
B-or 1.000 1.000 1.000 3.0
B-arrive_time.period_of_day 0.750 1.000 0.857 6.0
B-meal_description 1.000 0.900 0.947 10.0
I-cost_relative 1.000 0.667 0.800 3.0
I-airport_name 0.917 0.379 0.537 29.0
B-fare_amount 1.000 1.000 1.000 2.0
I-fare_amount 1.000 1.000 1.000 2.0
I-city_name 1.000 0.533 0.696 30.0
I-toloc.airport_name 0.750 1.000 0.857 3.0
B-transport_type 1.000 1.000 1.000 10.0
B-arrive_date.month_name 0.714 0.833 0.769 6.0
B-arrive_date.day_number 0.714 0.833 0.769 6.0
I-stoploc.city_name 1.000 1.000 1.000 10.0
B-meal 1.000 1.000 1.000 16.0
B-fromloc.state_code 1.000 1.000 1.000 23.0
B-depart_time.period_mod 1.000 1.000 1.000 5.0
B-connect 1.000 1.000 1.000 6.0
B-flight_days 1.000 1.000 1.000 10.0
B-toloc.airport_name 1.000 1.000 1.000 3.0
B-fromloc.state_name 0.944 1.000 0.971 17.0
B-airport_name 0.750 0.286 0.414 21.0
B-economy 1.000 1.000 1.000 6.0
I-flight_time 1.000 1.000 1.000 1.0
B-aircraft_code 1.000 0.848 0.918 33.0
B-mod 1.000 0.500 0.667 2.0
B-airport_code 1.000 0.222 0.364 9.0
B-depart_time.start_time 1.000 1.000 1.000 3.0
B-depart_time.end_time 1.000 0.667 0.800 3.0
B-depart_date.year 1.000 1.000 1.000 3.0
I-transport_type 0.000 0.000 0.000 1.0
B-restriction_code 1.000 0.500 0.667 4.0
B-arrive_time.start_time 1.000 1.000 1.000 8.0
B-toloc.airport_code 1.000 0.750 0.857 4.0
B-arrive_time.end_time 1.000 1.000 1.000 8.0
I-arrive_time.end_time 1.000 1.000 1.000 8.0
I-depart_time.end_time 1.000 0.667 0.800 3.0
I-flight_stop 0.000 0.000 0.000 0.0
B-fromloc.airport_code 0.625 1.000 0.769 5.0
I-restriction_code 1.000 1.000 1.000 3.0
I-depart_time.start_time 1.000 1.000 1.000 1.0
I-toloc.state_name 1.000 1.000 1.000 1.0
I-depart_date.today_relative 0.000 0.000 0.000 0.0
B-arrive_date.date_relative 1.000 1.000 1.000 2.0
I-flight_mod 1.000 0.167 0.286 6.0
I-economy 0.000 0.000 0.000 0.0
B-return_date.date_relative 1.000 0.333 0.500 3.0
I-fromloc.state_name 1.000 1.000 1.000 1.0
B-state_code 1.000 1.000 1.000 1.0
I-arrive_time.start_time 1.000 1.000 1.000 1.0
I-arrive_date.day_number 0.000 0.000 0.000 0.0
B-meal_code 0.000 0.000 0.000 1.0
I-depart_time.period_of_day 0.000 0.000 0.000 1.0
B-day_name 1.000 0.500 0.667 2.0
B-period_of_day 1.000 0.750 0.857 4.0
B-stoploc.state_code 0.000 0.000 0.000 0.0
B-return_date.month_name 0.000 0.000 0.000 0.0
B-return_date.day_number 0.000 0.000 0.000 0.0
B-arrive_time.period_mod 0.000 0.000 0.000 0.0
I-meal_code 0.000 0.000 0.000 0.0
B-toloc.country_name 1.000 1.000 1.000 1.0
B-days_code 0.000 0.000 0.000 1.0
I-arrive_time.period_of_day 0.000 0.000 0.000 0.0
I-today_relative 0.000 0.000 0.000 0.0
B-return_time.period_of_day 0.000 0.000 0.000 0.0
B-time 0.000 0.000 0.000 0.0
I-fare_basis_code 0.000 0.000 0.000 0.0
I-arrive_time.time_relative 0.000 0.000 0.000 4.0
I-depart_time.time_relative 0.000 0.000 0.000 1.0
B-today_relative 0.000 0.000 0.000 0.0
B-state_name 0.000 0.000 0.000 9.0
B-arrive_date.today_relative 0.000 0.000 0.000 0.0
B-return_time.period_mod 0.000 0.000 0.000 0.0
B-month_name 0.000 0.000 0.000 0.0
B-day_number 0.000 0.000 0.000 0.0
I-return_date.date_relative 1.000 0.667 0.800 3.0
I-return_date.today_relative 0.000 0.000 0.000 0.0
B-stoploc.airport_name 0.000 0.000 0.000 0.0
B-time_relative 0.000 0.000 0.000 0.0
I-time 0.000 0.000 0.000 0.0
I-return_date.day_number 0.000 0.000 0.000 0.0
I-meal_description 0.000 0.000 0.000 0.0
B-return_date.today_relative 0.000 0.000 0.000 0.0
B-return_date.day_name 0.000 0.000 0.000 2.0
micro avg 0.956 0.958 0.957 3657.0
macro avg 0.670 0.637 0.638 3657.0
weighted avg 0.959 0.958 0.953 3657.0
I0716 03:50:24.203364 139795883440000 interactiveshell.py:2882] Best Slot F1: 0.959
I0716 03:50:24.204905 139795883440000 estimator_training.py:186] Not using Distribute Coordinator.
I0716 03:50:24.207797 139795883440000 training.py:612] Running training and evaluation locally (non-distributed).
I0716 03:50:24.211139 139795883440000 training.py:700] Start train and evaluate loop. The evaluate will happen after every checkpoint. Checkpoint frequency is determined based on RunConfig arguments: save_checkpoints_steps 312 or save_checkpoints_secs None.
I0716 03:50:24.276541 139795883440000 estimator.py:1145] Calling model_fn.
I0716 03:50:25.731236 139795883440000 saver.py:1499] Saver not created because there are no variables in the graph to restore
I0716 03:50:26.477844 139795883440000 <ipython-input-6-d6747b40360b>:55]
[<tf.Variable 'embedding:0' shape=(750, 300) dtype=float32_ref>,
<tf.Variable 'module/aggregation/weights:0' shape=(3,) dtype=float32>,
<tf.Variable 'module/aggregation/scaling:0' shape=() dtype=float32>,
<tf.Variable 'dense/kernel:0' shape=(1324, 300) dtype=float32_ref>,
<tf.Variable 'dense/bias:0' shape=(300,) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/fw/gru_cell/gates/kernel:0' shape=(600, 600) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/fw/gru_cell/gates/bias:0' shape=(600,) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/fw/gru_cell/candidate/kernel:0' shape=(600, 300) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/fw/gru_cell/candidate/bias:0' shape=(300,) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/bw/gru_cell/gates/kernel:0' shape=(600, 600) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/bw/gru_cell/gates/bias:0' shape=(600,) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/bw/gru_cell/candidate/kernel:0' shape=(600, 300) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/bw/gru_cell/candidate/bias:0' shape=(300,) dtype=float32_ref>,
<tf.Variable 'dense_1/kernel:0' shape=(600, 23) dtype=float32_ref>,
<tf.Variable 'dense_1/bias:0' shape=(23,) dtype=float32_ref>,
<tf.Variable 'dense_2/kernel:0' shape=(600, 122) dtype=float32_ref>,
<tf.Variable 'dense_2/bias:0' shape=(122,) dtype=float32_ref>]
I0716 03:50:27.547219 139795883440000 estimator.py:1147] Done calling model_fn.
I0716 03:50:27.551077 139795883440000 basic_session_run_hooks.py:541] Create CheckpointSaverHook.
I0716 03:50:27.879000 139795883440000 monitored_session.py:240] Graph was finalized.
I0716 03:50:27.895506 139795883440000 saver.py:1280] Restoring parameters from ../model/elmo_bigru/model.ckpt-4368
I0716 03:50:28.932056 139795883440000 session_manager.py:500] Running local_init_op.
I0716 03:50:29.001270 139795883440000 session_manager.py:502] Done running local_init_op.
I0716 03:50:32.065008 139795883440000 basic_session_run_hooks.py:606] Saving checkpoints for 4368 into ../model/elmo_bigru/model.ckpt.
Reading ../data/atis.train.w-intent.iob
I0716 03:50:37.141797 139795883440000 basic_session_run_hooks.py:262] loss = 0.018032221, step = 4368 I0716 03:50:37.143822 139795883440000 basic_session_run_hooks.py:262] lr = 0.00018934443 I0716 03:51:04.711982 139795883440000 basic_session_run_hooks.py:692] global_step/sec: 3.627 I0716 03:51:04.719229 139795883440000 basic_session_run_hooks.py:260] loss = 0.026767438, step = 4468 (27.577 sec) I0716 03:51:04.721602 139795883440000 basic_session_run_hooks.py:260] lr = 0.00018735994 (27.578 sec) I0716 03:51:30.516970 139795883440000 basic_session_run_hooks.py:692] global_step/sec: 3.87521 I0716 03:51:30.519589 139795883440000 basic_session_run_hooks.py:260] loss = 0.021363582, step = 4568 (25.800 sec) I0716 03:51:30.523059 139795883440000 basic_session_run_hooks.py:260] lr = 0.00018539626 (25.801 sec) I0716 03:51:57.386001 139795883440000 basic_session_run_hooks.py:692] global_step/sec: 3.72176 I0716 03:51:57.389219 139795883440000 basic_session_run_hooks.py:260] loss = 0.029927757, step = 4668 (26.870 sec) I0716 03:51:57.393661 139795883440000 basic_session_run_hooks.py:260] lr = 0.00018345319 (26.871 sec) I0716 03:52:00.255527 139795883440000 basic_session_run_hooks.py:606] Saving checkpoints for 4680 into ../model/elmo_bigru/model.ckpt. I0716 03:52:02.224086 139795883440000 estimator.py:1145] Calling model_fn. I0716 03:52:03.564811 139795883440000 saver.py:1499] Saver not created because there are no variables in the graph to restore I0716 03:52:04.140324 139795883440000 estimator.py:1147] Done calling model_fn. I0716 03:52:04.169637 139795883440000 evaluation.py:255] Starting evaluation at 2019-07-16T03:52:04Z I0716 03:52:04.391069 139795883440000 monitored_session.py:240] Graph was finalized. I0716 03:52:04.406590 139795883440000 saver.py:1280] Restoring parameters from ../model/elmo_bigru/model.ckpt-4680 I0716 03:52:05.334580 139795883440000 session_manager.py:500] Running local_init_op. I0716 03:52:05.390577 139795883440000 session_manager.py:502] Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
I0716 03:53:04.278062 139795883440000 evaluation.py:275] Finished evaluation at 2019-07-16-03:53:04 I0716 03:53:04.279991 139795883440000 estimator.py:2039] Saving dict for global step 4680: global_step = 4680, loss = 0.28495464 I0716 03:53:04.287950 139795883440000 estimator.py:2099] Saving 'checkpoint_path' summary for global step 4680: ../model/elmo_bigru/model.ckpt-4680 I0716 03:53:04.370682 139795883440000 estimator.py:368] Loss for final step: 0.12527332.
Reading ../data/atis.test.w-intent.iob
I0716 03:53:04.963579 139795883440000 estimator.py:1145] Calling model_fn. I0716 03:53:06.610333 139795883440000 saver.py:1499] Saver not created because there are no variables in the graph to restore I0716 03:53:07.134688 139795883440000 estimator.py:1147] Done calling model_fn. I0716 03:53:07.528035 139795883440000 monitored_session.py:240] Graph was finalized. I0716 03:53:07.542665 139795883440000 saver.py:1280] Restoring parameters from ../model/elmo_bigru/model.ckpt-4680 I0716 03:53:08.440604 139795883440000 session_manager.py:500] Running local_init_op. I0716 03:53:08.489479 139795883440000 session_manager.py:502] Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.
'precision', 'predicted', average, warn_for)
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1439: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples.
'recall', 'true', average, warn_for)
I0716 03:54:06.450646 139795883440000 interactiveshell.py:2882]
precision recall f1-score support
atis_flight 0.991 0.992 0.991 632
atis_airfare 0.906 1.000 0.950 48
atis_ground_service 1.000 1.000 1.000 36
atis_airline 0.974 1.000 0.987 38
atis_abbreviation 0.943 1.000 0.971 33
atis_aircraft 1.000 0.889 0.941 9
atis_flight_time 0.500 1.000 0.667 1
atis_quantity 0.429 1.000 0.600 3
atis_flight#atis_airfare 0.857 0.500 0.632 12
atis_airport 1.000 0.944 0.971 18
atis_distance 1.000 1.000 1.000 10
atis_city 0.667 0.667 0.667 6
atis_ground_fare 1.000 0.857 0.923 7
atis_capacity 0.952 0.952 0.952 21
atis_flight_no 0.889 1.000 0.941 8
atis_meal 1.000 0.667 0.800 6
atis_restriction 0.000 0.000 0.000 0
atis_airline#atis_flight_no 0.000 0.000 0.000 0
atis_ground_service#atis_ground_fare 0.000 0.000 0.000 0
atis_airfare#atis_flight_time 0.000 0.000 0.000 0
atis_cheapest 0.000 0.000 0.000 0
atis_aircraft#atis_flight#atis_flight_no 0.000 0.000 0.000 0
micro avg 0.973 0.979 0.976 888
macro avg 0.641 0.658 0.636 888
weighted avg 0.976 0.979 0.976 888
I0716 03:54:06.487351 139795883440000 interactiveshell.py:2882]
precision recall f1-score support
O 0.000 0.000 0.000 0.0
B-toloc.city_name 0.977 0.993 0.985 716.0
B-fromloc.city_name 0.990 0.999 0.994 704.0
I-toloc.city_name 0.985 0.992 0.989 265.0
B-depart_date.day_name 0.991 0.991 0.991 212.0
B-airline_name 1.000 1.000 1.000 101.0
I-fromloc.city_name 0.983 0.994 0.989 177.0
B-depart_time.period_of_day 0.984 0.923 0.952 130.0
I-airline_name 0.985 1.000 0.992 65.0
B-depart_date.day_number 0.981 0.964 0.972 55.0
B-depart_date.month_name 0.981 0.946 0.964 56.0
B-depart_time.time 0.864 1.000 0.927 57.0
B-round_trip 1.000 1.000 1.000 73.0
B-cost_relative 0.973 0.973 0.973 37.0
I-round_trip 1.000 1.000 1.000 71.0
B-flight_mod 0.828 1.000 0.906 24.0
B-depart_time.time_relative 0.970 0.985 0.977 65.0
I-depart_time.time 0.945 1.000 0.972 52.0
B-stoploc.city_name 0.909 1.000 0.952 20.0
B-city_name 0.844 0.667 0.745 57.0
B-class_type 0.960 1.000 0.980 24.0
B-arrive_time.time 0.971 0.971 0.971 34.0
B-arrive_time.time_relative 0.909 0.968 0.937 31.0
I-class_type 1.000 1.000 1.000 17.0
B-flight_stop 1.000 1.000 1.000 21.0
I-arrive_time.time 1.000 0.971 0.986 35.0
B-airline_code 0.895 1.000 0.944 34.0
I-depart_date.day_number 1.000 0.933 0.966 15.0
I-fromloc.airport_name 0.429 1.000 0.600 15.0
B-fromloc.airport_name 0.400 1.000 0.571 12.0
B-arrive_date.day_name 0.786 1.000 0.880 11.0
B-toloc.state_code 1.000 1.000 1.000 18.0
B-depart_date.today_relative 1.000 0.889 0.941 9.0
B-flight_number 0.733 1.000 0.846 11.0
B-depart_date.date_relative 0.944 1.000 0.971 17.0
B-toloc.state_name 0.926 0.893 0.909 28.0
B-fare_basis_code 0.895 1.000 0.944 17.0
B-flight_time 1.000 1.000 1.000 1.0
B-or 1.000 1.000 1.000 3.0
B-arrive_time.period_of_day 0.750 1.000 0.857 6.0
B-meal_description 1.000 0.900 0.947 10.0
I-cost_relative 1.000 0.667 0.800 3.0
I-airport_name 0.846 0.379 0.524 29.0
B-fare_amount 1.000 1.000 1.000 2.0
I-fare_amount 1.000 1.000 1.000 2.0
I-city_name 1.000 0.667 0.800 30.0
I-toloc.airport_name 1.000 1.000 1.000 3.0
B-transport_type 1.000 1.000 1.000 10.0
B-arrive_date.month_name 0.625 0.833 0.714 6.0
B-arrive_date.day_number 0.625 0.833 0.714 6.0
I-stoploc.city_name 0.909 1.000 0.952 10.0
B-meal 0.941 1.000 0.970 16.0
B-fromloc.state_code 1.000 1.000 1.000 23.0
B-depart_time.period_mod 1.000 1.000 1.000 5.0
B-connect 1.000 1.000 1.000 6.0
B-flight_days 1.000 1.000 1.000 10.0
B-toloc.airport_name 1.000 1.000 1.000 3.0
B-fromloc.state_name 1.000 0.941 0.970 17.0
B-airport_name 0.583 0.333 0.424 21.0
B-economy 1.000 1.000 1.000 6.0
I-flight_time 1.000 1.000 1.000 1.0
B-aircraft_code 1.000 0.818 0.900 33.0
B-mod 1.000 0.500 0.667 2.0
B-airport_code 1.000 0.444 0.615 9.0
B-depart_time.start_time 1.000 0.667 0.800 3.0
B-depart_time.end_time 1.000 0.333 0.500 3.0
B-depart_date.year 1.000 1.000 1.000 3.0
I-transport_type 1.000 1.000 1.000 1.0
B-restriction_code 0.800 1.000 0.889 4.0
B-arrive_time.start_time 0.889 1.000 0.941 8.0
B-toloc.airport_code 1.000 0.750 0.857 4.0
B-arrive_time.end_time 0.889 1.000 0.941 8.0
I-arrive_time.end_time 0.889 1.000 0.941 8.0
I-depart_time.end_time 1.000 0.333 0.500 3.0
I-flight_stop 0.000 0.000 0.000 0.0
B-fromloc.airport_code 0.714 1.000 0.833 5.0
I-restriction_code 1.000 1.000 1.000 3.0
I-depart_time.start_time 1.000 1.000 1.000 1.0
I-toloc.state_name 1.000 1.000 1.000 1.0
I-depart_date.today_relative 0.000 0.000 0.000 0.0
B-arrive_date.date_relative 1.000 1.000 1.000 2.0
I-flight_mod 1.000 0.167 0.286 6.0
I-economy 0.000 0.000 0.000 0.0
B-return_date.date_relative 1.000 0.333 0.500 3.0
I-fromloc.state_name 1.000 1.000 1.000 1.0
B-state_code 1.000 1.000 1.000 1.0
I-arrive_time.start_time 1.000 1.000 1.000 1.0
I-arrive_date.day_number 0.000 0.000 0.000 0.0
B-meal_code 0.000 0.000 0.000 1.0
I-depart_time.period_of_day 0.000 0.000 0.000 1.0
B-day_name 1.000 0.500 0.667 2.0
B-period_of_day 1.000 0.750 0.857 4.0
B-stoploc.state_code 0.000 0.000 0.000 0.0
B-return_date.month_name 0.000 0.000 0.000 0.0
B-return_date.day_number 0.000 0.000 0.000 0.0
B-arrive_time.period_mod 0.000 0.000 0.000 0.0
I-meal_code 0.000 0.000 0.000 0.0
B-toloc.country_name 0.000 0.000 0.000 1.0
B-days_code 0.000 0.000 0.000 1.0
I-arrive_time.period_of_day 0.000 0.000 0.000 0.0
I-today_relative 0.000 0.000 0.000 0.0
B-return_time.period_of_day 0.000 0.000 0.000 0.0
B-time 0.000 0.000 0.000 0.0
I-fare_basis_code 0.000 0.000 0.000 0.0
I-arrive_time.time_relative 1.000 0.500 0.667 4.0
I-depart_time.time_relative 0.000 0.000 0.000 1.0
B-today_relative 0.000 0.000 0.000 0.0
B-state_name 0.000 0.000 0.000 9.0
B-arrive_date.today_relative 0.000 0.000 0.000 0.0
B-return_time.period_mod 0.000 0.000 0.000 0.0
B-month_name 0.000 0.000 0.000 0.0
B-day_number 0.000 0.000 0.000 0.0
I-return_date.date_relative 1.000 0.667 0.800 3.0
I-return_date.today_relative 0.000 0.000 0.000 0.0
B-stoploc.airport_name 0.000 0.000 0.000 0.0
B-time_relative 0.000 0.000 0.000 0.0
I-time 0.000 0.000 0.000 0.0
I-return_date.day_number 0.000 0.000 0.000 0.0
I-meal_description 0.000 0.000 0.000 0.0
B-return_date.today_relative 0.000 0.000 0.000 0.0
B-return_date.day_name 0.000 0.000 0.000 2.0
micro avg 0.958 0.960 0.959 3657.0
macro avg 0.674 0.639 0.641 3657.0
weighted avg 0.961 0.960 0.956 3657.0
I0716 03:54:06.507676 139795883440000 interactiveshell.py:2882] Best Slot F1: 0.959
I0716 03:54:06.509477 139795883440000 estimator_training.py:186] Not using Distribute Coordinator.
I0716 03:54:06.514797 139795883440000 training.py:612] Running training and evaluation locally (non-distributed).
I0716 03:54:06.517106 139795883440000 training.py:700] Start train and evaluate loop. The evaluate will happen after every checkpoint. Checkpoint frequency is determined based on RunConfig arguments: save_checkpoints_steps 312 or save_checkpoints_secs None.
I0716 03:54:06.581527 139795883440000 estimator.py:1145] Calling model_fn.
I0716 03:54:07.902871 139795883440000 saver.py:1499] Saver not created because there are no variables in the graph to restore
I0716 03:54:08.654480 139795883440000 <ipython-input-6-d6747b40360b>:55]
[<tf.Variable 'embedding:0' shape=(750, 300) dtype=float32_ref>,
<tf.Variable 'module/aggregation/weights:0' shape=(3,) dtype=float32>,
<tf.Variable 'module/aggregation/scaling:0' shape=() dtype=float32>,
<tf.Variable 'dense/kernel:0' shape=(1324, 300) dtype=float32_ref>,
<tf.Variable 'dense/bias:0' shape=(300,) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/fw/gru_cell/gates/kernel:0' shape=(600, 600) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/fw/gru_cell/gates/bias:0' shape=(600,) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/fw/gru_cell/candidate/kernel:0' shape=(600, 300) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/fw/gru_cell/candidate/bias:0' shape=(300,) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/bw/gru_cell/gates/kernel:0' shape=(600, 600) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/bw/gru_cell/gates/bias:0' shape=(600,) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/bw/gru_cell/candidate/kernel:0' shape=(600, 300) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/bw/gru_cell/candidate/bias:0' shape=(300,) dtype=float32_ref>,
<tf.Variable 'dense_1/kernel:0' shape=(600, 23) dtype=float32_ref>,
<tf.Variable 'dense_1/bias:0' shape=(23,) dtype=float32_ref>,
<tf.Variable 'dense_2/kernel:0' shape=(600, 122) dtype=float32_ref>,
<tf.Variable 'dense_2/bias:0' shape=(122,) dtype=float32_ref>]
I0716 03:54:09.724922 139795883440000 estimator.py:1147] Done calling model_fn.
I0716 03:54:09.729112 139795883440000 basic_session_run_hooks.py:541] Create CheckpointSaverHook.
I0716 03:54:10.054003 139795883440000 monitored_session.py:240] Graph was finalized.
I0716 03:54:10.073008 139795883440000 saver.py:1280] Restoring parameters from ../model/elmo_bigru/model.ckpt-4680
I0716 03:54:11.115986 139795883440000 session_manager.py:500] Running local_init_op.
I0716 03:54:11.183035 139795883440000 session_manager.py:502] Done running local_init_op.
I0716 03:54:14.195451 139795883440000 basic_session_run_hooks.py:606] Saving checkpoints for 4680 into ../model/elmo_bigru/model.ckpt.
Reading ../data/atis.train.w-intent.iob
I0716 03:54:18.893395 139795883440000 basic_session_run_hooks.py:262] loss = 0.023944816, step = 4680 I0716 03:54:18.895148 139795883440000 basic_session_run_hooks.py:262] lr = 0.00018322139 I0716 03:54:47.028027 139795883440000 basic_session_run_hooks.py:692] global_step/sec: 3.55429 I0716 03:54:47.030983 139795883440000 basic_session_run_hooks.py:260] loss = 0.018975822, step = 4780 (28.138 sec) I0716 03:54:47.036970 139795883440000 basic_session_run_hooks.py:260] lr = 0.00018130106 (28.142 sec) I0716 03:55:12.942830 139795883440000 basic_session_run_hooks.py:692] global_step/sec: 3.85878 I0716 03:55:12.945505 139795883440000 basic_session_run_hooks.py:260] loss = 0.015009847, step = 4880 (25.915 sec) I0716 03:55:12.950030 139795883440000 basic_session_run_hooks.py:260] lr = 0.00017940091 (25.913 sec) I0716 03:55:39.014466 139795883440000 basic_session_run_hooks.py:692] global_step/sec: 3.83559 I0716 03:55:39.021158 139795883440000 basic_session_run_hooks.py:260] loss = 0.01981283, step = 4980 (26.076 sec) I0716 03:55:39.023919 139795883440000 basic_session_run_hooks.py:260] lr = 0.00017752068 (26.074 sec) I0716 03:55:41.939601 139795883440000 basic_session_run_hooks.py:606] Saving checkpoints for 4992 into ../model/elmo_bigru/model.ckpt. I0716 03:55:44.422773 139795883440000 estimator.py:1145] Calling model_fn. I0716 03:55:45.824679 139795883440000 saver.py:1499] Saver not created because there are no variables in the graph to restore I0716 03:55:46.418750 139795883440000 estimator.py:1147] Done calling model_fn. I0716 03:55:46.448998 139795883440000 evaluation.py:255] Starting evaluation at 2019-07-16T03:55:46Z I0716 03:55:46.681682 139795883440000 monitored_session.py:240] Graph was finalized. I0716 03:55:46.696691 139795883440000 saver.py:1280] Restoring parameters from ../model/elmo_bigru/model.ckpt-4992 I0716 03:55:47.641804 139795883440000 session_manager.py:500] Running local_init_op. I0716 03:55:47.704630 139795883440000 session_manager.py:502] Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
I0716 03:56:46.573673 139795883440000 evaluation.py:275] Finished evaluation at 2019-07-16-03:56:46 I0716 03:56:46.575764 139795883440000 estimator.py:2039] Saving dict for global step 4992: global_step = 4992, loss = 0.31630886 I0716 03:56:46.581027 139795883440000 estimator.py:2099] Saving 'checkpoint_path' summary for global step 4992: ../model/elmo_bigru/model.ckpt-4992 I0716 03:56:46.659699 139795883440000 estimator.py:368] Loss for final step: 0.006171497.
Reading ../data/atis.test.w-intent.iob
I0716 03:56:47.247567 139795883440000 estimator.py:1145] Calling model_fn. I0716 03:56:48.900026 139795883440000 saver.py:1499] Saver not created because there are no variables in the graph to restore I0716 03:56:49.408294 139795883440000 estimator.py:1147] Done calling model_fn. I0716 03:56:49.634571 139795883440000 monitored_session.py:240] Graph was finalized. I0716 03:56:49.648227 139795883440000 saver.py:1280] Restoring parameters from ../model/elmo_bigru/model.ckpt-4992 I0716 03:56:50.536725 139795883440000 session_manager.py:500] Running local_init_op. I0716 03:56:50.585864 139795883440000 session_manager.py:502] Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.
'precision', 'predicted', average, warn_for)
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1439: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples.
'recall', 'true', average, warn_for)
I0716 03:57:48.300082 139795883440000 interactiveshell.py:2882]
precision recall f1-score support
atis_flight 0.984 0.992 0.988 632
atis_airfare 0.941 1.000 0.970 48
atis_ground_service 1.000 1.000 1.000 36
atis_airline 0.974 1.000 0.987 38
atis_abbreviation 0.943 1.000 0.971 33
atis_aircraft 1.000 0.889 0.941 9
atis_flight_time 0.333 1.000 0.500 1
atis_quantity 0.375 1.000 0.545 3
atis_flight#atis_airfare 0.833 0.417 0.556 12
atis_airport 1.000 0.944 0.971 18
atis_distance 1.000 1.000 1.000 10
atis_city 1.000 0.500 0.667 6
atis_ground_fare 1.000 0.857 0.923 7
atis_capacity 1.000 0.952 0.976 21
atis_flight_no 0.889 1.000 0.941 8
atis_meal 1.000 0.833 0.909 6
atis_restriction 0.000 0.000 0.000 0
atis_airline#atis_flight_no 0.000 0.000 0.000 0
atis_ground_service#atis_ground_fare 0.000 0.000 0.000 0
atis_airfare#atis_flight_time 0.000 0.000 0.000 0
atis_cheapest 0.000 0.000 0.000 0
atis_aircraft#atis_flight#atis_flight_no 0.000 0.000 0.000 0
micro avg 0.972 0.977 0.975 888
macro avg 0.649 0.654 0.629 888
weighted avg 0.976 0.977 0.974 888
I0716 03:57:48.337573 139795883440000 interactiveshell.py:2882]
precision recall f1-score support
O 0.000 0.000 0.000 0.0
B-toloc.city_name 0.974 0.994 0.984 716.0
B-fromloc.city_name 0.990 0.999 0.994 704.0
I-toloc.city_name 0.967 0.992 0.980 265.0
B-depart_date.day_name 0.991 0.991 0.991 212.0
B-airline_name 1.000 1.000 1.000 101.0
I-fromloc.city_name 0.978 0.994 0.986 177.0
B-depart_time.period_of_day 0.983 0.915 0.948 130.0
I-airline_name 1.000 1.000 1.000 65.0
B-depart_date.day_number 0.981 0.964 0.972 55.0
B-depart_date.month_name 0.981 0.946 0.964 56.0
B-depart_time.time 0.864 1.000 0.927 57.0
B-round_trip 1.000 1.000 1.000 73.0
B-cost_relative 0.973 0.973 0.973 37.0
I-round_trip 1.000 1.000 1.000 71.0
B-flight_mod 0.828 1.000 0.906 24.0
B-depart_time.time_relative 0.970 0.985 0.977 65.0
I-depart_time.time 0.929 1.000 0.963 52.0
B-stoploc.city_name 0.909 1.000 0.952 20.0
B-city_name 0.860 0.649 0.740 57.0
B-class_type 0.960 1.000 0.980 24.0
B-arrive_time.time 0.971 0.971 0.971 34.0
B-arrive_time.time_relative 0.857 0.968 0.909 31.0
I-class_type 1.000 1.000 1.000 17.0
B-flight_stop 1.000 1.000 1.000 21.0
I-arrive_time.time 1.000 0.971 0.986 35.0
B-airline_code 0.971 1.000 0.986 34.0
I-depart_date.day_number 1.000 0.933 0.966 15.0
I-fromloc.airport_name 0.417 1.000 0.588 15.0
B-fromloc.airport_name 0.429 1.000 0.600 12.0
B-arrive_date.day_name 0.786 1.000 0.880 11.0
B-toloc.state_code 1.000 1.000 1.000 18.0
B-depart_date.today_relative 1.000 0.889 0.941 9.0
B-flight_number 0.733 1.000 0.846 11.0
B-depart_date.date_relative 0.944 1.000 0.971 17.0
B-toloc.state_name 0.857 0.857 0.857 28.0
B-fare_basis_code 0.850 1.000 0.919 17.0
B-flight_time 1.000 1.000 1.000 1.0
B-or 1.000 1.000 1.000 3.0
B-arrive_time.period_of_day 0.750 1.000 0.857 6.0
B-meal_description 1.000 0.900 0.947 10.0
I-cost_relative 1.000 0.667 0.800 3.0
I-airport_name 0.833 0.345 0.488 29.0
B-fare_amount 1.000 1.000 1.000 2.0
I-fare_amount 1.000 1.000 1.000 2.0
I-city_name 1.000 0.567 0.723 30.0
I-toloc.airport_name 1.000 1.000 1.000 3.0
B-transport_type 1.000 1.000 1.000 10.0
B-arrive_date.month_name 0.625 0.833 0.714 6.0
B-arrive_date.day_number 0.714 0.833 0.769 6.0
I-stoploc.city_name 0.909 1.000 0.952 10.0
B-meal 1.000 1.000 1.000 16.0
B-fromloc.state_code 1.000 1.000 1.000 23.0
B-depart_time.period_mod 1.000 1.000 1.000 5.0
B-connect 1.000 1.000 1.000 6.0
B-flight_days 1.000 1.000 1.000 10.0
B-toloc.airport_name 1.000 1.000 1.000 3.0
B-fromloc.state_name 1.000 1.000 1.000 17.0
B-airport_name 0.636 0.333 0.437 21.0
B-economy 1.000 0.333 0.500 6.0
I-flight_time 1.000 1.000 1.000 1.0
B-aircraft_code 1.000 0.818 0.900 33.0
B-mod 1.000 0.500 0.667 2.0
B-airport_code 0.667 0.444 0.533 9.0
B-depart_time.start_time 1.000 0.667 0.800 3.0
B-depart_time.end_time 1.000 0.333 0.500 3.0
B-depart_date.year 1.000 1.000 1.000 3.0
I-transport_type 0.000 0.000 0.000 1.0
B-restriction_code 0.800 1.000 0.889 4.0
B-arrive_time.start_time 0.889 1.000 0.941 8.0
B-toloc.airport_code 1.000 0.750 0.857 4.0
B-arrive_time.end_time 0.889 1.000 0.941 8.0
I-arrive_time.end_time 0.889 1.000 0.941 8.0
I-depart_time.end_time 1.000 0.333 0.500 3.0
I-flight_stop 0.000 0.000 0.000 0.0
B-fromloc.airport_code 0.625 1.000 0.769 5.0
I-restriction_code 1.000 1.000 1.000 3.0
I-depart_time.start_time 1.000 1.000 1.000 1.0
I-toloc.state_name 1.000 1.000 1.000 1.0
I-depart_date.today_relative 0.000 0.000 0.000 0.0
B-arrive_date.date_relative 1.000 1.000 1.000 2.0
I-flight_mod 1.000 0.167 0.286 6.0
I-economy 0.000 0.000 0.000 0.0
B-return_date.date_relative 1.000 0.667 0.800 3.0
I-fromloc.state_name 1.000 1.000 1.000 1.0
B-state_code 1.000 1.000 1.000 1.0
I-arrive_time.start_time 1.000 1.000 1.000 1.0
I-arrive_date.day_number 0.000 0.000 0.000 0.0
B-meal_code 0.000 0.000 0.000 1.0
I-depart_time.period_of_day 0.000 0.000 0.000 1.0
B-day_name 1.000 0.500 0.667 2.0
B-period_of_day 1.000 0.750 0.857 4.0
B-stoploc.state_code 0.000 0.000 0.000 0.0
B-return_date.month_name 0.000 0.000 0.000 0.0
B-return_date.day_number 0.000 0.000 0.000 0.0
B-arrive_time.period_mod 0.000 0.000 0.000 0.0
I-meal_code 0.000 0.000 0.000 0.0
B-toloc.country_name 1.000 1.000 1.000 1.0
B-days_code 0.000 0.000 0.000 1.0
I-arrive_time.period_of_day 0.000 0.000 0.000 0.0
I-today_relative 0.000 0.000 0.000 0.0
B-return_time.period_of_day 0.000 0.000 0.000 0.0
B-time 0.000 0.000 0.000 0.0
I-fare_basis_code 0.000 0.000 0.000 0.0
I-arrive_time.time_relative 0.000 0.000 0.000 4.0
I-depart_time.time_relative 0.000 0.000 0.000 1.0
B-today_relative 0.000 0.000 0.000 0.0
B-state_name 0.000 0.000 0.000 9.0
B-arrive_date.today_relative 0.000 0.000 0.000 0.0
B-return_time.period_mod 0.000 0.000 0.000 0.0
B-month_name 0.000 0.000 0.000 0.0
B-day_number 0.000 0.000 0.000 0.0
I-return_date.date_relative 1.000 0.667 0.800 3.0
I-return_date.today_relative 0.000 0.000 0.000 0.0
B-stoploc.airport_name 0.000 0.000 0.000 0.0
B-time_relative 0.000 0.000 0.000 0.0
I-time 0.000 0.000 0.000 0.0
I-return_date.day_number 0.000 0.000 0.000 0.0
I-meal_description 0.000 0.000 0.000 0.0
B-return_date.today_relative 0.000 0.000 0.000 0.0
B-return_date.day_name 0.000 0.000 0.000 2.0
micro avg 0.955 0.957 0.956 3657.0
macro avg 0.663 0.631 0.632 3657.0
weighted avg 0.957 0.957 0.952 3657.0
I0716 03:57:48.358943 139795883440000 interactiveshell.py:2882] Best Slot F1: 0.959
I0716 03:57:48.360575 139795883440000 estimator_training.py:186] Not using Distribute Coordinator.
I0716 03:57:48.362465 139795883440000 training.py:612] Running training and evaluation locally (non-distributed).
I0716 03:57:48.364193 139795883440000 training.py:700] Start train and evaluate loop. The evaluate will happen after every checkpoint. Checkpoint frequency is determined based on RunConfig arguments: save_checkpoints_steps 312 or save_checkpoints_secs None.
I0716 03:57:48.430262 139795883440000 estimator.py:1145] Calling model_fn.
I0716 03:57:50.047309 139795883440000 saver.py:1499] Saver not created because there are no variables in the graph to restore
I0716 03:57:50.645043 139795883440000 <ipython-input-6-d6747b40360b>:55]
[<tf.Variable 'embedding:0' shape=(750, 300) dtype=float32_ref>,
<tf.Variable 'module/aggregation/weights:0' shape=(3,) dtype=float32>,
<tf.Variable 'module/aggregation/scaling:0' shape=() dtype=float32>,
<tf.Variable 'dense/kernel:0' shape=(1324, 300) dtype=float32_ref>,
<tf.Variable 'dense/bias:0' shape=(300,) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/fw/gru_cell/gates/kernel:0' shape=(600, 600) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/fw/gru_cell/gates/bias:0' shape=(600,) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/fw/gru_cell/candidate/kernel:0' shape=(600, 300) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/fw/gru_cell/candidate/bias:0' shape=(300,) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/bw/gru_cell/gates/kernel:0' shape=(600, 600) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/bw/gru_cell/gates/bias:0' shape=(600,) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/bw/gru_cell/candidate/kernel:0' shape=(600, 300) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/bw/gru_cell/candidate/bias:0' shape=(300,) dtype=float32_ref>,
<tf.Variable 'dense_1/kernel:0' shape=(600, 23) dtype=float32_ref>,
<tf.Variable 'dense_1/bias:0' shape=(23,) dtype=float32_ref>,
<tf.Variable 'dense_2/kernel:0' shape=(600, 122) dtype=float32_ref>,
<tf.Variable 'dense_2/bias:0' shape=(122,) dtype=float32_ref>]
I0716 03:57:51.722529 139795883440000 estimator.py:1147] Done calling model_fn.
I0716 03:57:51.725912 139795883440000 basic_session_run_hooks.py:541] Create CheckpointSaverHook.
I0716 03:57:52.252693 139795883440000 monitored_session.py:240] Graph was finalized.
I0716 03:57:52.268751 139795883440000 saver.py:1280] Restoring parameters from ../model/elmo_bigru/model.ckpt-4992
I0716 03:57:53.320537 139795883440000 session_manager.py:500] Running local_init_op.
I0716 03:57:53.384846 139795883440000 session_manager.py:502] Done running local_init_op.
I0716 03:57:56.176370 139795883440000 basic_session_run_hooks.py:606] Saving checkpoints for 4992 into ../model/elmo_bigru/model.ckpt.
Reading ../data/atis.train.w-intent.iob
I0716 03:58:00.755765 139795883440000 basic_session_run_hooks.py:262] loss = 0.018470936, step = 4992 I0716 03:58:00.757390 139795883440000 basic_session_run_hooks.py:262] lr = 0.00017729634 I0716 03:58:28.238779 139795883440000 basic_session_run_hooks.py:692] global_step/sec: 3.63854 I0716 03:58:28.243356 139795883440000 basic_session_run_hooks.py:260] loss = 0.011454404, step = 5092 (27.488 sec) I0716 03:58:28.247864 139795883440000 basic_session_run_hooks.py:260] lr = 0.00017543815 (27.490 sec) I0716 03:58:54.253058 139795883440000 basic_session_run_hooks.py:692] global_step/sec: 3.84406 I0716 03:58:54.259320 139795883440000 basic_session_run_hooks.py:260] loss = 0.016113179, step = 5192 (26.016 sec) I0716 03:58:54.261050 139795883440000 basic_session_run_hooks.py:260] lr = 0.00017359943 (26.013 sec) I0716 03:59:20.595207 139795883440000 basic_session_run_hooks.py:692] global_step/sec: 3.79619 I0716 03:59:20.600870 139795883440000 basic_session_run_hooks.py:260] loss = 0.014878462, step = 5292 (26.342 sec) I0716 03:59:20.602448 139795883440000 basic_session_run_hooks.py:260] lr = 0.00017177999 (26.341 sec) I0716 03:59:23.645135 139795883440000 basic_session_run_hooks.py:606] Saving checkpoints for 5304 into ../model/elmo_bigru/model.ckpt. I0716 03:59:25.506482 139795883440000 estimator.py:1145] Calling model_fn. I0716 03:59:26.849341 139795883440000 saver.py:1499] Saver not created because there are no variables in the graph to restore I0716 03:59:27.427505 139795883440000 estimator.py:1147] Done calling model_fn. I0716 03:59:27.457143 139795883440000 evaluation.py:255] Starting evaluation at 2019-07-16T03:59:27Z I0716 03:59:27.937038 139795883440000 monitored_session.py:240] Graph was finalized. I0716 03:59:27.952298 139795883440000 saver.py:1280] Restoring parameters from ../model/elmo_bigru/model.ckpt-5304 I0716 03:59:28.839442 139795883440000 session_manager.py:500] Running local_init_op. I0716 03:59:28.898953 139795883440000 session_manager.py:502] Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
I0716 04:00:27.577882 139795883440000 evaluation.py:275] Finished evaluation at 2019-07-16-04:00:27 I0716 04:00:27.579757 139795883440000 estimator.py:2039] Saving dict for global step 5304: global_step = 5304, loss = 0.3132215 I0716 04:00:27.585969 139795883440000 estimator.py:2099] Saving 'checkpoint_path' summary for global step 5304: ../model/elmo_bigru/model.ckpt-5304 I0716 04:00:27.668098 139795883440000 estimator.py:368] Loss for final step: 0.009373315.
Reading ../data/atis.test.w-intent.iob
I0716 04:00:28.264626 139795883440000 estimator.py:1145] Calling model_fn. I0716 04:00:29.294847 139795883440000 saver.py:1499] Saver not created because there are no variables in the graph to restore I0716 04:00:29.809007 139795883440000 estimator.py:1147] Done calling model_fn. I0716 04:00:30.021951 139795883440000 monitored_session.py:240] Graph was finalized. I0716 04:00:30.037668 139795883440000 saver.py:1280] Restoring parameters from ../model/elmo_bigru/model.ckpt-5304 I0716 04:00:30.928122 139795883440000 session_manager.py:500] Running local_init_op. I0716 04:00:30.974180 139795883440000 session_manager.py:502] Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.
'precision', 'predicted', average, warn_for)
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1439: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples.
'recall', 'true', average, warn_for)
I0716 04:01:29.005527 139795883440000 interactiveshell.py:2882]
precision recall f1-score support
atis_flight 0.986 0.992 0.989 632
atis_airfare 0.941 1.000 0.970 48
atis_ground_service 1.000 1.000 1.000 36
atis_airline 0.974 1.000 0.987 38
atis_abbreviation 0.943 1.000 0.971 33
atis_aircraft 1.000 0.889 0.941 9
atis_flight_time 0.500 1.000 0.667 1
atis_quantity 0.429 1.000 0.600 3
atis_flight#atis_airfare 0.833 0.417 0.556 12
atis_airport 1.000 0.944 0.971 18
atis_distance 1.000 1.000 1.000 10
atis_city 0.667 0.667 0.667 6
atis_ground_fare 1.000 0.857 0.923 7
atis_capacity 0.952 0.952 0.952 21
atis_flight_no 0.889 1.000 0.941 8
atis_meal 1.000 0.667 0.800 6
atis_restriction 0.000 0.000 0.000 0
atis_airline#atis_flight_no 0.000 0.000 0.000 0
atis_ground_service#atis_ground_fare 0.000 0.000 0.000 0
atis_airfare#atis_flight_time 0.000 0.000 0.000 0
atis_cheapest 0.000 0.000 0.000 0
atis_aircraft#atis_flight#atis_flight_no 0.000 0.000 0.000 0
micro avg 0.972 0.977 0.975 888
macro avg 0.642 0.654 0.633 888
weighted avg 0.974 0.977 0.974 888
I0716 04:01:29.041616 139795883440000 interactiveshell.py:2882]
precision recall f1-score support
O 0.000 0.000 0.000 0.0
B-toloc.city_name 0.977 0.993 0.985 716.0
B-fromloc.city_name 0.992 0.999 0.995 704.0
I-toloc.city_name 0.981 0.992 0.987 265.0
B-depart_date.day_name 0.991 0.991 0.991 212.0
B-airline_name 1.000 1.000 1.000 101.0
I-fromloc.city_name 0.983 1.000 0.992 177.0
B-depart_time.period_of_day 0.983 0.908 0.944 130.0
I-airline_name 1.000 1.000 1.000 65.0
B-depart_date.day_number 0.981 0.964 0.972 55.0
B-depart_date.month_name 0.981 0.946 0.964 56.0
B-depart_time.time 0.864 1.000 0.927 57.0
B-round_trip 1.000 1.000 1.000 73.0
B-cost_relative 0.973 0.973 0.973 37.0
I-round_trip 1.000 1.000 1.000 71.0
B-flight_mod 0.828 1.000 0.906 24.0
B-depart_time.time_relative 0.970 0.985 0.977 65.0
I-depart_time.time 0.912 1.000 0.954 52.0
B-stoploc.city_name 0.833 1.000 0.909 20.0
B-city_name 0.884 0.667 0.760 57.0
B-class_type 0.960 1.000 0.980 24.0
B-arrive_time.time 0.971 0.971 0.971 34.0
B-arrive_time.time_relative 0.882 0.968 0.923 31.0
I-class_type 1.000 1.000 1.000 17.0
B-flight_stop 1.000 1.000 1.000 21.0
I-arrive_time.time 1.000 0.971 0.986 35.0
B-airline_code 0.971 1.000 0.986 34.0
I-depart_date.day_number 1.000 0.933 0.966 15.0
I-fromloc.airport_name 0.441 1.000 0.612 15.0
B-fromloc.airport_name 0.444 1.000 0.615 12.0
B-arrive_date.day_name 0.786 1.000 0.880 11.0
B-toloc.state_code 1.000 1.000 1.000 18.0
B-depart_date.today_relative 1.000 0.889 0.941 9.0
B-flight_number 0.733 1.000 0.846 11.0
B-depart_date.date_relative 0.944 1.000 0.971 17.0
B-toloc.state_name 0.833 0.893 0.862 28.0
B-fare_basis_code 0.773 1.000 0.872 17.0
B-flight_time 1.000 1.000 1.000 1.0
B-or 1.000 1.000 1.000 3.0
B-arrive_time.period_of_day 0.750 1.000 0.857 6.0
B-meal_description 1.000 0.900 0.947 10.0
I-cost_relative 1.000 0.667 0.800 3.0
I-airport_name 0.846 0.379 0.524 29.0
B-fare_amount 1.000 1.000 1.000 2.0
I-fare_amount 1.000 1.000 1.000 2.0
I-city_name 1.000 0.633 0.776 30.0
I-toloc.airport_name 1.000 1.000 1.000 3.0
B-transport_type 1.000 1.000 1.000 10.0
B-arrive_date.month_name 0.625 0.833 0.714 6.0
B-arrive_date.day_number 0.714 0.833 0.769 6.0
I-stoploc.city_name 0.909 1.000 0.952 10.0
B-meal 1.000 1.000 1.000 16.0
B-fromloc.state_code 1.000 1.000 1.000 23.0
B-depart_time.period_mod 1.000 1.000 1.000 5.0
B-connect 1.000 1.000 1.000 6.0
B-flight_days 1.000 1.000 1.000 10.0
B-toloc.airport_name 1.000 1.000 1.000 3.0
B-fromloc.state_name 1.000 1.000 1.000 17.0
B-airport_name 0.636 0.333 0.437 21.0
B-economy 1.000 1.000 1.000 6.0
I-flight_time 1.000 1.000 1.000 1.0
B-aircraft_code 1.000 0.818 0.900 33.0
B-mod 1.000 0.500 0.667 2.0
B-airport_code 1.000 0.444 0.615 9.0
B-depart_time.start_time 1.000 0.667 0.800 3.0
B-depart_time.end_time 1.000 0.333 0.500 3.0
B-depart_date.year 1.000 1.000 1.000 3.0
I-transport_type 1.000 1.000 1.000 1.0
B-restriction_code 0.800 1.000 0.889 4.0
B-arrive_time.start_time 0.889 1.000 0.941 8.0
B-toloc.airport_code 1.000 0.750 0.857 4.0
B-arrive_time.end_time 0.889 1.000 0.941 8.0
I-arrive_time.end_time 0.889 1.000 0.941 8.0
I-depart_time.end_time 1.000 0.333 0.500 3.0
I-flight_stop 0.000 0.000 0.000 0.0
B-fromloc.airport_code 0.556 1.000 0.714 5.0
I-restriction_code 1.000 1.000 1.000 3.0
I-depart_time.start_time 1.000 1.000 1.000 1.0
I-toloc.state_name 1.000 1.000 1.000 1.0
I-depart_date.today_relative 0.000 0.000 0.000 0.0
B-arrive_date.date_relative 1.000 1.000 1.000 2.0
I-flight_mod 1.000 0.167 0.286 6.0
I-economy 0.000 0.000 0.000 0.0
B-return_date.date_relative 1.000 0.667 0.800 3.0
I-fromloc.state_name 1.000 1.000 1.000 1.0
B-state_code 1.000 1.000 1.000 1.0
I-arrive_time.start_time 1.000 1.000 1.000 1.0
I-arrive_date.day_number 0.000 0.000 0.000 0.0
B-meal_code 0.000 0.000 0.000 1.0
I-depart_time.period_of_day 0.000 0.000 0.000 1.0
B-day_name 1.000 0.500 0.667 2.0
B-period_of_day 1.000 0.750 0.857 4.0
B-stoploc.state_code 0.000 0.000 0.000 0.0
B-return_date.month_name 0.000 0.000 0.000 0.0
B-return_date.day_number 0.000 0.000 0.000 0.0
B-arrive_time.period_mod 0.000 0.000 0.000 0.0
I-meal_code 0.000 0.000 0.000 0.0
B-toloc.country_name 1.000 1.000 1.000 1.0
B-days_code 0.000 0.000 0.000 1.0
I-arrive_time.period_of_day 0.000 0.000 0.000 0.0
I-today_relative 0.000 0.000 0.000 0.0
B-return_time.period_of_day 0.000 0.000 0.000 0.0
B-time 0.000 0.000 0.000 0.0
I-fare_basis_code 0.000 0.000 0.000 0.0
I-arrive_time.time_relative 1.000 0.250 0.400 4.0
I-depart_time.time_relative 0.000 0.000 0.000 1.0
B-today_relative 0.000 0.000 0.000 0.0
B-state_name 0.000 0.000 0.000 9.0
B-arrive_date.today_relative 0.000 0.000 0.000 0.0
B-return_time.period_mod 0.000 0.000 0.000 0.0
B-month_name 0.000 0.000 0.000 0.0
B-day_number 0.000 0.000 0.000 0.0
I-return_date.date_relative 1.000 0.667 0.800 3.0
I-return_date.today_relative 0.000 0.000 0.000 0.0
B-stoploc.airport_name 0.000 0.000 0.000 0.0
B-time_relative 0.000 0.000 0.000 0.0
I-time 0.000 0.000 0.000 0.0
I-return_date.day_number 0.000 0.000 0.000 0.0
I-meal_description 0.000 0.000 0.000 0.0
B-return_date.today_relative 0.000 0.000 0.000 0.0
B-return_date.day_name 0.000 0.000 0.000 2.0
micro avg 0.958 0.960 0.959 3657.0
macro avg 0.681 0.648 0.649 3657.0
weighted avg 0.961 0.960 0.956 3657.0
I0716 04:01:29.061470 139795883440000 interactiveshell.py:2882] Best Slot F1: 0.959
I0716 04:01:29.063330 139795883440000 estimator_training.py:186] Not using Distribute Coordinator.
I0716 04:01:29.065515 139795883440000 training.py:612] Running training and evaluation locally (non-distributed).
I0716 04:01:29.068432 139795883440000 training.py:700] Start train and evaluate loop. The evaluate will happen after every checkpoint. Checkpoint frequency is determined based on RunConfig arguments: save_checkpoints_steps 312 or save_checkpoints_secs None.
I0716 04:01:29.132260 139795883440000 estimator.py:1145] Calling model_fn.
I0716 04:01:31.079405 139795883440000 saver.py:1499] Saver not created because there are no variables in the graph to restore
I0716 04:01:31.658530 139795883440000 <ipython-input-6-d6747b40360b>:55]
[<tf.Variable 'embedding:0' shape=(750, 300) dtype=float32_ref>,
<tf.Variable 'module/aggregation/weights:0' shape=(3,) dtype=float32>,
<tf.Variable 'module/aggregation/scaling:0' shape=() dtype=float32>,
<tf.Variable 'dense/kernel:0' shape=(1324, 300) dtype=float32_ref>,
<tf.Variable 'dense/bias:0' shape=(300,) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/fw/gru_cell/gates/kernel:0' shape=(600, 600) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/fw/gru_cell/gates/bias:0' shape=(600,) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/fw/gru_cell/candidate/kernel:0' shape=(600, 300) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/fw/gru_cell/candidate/bias:0' shape=(300,) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/bw/gru_cell/gates/kernel:0' shape=(600, 600) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/bw/gru_cell/gates/bias:0' shape=(600,) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/bw/gru_cell/candidate/kernel:0' shape=(600, 300) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/bw/gru_cell/candidate/bias:0' shape=(300,) dtype=float32_ref>,
<tf.Variable 'dense_1/kernel:0' shape=(600, 23) dtype=float32_ref>,
<tf.Variable 'dense_1/bias:0' shape=(23,) dtype=float32_ref>,
<tf.Variable 'dense_2/kernel:0' shape=(600, 122) dtype=float32_ref>,
<tf.Variable 'dense_2/bias:0' shape=(122,) dtype=float32_ref>]
I0716 04:01:32.732068 139795883440000 estimator.py:1147] Done calling model_fn.
I0716 04:01:32.735795 139795883440000 basic_session_run_hooks.py:541] Create CheckpointSaverHook.
I0716 04:01:33.267443 139795883440000 monitored_session.py:240] Graph was finalized.
I0716 04:01:33.287615 139795883440000 saver.py:1280] Restoring parameters from ../model/elmo_bigru/model.ckpt-5304
I0716 04:01:34.324961 139795883440000 session_manager.py:500] Running local_init_op.
I0716 04:01:34.390909 139795883440000 session_manager.py:502] Done running local_init_op.
I0716 04:01:37.174355 139795883440000 basic_session_run_hooks.py:606] Saving checkpoints for 5304 into ../model/elmo_bigru/model.ckpt.
Reading ../data/atis.train.w-intent.iob
I0716 04:01:41.832364 139795883440000 basic_session_run_hooks.py:262] loss = 0.01692254, step = 5304 I0716 04:01:41.834603 139795883440000 basic_session_run_hooks.py:262] lr = 0.00017156293 I0716 04:02:08.931746 139795883440000 basic_session_run_hooks.py:692] global_step/sec: 3.69005 I0716 04:02:08.937292 139795883440000 basic_session_run_hooks.py:260] loss = 0.016833542, step = 5404 (27.105 sec) I0716 04:02:08.938875 139795883440000 basic_session_run_hooks.py:260] lr = 0.00016976481 (27.104 sec) I0716 04:02:35.486347 139795883440000 basic_session_run_hooks.py:692] global_step/sec: 3.76582 I0716 04:02:35.491731 139795883440000 basic_session_run_hooks.py:260] loss = 0.017936949, step = 5504 (26.554 sec) I0716 04:02:35.493553 139795883440000 basic_session_run_hooks.py:260] lr = 0.00016798556 (26.555 sec) I0716 04:03:02.122225 139795883440000 basic_session_run_hooks.py:692] global_step/sec: 3.75434 I0716 04:03:02.127030 139795883440000 basic_session_run_hooks.py:260] loss = 0.013849929, step = 5604 (26.635 sec) I0716 04:03:02.130866 139795883440000 basic_session_run_hooks.py:260] lr = 0.00016622494 (26.637 sec) I0716 04:03:04.851873 139795883440000 basic_session_run_hooks.py:606] Saving checkpoints for 5616 into ../model/elmo_bigru/model.ckpt. I0716 04:03:06.795298 139795883440000 estimator.py:1145] Calling model_fn. I0716 04:03:08.144297 139795883440000 saver.py:1499] Saver not created because there are no variables in the graph to restore I0716 04:03:08.724595 139795883440000 estimator.py:1147] Done calling model_fn. I0716 04:03:08.752614 139795883440000 evaluation.py:255] Starting evaluation at 2019-07-16T04:03:08Z I0716 04:03:09.239857 139795883440000 monitored_session.py:240] Graph was finalized. I0716 04:03:09.260785 139795883440000 saver.py:1280] Restoring parameters from ../model/elmo_bigru/model.ckpt-5616 I0716 04:03:10.142755 139795883440000 session_manager.py:500] Running local_init_op. I0716 04:03:10.198426 139795883440000 session_manager.py:502] Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
I0716 04:04:08.997198 139795883440000 evaluation.py:275] Finished evaluation at 2019-07-16-04:04:08 I0716 04:04:08.999469 139795883440000 estimator.py:2039] Saving dict for global step 5616: global_step = 5616, loss = 0.32418722 I0716 04:04:09.004559 139795883440000 estimator.py:2099] Saving 'checkpoint_path' summary for global step 5616: ../model/elmo_bigru/model.ckpt-5616 I0716 04:04:09.090413 139795883440000 estimator.py:368] Loss for final step: 0.009361349.
Reading ../data/atis.test.w-intent.iob
I0716 04:04:09.697658 139795883440000 estimator.py:1145] Calling model_fn. I0716 04:04:10.716241 139795883440000 saver.py:1499] Saver not created because there are no variables in the graph to restore I0716 04:04:11.224436 139795883440000 estimator.py:1147] Done calling model_fn. I0716 04:04:11.439194 139795883440000 monitored_session.py:240] Graph was finalized. I0716 04:04:11.453939 139795883440000 saver.py:1280] Restoring parameters from ../model/elmo_bigru/model.ckpt-5616 I0716 04:04:12.335368 139795883440000 session_manager.py:500] Running local_init_op. I0716 04:04:12.383247 139795883440000 session_manager.py:502] Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.
'precision', 'predicted', average, warn_for)
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1439: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples.
'recall', 'true', average, warn_for)
I0716 04:05:10.217454 139795883440000 interactiveshell.py:2882]
precision recall f1-score support
atis_flight 0.984 0.992 0.988 632
atis_airfare 0.941 1.000 0.970 48
atis_ground_service 1.000 1.000 1.000 36
atis_airline 0.974 1.000 0.987 38
atis_abbreviation 0.943 1.000 0.971 33
atis_aircraft 1.000 0.889 0.941 9
atis_flight_time 0.500 1.000 0.667 1
atis_quantity 0.375 1.000 0.545 3
atis_flight#atis_airfare 0.833 0.417 0.556 12
atis_airport 1.000 0.944 0.971 18
atis_distance 1.000 1.000 1.000 10
atis_city 0.600 0.500 0.545 6
atis_ground_fare 1.000 0.857 0.923 7
atis_capacity 1.000 0.952 0.976 21
atis_flight_no 0.889 1.000 0.941 8
atis_meal 1.000 0.667 0.800 6
atis_restriction 0.000 0.000 0.000 0
atis_airline#atis_flight_no 0.000 0.000 0.000 0
atis_ground_service#atis_ground_fare 0.000 0.000 0.000 0
atis_airfare#atis_flight_time 0.000 0.000 0.000 0
atis_cheapest 0.000 0.000 0.000 0
atis_aircraft#atis_flight#atis_flight_no 0.000 0.000 0.000 0
micro avg 0.971 0.976 0.974 888
macro avg 0.638 0.646 0.626 888
weighted avg 0.974 0.976 0.973 888
I0716 04:05:10.252486 139795883440000 interactiveshell.py:2882]
precision recall f1-score support
O 0.000 0.000 0.000 0.0
B-toloc.city_name 0.978 0.990 0.984 716.0
B-fromloc.city_name 0.983 0.999 0.991 704.0
I-toloc.city_name 0.985 0.977 0.981 265.0
B-depart_date.day_name 0.991 0.991 0.991 212.0
B-airline_name 1.000 1.000 1.000 101.0
I-fromloc.city_name 0.957 0.994 0.975 177.0
B-depart_time.period_of_day 0.983 0.908 0.944 130.0
I-airline_name 1.000 1.000 1.000 65.0
B-depart_date.day_number 0.981 0.964 0.972 55.0
B-depart_date.month_name 0.982 0.964 0.973 56.0
B-depart_time.time 0.864 1.000 0.927 57.0
B-round_trip 1.000 1.000 1.000 73.0
B-cost_relative 0.973 0.973 0.973 37.0
I-round_trip 1.000 1.000 1.000 71.0
B-flight_mod 0.828 1.000 0.906 24.0
B-depart_time.time_relative 0.955 0.985 0.970 65.0
I-depart_time.time 0.912 1.000 0.954 52.0
B-stoploc.city_name 0.952 1.000 0.976 20.0
B-city_name 0.927 0.667 0.776 57.0
B-class_type 0.960 1.000 0.980 24.0
B-arrive_time.time 0.971 0.971 0.971 34.0
B-arrive_time.time_relative 0.879 0.935 0.906 31.0
I-class_type 1.000 1.000 1.000 17.0
B-flight_stop 1.000 1.000 1.000 21.0
I-arrive_time.time 1.000 0.971 0.986 35.0
B-airline_code 0.919 1.000 0.958 34.0
I-depart_date.day_number 1.000 0.933 0.966 15.0
I-fromloc.airport_name 0.417 1.000 0.588 15.0
B-fromloc.airport_name 0.414 1.000 0.585 12.0
B-arrive_date.day_name 0.786 1.000 0.880 11.0
B-toloc.state_code 1.000 1.000 1.000 18.0
B-depart_date.today_relative 1.000 0.889 0.941 9.0
B-flight_number 0.733 1.000 0.846 11.0
B-depart_date.date_relative 0.944 1.000 0.971 17.0
B-toloc.state_name 0.833 0.893 0.862 28.0
B-fare_basis_code 0.895 1.000 0.944 17.0
B-flight_time 1.000 1.000 1.000 1.0
B-or 1.000 1.000 1.000 3.0
B-arrive_time.period_of_day 0.750 1.000 0.857 6.0
B-meal_description 1.000 0.900 0.947 10.0
I-cost_relative 1.000 0.667 0.800 3.0
I-airport_name 0.769 0.345 0.476 29.0
B-fare_amount 1.000 1.000 1.000 2.0
I-fare_amount 1.000 1.000 1.000 2.0
I-city_name 1.000 0.567 0.723 30.0
I-toloc.airport_name 1.000 1.000 1.000 3.0
B-transport_type 1.000 1.000 1.000 10.0
B-arrive_date.month_name 0.714 0.833 0.769 6.0
B-arrive_date.day_number 0.714 0.833 0.769 6.0
I-stoploc.city_name 0.909 1.000 0.952 10.0
B-meal 1.000 1.000 1.000 16.0
B-fromloc.state_code 1.000 1.000 1.000 23.0
B-depart_time.period_mod 1.000 1.000 1.000 5.0
B-connect 1.000 1.000 1.000 6.0
B-flight_days 1.000 1.000 1.000 10.0
B-toloc.airport_name 1.000 1.000 1.000 3.0
B-fromloc.state_name 0.944 1.000 0.971 17.0
B-airport_name 0.636 0.333 0.437 21.0
B-economy 1.000 1.000 1.000 6.0
I-flight_time 1.000 1.000 1.000 1.0
B-aircraft_code 1.000 0.818 0.900 33.0
B-mod 1.000 0.500 0.667 2.0
B-airport_code 1.000 0.444 0.615 9.0
B-depart_time.start_time 1.000 1.000 1.000 3.0
B-depart_time.end_time 1.000 0.333 0.500 3.0
B-depart_date.year 1.000 1.000 1.000 3.0
I-transport_type 1.000 1.000 1.000 1.0
B-restriction_code 0.800 1.000 0.889 4.0
B-arrive_time.start_time 1.000 1.000 1.000 8.0
B-toloc.airport_code 1.000 0.750 0.857 4.0
B-arrive_time.end_time 0.889 1.000 0.941 8.0
I-arrive_time.end_time 0.889 1.000 0.941 8.0
I-depart_time.end_time 1.000 0.333 0.500 3.0
I-flight_stop 0.000 0.000 0.000 0.0
B-fromloc.airport_code 0.625 1.000 0.769 5.0
I-restriction_code 1.000 1.000 1.000 3.0
I-depart_time.start_time 1.000 1.000 1.000 1.0
I-toloc.state_name 1.000 1.000 1.000 1.0
I-depart_date.today_relative 0.000 0.000 0.000 0.0
B-arrive_date.date_relative 1.000 1.000 1.000 2.0
I-flight_mod 1.000 0.167 0.286 6.0
I-economy 0.000 0.000 0.000 0.0
B-return_date.date_relative 1.000 0.667 0.800 3.0
I-fromloc.state_name 1.000 1.000 1.000 1.0
B-state_code 1.000 1.000 1.000 1.0
I-arrive_time.start_time 1.000 1.000 1.000 1.0
I-arrive_date.day_number 0.000 0.000 0.000 0.0
B-meal_code 0.000 0.000 0.000 1.0
I-depart_time.period_of_day 0.000 0.000 0.000 1.0
B-day_name 1.000 0.500 0.667 2.0
B-period_of_day 1.000 0.750 0.857 4.0
B-stoploc.state_code 0.000 0.000 0.000 0.0
B-return_date.month_name 0.000 0.000 0.000 0.0
B-return_date.day_number 0.000 0.000 0.000 0.0
B-arrive_time.period_mod 0.000 0.000 0.000 0.0
I-meal_code 0.000 0.000 0.000 0.0
B-toloc.country_name 1.000 1.000 1.000 1.0
B-days_code 0.000 0.000 0.000 1.0
I-arrive_time.period_of_day 0.000 0.000 0.000 0.0
I-today_relative 0.000 0.000 0.000 0.0
B-return_time.period_of_day 0.000 0.000 0.000 0.0
B-time 0.000 0.000 0.000 0.0
I-fare_basis_code 0.000 0.000 0.000 0.0
I-arrive_time.time_relative 1.000 0.250 0.400 4.0
I-depart_time.time_relative 0.000 0.000 0.000 1.0
B-today_relative 0.000 0.000 0.000 0.0
B-state_name 0.000 0.000 0.000 9.0
B-arrive_date.today_relative 0.000 0.000 0.000 0.0
B-return_time.period_mod 0.000 0.000 0.000 0.0
B-month_name 0.000 0.000 0.000 0.0
B-day_number 0.000 0.000 0.000 0.0
I-return_date.date_relative 1.000 0.667 0.800 3.0
I-return_date.today_relative 0.000 0.000 0.000 0.0
B-stoploc.airport_name 0.000 0.000 0.000 0.0
B-time_relative 0.000 0.000 0.000 0.0
I-time 0.000 0.000 0.000 0.0
I-return_date.day_number 0.000 0.000 0.000 0.0
I-meal_description 0.000 0.000 0.000 0.0
B-return_date.today_relative 0.000 0.000 0.000 0.0
B-return_date.day_name 0.000 0.000 0.000 2.0
micro avg 0.956 0.957 0.956 3657.0
macro avg 0.683 0.650 0.651 3657.0
weighted avg 0.959 0.957 0.953 3657.0
I0716 04:05:10.271334 139795883440000 interactiveshell.py:2882] Best Slot F1: 0.959
I0716 04:05:10.273087 139795883440000 estimator_training.py:186] Not using Distribute Coordinator.
I0716 04:05:10.274667 139795883440000 training.py:612] Running training and evaluation locally (non-distributed).
I0716 04:05:10.277130 139795883440000 training.py:700] Start train and evaluate loop. The evaluate will happen after every checkpoint. Checkpoint frequency is determined based on RunConfig arguments: save_checkpoints_steps 312 or save_checkpoints_secs None.
I0716 04:05:10.341723 139795883440000 estimator.py:1145] Calling model_fn.
I0716 04:05:12.143309 139795883440000 saver.py:1499] Saver not created because there are no variables in the graph to restore
I0716 04:05:12.895039 139795883440000 <ipython-input-6-d6747b40360b>:55]
[<tf.Variable 'embedding:0' shape=(750, 300) dtype=float32_ref>,
<tf.Variable 'module/aggregation/weights:0' shape=(3,) dtype=float32>,
<tf.Variable 'module/aggregation/scaling:0' shape=() dtype=float32>,
<tf.Variable 'dense/kernel:0' shape=(1324, 300) dtype=float32_ref>,
<tf.Variable 'dense/bias:0' shape=(300,) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/fw/gru_cell/gates/kernel:0' shape=(600, 600) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/fw/gru_cell/gates/bias:0' shape=(600,) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/fw/gru_cell/candidate/kernel:0' shape=(600, 300) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/fw/gru_cell/candidate/bias:0' shape=(300,) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/bw/gru_cell/gates/kernel:0' shape=(600, 600) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/bw/gru_cell/gates/bias:0' shape=(600,) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/bw/gru_cell/candidate/kernel:0' shape=(600, 300) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/bw/gru_cell/candidate/bias:0' shape=(300,) dtype=float32_ref>,
<tf.Variable 'dense_1/kernel:0' shape=(600, 23) dtype=float32_ref>,
<tf.Variable 'dense_1/bias:0' shape=(23,) dtype=float32_ref>,
<tf.Variable 'dense_2/kernel:0' shape=(600, 122) dtype=float32_ref>,
<tf.Variable 'dense_2/bias:0' shape=(122,) dtype=float32_ref>]
I0716 04:05:13.963755 139795883440000 estimator.py:1147] Done calling model_fn.
I0716 04:05:13.967592 139795883440000 basic_session_run_hooks.py:541] Create CheckpointSaverHook.
I0716 04:05:14.292666 139795883440000 monitored_session.py:240] Graph was finalized.
I0716 04:05:14.311049 139795883440000 saver.py:1280] Restoring parameters from ../model/elmo_bigru/model.ckpt-5616
I0716 04:05:15.342847 139795883440000 session_manager.py:500] Running local_init_op.
I0716 04:05:15.411236 139795883440000 session_manager.py:502] Done running local_init_op.
I0716 04:05:18.447129 139795883440000 basic_session_run_hooks.py:606] Saving checkpoints for 5616 into ../model/elmo_bigru/model.ckpt.
Reading ../data/atis.train.w-intent.iob
I0716 04:05:23.309352 139795883440000 basic_session_run_hooks.py:262] loss = 0.024021134, step = 5616 I0716 04:05:23.311173 139795883440000 basic_session_run_hooks.py:262] lr = 0.00016601493 I0716 04:05:50.599553 139795883440000 basic_session_run_hooks.py:692] global_step/sec: 3.66423 I0716 04:05:50.605078 139795883440000 basic_session_run_hooks.py:260] loss = 0.021718727, step = 5716 (27.296 sec) I0716 04:05:50.606785 139795883440000 basic_session_run_hooks.py:260] lr = 0.00016427496 (27.296 sec) I0716 04:06:17.753937 139795883440000 basic_session_run_hooks.py:692] global_step/sec: 3.68265 I0716 04:06:17.759037 139795883440000 basic_session_run_hooks.py:260] loss = 0.018358195, step = 5816 (27.154 sec) I0716 04:06:17.761049 139795883440000 basic_session_run_hooks.py:260] lr = 0.00016255325 (27.154 sec) I0716 04:06:43.939122 139795883440000 basic_session_run_hooks.py:692] global_step/sec: 3.81895 I0716 04:06:43.942589 139795883440000 basic_session_run_hooks.py:260] loss = 0.014227215, step = 5916 (26.184 sec) I0716 04:06:43.945022 139795883440000 basic_session_run_hooks.py:260] lr = 0.00016084957 (26.184 sec) I0716 04:06:46.659544 139795883440000 basic_session_run_hooks.py:606] Saving checkpoints for 5928 into ../model/elmo_bigru/model.ckpt. I0716 04:06:48.588439 139795883440000 estimator.py:1145] Calling model_fn. I0716 04:06:49.920546 139795883440000 saver.py:1499] Saver not created because there are no variables in the graph to restore I0716 04:06:50.490733 139795883440000 estimator.py:1147] Done calling model_fn. I0716 04:06:50.519657 139795883440000 evaluation.py:255] Starting evaluation at 2019-07-16T04:06:50Z I0716 04:06:50.741883 139795883440000 monitored_session.py:240] Graph was finalized. I0716 04:06:50.756609 139795883440000 saver.py:1280] Restoring parameters from ../model/elmo_bigru/model.ckpt-5928 I0716 04:06:51.675721 139795883440000 session_manager.py:500] Running local_init_op. I0716 04:06:51.730380 139795883440000 session_manager.py:502] Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
I0716 04:07:50.468441 139795883440000 evaluation.py:275] Finished evaluation at 2019-07-16-04:07:50 I0716 04:07:50.470516 139795883440000 estimator.py:2039] Saving dict for global step 5928: global_step = 5928, loss = 0.31755933 I0716 04:07:50.476090 139795883440000 estimator.py:2099] Saving 'checkpoint_path' summary for global step 5928: ../model/elmo_bigru/model.ckpt-5928 I0716 04:07:50.557745 139795883440000 estimator.py:368] Loss for final step: 0.0038370441.
Reading ../data/atis.test.w-intent.iob
I0716 04:07:51.160962 139795883440000 estimator.py:1145] Calling model_fn. I0716 04:07:52.780646 139795883440000 saver.py:1499] Saver not created because there are no variables in the graph to restore I0716 04:07:53.467393 139795883440000 estimator.py:1147] Done calling model_fn. I0716 04:07:53.679161 139795883440000 monitored_session.py:240] Graph was finalized. I0716 04:07:53.694185 139795883440000 saver.py:1280] Restoring parameters from ../model/elmo_bigru/model.ckpt-5928 I0716 04:07:54.575456 139795883440000 session_manager.py:500] Running local_init_op. I0716 04:07:54.622498 139795883440000 session_manager.py:502] Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.
'precision', 'predicted', average, warn_for)
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1439: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples.
'recall', 'true', average, warn_for)
I0716 04:08:52.490920 139795883440000 interactiveshell.py:2882]
precision recall f1-score support
atis_flight 0.986 0.992 0.989 632
atis_airfare 0.941 1.000 0.970 48
atis_ground_service 0.973 1.000 0.986 36
atis_airline 1.000 1.000 1.000 38
atis_abbreviation 0.943 1.000 0.971 33
atis_aircraft 1.000 0.889 0.941 9
atis_flight_time 0.500 1.000 0.667 1
atis_quantity 0.429 1.000 0.600 3
atis_flight#atis_airfare 0.833 0.417 0.556 12
atis_airport 1.000 0.944 0.971 18
atis_distance 1.000 1.000 1.000 10
atis_city 0.667 0.667 0.667 6
atis_ground_fare 1.000 0.857 0.923 7
atis_capacity 0.952 0.952 0.952 21
atis_flight_no 0.889 1.000 0.941 8
atis_meal 1.000 0.667 0.800 6
atis_restriction 0.000 0.000 0.000 0
atis_airline#atis_flight_no 0.000 0.000 0.000 0
atis_ground_service#atis_ground_fare 0.000 0.000 0.000 0
atis_airfare#atis_flight_time 0.000 0.000 0.000 0
atis_cheapest 0.000 0.000 0.000 0
atis_aircraft#atis_flight#atis_flight_no 0.000 0.000 0.000 0
micro avg 0.972 0.977 0.975 888
macro avg 0.641 0.654 0.633 888
weighted avg 0.974 0.977 0.974 888
I0716 04:08:52.525501 139795883440000 interactiveshell.py:2882]
precision recall f1-score support
O 0.000 0.000 0.000 0.0
B-toloc.city_name 0.977 0.993 0.985 716.0
B-fromloc.city_name 0.989 0.999 0.994 704.0
I-toloc.city_name 0.985 0.992 0.989 265.0
B-depart_date.day_name 0.991 0.991 0.991 212.0
B-airline_name 1.000 1.000 1.000 101.0
I-fromloc.city_name 0.983 0.994 0.989 177.0
B-depart_time.period_of_day 0.992 0.900 0.944 130.0
I-airline_name 1.000 1.000 1.000 65.0
B-depart_date.day_number 0.981 0.945 0.963 55.0
B-depart_date.month_name 0.981 0.946 0.964 56.0
B-depart_time.time 0.862 0.982 0.918 57.0
B-round_trip 1.000 1.000 1.000 73.0
B-cost_relative 0.973 0.973 0.973 37.0
I-round_trip 1.000 1.000 1.000 71.0
B-flight_mod 0.828 1.000 0.906 24.0
B-depart_time.time_relative 0.984 0.954 0.969 65.0
I-depart_time.time 0.912 1.000 0.954 52.0
B-stoploc.city_name 0.909 1.000 0.952 20.0
B-city_name 0.905 0.667 0.768 57.0
B-class_type 0.960 1.000 0.980 24.0
B-arrive_time.time 0.943 0.971 0.957 34.0
B-arrive_time.time_relative 0.833 0.968 0.896 31.0
I-class_type 1.000 1.000 1.000 17.0
B-flight_stop 1.000 1.000 1.000 21.0
I-arrive_time.time 1.000 0.971 0.986 35.0
B-airline_code 0.971 1.000 0.986 34.0
I-depart_date.day_number 1.000 0.933 0.966 15.0
I-fromloc.airport_name 0.429 1.000 0.600 15.0
B-fromloc.airport_name 0.429 1.000 0.600 12.0
B-arrive_date.day_name 0.786 1.000 0.880 11.0
B-toloc.state_code 1.000 1.000 1.000 18.0
B-depart_date.today_relative 1.000 0.889 0.941 9.0
B-flight_number 0.733 1.000 0.846 11.0
B-depart_date.date_relative 0.944 1.000 0.971 17.0
B-toloc.state_name 0.833 0.893 0.862 28.0
B-fare_basis_code 0.850 1.000 0.919 17.0
B-flight_time 1.000 1.000 1.000 1.0
B-or 1.000 1.000 1.000 3.0
B-arrive_time.period_of_day 0.667 1.000 0.800 6.0
B-meal_description 1.000 0.900 0.947 10.0
I-cost_relative 1.000 0.667 0.800 3.0
I-airport_name 0.846 0.379 0.524 29.0
B-fare_amount 1.000 1.000 1.000 2.0
I-fare_amount 1.000 1.000 1.000 2.0
I-city_name 1.000 0.667 0.800 30.0
I-toloc.airport_name 1.000 1.000 1.000 3.0
B-transport_type 1.000 1.000 1.000 10.0
B-arrive_date.month_name 0.625 0.833 0.714 6.0
B-arrive_date.day_number 0.625 0.833 0.714 6.0
I-stoploc.city_name 0.909 1.000 0.952 10.0
B-meal 1.000 1.000 1.000 16.0
B-fromloc.state_code 1.000 1.000 1.000 23.0
B-depart_time.period_mod 0.833 1.000 0.909 5.0
B-connect 1.000 1.000 1.000 6.0
B-flight_days 1.000 1.000 1.000 10.0
B-toloc.airport_name 1.000 1.000 1.000 3.0
B-fromloc.state_name 1.000 1.000 1.000 17.0
B-airport_name 0.636 0.333 0.437 21.0
B-economy 1.000 1.000 1.000 6.0
I-flight_time 1.000 1.000 1.000 1.0
B-aircraft_code 1.000 0.879 0.935 33.0
B-mod 1.000 0.500 0.667 2.0
B-airport_code 1.000 0.556 0.714 9.0
B-depart_time.start_time 1.000 0.667 0.800 3.0
B-depart_time.end_time 1.000 0.333 0.500 3.0
B-depart_date.year 1.000 1.000 1.000 3.0
I-transport_type 1.000 1.000 1.000 1.0
B-restriction_code 0.800 1.000 0.889 4.0
B-arrive_time.start_time 0.889 1.000 0.941 8.0
B-toloc.airport_code 1.000 0.750 0.857 4.0
B-arrive_time.end_time 0.889 1.000 0.941 8.0
I-arrive_time.end_time 0.889 1.000 0.941 8.0
I-depart_time.end_time 1.000 0.333 0.500 3.0
I-flight_stop 0.000 0.000 0.000 0.0
B-fromloc.airport_code 0.714 1.000 0.833 5.0
I-restriction_code 1.000 1.000 1.000 3.0
I-depart_time.start_time 1.000 1.000 1.000 1.0
I-toloc.state_name 1.000 1.000 1.000 1.0
I-depart_date.today_relative 0.000 0.000 0.000 0.0
B-arrive_date.date_relative 1.000 1.000 1.000 2.0
I-flight_mod 1.000 0.167 0.286 6.0
I-economy 0.000 0.000 0.000 0.0
B-return_date.date_relative 1.000 0.333 0.500 3.0
I-fromloc.state_name 1.000 1.000 1.000 1.0
B-state_code 1.000 1.000 1.000 1.0
I-arrive_time.start_time 1.000 1.000 1.000 1.0
I-arrive_date.day_number 0.000 0.000 0.000 0.0
B-meal_code 0.000 0.000 0.000 1.0
I-depart_time.period_of_day 1.000 1.000 1.000 1.0
B-day_name 1.000 0.500 0.667 2.0
B-period_of_day 1.000 0.750 0.857 4.0
B-stoploc.state_code 0.000 0.000 0.000 0.0
B-return_date.month_name 0.000 0.000 0.000 0.0
B-return_date.day_number 0.000 0.000 0.000 0.0
B-arrive_time.period_mod 0.000 0.000 0.000 0.0
I-meal_code 0.000 0.000 0.000 0.0
B-toloc.country_name 1.000 1.000 1.000 1.0
B-days_code 0.000 0.000 0.000 1.0
I-arrive_time.period_of_day 0.000 0.000 0.000 0.0
I-today_relative 0.000 0.000 0.000 0.0
B-return_time.period_of_day 0.000 0.000 0.000 0.0
B-time 0.000 0.000 0.000 0.0
I-fare_basis_code 0.000 0.000 0.000 0.0
I-arrive_time.time_relative 0.000 0.000 0.000 4.0
I-depart_time.time_relative 0.000 0.000 0.000 1.0
B-today_relative 0.000 0.000 0.000 0.0
B-state_name 0.000 0.000 0.000 9.0
B-arrive_date.today_relative 0.000 0.000 0.000 0.0
B-return_time.period_mod 0.000 0.000 0.000 0.0
B-month_name 0.000 0.000 0.000 0.0
B-day_number 0.000 0.000 0.000 0.0
I-return_date.date_relative 0.667 0.667 0.667 3.0
I-return_date.today_relative 0.000 0.000 0.000 0.0
B-stoploc.airport_name 0.000 0.000 0.000 0.0
B-time_relative 0.000 0.000 0.000 0.0
I-time 0.000 0.000 0.000 0.0
I-return_date.day_number 0.000 0.000 0.000 0.0
I-meal_description 0.000 0.000 0.000 0.0
B-return_date.today_relative 0.000 0.000 0.000 0.0
B-return_date.day_name 0.000 0.000 0.000 2.0
micro avg 0.957 0.959 0.958 3657.0
macro avg 0.677 0.653 0.651 3657.0
weighted avg 0.960 0.959 0.955 3657.0
I0716 04:08:52.544500 139795883440000 interactiveshell.py:2882] Best Slot F1: 0.959
I0716 04:08:52.546095 139795883440000 estimator_training.py:186] Not using Distribute Coordinator.
I0716 04:08:52.551183 139795883440000 training.py:612] Running training and evaluation locally (non-distributed).
I0716 04:08:52.552832 139795883440000 training.py:700] Start train and evaluate loop. The evaluate will happen after every checkpoint. Checkpoint frequency is determined based on RunConfig arguments: save_checkpoints_steps 312 or save_checkpoints_secs None.
I0716 04:08:52.613224 139795883440000 estimator.py:1145] Calling model_fn.
I0716 04:08:54.098511 139795883440000 saver.py:1499] Saver not created because there are no variables in the graph to restore
I0716 04:08:54.677569 139795883440000 <ipython-input-6-d6747b40360b>:55]
[<tf.Variable 'embedding:0' shape=(750, 300) dtype=float32_ref>,
<tf.Variable 'module/aggregation/weights:0' shape=(3,) dtype=float32>,
<tf.Variable 'module/aggregation/scaling:0' shape=() dtype=float32>,
<tf.Variable 'dense/kernel:0' shape=(1324, 300) dtype=float32_ref>,
<tf.Variable 'dense/bias:0' shape=(300,) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/fw/gru_cell/gates/kernel:0' shape=(600, 600) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/fw/gru_cell/gates/bias:0' shape=(600,) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/fw/gru_cell/candidate/kernel:0' shape=(600, 300) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/fw/gru_cell/candidate/bias:0' shape=(300,) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/bw/gru_cell/gates/kernel:0' shape=(600, 600) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/bw/gru_cell/gates/bias:0' shape=(600,) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/bw/gru_cell/candidate/kernel:0' shape=(600, 300) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/bw/gru_cell/candidate/bias:0' shape=(300,) dtype=float32_ref>,
<tf.Variable 'dense_1/kernel:0' shape=(600, 23) dtype=float32_ref>,
<tf.Variable 'dense_1/bias:0' shape=(23,) dtype=float32_ref>,
<tf.Variable 'dense_2/kernel:0' shape=(600, 122) dtype=float32_ref>,
<tf.Variable 'dense_2/bias:0' shape=(122,) dtype=float32_ref>]
I0716 04:08:55.731200 139795883440000 estimator.py:1147] Done calling model_fn.
I0716 04:08:55.735250 139795883440000 basic_session_run_hooks.py:541] Create CheckpointSaverHook.
I0716 04:08:56.253782 139795883440000 monitored_session.py:240] Graph was finalized.
I0716 04:08:56.271320 139795883440000 saver.py:1280] Restoring parameters from ../model/elmo_bigru/model.ckpt-5928
I0716 04:08:57.309556 139795883440000 session_manager.py:500] Running local_init_op.
I0716 04:08:57.385339 139795883440000 session_manager.py:502] Done running local_init_op.
I0716 04:09:00.167442 139795883440000 basic_session_run_hooks.py:606] Saving checkpoints for 5928 into ../model/elmo_bigru/model.ckpt.
Reading ../data/atis.train.w-intent.iob
I0716 04:09:04.799297 139795883440000 basic_session_run_hooks.py:262] loss = 0.013556904, step = 5928 I0716 04:09:04.801827 139795883440000 basic_session_run_hooks.py:262] lr = 0.0001606463 I0716 04:09:32.558158 139795883440000 basic_session_run_hooks.py:692] global_step/sec: 3.60234 I0716 04:09:32.561326 139795883440000 basic_session_run_hooks.py:260] loss = 0.013189382, step = 6028 (27.762 sec) I0716 04:09:32.565477 139795883440000 basic_session_run_hooks.py:260] lr = 0.00015896263 (27.764 sec) I0716 04:09:59.291026 139795883440000 basic_session_run_hooks.py:692] global_step/sec: 3.74072 I0716 04:09:59.293781 139795883440000 basic_session_run_hooks.py:260] loss = 0.024098143, step = 6128 (26.732 sec) I0716 04:09:59.297652 139795883440000 basic_session_run_hooks.py:260] lr = 0.00015729658 (26.732 sec) I0716 04:10:25.366129 139795883440000 basic_session_run_hooks.py:692] global_step/sec: 3.83507 I0716 04:10:25.371350 139795883440000 basic_session_run_hooks.py:260] loss = 0.016176166, step = 6228 (26.078 sec) I0716 04:10:25.373055 139795883440000 basic_session_run_hooks.py:260] lr = 0.00015564801 (26.075 sec) I0716 04:10:28.079471 139795883440000 basic_session_run_hooks.py:606] Saving checkpoints for 6240 into ../model/elmo_bigru/model.ckpt. I0716 04:10:30.156975 139795883440000 estimator.py:1145] Calling model_fn. I0716 04:10:31.521165 139795883440000 saver.py:1499] Saver not created because there are no variables in the graph to restore I0716 04:10:32.103361 139795883440000 estimator.py:1147] Done calling model_fn. I0716 04:10:32.132641 139795883440000 evaluation.py:255] Starting evaluation at 2019-07-16T04:10:32Z I0716 04:10:32.350379 139795883440000 monitored_session.py:240] Graph was finalized. I0716 04:10:32.365226 139795883440000 saver.py:1280] Restoring parameters from ../model/elmo_bigru/model.ckpt-6240 I0716 04:10:33.253172 139795883440000 session_manager.py:500] Running local_init_op. I0716 04:10:33.307003 139795883440000 session_manager.py:502] Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
I0716 04:11:32.190853 139795883440000 evaluation.py:275] Finished evaluation at 2019-07-16-04:11:32 I0716 04:11:32.192710 139795883440000 estimator.py:2039] Saving dict for global step 6240: global_step = 6240, loss = 0.32681456 I0716 04:11:32.198311 139795883440000 estimator.py:2099] Saving 'checkpoint_path' summary for global step 6240: ../model/elmo_bigru/model.ckpt-6240 I0716 04:11:32.275698 139795883440000 estimator.py:368] Loss for final step: 0.009180053.
Reading ../data/atis.test.w-intent.iob
I0716 04:11:32.887591 139795883440000 estimator.py:1145] Calling model_fn. I0716 04:11:34.520857 139795883440000 saver.py:1499] Saver not created because there are no variables in the graph to restore I0716 04:11:35.211205 139795883440000 estimator.py:1147] Done calling model_fn. I0716 04:11:35.421146 139795883440000 monitored_session.py:240] Graph was finalized. I0716 04:11:35.437683 139795883440000 saver.py:1280] Restoring parameters from ../model/elmo_bigru/model.ckpt-6240 I0716 04:11:36.325077 139795883440000 session_manager.py:500] Running local_init_op. I0716 04:11:36.370754 139795883440000 session_manager.py:502] Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.
'precision', 'predicted', average, warn_for)
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1439: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples.
'recall', 'true', average, warn_for)
I0716 04:12:33.999751 139795883440000 interactiveshell.py:2882]
precision recall f1-score support
atis_flight 0.983 0.992 0.987 632
atis_airfare 0.941 1.000 0.970 48
atis_ground_service 1.000 1.000 1.000 36
atis_airline 1.000 1.000 1.000 38
atis_abbreviation 0.943 1.000 0.971 33
atis_aircraft 1.000 0.889 0.941 9
atis_flight_time 0.500 1.000 0.667 1
atis_quantity 0.429 1.000 0.600 3
atis_flight#atis_airfare 0.833 0.417 0.556 12
atis_airport 1.000 1.000 1.000 18
atis_distance 1.000 1.000 1.000 10
atis_city 1.000 0.500 0.667 6
atis_ground_fare 1.000 0.857 0.923 7
atis_capacity 0.952 0.952 0.952 21
atis_flight_no 0.889 1.000 0.941 8
atis_meal 1.000 0.833 0.909 6
atis_restriction 0.000 0.000 0.000 0
atis_airline#atis_flight_no 0.000 0.000 0.000 0
atis_ground_service#atis_ground_fare 0.000 0.000 0.000 0
atis_airfare#atis_flight_time 0.000 0.000 0.000 0
atis_cheapest 0.000 0.000 0.000 0
atis_aircraft#atis_flight#atis_flight_no 0.000 0.000 0.000 0
micro avg 0.973 0.979 0.976 888
macro avg 0.658 0.656 0.640 888
weighted avg 0.976 0.979 0.975 888
I0716 04:12:34.034637 139795883440000 interactiveshell.py:2882]
precision recall f1-score support
O 0.000 0.000 0.000 0.0
B-toloc.city_name 0.974 0.993 0.983 716.0
B-fromloc.city_name 0.987 1.000 0.994 704.0
I-toloc.city_name 0.981 0.985 0.983 265.0
B-depart_date.day_name 0.991 0.991 0.991 212.0
B-airline_name 1.000 1.000 1.000 101.0
I-fromloc.city_name 0.973 1.000 0.986 177.0
B-depart_time.period_of_day 0.984 0.923 0.952 130.0
I-airline_name 1.000 1.000 1.000 65.0
B-depart_date.day_number 0.981 0.964 0.972 55.0
B-depart_date.month_name 0.982 0.964 0.973 56.0
B-depart_time.time 0.864 1.000 0.927 57.0
B-round_trip 1.000 1.000 1.000 73.0
B-cost_relative 0.973 0.973 0.973 37.0
I-round_trip 1.000 1.000 1.000 71.0
B-flight_mod 0.828 1.000 0.906 24.0
B-depart_time.time_relative 0.955 0.985 0.970 65.0
I-depart_time.time 0.945 1.000 0.972 52.0
B-stoploc.city_name 0.952 1.000 0.976 20.0
B-city_name 0.925 0.649 0.763 57.0
B-class_type 0.960 1.000 0.980 24.0
B-arrive_time.time 0.971 0.971 0.971 34.0
B-arrive_time.time_relative 0.967 0.935 0.951 31.0
I-class_type 1.000 1.000 1.000 17.0
B-flight_stop 1.000 1.000 1.000 21.0
I-arrive_time.time 1.000 0.971 0.986 35.0
B-airline_code 0.919 1.000 0.958 34.0
I-depart_date.day_number 1.000 0.933 0.966 15.0
I-fromloc.airport_name 0.429 1.000 0.600 15.0
B-fromloc.airport_name 0.429 1.000 0.600 12.0
B-arrive_date.day_name 0.786 1.000 0.880 11.0
B-toloc.state_code 1.000 1.000 1.000 18.0
B-depart_date.today_relative 1.000 0.889 0.941 9.0
B-flight_number 0.733 1.000 0.846 11.0
B-depart_date.date_relative 0.944 1.000 0.971 17.0
B-toloc.state_name 0.828 0.857 0.842 28.0
B-fare_basis_code 0.850 1.000 0.919 17.0
B-flight_time 1.000 1.000 1.000 1.0
B-or 1.000 1.000 1.000 3.0
B-arrive_time.period_of_day 0.750 1.000 0.857 6.0
B-meal_description 1.000 0.900 0.947 10.0
I-cost_relative 1.000 0.667 0.800 3.0
I-airport_name 0.818 0.310 0.450 29.0
B-fare_amount 1.000 1.000 1.000 2.0
I-fare_amount 1.000 1.000 1.000 2.0
I-city_name 0.947 0.600 0.735 30.0
I-toloc.airport_name 1.000 1.000 1.000 3.0
B-transport_type 1.000 1.000 1.000 10.0
B-arrive_date.month_name 0.714 0.833 0.769 6.0
B-arrive_date.day_number 0.714 0.833 0.769 6.0
I-stoploc.city_name 0.909 1.000 0.952 10.0
B-meal 1.000 1.000 1.000 16.0
B-fromloc.state_code 1.000 1.000 1.000 23.0
B-depart_time.period_mod 1.000 1.000 1.000 5.0
B-connect 1.000 1.000 1.000 6.0
B-flight_days 1.000 1.000 1.000 10.0
B-toloc.airport_name 1.000 1.000 1.000 3.0
B-fromloc.state_name 0.944 1.000 0.971 17.0
B-airport_name 0.636 0.333 0.437 21.0
B-economy 1.000 1.000 1.000 6.0
I-flight_time 1.000 1.000 1.000 1.0
B-aircraft_code 1.000 0.818 0.900 33.0
B-mod 1.000 0.500 0.667 2.0
B-airport_code 1.000 0.444 0.615 9.0
B-depart_time.start_time 1.000 0.667 0.800 3.0
B-depart_time.end_time 1.000 0.333 0.500 3.0
B-depart_date.year 1.000 1.000 1.000 3.0
I-transport_type 0.000 0.000 0.000 1.0
B-restriction_code 0.800 1.000 0.889 4.0
B-arrive_time.start_time 0.889 1.000 0.941 8.0
B-toloc.airport_code 1.000 0.750 0.857 4.0
B-arrive_time.end_time 0.889 1.000 0.941 8.0
I-arrive_time.end_time 0.889 1.000 0.941 8.0
I-depart_time.end_time 1.000 0.333 0.500 3.0
I-flight_stop 0.000 0.000 0.000 0.0
B-fromloc.airport_code 0.625 1.000 0.769 5.0
I-restriction_code 1.000 1.000 1.000 3.0
I-depart_time.start_time 1.000 1.000 1.000 1.0
I-toloc.state_name 1.000 1.000 1.000 1.0
I-depart_date.today_relative 0.000 0.000 0.000 0.0
B-arrive_date.date_relative 1.000 1.000 1.000 2.0
I-flight_mod 1.000 0.167 0.286 6.0
I-economy 0.000 0.000 0.000 0.0
B-return_date.date_relative 1.000 0.333 0.500 3.0
I-fromloc.state_name 1.000 1.000 1.000 1.0
B-state_code 1.000 1.000 1.000 1.0
I-arrive_time.start_time 1.000 1.000 1.000 1.0
I-arrive_date.day_number 0.000 0.000 0.000 0.0
B-meal_code 0.000 0.000 0.000 1.0
I-depart_time.period_of_day 0.000 0.000 0.000 1.0
B-day_name 1.000 0.500 0.667 2.0
B-period_of_day 1.000 0.750 0.857 4.0
B-stoploc.state_code 0.000 0.000 0.000 0.0
B-return_date.month_name 0.000 0.000 0.000 0.0
B-return_date.day_number 0.000 0.000 0.000 0.0
B-arrive_time.period_mod 0.000 0.000 0.000 0.0
I-meal_code 0.000 0.000 0.000 0.0
B-toloc.country_name 1.000 1.000 1.000 1.0
B-days_code 0.000 0.000 0.000 1.0
I-arrive_time.period_of_day 0.000 0.000 0.000 0.0
I-today_relative 0.000 0.000 0.000 0.0
B-return_time.period_of_day 0.000 0.000 0.000 0.0
B-time 0.000 0.000 0.000 0.0
I-fare_basis_code 0.000 0.000 0.000 0.0
I-arrive_time.time_relative 1.000 1.000 1.000 4.0
I-depart_time.time_relative 0.000 0.000 0.000 1.0
B-today_relative 0.000 0.000 0.000 0.0
B-state_name 0.000 0.000 0.000 9.0
B-arrive_date.today_relative 0.000 0.000 0.000 0.0
B-return_time.period_mod 0.000 0.000 0.000 0.0
B-month_name 0.000 0.000 0.000 0.0
B-day_number 0.000 0.000 0.000 0.0
I-return_date.date_relative 0.667 0.667 0.667 3.0
I-return_date.today_relative 0.000 0.000 0.000 0.0
B-stoploc.airport_name 0.000 0.000 0.000 0.0
B-time_relative 0.000 0.000 0.000 0.0
I-time 0.000 0.000 0.000 0.0
I-return_date.day_number 0.000 0.000 0.000 0.0
I-meal_description 0.000 0.000 0.000 0.0
B-return_date.today_relative 0.000 0.000 0.000 0.0
B-return_date.day_name 0.000 0.000 0.000 2.0
micro avg 0.957 0.959 0.958 3657.0
macro avg 0.672 0.642 0.642 3657.0
weighted avg 0.960 0.959 0.955 3657.0
I0716 04:12:34.054486 139795883440000 interactiveshell.py:2882] Best Slot F1: 0.959
I0716 04:12:34.056142 139795883440000 estimator_training.py:186] Not using Distribute Coordinator.
I0716 04:12:34.058927 139795883440000 training.py:612] Running training and evaluation locally (non-distributed).
I0716 04:12:34.061445 139795883440000 training.py:700] Start train and evaluate loop. The evaluate will happen after every checkpoint. Checkpoint frequency is determined based on RunConfig arguments: save_checkpoints_steps 312 or save_checkpoints_secs None.
I0716 04:12:34.123326 139795883440000 estimator.py:1145] Calling model_fn.
I0716 04:12:35.612440 139795883440000 saver.py:1499] Saver not created because there are no variables in the graph to restore
I0716 04:12:36.184969 139795883440000 <ipython-input-6-d6747b40360b>:55]
[<tf.Variable 'embedding:0' shape=(750, 300) dtype=float32_ref>,
<tf.Variable 'module/aggregation/weights:0' shape=(3,) dtype=float32>,
<tf.Variable 'module/aggregation/scaling:0' shape=() dtype=float32>,
<tf.Variable 'dense/kernel:0' shape=(1324, 300) dtype=float32_ref>,
<tf.Variable 'dense/bias:0' shape=(300,) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/fw/gru_cell/gates/kernel:0' shape=(600, 600) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/fw/gru_cell/gates/bias:0' shape=(600,) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/fw/gru_cell/candidate/kernel:0' shape=(600, 300) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/fw/gru_cell/candidate/bias:0' shape=(300,) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/bw/gru_cell/gates/kernel:0' shape=(600, 600) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/bw/gru_cell/gates/bias:0' shape=(600,) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/bw/gru_cell/candidate/kernel:0' shape=(600, 300) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/bw/gru_cell/candidate/bias:0' shape=(300,) dtype=float32_ref>,
<tf.Variable 'dense_1/kernel:0' shape=(600, 23) dtype=float32_ref>,
<tf.Variable 'dense_1/bias:0' shape=(23,) dtype=float32_ref>,
<tf.Variable 'dense_2/kernel:0' shape=(600, 122) dtype=float32_ref>,
<tf.Variable 'dense_2/bias:0' shape=(122,) dtype=float32_ref>]
I0716 04:12:37.262094 139795883440000 estimator.py:1147] Done calling model_fn.
I0716 04:12:37.266188 139795883440000 basic_session_run_hooks.py:541] Create CheckpointSaverHook.
I0716 04:12:37.794167 139795883440000 monitored_session.py:240] Graph was finalized.
I0716 04:12:37.813221 139795883440000 saver.py:1280] Restoring parameters from ../model/elmo_bigru/model.ckpt-6240
I0716 04:12:38.844015 139795883440000 session_manager.py:500] Running local_init_op.
I0716 04:12:38.915020 139795883440000 session_manager.py:502] Done running local_init_op.
I0716 04:12:41.675868 139795883440000 basic_session_run_hooks.py:606] Saving checkpoints for 6240 into ../model/elmo_bigru/model.ckpt.
Reading ../data/atis.train.w-intent.iob
I0716 04:12:46.442596 139795883440000 basic_session_run_hooks.py:262] loss = 0.015512767, step = 6240 I0716 04:12:46.444594 139795883440000 basic_session_run_hooks.py:262] lr = 0.00015545134 I0716 04:13:13.395112 139795883440000 basic_session_run_hooks.py:692] global_step/sec: 3.71013 I0716 04:13:13.400246 139795883440000 basic_session_run_hooks.py:260] loss = 0.022132397, step = 6340 (26.958 sec) I0716 04:13:13.401929 139795883440000 basic_session_run_hooks.py:260] lr = 0.00015382208 (26.957 sec) I0716 04:13:39.564567 139795883440000 basic_session_run_hooks.py:692] global_step/sec: 3.82124 I0716 04:13:39.569588 139795883440000 basic_session_run_hooks.py:260] loss = 0.016259994, step = 6440 (26.169 sec) I0716 04:13:39.574711 139795883440000 basic_session_run_hooks.py:260] lr = 0.00015220992 (26.173 sec) I0716 04:14:06.450009 139795883440000 basic_session_run_hooks.py:692] global_step/sec: 3.71949 I0716 04:14:06.456068 139795883440000 basic_session_run_hooks.py:260] loss = 0.019697318, step = 6540 (26.886 sec) I0716 04:14:06.457782 139795883440000 basic_session_run_hooks.py:260] lr = 0.00015061465 (26.883 sec) I0716 04:14:09.685927 139795883440000 basic_session_run_hooks.py:606] Saving checkpoints for 6552 into ../model/elmo_bigru/model.ckpt. I0716 04:14:11.658162 139795883440000 estimator.py:1145] Calling model_fn. I0716 04:14:13.006481 139795883440000 saver.py:1499] Saver not created because there are no variables in the graph to restore I0716 04:14:13.593723 139795883440000 estimator.py:1147] Done calling model_fn. I0716 04:14:13.623715 139795883440000 evaluation.py:255] Starting evaluation at 2019-07-16T04:14:13Z I0716 04:14:14.108688 139795883440000 monitored_session.py:240] Graph was finalized. I0716 04:14:14.124477 139795883440000 saver.py:1280] Restoring parameters from ../model/elmo_bigru/model.ckpt-6552 I0716 04:14:15.020569 139795883440000 session_manager.py:500] Running local_init_op. I0716 04:14:15.077372 139795883440000 session_manager.py:502] Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
I0716 04:15:13.894021 139795883440000 evaluation.py:275] Finished evaluation at 2019-07-16-04:15:13 I0716 04:15:13.895933 139795883440000 estimator.py:2039] Saving dict for global step 6552: global_step = 6552, loss = 0.3396054 I0716 04:15:13.901288 139795883440000 estimator.py:2099] Saving 'checkpoint_path' summary for global step 6552: ../model/elmo_bigru/model.ckpt-6552 I0716 04:15:13.987282 139795883440000 estimator.py:368] Loss for final step: 0.005030209.
Reading ../data/atis.test.w-intent.iob
I0716 04:15:14.588431 139795883440000 estimator.py:1145] Calling model_fn. I0716 04:15:15.618718 139795883440000 saver.py:1499] Saver not created because there are no variables in the graph to restore I0716 04:15:16.139860 139795883440000 estimator.py:1147] Done calling model_fn. I0716 04:15:16.355150 139795883440000 monitored_session.py:240] Graph was finalized. I0716 04:15:16.373098 139795883440000 saver.py:1280] Restoring parameters from ../model/elmo_bigru/model.ckpt-6552 I0716 04:15:17.265295 139795883440000 session_manager.py:500] Running local_init_op. I0716 04:15:17.321921 139795883440000 session_manager.py:502] Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.
'precision', 'predicted', average, warn_for)
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1439: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples.
'recall', 'true', average, warn_for)
I0716 04:16:15.382930 139795883440000 interactiveshell.py:2882]
precision recall f1-score support
atis_flight 0.983 0.992 0.987 632
atis_airfare 0.941 1.000 0.970 48
atis_ground_service 1.000 1.000 1.000 36
atis_airline 0.974 1.000 0.987 38
atis_abbreviation 0.943 1.000 0.971 33
atis_aircraft 1.000 0.889 0.941 9
atis_flight_time 0.500 1.000 0.667 1
atis_quantity 0.375 1.000 0.545 3
atis_flight#atis_airfare 0.833 0.417 0.556 12
atis_airport 1.000 0.944 0.971 18
atis_distance 1.000 1.000 1.000 10
atis_city 1.000 0.500 0.667 6
atis_ground_fare 1.000 0.857 0.923 7
atis_capacity 1.000 0.952 0.976 21
atis_flight_no 0.889 1.000 0.941 8
atis_meal 1.000 0.833 0.909 6
atis_restriction 0.000 0.000 0.000 0
atis_airline#atis_flight_no 0.000 0.000 0.000 0
atis_ground_service#atis_ground_fare 0.000 0.000 0.000 0
atis_airfare#atis_flight_time 0.000 0.000 0.000 0
atis_cheapest 0.000 0.000 0.000 0
atis_aircraft#atis_flight#atis_flight_no 0.000 0.000 0.000 0
micro avg 0.972 0.977 0.975 888
macro avg 0.656 0.654 0.637 888
weighted avg 0.975 0.977 0.974 888
I0716 04:16:15.417814 139795883440000 interactiveshell.py:2882]
precision recall f1-score support
O 0.000 0.000 0.000 0.0
B-toloc.city_name 0.975 0.992 0.983 716.0
B-fromloc.city_name 0.989 0.999 0.994 704.0
I-toloc.city_name 0.978 0.992 0.985 265.0
B-depart_date.day_name 0.991 0.991 0.991 212.0
B-airline_name 1.000 1.000 1.000 101.0
I-fromloc.city_name 0.978 1.000 0.989 177.0
B-depart_time.period_of_day 0.992 0.923 0.956 130.0
I-airline_name 1.000 1.000 1.000 65.0
B-depart_date.day_number 0.981 0.964 0.972 55.0
B-depart_date.month_name 0.981 0.946 0.964 56.0
B-depart_time.time 0.864 1.000 0.927 57.0
B-round_trip 1.000 1.000 1.000 73.0
B-cost_relative 0.973 0.973 0.973 37.0
I-round_trip 1.000 1.000 1.000 71.0
B-flight_mod 0.828 1.000 0.906 24.0
B-depart_time.time_relative 0.970 0.985 0.977 65.0
I-depart_time.time 0.945 1.000 0.972 52.0
B-stoploc.city_name 0.870 1.000 0.930 20.0
B-city_name 0.902 0.649 0.755 57.0
B-class_type 0.960 1.000 0.980 24.0
B-arrive_time.time 0.971 0.971 0.971 34.0
B-arrive_time.time_relative 0.906 0.935 0.921 31.0
I-class_type 1.000 1.000 1.000 17.0
B-flight_stop 1.000 1.000 1.000 21.0
I-arrive_time.time 1.000 0.971 0.986 35.0
B-airline_code 0.850 1.000 0.919 34.0
I-depart_date.day_number 1.000 0.933 0.966 15.0
I-fromloc.airport_name 0.441 1.000 0.612 15.0
B-fromloc.airport_name 0.444 1.000 0.615 12.0
B-arrive_date.day_name 0.786 1.000 0.880 11.0
B-toloc.state_code 1.000 1.000 1.000 18.0
B-depart_date.today_relative 1.000 0.889 0.941 9.0
B-flight_number 0.733 1.000 0.846 11.0
B-depart_date.date_relative 0.944 1.000 0.971 17.0
B-toloc.state_name 0.833 0.893 0.862 28.0
B-fare_basis_code 0.895 1.000 0.944 17.0
B-flight_time 1.000 1.000 1.000 1.0
B-or 1.000 1.000 1.000 3.0
B-arrive_time.period_of_day 0.750 1.000 0.857 6.0
B-meal_description 1.000 0.900 0.947 10.0
I-cost_relative 1.000 0.667 0.800 3.0
I-airport_name 0.833 0.345 0.488 29.0
B-fare_amount 1.000 1.000 1.000 2.0
I-fare_amount 1.000 1.000 1.000 2.0
I-city_name 0.947 0.600 0.735 30.0
I-toloc.airport_name 1.000 1.000 1.000 3.0
B-transport_type 1.000 1.000 1.000 10.0
B-arrive_date.month_name 0.625 0.833 0.714 6.0
B-arrive_date.day_number 0.714 0.833 0.769 6.0
I-stoploc.city_name 0.909 1.000 0.952 10.0
B-meal 1.000 1.000 1.000 16.0
B-fromloc.state_code 1.000 1.000 1.000 23.0
B-depart_time.period_mod 1.000 1.000 1.000 5.0
B-connect 1.000 1.000 1.000 6.0
B-flight_days 1.000 1.000 1.000 10.0
B-toloc.airport_name 1.000 1.000 1.000 3.0
B-fromloc.state_name 0.944 1.000 0.971 17.0
B-airport_name 0.636 0.333 0.437 21.0
B-economy 1.000 1.000 1.000 6.0
I-flight_time 1.000 1.000 1.000 1.0
B-aircraft_code 1.000 0.818 0.900 33.0
B-mod 1.000 0.500 0.667 2.0
B-airport_code 1.000 0.444 0.615 9.0
B-depart_time.start_time 1.000 0.667 0.800 3.0
B-depart_time.end_time 1.000 0.333 0.500 3.0
B-depart_date.year 1.000 1.000 1.000 3.0
I-transport_type 1.000 1.000 1.000 1.0
B-restriction_code 1.000 0.500 0.667 4.0
B-arrive_time.start_time 0.889 1.000 0.941 8.0
B-toloc.airport_code 1.000 0.750 0.857 4.0
B-arrive_time.end_time 0.889 1.000 0.941 8.0
I-arrive_time.end_time 0.889 1.000 0.941 8.0
I-depart_time.end_time 1.000 0.333 0.500 3.0
I-flight_stop 0.000 0.000 0.000 0.0
B-fromloc.airport_code 0.625 1.000 0.769 5.0
I-restriction_code 1.000 1.000 1.000 3.0
I-depart_time.start_time 1.000 1.000 1.000 1.0
I-toloc.state_name 1.000 1.000 1.000 1.0
I-depart_date.today_relative 0.000 0.000 0.000 0.0
B-arrive_date.date_relative 1.000 1.000 1.000 2.0
I-flight_mod 1.000 0.167 0.286 6.0
I-economy 0.000 0.000 0.000 0.0
B-return_date.date_relative 1.000 0.333 0.500 3.0
I-fromloc.state_name 1.000 1.000 1.000 1.0
B-state_code 1.000 1.000 1.000 1.0
I-arrive_time.start_time 1.000 1.000 1.000 1.0
I-arrive_date.day_number 0.000 0.000 0.000 0.0
B-meal_code 0.000 0.000 0.000 1.0
I-depart_time.period_of_day 1.000 1.000 1.000 1.0
B-day_name 1.000 0.500 0.667 2.0
B-period_of_day 1.000 0.750 0.857 4.0
B-stoploc.state_code 0.000 0.000 0.000 0.0
B-return_date.month_name 0.000 0.000 0.000 0.0
B-return_date.day_number 0.000 0.000 0.000 0.0
B-arrive_time.period_mod 0.000 0.000 0.000 0.0
I-meal_code 0.000 0.000 0.000 0.0
B-toloc.country_name 1.000 1.000 1.000 1.0
B-days_code 1.000 1.000 1.000 1.0
I-arrive_time.period_of_day 0.000 0.000 0.000 0.0
I-today_relative 0.000 0.000 0.000 0.0
B-return_time.period_of_day 0.000 0.000 0.000 0.0
B-time 0.000 0.000 0.000 0.0
I-fare_basis_code 0.000 0.000 0.000 0.0
I-arrive_time.time_relative 1.000 0.500 0.667 4.0
I-depart_time.time_relative 0.000 0.000 0.000 1.0
B-today_relative 0.000 0.000 0.000 0.0
B-state_name 0.000 0.000 0.000 9.0
B-arrive_date.today_relative 0.000 0.000 0.000 0.0
B-return_time.period_mod 0.000 0.000 0.000 0.0
B-month_name 0.000 0.000 0.000 0.0
B-day_number 0.000 0.000 0.000 0.0
I-return_date.date_relative 0.667 0.667 0.667 3.0
I-return_date.today_relative 0.000 0.000 0.000 0.0
B-stoploc.airport_name 0.000 0.000 0.000 0.0
B-time_relative 0.000 0.000 0.000 0.0
I-time 0.000 0.000 0.000 0.0
I-return_date.day_number 0.000 0.000 0.000 0.0
I-meal_description 0.000 0.000 0.000 0.0
B-return_date.today_relative 0.000 0.000 0.000 0.0
B-return_date.day_name 0.000 0.000 0.000 2.0
micro avg 0.957 0.959 0.958 3657.0
macro avg 0.697 0.659 0.662 3657.0
weighted avg 0.961 0.959 0.955 3657.0
I0716 04:16:15.437776 139795883440000 interactiveshell.py:2882] Best Slot F1: 0.959
I0716 04:16:15.439440 139795883440000 estimator_training.py:186] Not using Distribute Coordinator.
I0716 04:16:15.441732 139795883440000 training.py:612] Running training and evaluation locally (non-distributed).
I0716 04:16:15.446295 139795883440000 training.py:700] Start train and evaluate loop. The evaluate will happen after every checkpoint. Checkpoint frequency is determined based on RunConfig arguments: save_checkpoints_steps 312 or save_checkpoints_secs None.
I0716 04:16:15.507980 139795883440000 estimator.py:1145] Calling model_fn.
I0716 04:16:17.302391 139795883440000 saver.py:1499] Saver not created because there are no variables in the graph to restore
I0716 04:16:18.061816 139795883440000 <ipython-input-6-d6747b40360b>:55]
[<tf.Variable 'embedding:0' shape=(750, 300) dtype=float32_ref>,
<tf.Variable 'module/aggregation/weights:0' shape=(3,) dtype=float32>,
<tf.Variable 'module/aggregation/scaling:0' shape=() dtype=float32>,
<tf.Variable 'dense/kernel:0' shape=(1324, 300) dtype=float32_ref>,
<tf.Variable 'dense/bias:0' shape=(300,) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/fw/gru_cell/gates/kernel:0' shape=(600, 600) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/fw/gru_cell/gates/bias:0' shape=(600,) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/fw/gru_cell/candidate/kernel:0' shape=(600, 300) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/fw/gru_cell/candidate/bias:0' shape=(300,) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/bw/gru_cell/gates/kernel:0' shape=(600, 600) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/bw/gru_cell/gates/bias:0' shape=(600,) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/bw/gru_cell/candidate/kernel:0' shape=(600, 300) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/bw/gru_cell/candidate/bias:0' shape=(300,) dtype=float32_ref>,
<tf.Variable 'dense_1/kernel:0' shape=(600, 23) dtype=float32_ref>,
<tf.Variable 'dense_1/bias:0' shape=(23,) dtype=float32_ref>,
<tf.Variable 'dense_2/kernel:0' shape=(600, 122) dtype=float32_ref>,
<tf.Variable 'dense_2/bias:0' shape=(122,) dtype=float32_ref>]
I0716 04:16:19.134381 139795883440000 estimator.py:1147] Done calling model_fn.
I0716 04:16:19.138150 139795883440000 basic_session_run_hooks.py:541] Create CheckpointSaverHook.
I0716 04:16:19.467717 139795883440000 monitored_session.py:240] Graph was finalized.
I0716 04:16:19.484432 139795883440000 saver.py:1280] Restoring parameters from ../model/elmo_bigru/model.ckpt-6552
I0716 04:16:20.499225 139795883440000 session_manager.py:500] Running local_init_op.
I0716 04:16:20.567731 139795883440000 session_manager.py:502] Done running local_init_op.
I0716 04:16:23.578376 139795883440000 basic_session_run_hooks.py:606] Saving checkpoints for 6552 into ../model/elmo_bigru/model.ckpt.
Reading ../data/atis.train.w-intent.iob
I0716 04:16:28.719417 139795883440000 basic_session_run_hooks.py:262] loss = 0.018550156, step = 6552 I0716 04:16:28.725748 139795883440000 basic_session_run_hooks.py:262] lr = 0.00015042433 I0716 04:16:56.900079 139795883440000 basic_session_run_hooks.py:692] global_step/sec: 3.54847 I0716 04:16:56.905190 139795883440000 basic_session_run_hooks.py:260] loss = 0.015438979, step = 6652 (28.186 sec) I0716 04:16:56.908043 139795883440000 basic_session_run_hooks.py:260] lr = 0.00014884777 (28.182 sec) I0716 04:17:23.714182 139795883440000 basic_session_run_hooks.py:692] global_step/sec: 3.72937 I0716 04:17:23.717083 139795883440000 basic_session_run_hooks.py:260] loss = 0.015352417, step = 6752 (26.812 sec) I0716 04:17:23.719132 139795883440000 basic_session_run_hooks.py:260] lr = 0.00014728773 (26.811 sec) I0716 04:17:49.155670 139795883440000 basic_session_run_hooks.py:692] global_step/sec: 3.93059 I0716 04:17:49.160705 139795883440000 basic_session_run_hooks.py:260] loss = 0.023017865, step = 6852 (25.444 sec) I0716 04:17:49.164227 139795883440000 basic_session_run_hooks.py:260] lr = 0.00014574405 (25.445 sec) I0716 04:17:51.754682 139795883440000 basic_session_run_hooks.py:606] Saving checkpoints for 6864 into ../model/elmo_bigru/model.ckpt. I0716 04:17:53.686567 139795883440000 estimator.py:1145] Calling model_fn. I0716 04:17:55.038545 139795883440000 saver.py:1499] Saver not created because there are no variables in the graph to restore I0716 04:17:55.623387 139795883440000 estimator.py:1147] Done calling model_fn. I0716 04:17:55.653599 139795883440000 evaluation.py:255] Starting evaluation at 2019-07-16T04:17:55Z I0716 04:17:55.876722 139795883440000 monitored_session.py:240] Graph was finalized. I0716 04:17:55.893433 139795883440000 saver.py:1280] Restoring parameters from ../model/elmo_bigru/model.ckpt-6864 I0716 04:17:56.822394 139795883440000 session_manager.py:500] Running local_init_op. I0716 04:17:56.880496 139795883440000 session_manager.py:502] Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
I0716 04:18:55.726617 139795883440000 evaluation.py:275] Finished evaluation at 2019-07-16-04:18:55 I0716 04:18:55.728466 139795883440000 estimator.py:2039] Saving dict for global step 6864: global_step = 6864, loss = 0.33906943 I0716 04:18:55.733350 139795883440000 estimator.py:2099] Saving 'checkpoint_path' summary for global step 6864: ../model/elmo_bigru/model.ckpt-6864 I0716 04:18:55.811212 139795883440000 estimator.py:368] Loss for final step: 0.0032379918.
Reading ../data/atis.test.w-intent.iob
I0716 04:18:56.430192 139795883440000 estimator.py:1145] Calling model_fn. I0716 04:18:58.101979 139795883440000 saver.py:1499] Saver not created because there are no variables in the graph to restore I0716 04:18:58.792852 139795883440000 estimator.py:1147] Done calling model_fn. I0716 04:18:59.017447 139795883440000 monitored_session.py:240] Graph was finalized. I0716 04:18:59.033003 139795883440000 saver.py:1280] Restoring parameters from ../model/elmo_bigru/model.ckpt-6864 I0716 04:18:59.925140 139795883440000 session_manager.py:500] Running local_init_op. I0716 04:18:59.971752 139795883440000 session_manager.py:502] Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.
'precision', 'predicted', average, warn_for)
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1439: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples.
'recall', 'true', average, warn_for)
I0716 04:19:57.884124 139795883440000 interactiveshell.py:2882]
precision recall f1-score support
atis_flight 0.986 0.992 0.989 632
atis_airfare 0.941 1.000 0.970 48
atis_ground_service 0.973 1.000 0.986 36
atis_airline 1.000 1.000 1.000 38
atis_abbreviation 0.943 1.000 0.971 33
atis_aircraft 1.000 0.889 0.941 9
atis_flight_time 0.500 1.000 0.667 1
atis_quantity 0.429 1.000 0.600 3
atis_flight#atis_airfare 0.833 0.417 0.556 12
atis_airport 1.000 0.944 0.971 18
atis_distance 1.000 1.000 1.000 10
atis_city 0.667 0.667 0.667 6
atis_ground_fare 1.000 0.857 0.923 7
atis_capacity 0.952 0.952 0.952 21
atis_flight_no 0.889 1.000 0.941 8
atis_meal 1.000 0.667 0.800 6
atis_restriction 0.000 0.000 0.000 0
atis_airline#atis_flight_no 0.000 0.000 0.000 0
atis_ground_service#atis_ground_fare 0.000 0.000 0.000 0
atis_airfare#atis_flight_time 0.000 0.000 0.000 0
atis_cheapest 0.000 0.000 0.000 0
atis_aircraft#atis_flight#atis_flight_no 0.000 0.000 0.000 0
micro avg 0.972 0.977 0.975 888
macro avg 0.641 0.654 0.633 888
weighted avg 0.974 0.977 0.974 888
I0716 04:19:57.920375 139795883440000 interactiveshell.py:2882]
precision recall f1-score support
O 0.000 0.000 0.000 0.0
B-toloc.city_name 0.977 0.994 0.985 716.0
B-fromloc.city_name 0.992 0.999 0.995 704.0
I-toloc.city_name 0.981 0.992 0.987 265.0
B-depart_date.day_name 0.991 0.991 0.991 212.0
B-airline_name 1.000 1.000 1.000 101.0
I-fromloc.city_name 0.983 1.000 0.992 177.0
B-depart_time.period_of_day 0.984 0.923 0.952 130.0
I-airline_name 0.985 1.000 0.992 65.0
B-depart_date.day_number 0.981 0.964 0.972 55.0
B-depart_date.month_name 0.982 0.964 0.973 56.0
B-depart_time.time 0.864 1.000 0.927 57.0
B-round_trip 1.000 1.000 1.000 73.0
B-cost_relative 0.973 0.973 0.973 37.0
I-round_trip 1.000 1.000 1.000 71.0
B-flight_mod 0.828 1.000 0.906 24.0
B-depart_time.time_relative 0.955 0.969 0.962 65.0
I-depart_time.time 0.945 1.000 0.972 52.0
B-stoploc.city_name 0.870 1.000 0.930 20.0
B-city_name 0.902 0.649 0.755 57.0
B-class_type 0.960 1.000 0.980 24.0
B-arrive_time.time 0.971 0.971 0.971 34.0
B-arrive_time.time_relative 0.967 0.935 0.951 31.0
I-class_type 1.000 1.000 1.000 17.0
B-flight_stop 1.000 1.000 1.000 21.0
I-arrive_time.time 1.000 0.971 0.986 35.0
B-airline_code 0.895 1.000 0.944 34.0
I-depart_date.day_number 1.000 0.933 0.966 15.0
I-fromloc.airport_name 0.441 1.000 0.612 15.0
B-fromloc.airport_name 0.429 1.000 0.600 12.0
B-arrive_date.day_name 0.786 1.000 0.880 11.0
B-toloc.state_code 1.000 1.000 1.000 18.0
B-depart_date.today_relative 1.000 0.889 0.941 9.0
B-flight_number 0.733 1.000 0.846 11.0
B-depart_date.date_relative 0.944 1.000 0.971 17.0
B-toloc.state_name 0.839 0.929 0.881 28.0
B-fare_basis_code 0.850 1.000 0.919 17.0
B-flight_time 1.000 1.000 1.000 1.0
B-or 1.000 1.000 1.000 3.0
B-arrive_time.period_of_day 0.750 1.000 0.857 6.0
B-meal_description 1.000 0.900 0.947 10.0
I-cost_relative 1.000 0.667 0.800 3.0
I-airport_name 0.833 0.345 0.488 29.0
B-fare_amount 1.000 1.000 1.000 2.0
I-fare_amount 1.000 1.000 1.000 2.0
I-city_name 0.950 0.633 0.760 30.0
I-toloc.airport_name 1.000 1.000 1.000 3.0
B-transport_type 1.000 1.000 1.000 10.0
B-arrive_date.month_name 0.714 0.833 0.769 6.0
B-arrive_date.day_number 0.714 0.833 0.769 6.0
I-stoploc.city_name 1.000 1.000 1.000 10.0
B-meal 1.000 1.000 1.000 16.0
B-fromloc.state_code 1.000 1.000 1.000 23.0
B-depart_time.period_mod 0.833 1.000 0.909 5.0
B-connect 1.000 1.000 1.000 6.0
B-flight_days 1.000 1.000 1.000 10.0
B-toloc.airport_name 1.000 1.000 1.000 3.0
B-fromloc.state_name 1.000 1.000 1.000 17.0
B-airport_name 0.636 0.333 0.437 21.0
B-economy 1.000 1.000 1.000 6.0
I-flight_time 1.000 1.000 1.000 1.0
B-aircraft_code 1.000 0.818 0.900 33.0
B-mod 1.000 0.500 0.667 2.0
B-airport_code 1.000 0.556 0.714 9.0
B-depart_time.start_time 1.000 0.667 0.800 3.0
B-depart_time.end_time 1.000 0.333 0.500 3.0
B-depart_date.year 1.000 1.000 1.000 3.0
I-transport_type 1.000 1.000 1.000 1.0
B-restriction_code 1.000 1.000 1.000 4.0
B-arrive_time.start_time 0.889 1.000 0.941 8.0
B-toloc.airport_code 1.000 0.750 0.857 4.0
B-arrive_time.end_time 0.889 1.000 0.941 8.0
I-arrive_time.end_time 0.889 1.000 0.941 8.0
I-depart_time.end_time 1.000 0.333 0.500 3.0
I-flight_stop 0.000 0.000 0.000 0.0
B-fromloc.airport_code 0.714 1.000 0.833 5.0
I-restriction_code 1.000 1.000 1.000 3.0
I-depart_time.start_time 1.000 1.000 1.000 1.0
I-toloc.state_name 1.000 1.000 1.000 1.0
I-depart_date.today_relative 0.000 0.000 0.000 0.0
B-arrive_date.date_relative 1.000 1.000 1.000 2.0
I-flight_mod 1.000 0.167 0.286 6.0
I-economy 0.000 0.000 0.000 0.0
B-return_date.date_relative 1.000 0.333 0.500 3.0
I-fromloc.state_name 1.000 1.000 1.000 1.0
B-state_code 1.000 1.000 1.000 1.0
I-arrive_time.start_time 1.000 1.000 1.000 1.0
I-arrive_date.day_number 0.000 0.000 0.000 0.0
B-meal_code 0.000 0.000 0.000 1.0
I-depart_time.period_of_day 0.000 0.000 0.000 1.0
B-day_name 1.000 0.500 0.667 2.0
B-period_of_day 1.000 0.750 0.857 4.0
B-stoploc.state_code 0.000 0.000 0.000 0.0
B-return_date.month_name 0.000 0.000 0.000 0.0
B-return_date.day_number 0.000 0.000 0.000 0.0
B-arrive_time.period_mod 0.000 0.000 0.000 0.0
I-meal_code 0.000 0.000 0.000 0.0
B-toloc.country_name 0.000 0.000 0.000 1.0
B-days_code 0.000 0.000 0.000 1.0
I-arrive_time.period_of_day 0.000 0.000 0.000 0.0
I-today_relative 0.000 0.000 0.000 0.0
B-return_time.period_of_day 0.000 0.000 0.000 0.0
B-time 0.000 0.000 0.000 0.0
I-fare_basis_code 0.000 0.000 0.000 0.0
I-arrive_time.time_relative 1.000 1.000 1.000 4.0
I-depart_time.time_relative 0.000 0.000 0.000 1.0
B-today_relative 0.000 0.000 0.000 0.0
B-state_name 0.000 0.000 0.000 9.0
B-arrive_date.today_relative 0.000 0.000 0.000 0.0
B-return_time.period_mod 0.000 0.000 0.000 0.0
B-month_name 0.000 0.000 0.000 0.0
B-day_number 0.000 0.000 0.000 0.0
I-return_date.date_relative 0.667 0.667 0.667 3.0
I-return_date.today_relative 0.000 0.000 0.000 0.0
B-stoploc.airport_name 0.000 0.000 0.000 0.0
B-time_relative 0.000 0.000 0.000 0.0
I-time 0.000 0.000 0.000 0.0
I-return_date.day_number 0.000 0.000 0.000 0.0
I-meal_description 0.000 0.000 0.000 0.0
B-return_date.today_relative 0.000 0.000 0.000 0.0
B-return_date.day_name 0.000 0.000 0.000 2.0
micro avg 0.959 0.960 0.960 3657.0
macro avg 0.673 0.644 0.645 3657.0
weighted avg 0.962 0.960 0.957 3657.0
I0716 04:19:57.940826 139795883440000 interactiveshell.py:2882] Best Slot F1: 0.960
I0716 04:19:57.942459 139795883440000 estimator_training.py:186] Not using Distribute Coordinator.
I0716 04:19:57.943920 139795883440000 training.py:612] Running training and evaluation locally (non-distributed).
I0716 04:19:57.947064 139795883440000 training.py:700] Start train and evaluate loop. The evaluate will happen after every checkpoint. Checkpoint frequency is determined based on RunConfig arguments: save_checkpoints_steps 312 or save_checkpoints_secs None.
I0716 04:19:58.011204 139795883440000 estimator.py:1145] Calling model_fn.
I0716 04:19:59.334115 139795883440000 saver.py:1499] Saver not created because there are no variables in the graph to restore
I0716 04:20:00.090726 139795883440000 <ipython-input-6-d6747b40360b>:55]
[<tf.Variable 'embedding:0' shape=(750, 300) dtype=float32_ref>,
<tf.Variable 'module/aggregation/weights:0' shape=(3,) dtype=float32>,
<tf.Variable 'module/aggregation/scaling:0' shape=() dtype=float32>,
<tf.Variable 'dense/kernel:0' shape=(1324, 300) dtype=float32_ref>,
<tf.Variable 'dense/bias:0' shape=(300,) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/fw/gru_cell/gates/kernel:0' shape=(600, 600) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/fw/gru_cell/gates/bias:0' shape=(600,) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/fw/gru_cell/candidate/kernel:0' shape=(600, 300) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/fw/gru_cell/candidate/bias:0' shape=(300,) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/bw/gru_cell/gates/kernel:0' shape=(600, 600) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/bw/gru_cell/gates/bias:0' shape=(600,) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/bw/gru_cell/candidate/kernel:0' shape=(600, 300) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/bw/gru_cell/candidate/bias:0' shape=(300,) dtype=float32_ref>,
<tf.Variable 'dense_1/kernel:0' shape=(600, 23) dtype=float32_ref>,
<tf.Variable 'dense_1/bias:0' shape=(23,) dtype=float32_ref>,
<tf.Variable 'dense_2/kernel:0' shape=(600, 122) dtype=float32_ref>,
<tf.Variable 'dense_2/bias:0' shape=(122,) dtype=float32_ref>]
I0716 04:20:01.162770 139795883440000 estimator.py:1147] Done calling model_fn.
I0716 04:20:01.166483 139795883440000 basic_session_run_hooks.py:541] Create CheckpointSaverHook.
I0716 04:20:01.490742 139795883440000 monitored_session.py:240] Graph was finalized.
I0716 04:20:01.508144 139795883440000 saver.py:1280] Restoring parameters from ../model/elmo_bigru/model.ckpt-6864
I0716 04:20:02.552333 139795883440000 session_manager.py:500] Running local_init_op.
I0716 04:20:02.620016 139795883440000 session_manager.py:502] Done running local_init_op.
I0716 04:20:05.599907 139795883440000 basic_session_run_hooks.py:606] Saving checkpoints for 6864 into ../model/elmo_bigru/model.ckpt.
Reading ../data/atis.train.w-intent.iob
I0716 04:20:10.308010 139795883440000 basic_session_run_hooks.py:262] loss = 0.016606176, step = 6864 I0716 04:20:10.309966 139795883440000 basic_session_run_hooks.py:262] lr = 0.0001455599 I0716 04:20:37.941588 139795883440000 basic_session_run_hooks.py:692] global_step/sec: 3.6187 I0716 04:20:37.946143 139795883440000 basic_session_run_hooks.py:260] loss = 0.018260527, step = 6964 (27.638 sec) I0716 04:20:37.947947 139795883440000 basic_session_run_hooks.py:260] lr = 0.00014403433 (27.638 sec) I0716 04:21:04.987621 139795883440000 basic_session_run_hooks.py:692] global_step/sec: 3.69741 I0716 04:21:04.993425 139795883440000 basic_session_run_hooks.py:260] loss = 0.013289442, step = 7064 (27.047 sec) I0716 04:21:04.996516 139795883440000 basic_session_run_hooks.py:260] lr = 0.00014252475 (27.049 sec) I0716 04:21:31.147312 139795883440000 basic_session_run_hooks.py:692] global_step/sec: 3.82267 I0716 04:21:31.152855 139795883440000 basic_session_run_hooks.py:260] loss = 0.012278922, step = 7164 (26.159 sec) I0716 04:21:31.154504 139795883440000 basic_session_run_hooks.py:260] lr = 0.00014103096 (26.158 sec) I0716 04:21:33.702331 139795883440000 basic_session_run_hooks.py:606] Saving checkpoints for 7176 into ../model/elmo_bigru/model.ckpt. I0716 04:21:35.939595 139795883440000 estimator.py:1145] Calling model_fn. I0716 04:21:37.311266 139795883440000 saver.py:1499] Saver not created because there are no variables in the graph to restore I0716 04:21:37.893917 139795883440000 estimator.py:1147] Done calling model_fn. I0716 04:21:37.924747 139795883440000 evaluation.py:255] Starting evaluation at 2019-07-16T04:21:37Z I0716 04:21:38.176059 139795883440000 monitored_session.py:240] Graph was finalized. I0716 04:21:38.188971 139795883440000 saver.py:1280] Restoring parameters from ../model/elmo_bigru/model.ckpt-7176 I0716 04:21:39.124279 139795883440000 session_manager.py:500] Running local_init_op. I0716 04:21:39.181013 139795883440000 session_manager.py:502] Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
I0716 04:22:38.135489 139795883440000 evaluation.py:275] Finished evaluation at 2019-07-16-04:22:38 I0716 04:22:38.137200 139795883440000 estimator.py:2039] Saving dict for global step 7176: global_step = 7176, loss = 0.3438473 I0716 04:22:38.143747 139795883440000 estimator.py:2099] Saving 'checkpoint_path' summary for global step 7176: ../model/elmo_bigru/model.ckpt-7176 I0716 04:22:38.224134 139795883440000 estimator.py:368] Loss for final step: 0.0030283371.
Reading ../data/atis.test.w-intent.iob
I0716 04:22:38.820212 139795883440000 estimator.py:1145] Calling model_fn. I0716 04:22:40.462757 139795883440000 saver.py:1499] Saver not created because there are no variables in the graph to restore I0716 04:22:40.978199 139795883440000 estimator.py:1147] Done calling model_fn. I0716 04:22:41.382744 139795883440000 monitored_session.py:240] Graph was finalized. I0716 04:22:41.397099 139795883440000 saver.py:1280] Restoring parameters from ../model/elmo_bigru/model.ckpt-7176 I0716 04:22:42.377617 139795883440000 session_manager.py:500] Running local_init_op. I0716 04:22:42.428126 139795883440000 session_manager.py:502] Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.
'precision', 'predicted', average, warn_for)
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1439: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples.
'recall', 'true', average, warn_for)
I0716 04:23:40.307400 139795883440000 interactiveshell.py:2882]
precision recall f1-score support
atis_flight 0.986 0.992 0.989 632
atis_airfare 0.941 1.000 0.970 48
atis_ground_service 1.000 1.000 1.000 36
atis_airline 1.000 1.000 1.000 38
atis_abbreviation 0.943 1.000 0.971 33
atis_aircraft 1.000 0.889 0.941 9
atis_flight_time 0.500 1.000 0.667 1
atis_quantity 0.375 1.000 0.545 3
atis_flight#atis_airfare 0.833 0.417 0.556 12
atis_airport 1.000 1.000 1.000 18
atis_distance 1.000 1.000 1.000 10
atis_city 0.667 0.667 0.667 6
atis_ground_fare 1.000 0.857 0.923 7
atis_capacity 1.000 0.952 0.976 21
atis_flight_no 0.889 1.000 0.941 8
atis_meal 1.000 0.667 0.800 6
atis_restriction 0.000 0.000 0.000 0
atis_airline#atis_flight_no 0.000 0.000 0.000 0
atis_ground_service#atis_ground_fare 0.000 0.000 0.000 0
atis_airfare#atis_flight_time 0.000 0.000 0.000 0
atis_cheapest 0.000 0.000 0.000 0
atis_aircraft#atis_flight#atis_flight_no 0.000 0.000 0.000 0
micro avg 0.973 0.979 0.976 888
macro avg 0.642 0.656 0.634 888
weighted avg 0.976 0.979 0.976 888
I0716 04:23:40.341473 139795883440000 interactiveshell.py:2882]
precision recall f1-score support
O 0.000 0.000 0.000 0.0
B-toloc.city_name 0.978 0.993 0.985 716.0
B-fromloc.city_name 0.992 0.999 0.995 704.0
I-toloc.city_name 0.978 0.992 0.985 265.0
B-depart_date.day_name 0.991 0.991 0.991 212.0
B-airline_name 1.000 1.000 1.000 101.0
I-fromloc.city_name 0.983 1.000 0.992 177.0
B-depart_time.period_of_day 0.992 0.923 0.956 130.0
I-airline_name 1.000 1.000 1.000 65.0
B-depart_date.day_number 0.981 0.964 0.972 55.0
B-depart_date.month_name 0.982 0.964 0.973 56.0
B-depart_time.time 0.864 1.000 0.927 57.0
B-round_trip 1.000 1.000 1.000 73.0
B-cost_relative 0.973 0.973 0.973 37.0
I-round_trip 1.000 1.000 1.000 71.0
B-flight_mod 0.828 1.000 0.906 24.0
B-depart_time.time_relative 0.955 0.985 0.970 65.0
I-depart_time.time 0.945 1.000 0.972 52.0
B-stoploc.city_name 0.833 1.000 0.909 20.0
B-city_name 0.905 0.667 0.768 57.0
B-class_type 0.960 1.000 0.980 24.0
B-arrive_time.time 0.971 0.971 0.971 34.0
B-arrive_time.time_relative 0.967 0.935 0.951 31.0
I-class_type 1.000 1.000 1.000 17.0
B-flight_stop 1.000 1.000 1.000 21.0
I-arrive_time.time 1.000 0.971 0.986 35.0
B-airline_code 0.895 1.000 0.944 34.0
I-depart_date.day_number 1.000 0.933 0.966 15.0
I-fromloc.airport_name 0.441 1.000 0.612 15.0
B-fromloc.airport_name 0.429 1.000 0.600 12.0
B-arrive_date.day_name 0.786 1.000 0.880 11.0
B-toloc.state_code 1.000 1.000 1.000 18.0
B-depart_date.today_relative 1.000 0.889 0.941 9.0
B-flight_number 0.733 1.000 0.846 11.0
B-depart_date.date_relative 0.944 1.000 0.971 17.0
B-toloc.state_name 0.839 0.929 0.881 28.0
B-fare_basis_code 0.895 1.000 0.944 17.0
B-flight_time 1.000 1.000 1.000 1.0
B-or 1.000 1.000 1.000 3.0
B-arrive_time.period_of_day 0.750 1.000 0.857 6.0
B-meal_description 1.000 0.900 0.947 10.0
I-cost_relative 1.000 0.667 0.800 3.0
I-airport_name 0.846 0.379 0.524 29.0
B-fare_amount 1.000 1.000 1.000 2.0
I-fare_amount 1.000 1.000 1.000 2.0
I-city_name 1.000 0.600 0.750 30.0
I-toloc.airport_name 1.000 1.000 1.000 3.0
B-transport_type 1.000 1.000 1.000 10.0
B-arrive_date.month_name 0.714 0.833 0.769 6.0
B-arrive_date.day_number 0.714 0.833 0.769 6.0
I-stoploc.city_name 1.000 1.000 1.000 10.0
B-meal 1.000 1.000 1.000 16.0
B-fromloc.state_code 1.000 1.000 1.000 23.0
B-depart_time.period_mod 1.000 1.000 1.000 5.0
B-connect 1.000 1.000 1.000 6.0
B-flight_days 1.000 1.000 1.000 10.0
B-toloc.airport_name 1.000 1.000 1.000 3.0
B-fromloc.state_name 1.000 1.000 1.000 17.0
B-airport_name 0.636 0.333 0.437 21.0
B-economy 1.000 1.000 1.000 6.0
I-flight_time 1.000 1.000 1.000 1.0
B-aircraft_code 1.000 0.818 0.900 33.0
B-mod 1.000 0.500 0.667 2.0
B-airport_code 1.000 0.444 0.615 9.0
B-depart_time.start_time 1.000 0.667 0.800 3.0
B-depart_time.end_time 1.000 0.333 0.500 3.0
B-depart_date.year 1.000 1.000 1.000 3.0
I-transport_type 0.000 0.000 0.000 1.0
B-restriction_code 1.000 1.000 1.000 4.0
B-arrive_time.start_time 0.889 1.000 0.941 8.0
B-toloc.airport_code 1.000 0.750 0.857 4.0
B-arrive_time.end_time 0.889 1.000 0.941 8.0
I-arrive_time.end_time 0.889 1.000 0.941 8.0
I-depart_time.end_time 1.000 0.333 0.500 3.0
I-flight_stop 0.000 0.000 0.000 0.0
B-fromloc.airport_code 0.625 1.000 0.769 5.0
I-restriction_code 1.000 1.000 1.000 3.0
I-depart_time.start_time 1.000 1.000 1.000 1.0
I-toloc.state_name 1.000 1.000 1.000 1.0
I-depart_date.today_relative 0.000 0.000 0.000 0.0
B-arrive_date.date_relative 1.000 1.000 1.000 2.0
I-flight_mod 1.000 0.167 0.286 6.0
I-economy 0.000 0.000 0.000 0.0
B-return_date.date_relative 1.000 0.667 0.800 3.0
I-fromloc.state_name 1.000 1.000 1.000 1.0
B-state_code 1.000 1.000 1.000 1.0
I-arrive_time.start_time 1.000 1.000 1.000 1.0
I-arrive_date.day_number 0.000 0.000 0.000 0.0
B-meal_code 0.000 0.000 0.000 1.0
I-depart_time.period_of_day 1.000 1.000 1.000 1.0
B-day_name 1.000 0.500 0.667 2.0
B-period_of_day 1.000 0.750 0.857 4.0
B-stoploc.state_code 0.000 0.000 0.000 0.0
B-return_date.month_name 0.000 0.000 0.000 0.0
B-return_date.day_number 0.000 0.000 0.000 0.0
B-arrive_time.period_mod 0.000 0.000 0.000 0.0
I-meal_code 0.000 0.000 0.000 0.0
B-toloc.country_name 1.000 1.000 1.000 1.0
B-days_code 1.000 1.000 1.000 1.0
I-arrive_time.period_of_day 0.000 0.000 0.000 0.0
I-today_relative 0.000 0.000 0.000 0.0
B-return_time.period_of_day 0.000 0.000 0.000 0.0
B-time 0.000 0.000 0.000 0.0
I-fare_basis_code 0.000 0.000 0.000 0.0
I-arrive_time.time_relative 1.000 1.000 1.000 4.0
I-depart_time.time_relative 0.000 0.000 0.000 1.0
B-today_relative 0.000 0.000 0.000 0.0
B-state_name 0.000 0.000 0.000 9.0
B-arrive_date.today_relative 0.000 0.000 0.000 0.0
B-return_time.period_mod 0.000 0.000 0.000 0.0
B-month_name 0.000 0.000 0.000 0.0
B-day_number 0.000 0.000 0.000 0.0
I-return_date.date_relative 1.000 0.667 0.800 3.0
I-return_date.today_relative 0.000 0.000 0.000 0.0
B-stoploc.airport_name 0.000 0.000 0.000 0.0
B-time_relative 0.000 0.000 0.000 0.0
I-time 0.000 0.000 0.000 0.0
I-return_date.day_number 0.000 0.000 0.000 0.0
I-meal_description 0.000 0.000 0.000 0.0
B-return_date.today_relative 0.000 0.000 0.000 0.0
B-return_date.day_name 0.000 0.000 0.000 2.0
micro avg 0.960 0.961 0.960 3657.0
macro avg 0.694 0.663 0.665 3657.0
weighted avg 0.964 0.961 0.958 3657.0
I0716 04:23:40.360459 139795883440000 interactiveshell.py:2882] Best Slot F1: 0.960
I0716 04:23:40.362138 139795883440000 estimator_training.py:186] Not using Distribute Coordinator.
I0716 04:23:40.364122 139795883440000 training.py:612] Running training and evaluation locally (non-distributed).
I0716 04:23:40.365844 139795883440000 training.py:700] Start train and evaluate loop. The evaluate will happen after every checkpoint. Checkpoint frequency is determined based on RunConfig arguments: save_checkpoints_steps 312 or save_checkpoints_secs None.
I0716 04:23:40.436918 139795883440000 estimator.py:1145] Calling model_fn.
I0716 04:23:41.739951 139795883440000 saver.py:1499] Saver not created because there are no variables in the graph to restore
I0716 04:23:42.492683 139795883440000 <ipython-input-6-d6747b40360b>:55]
[<tf.Variable 'embedding:0' shape=(750, 300) dtype=float32_ref>,
<tf.Variable 'module/aggregation/weights:0' shape=(3,) dtype=float32>,
<tf.Variable 'module/aggregation/scaling:0' shape=() dtype=float32>,
<tf.Variable 'dense/kernel:0' shape=(1324, 300) dtype=float32_ref>,
<tf.Variable 'dense/bias:0' shape=(300,) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/fw/gru_cell/gates/kernel:0' shape=(600, 600) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/fw/gru_cell/gates/bias:0' shape=(600,) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/fw/gru_cell/candidate/kernel:0' shape=(600, 300) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/fw/gru_cell/candidate/bias:0' shape=(300,) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/bw/gru_cell/gates/kernel:0' shape=(600, 600) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/bw/gru_cell/gates/bias:0' shape=(600,) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/bw/gru_cell/candidate/kernel:0' shape=(600, 300) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/bw/gru_cell/candidate/bias:0' shape=(300,) dtype=float32_ref>,
<tf.Variable 'dense_1/kernel:0' shape=(600, 23) dtype=float32_ref>,
<tf.Variable 'dense_1/bias:0' shape=(23,) dtype=float32_ref>,
<tf.Variable 'dense_2/kernel:0' shape=(600, 122) dtype=float32_ref>,
<tf.Variable 'dense_2/bias:0' shape=(122,) dtype=float32_ref>]
I0716 04:23:43.572311 139795883440000 estimator.py:1147] Done calling model_fn.
I0716 04:23:43.579196 139795883440000 basic_session_run_hooks.py:541] Create CheckpointSaverHook.
I0716 04:23:43.895811 139795883440000 monitored_session.py:240] Graph was finalized.
I0716 04:23:43.913872 139795883440000 saver.py:1280] Restoring parameters from ../model/elmo_bigru/model.ckpt-7176
I0716 04:23:44.946667 139795883440000 session_manager.py:500] Running local_init_op.
I0716 04:23:45.012770 139795883440000 session_manager.py:502] Done running local_init_op.
I0716 04:23:47.791606 139795883440000 basic_session_run_hooks.py:606] Saving checkpoints for 7176 into ../model/elmo_bigru/model.ckpt.
Reading ../data/atis.train.w-intent.iob
I0716 04:23:52.395349 139795883440000 basic_session_run_hooks.py:262] loss = 0.012449772, step = 7176 I0716 04:23:52.397220 139795883440000 basic_session_run_hooks.py:262] lr = 0.00014085279 I0716 04:24:19.545236 139795883440000 basic_session_run_hooks.py:692] global_step/sec: 3.68319 I0716 04:24:19.550419 139795883440000 basic_session_run_hooks.py:260] loss = 0.012549496, step = 7276 (27.155 sec) I0716 04:24:19.552397 139795883440000 basic_session_run_hooks.py:260] lr = 0.00013937654 (27.155 sec) I0716 04:24:45.943143 139795883440000 basic_session_run_hooks.py:692] global_step/sec: 3.78818 I0716 04:24:45.951433 139795883440000 basic_session_run_hooks.py:260] loss = 0.016497286, step = 7376 (26.401 sec) I0716 04:24:45.953580 139795883440000 basic_session_run_hooks.py:260] lr = 0.00013791576 (26.401 sec) I0716 04:25:11.899698 139795883440000 basic_session_run_hooks.py:692] global_step/sec: 3.85259 I0716 04:25:11.905721 139795883440000 basic_session_run_hooks.py:260] loss = 0.011051845, step = 7476 (25.954 sec) I0716 04:25:11.907590 139795883440000 basic_session_run_hooks.py:260] lr = 0.0001364703 (25.954 sec) I0716 04:25:14.569300 139795883440000 basic_session_run_hooks.py:606] Saving checkpoints for 7488 into ../model/elmo_bigru/model.ckpt. I0716 04:25:16.719005 139795883440000 estimator.py:1145] Calling model_fn. I0716 04:25:18.079175 139795883440000 saver.py:1499] Saver not created because there are no variables in the graph to restore I0716 04:25:18.678386 139795883440000 estimator.py:1147] Done calling model_fn. I0716 04:25:18.706707 139795883440000 evaluation.py:255] Starting evaluation at 2019-07-16T04:25:18Z I0716 04:25:18.926704 139795883440000 monitored_session.py:240] Graph was finalized. I0716 04:25:18.943488 139795883440000 saver.py:1280] Restoring parameters from ../model/elmo_bigru/model.ckpt-7488 I0716 04:25:19.838812 139795883440000 session_manager.py:500] Running local_init_op. I0716 04:25:19.898500 139795883440000 session_manager.py:502] Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
I0716 04:26:19.402054 139795883440000 evaluation.py:275] Finished evaluation at 2019-07-16-04:26:19 I0716 04:26:19.403770 139795883440000 estimator.py:2039] Saving dict for global step 7488: global_step = 7488, loss = 0.35254303 I0716 04:26:19.412193 139795883440000 estimator.py:2099] Saving 'checkpoint_path' summary for global step 7488: ../model/elmo_bigru/model.ckpt-7488 I0716 04:26:19.497845 139795883440000 estimator.py:368] Loss for final step: 0.005028704.
Reading ../data/atis.test.w-intent.iob
I0716 04:26:20.104184 139795883440000 estimator.py:1145] Calling model_fn. I0716 04:26:21.773334 139795883440000 saver.py:1499] Saver not created because there are no variables in the graph to restore I0716 04:26:22.289934 139795883440000 estimator.py:1147] Done calling model_fn. I0716 04:26:22.500285 139795883440000 monitored_session.py:240] Graph was finalized. I0716 04:26:22.516782 139795883440000 saver.py:1280] Restoring parameters from ../model/elmo_bigru/model.ckpt-7488 I0716 04:26:23.414255 139795883440000 session_manager.py:500] Running local_init_op. I0716 04:26:23.474438 139795883440000 session_manager.py:502] Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.
'precision', 'predicted', average, warn_for)
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1439: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples.
'recall', 'true', average, warn_for)
I0716 04:27:21.630802 139795883440000 interactiveshell.py:2882]
precision recall f1-score support
atis_flight 0.986 0.992 0.989 632
atis_airfare 0.941 1.000 0.970 48
atis_ground_service 1.000 1.000 1.000 36
atis_airline 0.974 1.000 0.987 38
atis_abbreviation 0.943 1.000 0.971 33
atis_aircraft 1.000 0.889 0.941 9
atis_flight_time 0.500 1.000 0.667 1
atis_quantity 0.429 1.000 0.600 3
atis_flight#atis_airfare 0.833 0.417 0.556 12
atis_airport 1.000 0.944 0.971 18
atis_distance 1.000 1.000 1.000 10
atis_city 0.667 0.667 0.667 6
atis_ground_fare 1.000 0.857 0.923 7
atis_capacity 0.952 0.952 0.952 21
atis_flight_no 0.889 1.000 0.941 8
atis_meal 1.000 0.667 0.800 6
atis_restriction 0.000 0.000 0.000 0
atis_airline#atis_flight_no 0.000 0.000 0.000 0
atis_ground_service#atis_ground_fare 0.000 0.000 0.000 0
atis_airfare#atis_flight_time 0.000 0.000 0.000 0
atis_cheapest 0.000 0.000 0.000 0
atis_aircraft#atis_flight#atis_flight_no 0.000 0.000 0.000 0
micro avg 0.972 0.977 0.975 888
macro avg 0.642 0.654 0.633 888
weighted avg 0.974 0.977 0.974 888
I0716 04:27:21.669584 139795883440000 interactiveshell.py:2882]
precision recall f1-score support
O 0.000 0.000 0.000 0.0
B-toloc.city_name 0.975 0.992 0.983 716.0
B-fromloc.city_name 0.986 0.999 0.992 704.0
I-toloc.city_name 0.978 0.992 0.985 265.0
B-depart_date.day_name 0.991 0.991 0.991 212.0
B-airline_name 1.000 1.000 1.000 101.0
I-fromloc.city_name 0.983 1.000 0.992 177.0
B-depart_time.period_of_day 0.992 0.923 0.956 130.0
I-airline_name 1.000 1.000 1.000 65.0
B-depart_date.day_number 0.981 0.964 0.972 55.0
B-depart_date.month_name 0.982 0.964 0.973 56.0
B-depart_time.time 0.864 1.000 0.927 57.0
B-round_trip 1.000 1.000 1.000 73.0
B-cost_relative 0.973 0.973 0.973 37.0
I-round_trip 1.000 1.000 1.000 71.0
B-flight_mod 0.828 1.000 0.906 24.0
B-depart_time.time_relative 0.955 0.969 0.962 65.0
I-depart_time.time 0.945 1.000 0.972 52.0
B-stoploc.city_name 0.909 1.000 0.952 20.0
B-city_name 0.881 0.649 0.747 57.0
B-class_type 0.960 1.000 0.980 24.0
B-arrive_time.time 0.971 0.971 0.971 34.0
B-arrive_time.time_relative 0.906 0.935 0.921 31.0
I-class_type 1.000 1.000 1.000 17.0
B-flight_stop 1.000 1.000 1.000 21.0
I-arrive_time.time 1.000 0.971 0.986 35.0
B-airline_code 0.944 1.000 0.971 34.0
I-depart_date.day_number 1.000 0.933 0.966 15.0
I-fromloc.airport_name 0.429 1.000 0.600 15.0
B-fromloc.airport_name 0.429 1.000 0.600 12.0
B-arrive_date.day_name 0.786 1.000 0.880 11.0
B-toloc.state_code 1.000 1.000 1.000 18.0
B-depart_date.today_relative 1.000 0.889 0.941 9.0
B-flight_number 0.733 1.000 0.846 11.0
B-depart_date.date_relative 0.944 1.000 0.971 17.0
B-toloc.state_name 0.862 0.893 0.877 28.0
B-fare_basis_code 0.773 1.000 0.872 17.0
B-flight_time 1.000 1.000 1.000 1.0
B-or 1.000 1.000 1.000 3.0
B-arrive_time.period_of_day 0.750 1.000 0.857 6.0
B-meal_description 1.000 0.900 0.947 10.0
I-cost_relative 1.000 0.667 0.800 3.0
I-airport_name 0.818 0.310 0.450 29.0
B-fare_amount 1.000 1.000 1.000 2.0
I-fare_amount 1.000 1.000 1.000 2.0
I-city_name 0.944 0.567 0.708 30.0
I-toloc.airport_name 1.000 1.000 1.000 3.0
B-transport_type 1.000 1.000 1.000 10.0
B-arrive_date.month_name 0.714 0.833 0.769 6.0
B-arrive_date.day_number 0.714 0.833 0.769 6.0
I-stoploc.city_name 1.000 1.000 1.000 10.0
B-meal 1.000 1.000 1.000 16.0
B-fromloc.state_code 1.000 1.000 1.000 23.0
B-depart_time.period_mod 0.833 1.000 0.909 5.0
B-connect 1.000 1.000 1.000 6.0
B-flight_days 1.000 1.000 1.000 10.0
B-toloc.airport_name 1.000 1.000 1.000 3.0
B-fromloc.state_name 0.944 1.000 0.971 17.0
B-airport_name 0.600 0.286 0.387 21.0
B-economy 1.000 1.000 1.000 6.0
I-flight_time 1.000 1.000 1.000 1.0
B-aircraft_code 1.000 0.818 0.900 33.0
B-mod 1.000 0.500 0.667 2.0
B-airport_code 1.000 0.444 0.615 9.0
B-depart_time.start_time 1.000 0.667 0.800 3.0
B-depart_time.end_time 1.000 0.333 0.500 3.0
B-depart_date.year 1.000 1.000 1.000 3.0
I-transport_type 0.000 0.000 0.000 1.0
B-restriction_code 1.000 1.000 1.000 4.0
B-arrive_time.start_time 0.889 1.000 0.941 8.0
B-toloc.airport_code 1.000 0.750 0.857 4.0
B-arrive_time.end_time 0.889 1.000 0.941 8.0
I-arrive_time.end_time 0.889 1.000 0.941 8.0
I-depart_time.end_time 1.000 0.333 0.500 3.0
I-flight_stop 0.000 0.000 0.000 0.0
B-fromloc.airport_code 0.625 1.000 0.769 5.0
I-restriction_code 1.000 1.000 1.000 3.0
I-depart_time.start_time 1.000 1.000 1.000 1.0
I-toloc.state_name 1.000 1.000 1.000 1.0
I-depart_date.today_relative 0.000 0.000 0.000 0.0
B-arrive_date.date_relative 1.000 1.000 1.000 2.0
I-flight_mod 1.000 0.167 0.286 6.0
I-economy 0.000 0.000 0.000 0.0
B-return_date.date_relative 1.000 0.667 0.800 3.0
I-fromloc.state_name 1.000 1.000 1.000 1.0
B-state_code 1.000 1.000 1.000 1.0
I-arrive_time.start_time 1.000 1.000 1.000 1.0
I-arrive_date.day_number 0.000 0.000 0.000 0.0
B-meal_code 0.000 0.000 0.000 1.0
I-depart_time.period_of_day 1.000 1.000 1.000 1.0
B-day_name 1.000 0.500 0.667 2.0
B-period_of_day 1.000 0.750 0.857 4.0
B-stoploc.state_code 0.000 0.000 0.000 0.0
B-return_date.month_name 0.000 0.000 0.000 0.0
B-return_date.day_number 0.000 0.000 0.000 0.0
B-arrive_time.period_mod 0.000 0.000 0.000 0.0
I-meal_code 0.000 0.000 0.000 0.0
B-toloc.country_name 1.000 1.000 1.000 1.0
B-days_code 0.000 0.000 0.000 1.0
I-arrive_time.period_of_day 0.000 0.000 0.000 0.0
I-today_relative 0.000 0.000 0.000 0.0
B-return_time.period_of_day 0.000 0.000 0.000 0.0
B-time 0.000 0.000 0.000 0.0
I-fare_basis_code 0.000 0.000 0.000 0.0
I-arrive_time.time_relative 1.000 0.500 0.667 4.0
I-depart_time.time_relative 0.000 0.000 0.000 1.0
B-today_relative 0.000 0.000 0.000 0.0
B-state_name 0.000 0.000 0.000 9.0
B-arrive_date.today_relative 0.000 0.000 0.000 0.0
B-return_time.period_mod 0.000 0.000 0.000 0.0
B-month_name 0.000 0.000 0.000 0.0
B-day_number 0.000 0.000 0.000 0.0
I-return_date.date_relative 1.000 0.667 0.800 3.0
I-return_date.today_relative 0.000 0.000 0.000 0.0
B-stoploc.airport_name 0.000 0.000 0.000 0.0
B-time_relative 0.000 0.000 0.000 0.0
I-time 0.000 0.000 0.000 0.0
I-return_date.day_number 0.000 0.000 0.000 0.0
I-meal_description 0.000 0.000 0.000 0.0
B-return_date.today_relative 0.000 0.000 0.000 0.0
B-return_date.day_name 0.000 0.000 0.000 2.0
micro avg 0.957 0.958 0.957 3657.0
macro avg 0.682 0.649 0.651 3657.0
weighted avg 0.960 0.958 0.954 3657.0
I0716 04:27:21.691767 139795883440000 interactiveshell.py:2882] Best Slot F1: 0.960
I0716 04:27:21.693325 139795883440000 estimator_training.py:186] Not using Distribute Coordinator.
I0716 04:27:21.695086 139795883440000 training.py:612] Running training and evaluation locally (non-distributed).
I0716 04:27:21.697174 139795883440000 training.py:700] Start train and evaluate loop. The evaluate will happen after every checkpoint. Checkpoint frequency is determined based on RunConfig arguments: save_checkpoints_steps 312 or save_checkpoints_secs None.
I0716 04:27:21.766440 139795883440000 estimator.py:1145] Calling model_fn.
I0716 04:27:23.241953 139795883440000 saver.py:1499] Saver not created because there are no variables in the graph to restore
I0716 04:27:23.843529 139795883440000 <ipython-input-6-d6747b40360b>:55]
[<tf.Variable 'embedding:0' shape=(750, 300) dtype=float32_ref>,
<tf.Variable 'module/aggregation/weights:0' shape=(3,) dtype=float32>,
<tf.Variable 'module/aggregation/scaling:0' shape=() dtype=float32>,
<tf.Variable 'dense/kernel:0' shape=(1324, 300) dtype=float32_ref>,
<tf.Variable 'dense/bias:0' shape=(300,) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/fw/gru_cell/gates/kernel:0' shape=(600, 600) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/fw/gru_cell/gates/bias:0' shape=(600,) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/fw/gru_cell/candidate/kernel:0' shape=(600, 300) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/fw/gru_cell/candidate/bias:0' shape=(300,) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/bw/gru_cell/gates/kernel:0' shape=(600, 600) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/bw/gru_cell/gates/bias:0' shape=(600,) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/bw/gru_cell/candidate/kernel:0' shape=(600, 300) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/bw/gru_cell/candidate/bias:0' shape=(300,) dtype=float32_ref>,
<tf.Variable 'dense_1/kernel:0' shape=(600, 23) dtype=float32_ref>,
<tf.Variable 'dense_1/bias:0' shape=(23,) dtype=float32_ref>,
<tf.Variable 'dense_2/kernel:0' shape=(600, 122) dtype=float32_ref>,
<tf.Variable 'dense_2/bias:0' shape=(122,) dtype=float32_ref>]
I0716 04:27:25.118303 139795883440000 estimator.py:1147] Done calling model_fn.
I0716 04:27:25.121737 139795883440000 basic_session_run_hooks.py:541] Create CheckpointSaverHook.
I0716 04:27:25.438082 139795883440000 monitored_session.py:240] Graph was finalized.
I0716 04:27:25.456156 139795883440000 saver.py:1280] Restoring parameters from ../model/elmo_bigru/model.ckpt-7488
I0716 04:27:26.499037 139795883440000 session_manager.py:500] Running local_init_op.
I0716 04:27:26.571258 139795883440000 session_manager.py:502] Done running local_init_op.
I0716 04:27:29.367456 139795883440000 basic_session_run_hooks.py:606] Saving checkpoints for 7488 into ../model/elmo_bigru/model.ckpt.
Reading ../data/atis.train.w-intent.iob
I0716 04:27:34.189820 139795883440000 basic_session_run_hooks.py:262] loss = 0.020746749, step = 7488 I0716 04:27:34.191736 139795883440000 basic_session_run_hooks.py:262] lr = 0.00013629787 I0716 04:28:01.006389 139795883440000 basic_session_run_hooks.py:692] global_step/sec: 3.72894 I0716 04:28:01.014677 139795883440000 basic_session_run_hooks.py:260] loss = 0.014712622, step = 7588 (26.825 sec) I0716 04:28:01.016867 139795883440000 basic_session_run_hooks.py:260] lr = 0.00013486936 (26.825 sec) I0716 04:28:26.477072 139795883440000 basic_session_run_hooks.py:692] global_step/sec: 3.92609 I0716 04:28:26.480191 139795883440000 basic_session_run_hooks.py:260] loss = 0.012815632, step = 7688 (25.466 sec) I0716 04:28:26.481969 139795883440000 basic_session_run_hooks.py:260] lr = 0.00013345583 (25.465 sec) I0716 04:28:54.003954 139795883440000 basic_session_run_hooks.py:692] global_step/sec: 3.63281 I0716 04:28:54.011584 139795883440000 basic_session_run_hooks.py:260] loss = 0.015086541, step = 7788 (27.531 sec) I0716 04:28:54.014548 139795883440000 basic_session_run_hooks.py:260] lr = 0.00013205713 (27.533 sec) I0716 04:28:57.070349 139795883440000 basic_session_run_hooks.py:606] Saving checkpoints for 7800 into ../model/elmo_bigru/model.ckpt. I0716 04:28:59.224375 139795883440000 estimator.py:1145] Calling model_fn. I0716 04:29:00.839649 139795883440000 saver.py:1499] Saver not created because there are no variables in the graph to restore I0716 04:29:01.399063 139795883440000 estimator.py:1147] Done calling model_fn. I0716 04:29:01.428005 139795883440000 evaluation.py:255] Starting evaluation at 2019-07-16T04:29:01Z I0716 04:29:01.650110 139795883440000 monitored_session.py:240] Graph was finalized. I0716 04:29:01.671104 139795883440000 saver.py:1280] Restoring parameters from ../model/elmo_bigru/model.ckpt-7800 I0716 04:29:02.555532 139795883440000 session_manager.py:500] Running local_init_op. I0716 04:29:02.610178 139795883440000 session_manager.py:502] Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
I0716 04:30:01.353968 139795883440000 evaluation.py:275] Finished evaluation at 2019-07-16-04:30:01 I0716 04:30:01.355541 139795883440000 estimator.py:2039] Saving dict for global step 7800: global_step = 7800, loss = 0.34348074 I0716 04:30:01.362978 139795883440000 estimator.py:2099] Saving 'checkpoint_path' summary for global step 7800: ../model/elmo_bigru/model.ckpt-7800 I0716 04:30:01.447452 139795883440000 estimator.py:368] Loss for final step: 0.009322032.
Reading ../data/atis.test.w-intent.iob
I0716 04:30:02.041821 139795883440000 estimator.py:1145] Calling model_fn. I0716 04:30:03.569986 139795883440000 saver.py:1499] Saver not created because there are no variables in the graph to restore I0716 04:30:04.087395 139795883440000 estimator.py:1147] Done calling model_fn. I0716 04:30:04.313645 139795883440000 monitored_session.py:240] Graph was finalized. I0716 04:30:04.329914 139795883440000 saver.py:1280] Restoring parameters from ../model/elmo_bigru/model.ckpt-7800 I0716 04:30:05.232841 139795883440000 session_manager.py:500] Running local_init_op. I0716 04:30:05.289027 139795883440000 session_manager.py:502] Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.
'precision', 'predicted', average, warn_for)
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1439: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples.
'recall', 'true', average, warn_for)
I0716 04:31:02.991110 139795883440000 interactiveshell.py:2882]
precision recall f1-score support
atis_flight 0.987 0.992 0.990 632
atis_airfare 0.941 1.000 0.970 48
atis_ground_service 1.000 1.000 1.000 36
atis_airline 0.974 1.000 0.987 38
atis_abbreviation 0.943 1.000 0.971 33
atis_aircraft 1.000 0.889 0.941 9
atis_flight_time 0.333 1.000 0.500 1
atis_quantity 0.429 1.000 0.600 3
atis_flight#atis_airfare 0.833 0.417 0.556 12
atis_airport 1.000 0.944 0.971 18
atis_distance 1.000 1.000 1.000 10
atis_city 0.800 0.667 0.727 6
atis_ground_fare 1.000 0.857 0.923 7
atis_capacity 0.952 0.952 0.952 21
atis_flight_no 0.889 1.000 0.941 8
atis_meal 1.000 0.833 0.909 6
atis_restriction 0.000 0.000 0.000 0
atis_airline#atis_flight_no 0.000 0.000 0.000 0
atis_ground_service#atis_ground_fare 0.000 0.000 0.000 0
atis_airfare#atis_flight_time 0.000 0.000 0.000 0
atis_cheapest 0.000 0.000 0.000 0
atis_aircraft#atis_flight#atis_flight_no 0.000 0.000 0.000 0
micro avg 0.973 0.979 0.976 888
macro avg 0.640 0.661 0.634 888
weighted avg 0.976 0.979 0.976 888
I0716 04:31:03.029418 139795883440000 interactiveshell.py:2882]
precision recall f1-score support
O 0.000 0.000 0.000 0.0
B-toloc.city_name 0.978 0.994 0.986 716.0
B-fromloc.city_name 0.992 0.999 0.995 704.0
I-toloc.city_name 0.978 0.992 0.985 265.0
B-depart_date.day_name 0.991 0.991 0.991 212.0
B-airline_name 1.000 1.000 1.000 101.0
I-fromloc.city_name 0.983 1.000 0.992 177.0
B-depart_time.period_of_day 0.992 0.923 0.956 130.0
I-airline_name 0.985 1.000 0.992 65.0
B-depart_date.day_number 0.981 0.964 0.972 55.0
B-depart_date.month_name 0.982 0.964 0.973 56.0
B-depart_time.time 0.864 1.000 0.927 57.0
B-round_trip 1.000 1.000 1.000 73.0
B-cost_relative 0.973 0.973 0.973 37.0
I-round_trip 1.000 1.000 1.000 71.0
B-flight_mod 0.828 1.000 0.906 24.0
B-depart_time.time_relative 0.970 0.985 0.977 65.0
I-depart_time.time 0.945 1.000 0.972 52.0
B-stoploc.city_name 0.870 1.000 0.930 20.0
B-city_name 0.864 0.667 0.752 57.0
B-class_type 0.960 1.000 0.980 24.0
B-arrive_time.time 0.971 0.971 0.971 34.0
B-arrive_time.time_relative 0.906 0.935 0.921 31.0
I-class_type 1.000 1.000 1.000 17.0
B-flight_stop 1.000 1.000 1.000 21.0
I-arrive_time.time 1.000 0.971 0.986 35.0
B-airline_code 0.895 1.000 0.944 34.0
I-depart_date.day_number 1.000 0.933 0.966 15.0
I-fromloc.airport_name 0.441 1.000 0.612 15.0
B-fromloc.airport_name 0.444 1.000 0.615 12.0
B-arrive_date.day_name 0.786 1.000 0.880 11.0
B-toloc.state_code 1.000 1.000 1.000 18.0
B-depart_date.today_relative 1.000 0.889 0.941 9.0
B-flight_number 0.733 1.000 0.846 11.0
B-depart_date.date_relative 0.944 1.000 0.971 17.0
B-toloc.state_name 0.788 0.929 0.852 28.0
B-fare_basis_code 0.850 1.000 0.919 17.0
B-flight_time 1.000 1.000 1.000 1.0
B-or 1.000 1.000 1.000 3.0
B-arrive_time.period_of_day 0.750 1.000 0.857 6.0
B-meal_description 1.000 0.900 0.947 10.0
I-cost_relative 1.000 0.667 0.800 3.0
I-airport_name 0.846 0.379 0.524 29.0
B-fare_amount 1.000 1.000 1.000 2.0
I-fare_amount 1.000 1.000 1.000 2.0
I-city_name 1.000 0.600 0.750 30.0
I-toloc.airport_name 1.000 1.000 1.000 3.0
B-transport_type 1.000 1.000 1.000 10.0
B-arrive_date.month_name 0.714 0.833 0.769 6.0
B-arrive_date.day_number 0.714 0.833 0.769 6.0
I-stoploc.city_name 1.000 1.000 1.000 10.0
B-meal 0.941 1.000 0.970 16.0
B-fromloc.state_code 1.000 1.000 1.000 23.0
B-depart_time.period_mod 1.000 1.000 1.000 5.0
B-connect 1.000 1.000 1.000 6.0
B-flight_days 1.000 1.000 1.000 10.0
B-toloc.airport_name 1.000 1.000 1.000 3.0
B-fromloc.state_name 1.000 1.000 1.000 17.0
B-airport_name 0.750 0.286 0.414 21.0
B-economy 1.000 1.000 1.000 6.0
I-flight_time 1.000 1.000 1.000 1.0
B-aircraft_code 1.000 0.818 0.900 33.0
B-mod 1.000 0.500 0.667 2.0
B-airport_code 1.000 0.444 0.615 9.0
B-depart_time.start_time 1.000 0.667 0.800 3.0
B-depart_time.end_time 1.000 0.333 0.500 3.0
B-depart_date.year 1.000 1.000 1.000 3.0
I-transport_type 0.000 0.000 0.000 1.0
B-restriction_code 1.000 1.000 1.000 4.0
B-arrive_time.start_time 0.889 1.000 0.941 8.0
B-toloc.airport_code 1.000 0.750 0.857 4.0
B-arrive_time.end_time 0.889 1.000 0.941 8.0
I-arrive_time.end_time 0.889 1.000 0.941 8.0
I-depart_time.end_time 1.000 0.333 0.500 3.0
I-flight_stop 0.000 0.000 0.000 0.0
B-fromloc.airport_code 0.625 1.000 0.769 5.0
I-restriction_code 1.000 1.000 1.000 3.0
I-depart_time.start_time 1.000 1.000 1.000 1.0
I-toloc.state_name 1.000 1.000 1.000 1.0
I-depart_date.today_relative 0.000 0.000 0.000 0.0
B-arrive_date.date_relative 1.000 1.000 1.000 2.0
I-flight_mod 1.000 0.167 0.286 6.0
I-economy 0.000 0.000 0.000 0.0
B-return_date.date_relative 1.000 0.667 0.800 3.0
I-fromloc.state_name 1.000 1.000 1.000 1.0
B-state_code 1.000 1.000 1.000 1.0
I-arrive_time.start_time 1.000 1.000 1.000 1.0
I-arrive_date.day_number 0.000 0.000 0.000 0.0
B-meal_code 0.000 0.000 0.000 1.0
I-depart_time.period_of_day 1.000 1.000 1.000 1.0
B-day_name 1.000 0.500 0.667 2.0
B-period_of_day 1.000 0.750 0.857 4.0
B-stoploc.state_code 0.000 0.000 0.000 0.0
B-return_date.month_name 0.000 0.000 0.000 0.0
B-return_date.day_number 0.000 0.000 0.000 0.0
B-arrive_time.period_mod 0.000 0.000 0.000 0.0
I-meal_code 0.000 0.000 0.000 0.0
B-toloc.country_name 0.000 0.000 0.000 1.0
B-days_code 0.000 0.000 0.000 1.0
I-arrive_time.period_of_day 0.000 0.000 0.000 0.0
I-today_relative 0.000 0.000 0.000 0.0
B-return_time.period_of_day 0.000 0.000 0.000 0.0
B-time 0.000 0.000 0.000 0.0
I-fare_basis_code 0.000 0.000 0.000 0.0
I-arrive_time.time_relative 1.000 0.500 0.667 4.0
I-depart_time.time_relative 0.000 0.000 0.000 1.0
B-today_relative 0.000 0.000 0.000 0.0
B-state_name 0.000 0.000 0.000 9.0
B-arrive_date.today_relative 0.000 0.000 0.000 0.0
B-return_time.period_mod 0.000 0.000 0.000 0.0
B-month_name 0.000 0.000 0.000 0.0
B-day_number 0.000 0.000 0.000 0.0
I-return_date.date_relative 1.000 0.667 0.800 3.0
I-return_date.today_relative 0.000 0.000 0.000 0.0
B-stoploc.airport_name 0.000 0.000 0.000 0.0
B-time_relative 0.000 0.000 0.000 0.0
I-time 0.000 0.000 0.000 0.0
I-return_date.day_number 0.000 0.000 0.000 0.0
I-meal_description 0.000 0.000 0.000 0.0
B-return_date.today_relative 0.000 0.000 0.000 0.0
B-return_date.day_name 0.000 0.000 0.000 2.0
micro avg 0.959 0.960 0.959 3657.0
macro avg 0.677 0.642 0.645 3657.0
weighted avg 0.962 0.960 0.956 3657.0
I0716 04:31:03.051072 139795883440000 interactiveshell.py:2882] Best Slot F1: 0.960
I0716 04:31:03.052510 139795883440000 estimator_training.py:186] Not using Distribute Coordinator.
I0716 04:31:03.054505 139795883440000 training.py:612] Running training and evaluation locally (non-distributed).
I0716 04:31:03.056043 139795883440000 training.py:700] Start train and evaluate loop. The evaluate will happen after every checkpoint. Checkpoint frequency is determined based on RunConfig arguments: save_checkpoints_steps 312 or save_checkpoints_secs None.
I0716 04:31:03.125035 139795883440000 estimator.py:1145] Calling model_fn.
I0716 04:31:04.600318 139795883440000 saver.py:1499] Saver not created because there are no variables in the graph to restore
I0716 04:31:05.180009 139795883440000 <ipython-input-6-d6747b40360b>:55]
[<tf.Variable 'embedding:0' shape=(750, 300) dtype=float32_ref>,
<tf.Variable 'module/aggregation/weights:0' shape=(3,) dtype=float32>,
<tf.Variable 'module/aggregation/scaling:0' shape=() dtype=float32>,
<tf.Variable 'dense/kernel:0' shape=(1324, 300) dtype=float32_ref>,
<tf.Variable 'dense/bias:0' shape=(300,) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/fw/gru_cell/gates/kernel:0' shape=(600, 600) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/fw/gru_cell/gates/bias:0' shape=(600,) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/fw/gru_cell/candidate/kernel:0' shape=(600, 300) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/fw/gru_cell/candidate/bias:0' shape=(300,) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/bw/gru_cell/gates/kernel:0' shape=(600, 600) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/bw/gru_cell/gates/bias:0' shape=(600,) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/bw/gru_cell/candidate/kernel:0' shape=(600, 300) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/bw/gru_cell/candidate/bias:0' shape=(300,) dtype=float32_ref>,
<tf.Variable 'dense_1/kernel:0' shape=(600, 23) dtype=float32_ref>,
<tf.Variable 'dense_1/bias:0' shape=(23,) dtype=float32_ref>,
<tf.Variable 'dense_2/kernel:0' shape=(600, 122) dtype=float32_ref>,
<tf.Variable 'dense_2/bias:0' shape=(122,) dtype=float32_ref>]
I0716 04:31:06.442768 139795883440000 estimator.py:1147] Done calling model_fn.
I0716 04:31:06.446258 139795883440000 basic_session_run_hooks.py:541] Create CheckpointSaverHook.
I0716 04:31:06.777996 139795883440000 monitored_session.py:240] Graph was finalized.
I0716 04:31:06.797566 139795883440000 saver.py:1280] Restoring parameters from ../model/elmo_bigru/model.ckpt-7800
I0716 04:31:07.846698 139795883440000 session_manager.py:500] Running local_init_op.
I0716 04:31:07.916826 139795883440000 session_manager.py:502] Done running local_init_op.
I0716 04:31:10.738055 139795883440000 basic_session_run_hooks.py:606] Saving checkpoints for 7800 into ../model/elmo_bigru/model.ckpt.
Reading ../data/atis.train.w-intent.iob
I0716 04:31:15.473568 139795883440000 basic_session_run_hooks.py:262] loss = 0.010955091, step = 7800 I0716 04:31:15.475627 139795883440000 basic_session_run_hooks.py:262] lr = 0.00013189027 I0716 04:31:42.469512 139795883440000 basic_session_run_hooks.py:692] global_step/sec: 3.7042 I0716 04:31:42.475643 139795883440000 basic_session_run_hooks.py:260] loss = 0.016697695, step = 7900 (27.002 sec) I0716 04:31:42.478532 139795883440000 basic_session_run_hooks.py:260] lr = 0.00013050795 (27.003 sec) I0716 04:32:09.090202 139795883440000 basic_session_run_hooks.py:692] global_step/sec: 3.75649 I0716 04:32:09.097246 139795883440000 basic_session_run_hooks.py:260] loss = 0.017720591, step = 8000 (26.622 sec) I0716 04:32:09.100723 139795883440000 basic_session_run_hooks.py:260] lr = 0.00012914014 (26.622 sec) I0716 04:32:36.386731 139795883440000 basic_session_run_hooks.py:692] global_step/sec: 3.66345 I0716 04:32:36.389632 139795883440000 basic_session_run_hooks.py:260] loss = 0.00943367, step = 8100 (27.292 sec) I0716 04:32:36.396331 139795883440000 basic_session_run_hooks.py:260] lr = 0.00012778665 (27.296 sec) I0716 04:32:39.250842 139795883440000 basic_session_run_hooks.py:606] Saving checkpoints for 8112 into ../model/elmo_bigru/model.ckpt. I0716 04:32:41.199497 139795883440000 estimator.py:1145] Calling model_fn. I0716 04:32:42.815036 139795883440000 saver.py:1499] Saver not created because there are no variables in the graph to restore I0716 04:32:43.378526 139795883440000 estimator.py:1147] Done calling model_fn. I0716 04:32:43.407763 139795883440000 evaluation.py:255] Starting evaluation at 2019-07-16T04:32:43Z I0716 04:32:43.633741 139795883440000 monitored_session.py:240] Graph was finalized. I0716 04:32:43.653753 139795883440000 saver.py:1280] Restoring parameters from ../model/elmo_bigru/model.ckpt-8112 I0716 04:32:44.570884 139795883440000 session_manager.py:500] Running local_init_op. I0716 04:32:44.623782 139795883440000 session_manager.py:502] Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
I0716 04:33:43.256713 139795883440000 evaluation.py:275] Finished evaluation at 2019-07-16-04:33:43 I0716 04:33:43.258413 139795883440000 estimator.py:2039] Saving dict for global step 8112: global_step = 8112, loss = 0.3477777 I0716 04:33:43.267474 139795883440000 estimator.py:2099] Saving 'checkpoint_path' summary for global step 8112: ../model/elmo_bigru/model.ckpt-8112 I0716 04:33:43.352654 139795883440000 estimator.py:368] Loss for final step: 0.011134683.
Reading ../data/atis.test.w-intent.iob
I0716 04:33:43.942719 139795883440000 estimator.py:1145] Calling model_fn. I0716 04:33:45.503614 139795883440000 saver.py:1499] Saver not created because there are no variables in the graph to restore I0716 04:33:46.019409 139795883440000 estimator.py:1147] Done calling model_fn. I0716 04:33:46.249564 139795883440000 monitored_session.py:240] Graph was finalized. I0716 04:33:46.265639 139795883440000 saver.py:1280] Restoring parameters from ../model/elmo_bigru/model.ckpt-8112 I0716 04:33:47.177340 139795883440000 session_manager.py:500] Running local_init_op. I0716 04:33:47.228640 139795883440000 session_manager.py:502] Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.
'precision', 'predicted', average, warn_for)
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1439: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples.
'recall', 'true', average, warn_for)
I0716 04:34:45.102535 139795883440000 interactiveshell.py:2882]
precision recall f1-score support
atis_flight 0.984 0.992 0.988 632
atis_airfare 0.941 1.000 0.970 48
atis_ground_service 1.000 1.000 1.000 36
atis_airline 1.000 1.000 1.000 38
atis_abbreviation 0.943 1.000 0.971 33
atis_aircraft 1.000 0.889 0.941 9
atis_flight_time 0.500 1.000 0.667 1
atis_quantity 0.429 1.000 0.600 3
atis_flight#atis_airfare 0.833 0.417 0.556 12
atis_airport 1.000 1.000 1.000 18
atis_distance 1.000 1.000 1.000 10
atis_city 0.750 0.500 0.600 6
atis_ground_fare 1.000 0.857 0.923 7
atis_capacity 0.952 0.952 0.952 21
atis_flight_no 0.889 1.000 0.941 8
atis_meal 1.000 0.833 0.909 6
atis_restriction 0.000 0.000 0.000 0
atis_airline#atis_flight_no 0.000 0.000 0.000 0
atis_ground_service#atis_ground_fare 0.000 0.000 0.000 0
atis_airfare#atis_flight_time 0.000 0.000 0.000 0
atis_cheapest 0.000 0.000 0.000 0
atis_aircraft#atis_flight#atis_flight_no 0.000 0.000 0.000 0
micro avg 0.973 0.979 0.976 888
macro avg 0.646 0.656 0.637 888
weighted avg 0.975 0.979 0.975 888
I0716 04:34:45.141074 139795883440000 interactiveshell.py:2882]
precision recall f1-score support
O 0.000 0.000 0.000 0.0
B-toloc.city_name 0.978 0.990 0.984 716.0
B-fromloc.city_name 0.983 1.000 0.992 704.0
I-toloc.city_name 0.981 0.977 0.979 265.0
B-depart_date.day_name 0.991 0.991 0.991 212.0
B-airline_name 1.000 1.000 1.000 101.0
I-fromloc.city_name 0.957 1.000 0.978 177.0
B-depart_time.period_of_day 0.992 0.923 0.956 130.0
I-airline_name 1.000 1.000 1.000 65.0
B-depart_date.day_number 0.981 0.964 0.972 55.0
B-depart_date.month_name 0.982 0.964 0.973 56.0
B-depart_time.time 0.864 1.000 0.927 57.0
B-round_trip 1.000 1.000 1.000 73.0
B-cost_relative 0.973 0.973 0.973 37.0
I-round_trip 1.000 1.000 1.000 71.0
B-flight_mod 0.828 1.000 0.906 24.0
B-depart_time.time_relative 0.955 0.985 0.970 65.0
I-depart_time.time 0.945 1.000 0.972 52.0
B-stoploc.city_name 0.952 1.000 0.976 20.0
B-city_name 0.905 0.667 0.768 57.0
B-class_type 0.960 1.000 0.980 24.0
B-arrive_time.time 0.971 0.971 0.971 34.0
B-arrive_time.time_relative 0.967 0.935 0.951 31.0
I-class_type 1.000 1.000 1.000 17.0
B-flight_stop 1.000 1.000 1.000 21.0
I-arrive_time.time 1.000 0.971 0.986 35.0
B-airline_code 0.895 1.000 0.944 34.0
I-depart_date.day_number 1.000 0.933 0.966 15.0
I-fromloc.airport_name 0.429 1.000 0.600 15.0
B-fromloc.airport_name 0.429 1.000 0.600 12.0
B-arrive_date.day_name 0.786 1.000 0.880 11.0
B-toloc.state_code 1.000 1.000 1.000 18.0
B-depart_date.today_relative 1.000 0.889 0.941 9.0
B-flight_number 0.733 1.000 0.846 11.0
B-depart_date.date_relative 0.944 1.000 0.971 17.0
B-toloc.state_name 0.893 0.893 0.893 28.0
B-fare_basis_code 0.850 1.000 0.919 17.0
B-flight_time 1.000 1.000 1.000 1.0
B-or 1.000 1.000 1.000 3.0
B-arrive_time.period_of_day 0.750 1.000 0.857 6.0
B-meal_description 1.000 0.900 0.947 10.0
I-cost_relative 1.000 0.667 0.800 3.0
I-airport_name 0.833 0.345 0.488 29.0
B-fare_amount 1.000 1.000 1.000 2.0
I-fare_amount 1.000 1.000 1.000 2.0
I-city_name 1.000 0.567 0.723 30.0
I-toloc.airport_name 1.000 1.000 1.000 3.0
B-transport_type 1.000 1.000 1.000 10.0
B-arrive_date.month_name 0.714 0.833 0.769 6.0
B-arrive_date.day_number 0.714 0.833 0.769 6.0
I-stoploc.city_name 1.000 1.000 1.000 10.0
B-meal 0.941 1.000 0.970 16.0
B-fromloc.state_code 1.000 1.000 1.000 23.0
B-depart_time.period_mod 1.000 1.000 1.000 5.0
B-connect 1.000 1.000 1.000 6.0
B-flight_days 1.000 1.000 1.000 10.0
B-toloc.airport_name 1.000 1.000 1.000 3.0
B-fromloc.state_name 0.944 1.000 0.971 17.0
B-airport_name 0.583 0.333 0.424 21.0
B-economy 1.000 1.000 1.000 6.0
I-flight_time 1.000 1.000 1.000 1.0
B-aircraft_code 1.000 0.818 0.900 33.0
B-mod 1.000 0.500 0.667 2.0
B-airport_code 1.000 0.444 0.615 9.0
B-depart_time.start_time 1.000 0.667 0.800 3.0
B-depart_time.end_time 1.000 0.333 0.500 3.0
B-depart_date.year 1.000 1.000 1.000 3.0
I-transport_type 1.000 1.000 1.000 1.0
B-restriction_code 1.000 1.000 1.000 4.0
B-arrive_time.start_time 0.889 1.000 0.941 8.0
B-toloc.airport_code 1.000 0.750 0.857 4.0
B-arrive_time.end_time 0.889 1.000 0.941 8.0
I-arrive_time.end_time 0.889 1.000 0.941 8.0
I-depart_time.end_time 1.000 0.333 0.500 3.0
I-flight_stop 0.000 0.000 0.000 0.0
B-fromloc.airport_code 0.625 1.000 0.769 5.0
I-restriction_code 1.000 1.000 1.000 3.0
I-depart_time.start_time 1.000 1.000 1.000 1.0
I-toloc.state_name 1.000 1.000 1.000 1.0
I-depart_date.today_relative 0.000 0.000 0.000 0.0
B-arrive_date.date_relative 1.000 1.000 1.000 2.0
I-flight_mod 1.000 0.167 0.286 6.0
I-economy 0.000 0.000 0.000 0.0
B-return_date.date_relative 1.000 0.667 0.800 3.0
I-fromloc.state_name 1.000 1.000 1.000 1.0
B-state_code 1.000 1.000 1.000 1.0
I-arrive_time.start_time 1.000 1.000 1.000 1.0
I-arrive_date.day_number 0.000 0.000 0.000 0.0
B-meal_code 0.000 0.000 0.000 1.0
I-depart_time.period_of_day 1.000 1.000 1.000 1.0
B-day_name 1.000 0.500 0.667 2.0
B-period_of_day 1.000 0.750 0.857 4.0
B-stoploc.state_code 0.000 0.000 0.000 0.0
B-return_date.month_name 0.000 0.000 0.000 0.0
B-return_date.day_number 0.000 0.000 0.000 0.0
B-arrive_time.period_mod 0.000 0.000 0.000 0.0
I-meal_code 0.000 0.000 0.000 0.0
B-toloc.country_name 1.000 1.000 1.000 1.0
B-days_code 0.000 0.000 0.000 1.0
I-arrive_time.period_of_day 0.000 0.000 0.000 0.0
I-today_relative 0.000 0.000 0.000 0.0
B-return_time.period_of_day 0.000 0.000 0.000 0.0
B-time 0.000 0.000 0.000 0.0
I-fare_basis_code 0.000 0.000 0.000 0.0
I-arrive_time.time_relative 1.000 1.000 1.000 4.0
I-depart_time.time_relative 0.000 0.000 0.000 1.0
B-today_relative 0.000 0.000 0.000 0.0
B-state_name 0.000 0.000 0.000 9.0
B-arrive_date.today_relative 0.000 0.000 0.000 0.0
B-return_time.period_mod 0.000 0.000 0.000 0.0
B-month_name 0.000 0.000 0.000 0.0
B-day_number 0.000 0.000 0.000 0.0
I-return_date.date_relative 1.000 0.667 0.800 3.0
I-return_date.today_relative 0.000 0.000 0.000 0.0
B-stoploc.airport_name 0.000 0.000 0.000 0.0
B-time_relative 0.000 0.000 0.000 0.0
I-time 0.000 0.000 0.000 0.0
I-return_date.day_number 0.000 0.000 0.000 0.0
I-meal_description 0.000 0.000 0.000 0.0
B-return_date.today_relative 0.000 0.000 0.000 0.0
B-return_date.day_name 0.000 0.000 0.000 2.0
micro avg 0.957 0.959 0.958 3657.0
macro avg 0.693 0.662 0.664 3657.0
weighted avg 0.961 0.959 0.955 3657.0
I0716 04:34:45.163522 139795883440000 interactiveshell.py:2882] Best Slot F1: 0.960
I0716 04:34:45.165222 139795883440000 estimator_training.py:186] Not using Distribute Coordinator.
I0716 04:34:45.167124 139795883440000 training.py:612] Running training and evaluation locally (non-distributed).
I0716 04:34:45.168864 139795883440000 training.py:700] Start train and evaluate loop. The evaluate will happen after every checkpoint. Checkpoint frequency is determined based on RunConfig arguments: save_checkpoints_steps 312 or save_checkpoints_secs None.
I0716 04:34:45.237806 139795883440000 estimator.py:1145] Calling model_fn.
I0716 04:34:46.711839 139795883440000 saver.py:1499] Saver not created because there are no variables in the graph to restore
I0716 04:34:47.297554 139795883440000 <ipython-input-6-d6747b40360b>:55]
[<tf.Variable 'embedding:0' shape=(750, 300) dtype=float32_ref>,
<tf.Variable 'module/aggregation/weights:0' shape=(3,) dtype=float32>,
<tf.Variable 'module/aggregation/scaling:0' shape=() dtype=float32>,
<tf.Variable 'dense/kernel:0' shape=(1324, 300) dtype=float32_ref>,
<tf.Variable 'dense/bias:0' shape=(300,) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/fw/gru_cell/gates/kernel:0' shape=(600, 600) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/fw/gru_cell/gates/bias:0' shape=(600,) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/fw/gru_cell/candidate/kernel:0' shape=(600, 300) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/fw/gru_cell/candidate/bias:0' shape=(300,) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/bw/gru_cell/gates/kernel:0' shape=(600, 600) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/bw/gru_cell/gates/bias:0' shape=(600,) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/bw/gru_cell/candidate/kernel:0' shape=(600, 300) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/bw/gru_cell/candidate/bias:0' shape=(300,) dtype=float32_ref>,
<tf.Variable 'dense_1/kernel:0' shape=(600, 23) dtype=float32_ref>,
<tf.Variable 'dense_1/bias:0' shape=(23,) dtype=float32_ref>,
<tf.Variable 'dense_2/kernel:0' shape=(600, 122) dtype=float32_ref>,
<tf.Variable 'dense_2/bias:0' shape=(122,) dtype=float32_ref>]
I0716 04:34:48.563750 139795883440000 estimator.py:1147] Done calling model_fn.
I0716 04:34:48.567470 139795883440000 basic_session_run_hooks.py:541] Create CheckpointSaverHook.
I0716 04:34:48.897768 139795883440000 monitored_session.py:240] Graph was finalized.
I0716 04:34:48.916313 139795883440000 saver.py:1280] Restoring parameters from ../model/elmo_bigru/model.ckpt-8112
I0716 04:34:49.950185 139795883440000 session_manager.py:500] Running local_init_op.
I0716 04:34:50.025686 139795883440000 session_manager.py:502] Done running local_init_op.
I0716 04:34:52.815325 139795883440000 basic_session_run_hooks.py:606] Saving checkpoints for 8112 into ../model/elmo_bigru/model.ckpt.
Reading ../data/atis.train.w-intent.iob
I0716 04:34:57.998856 139795883440000 basic_session_run_hooks.py:262] loss = 0.021866146, step = 8112 I0716 04:34:58.000585 139795883440000 basic_session_run_hooks.py:262] lr = 0.00012762519 I0716 04:35:25.058067 139795883440000 basic_session_run_hooks.py:692] global_step/sec: 3.69552 I0716 04:35:25.061380 139795883440000 basic_session_run_hooks.py:260] loss = 0.011125647, step = 8212 (27.063 sec) I0716 04:35:25.066845 139795883440000 basic_session_run_hooks.py:260] lr = 0.00012628757 (27.066 sec) I0716 04:35:51.005081 139795883440000 basic_session_run_hooks.py:692] global_step/sec: 3.85401 I0716 04:35:51.012858 139795883440000 basic_session_run_hooks.py:260] loss = 0.018105546, step = 8312 (25.951 sec) I0716 04:35:51.015550 139795883440000 basic_session_run_hooks.py:260] lr = 0.00012496399 (25.949 sec) I0716 04:36:18.147511 139795883440000 basic_session_run_hooks.py:692] global_step/sec: 3.68427 I0716 04:36:18.155609 139795883440000 basic_session_run_hooks.py:260] loss = 0.010910708, step = 8412 (27.143 sec) I0716 04:36:18.158275 139795883440000 basic_session_run_hooks.py:260] lr = 0.00012365427 (27.143 sec) I0716 04:36:20.829231 139795883440000 basic_session_run_hooks.py:606] Saving checkpoints for 8424 into ../model/elmo_bigru/model.ckpt. I0716 04:36:22.884379 139795883440000 estimator.py:1145] Calling model_fn. I0716 04:36:24.508466 139795883440000 saver.py:1499] Saver not created because there are no variables in the graph to restore I0716 04:36:25.108688 139795883440000 estimator.py:1147] Done calling model_fn. I0716 04:36:25.139392 139795883440000 evaluation.py:255] Starting evaluation at 2019-07-16T04:36:25Z I0716 04:36:25.361651 139795883440000 monitored_session.py:240] Graph was finalized. I0716 04:36:25.378874 139795883440000 saver.py:1280] Restoring parameters from ../model/elmo_bigru/model.ckpt-8424 I0716 04:36:26.279739 139795883440000 session_manager.py:500] Running local_init_op. I0716 04:36:26.334746 139795883440000 session_manager.py:502] Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
I0716 04:37:25.359095 139795883440000 evaluation.py:275] Finished evaluation at 2019-07-16-04:37:25 I0716 04:37:25.361010 139795883440000 estimator.py:2039] Saving dict for global step 8424: global_step = 8424, loss = 0.35378975 I0716 04:37:25.370392 139795883440000 estimator.py:2099] Saving 'checkpoint_path' summary for global step 8424: ../model/elmo_bigru/model.ckpt-8424 I0716 04:37:25.453804 139795883440000 estimator.py:368] Loss for final step: 0.0015609998.
Reading ../data/atis.test.w-intent.iob
I0716 04:37:26.045480 139795883440000 estimator.py:1145] Calling model_fn. I0716 04:37:27.562336 139795883440000 saver.py:1499] Saver not created because there are no variables in the graph to restore I0716 04:37:28.074602 139795883440000 estimator.py:1147] Done calling model_fn. I0716 04:37:28.298694 139795883440000 monitored_session.py:240] Graph was finalized. I0716 04:37:28.319315 139795883440000 saver.py:1280] Restoring parameters from ../model/elmo_bigru/model.ckpt-8424 I0716 04:37:29.206162 139795883440000 session_manager.py:500] Running local_init_op. I0716 04:37:29.255664 139795883440000 session_manager.py:502] Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.
'precision', 'predicted', average, warn_for)
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1439: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples.
'recall', 'true', average, warn_for)
I0716 04:38:27.078902 139795883440000 interactiveshell.py:2882]
precision recall f1-score support
atis_flight 0.984 0.992 0.988 632
atis_airfare 0.941 1.000 0.970 48
atis_ground_service 1.000 1.000 1.000 36
atis_airline 1.000 1.000 1.000 38
atis_abbreviation 0.943 1.000 0.971 33
atis_aircraft 1.000 0.889 0.941 9
atis_flight_time 0.500 1.000 0.667 1
atis_quantity 0.429 1.000 0.600 3
atis_flight#atis_airfare 0.833 0.417 0.556 12
atis_airport 1.000 1.000 1.000 18
atis_distance 1.000 1.000 1.000 10
atis_city 0.750 0.500 0.600 6
atis_ground_fare 1.000 0.857 0.923 7
atis_capacity 0.952 0.952 0.952 21
atis_flight_no 0.889 1.000 0.941 8
atis_meal 1.000 0.833 0.909 6
atis_restriction 0.000 0.000 0.000 0
atis_airline#atis_flight_no 0.000 0.000 0.000 0
atis_ground_service#atis_ground_fare 0.000 0.000 0.000 0
atis_airfare#atis_flight_time 0.000 0.000 0.000 0
atis_cheapest 0.000 0.000 0.000 0
atis_aircraft#atis_flight#atis_flight_no 0.000 0.000 0.000 0
micro avg 0.973 0.979 0.976 888
macro avg 0.646 0.656 0.637 888
weighted avg 0.975 0.979 0.975 888
I0716 04:38:27.119267 139795883440000 interactiveshell.py:2882]
precision recall f1-score support
O 0.000 0.000 0.000 0.0
B-toloc.city_name 0.977 0.993 0.985 716.0
B-fromloc.city_name 0.992 0.999 0.995 704.0
I-toloc.city_name 0.981 0.992 0.987 265.0
B-depart_date.day_name 0.991 0.991 0.991 212.0
B-airline_name 1.000 1.000 1.000 101.0
I-fromloc.city_name 0.978 1.000 0.989 177.0
B-depart_time.period_of_day 0.992 0.915 0.952 130.0
I-airline_name 1.000 1.000 1.000 65.0
B-depart_date.day_number 0.981 0.964 0.972 55.0
B-depart_date.month_name 0.982 0.964 0.973 56.0
B-depart_time.time 0.864 1.000 0.927 57.0
B-round_trip 1.000 1.000 1.000 73.0
B-cost_relative 0.973 0.973 0.973 37.0
I-round_trip 1.000 1.000 1.000 71.0
B-flight_mod 0.828 1.000 0.906 24.0
B-depart_time.time_relative 0.970 0.985 0.977 65.0
I-depart_time.time 0.929 1.000 0.963 52.0
B-stoploc.city_name 0.833 1.000 0.909 20.0
B-city_name 0.844 0.667 0.745 57.0
B-class_type 0.960 1.000 0.980 24.0
B-arrive_time.time 0.971 0.971 0.971 34.0
B-arrive_time.time_relative 0.906 0.935 0.921 31.0
I-class_type 1.000 1.000 1.000 17.0
B-flight_stop 1.000 1.000 1.000 21.0
I-arrive_time.time 1.000 0.971 0.986 35.0
B-airline_code 0.944 1.000 0.971 34.0
I-depart_date.day_number 1.000 0.933 0.966 15.0
I-fromloc.airport_name 0.441 1.000 0.612 15.0
B-fromloc.airport_name 0.462 1.000 0.632 12.0
B-arrive_date.day_name 0.786 1.000 0.880 11.0
B-toloc.state_code 1.000 1.000 1.000 18.0
B-depart_date.today_relative 1.000 0.889 0.941 9.0
B-flight_number 0.733 1.000 0.846 11.0
B-depart_date.date_relative 0.944 1.000 0.971 17.0
B-toloc.state_name 0.806 0.893 0.847 28.0
B-fare_basis_code 0.850 1.000 0.919 17.0
B-flight_time 1.000 1.000 1.000 1.0
B-or 1.000 1.000 1.000 3.0
B-arrive_time.period_of_day 0.750 1.000 0.857 6.0
B-meal_description 1.000 0.900 0.947 10.0
I-cost_relative 1.000 0.667 0.800 3.0
I-airport_name 0.846 0.379 0.524 29.0
B-fare_amount 1.000 1.000 1.000 2.0
I-fare_amount 1.000 1.000 1.000 2.0
I-city_name 1.000 0.600 0.750 30.0
I-toloc.airport_name 1.000 1.000 1.000 3.0
B-transport_type 1.000 1.000 1.000 10.0
B-arrive_date.month_name 0.714 0.833 0.769 6.0
B-arrive_date.day_number 0.714 0.833 0.769 6.0
I-stoploc.city_name 1.000 1.000 1.000 10.0
B-meal 0.941 1.000 0.970 16.0
B-fromloc.state_code 1.000 1.000 1.000 23.0
B-depart_time.period_mod 1.000 1.000 1.000 5.0
B-connect 1.000 1.000 1.000 6.0
B-flight_days 1.000 1.000 1.000 10.0
B-toloc.airport_name 1.000 1.000 1.000 3.0
B-fromloc.state_name 1.000 1.000 1.000 17.0
B-airport_name 0.750 0.286 0.414 21.0
B-economy 1.000 1.000 1.000 6.0
I-flight_time 1.000 1.000 1.000 1.0
B-aircraft_code 1.000 0.879 0.935 33.0
B-mod 1.000 0.500 0.667 2.0
B-airport_code 1.000 0.444 0.615 9.0
B-depart_time.start_time 1.000 0.667 0.800 3.0
B-depart_time.end_time 1.000 0.333 0.500 3.0
B-depart_date.year 1.000 1.000 1.000 3.0
I-transport_type 0.000 0.000 0.000 1.0
B-restriction_code 1.000 1.000 1.000 4.0
B-arrive_time.start_time 0.889 1.000 0.941 8.0
B-toloc.airport_code 1.000 0.750 0.857 4.0
B-arrive_time.end_time 0.889 1.000 0.941 8.0
I-arrive_time.end_time 0.889 1.000 0.941 8.0
I-depart_time.end_time 1.000 0.333 0.500 3.0
I-flight_stop 0.000 0.000 0.000 0.0
B-fromloc.airport_code 0.556 1.000 0.714 5.0
I-restriction_code 1.000 1.000 1.000 3.0
I-depart_time.start_time 1.000 1.000 1.000 1.0
I-toloc.state_name 1.000 1.000 1.000 1.0
I-depart_date.today_relative 0.000 0.000 0.000 0.0
B-arrive_date.date_relative 1.000 1.000 1.000 2.0
I-flight_mod 1.000 0.167 0.286 6.0
I-economy 0.000 0.000 0.000 0.0
B-return_date.date_relative 1.000 0.667 0.800 3.0
I-fromloc.state_name 1.000 1.000 1.000 1.0
B-state_code 1.000 1.000 1.000 1.0
I-arrive_time.start_time 1.000 1.000 1.000 1.0
I-arrive_date.day_number 0.000 0.000 0.000 0.0
B-meal_code 0.000 0.000 0.000 1.0
I-depart_time.period_of_day 1.000 1.000 1.000 1.0
B-day_name 1.000 0.500 0.667 2.0
B-period_of_day 1.000 0.750 0.857 4.0
B-stoploc.state_code 0.000 0.000 0.000 0.0
B-return_date.month_name 0.000 0.000 0.000 0.0
B-return_date.day_number 0.000 0.000 0.000 0.0
B-arrive_time.period_mod 0.000 0.000 0.000 0.0
I-meal_code 0.000 0.000 0.000 0.0
B-toloc.country_name 1.000 1.000 1.000 1.0
B-days_code 0.000 0.000 0.000 1.0
I-arrive_time.period_of_day 0.000 0.000 0.000 0.0
I-today_relative 0.000 0.000 0.000 0.0
B-return_time.period_of_day 0.000 0.000 0.000 0.0
B-time 0.000 0.000 0.000 0.0
I-fare_basis_code 0.000 0.000 0.000 0.0
I-arrive_time.time_relative 1.000 0.500 0.667 4.0
I-depart_time.time_relative 0.000 0.000 0.000 1.0
B-today_relative 0.000 0.000 0.000 0.0
B-state_name 0.000 0.000 0.000 9.0
B-arrive_date.today_relative 0.000 0.000 0.000 0.0
B-return_time.period_mod 0.000 0.000 0.000 0.0
B-month_name 0.000 0.000 0.000 0.0
B-day_number 0.000 0.000 0.000 0.0
I-return_date.date_relative 1.000 0.667 0.800 3.0
I-return_date.today_relative 0.000 0.000 0.000 0.0
B-stoploc.airport_name 0.000 0.000 0.000 0.0
B-time_relative 0.000 0.000 0.000 0.0
I-time 0.000 0.000 0.000 0.0
I-return_date.day_number 0.000 0.000 0.000 0.0
I-meal_description 0.000 0.000 0.000 0.0
B-return_date.today_relative 0.000 0.000 0.000 0.0
B-return_date.day_name 0.000 0.000 0.000 2.0
micro avg 0.959 0.960 0.959 3657.0
macro avg 0.685 0.650 0.653 3657.0
weighted avg 0.962 0.960 0.956 3657.0
I0716 04:38:27.141413 139795883440000 interactiveshell.py:2882] Best Slot F1: 0.960
I0716 04:38:27.142925 139795883440000 estimator_training.py:186] Not using Distribute Coordinator.
I0716 04:38:27.144620 139795883440000 training.py:612] Running training and evaluation locally (non-distributed).
I0716 04:38:27.146125 139795883440000 training.py:700] Start train and evaluate loop. The evaluate will happen after every checkpoint. Checkpoint frequency is determined based on RunConfig arguments: save_checkpoints_steps 312 or save_checkpoints_secs None.
I0716 04:38:27.216298 139795883440000 estimator.py:1145] Calling model_fn.
I0716 04:38:28.712872 139795883440000 saver.py:1499] Saver not created because there are no variables in the graph to restore
I0716 04:38:29.477923 139795883440000 <ipython-input-6-d6747b40360b>:55]
[<tf.Variable 'embedding:0' shape=(750, 300) dtype=float32_ref>,
<tf.Variable 'module/aggregation/weights:0' shape=(3,) dtype=float32>,
<tf.Variable 'module/aggregation/scaling:0' shape=() dtype=float32>,
<tf.Variable 'dense/kernel:0' shape=(1324, 300) dtype=float32_ref>,
<tf.Variable 'dense/bias:0' shape=(300,) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/fw/gru_cell/gates/kernel:0' shape=(600, 600) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/fw/gru_cell/gates/bias:0' shape=(600,) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/fw/gru_cell/candidate/kernel:0' shape=(600, 300) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/fw/gru_cell/candidate/bias:0' shape=(300,) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/bw/gru_cell/gates/kernel:0' shape=(600, 600) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/bw/gru_cell/gates/bias:0' shape=(600,) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/bw/gru_cell/candidate/kernel:0' shape=(600, 300) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/bw/gru_cell/candidate/bias:0' shape=(300,) dtype=float32_ref>,
<tf.Variable 'dense_1/kernel:0' shape=(600, 23) dtype=float32_ref>,
<tf.Variable 'dense_1/bias:0' shape=(23,) dtype=float32_ref>,
<tf.Variable 'dense_2/kernel:0' shape=(600, 122) dtype=float32_ref>,
<tf.Variable 'dense_2/bias:0' shape=(122,) dtype=float32_ref>]
I0716 04:38:30.564863 139795883440000 estimator.py:1147] Done calling model_fn.
I0716 04:38:30.569037 139795883440000 basic_session_run_hooks.py:541] Create CheckpointSaverHook.
I0716 04:38:30.901015 139795883440000 monitored_session.py:240] Graph was finalized.
I0716 04:38:30.921924 139795883440000 saver.py:1280] Restoring parameters from ../model/elmo_bigru/model.ckpt-8424
I0716 04:38:31.968199 139795883440000 session_manager.py:500] Running local_init_op.
I0716 04:38:32.039877 139795883440000 session_manager.py:502] Done running local_init_op.
I0716 04:38:35.075495 139795883440000 basic_session_run_hooks.py:606] Saving checkpoints for 8424 into ../model/elmo_bigru/model.ckpt.
Reading ../data/atis.train.w-intent.iob
I0716 04:38:39.802695 139795883440000 basic_session_run_hooks.py:262] loss = 0.019132338, step = 8424 I0716 04:38:39.804921 139795883440000 basic_session_run_hooks.py:262] lr = 0.00012349803 I0716 04:39:07.285551 139795883440000 basic_session_run_hooks.py:692] global_step/sec: 3.63856 I0716 04:39:07.297929 139795883440000 basic_session_run_hooks.py:260] loss = 0.015347525, step = 8524 (27.495 sec) I0716 04:39:07.299650 139795883440000 basic_session_run_hooks.py:260] lr = 0.00012220368 (27.495 sec) I0716 04:39:34.417328 139795883440000 basic_session_run_hooks.py:692] global_step/sec: 3.68571 I0716 04:39:34.422420 139795883440000 basic_session_run_hooks.py:260] loss = 0.013917305, step = 8624 (27.124 sec) I0716 04:39:34.427424 139795883440000 basic_session_run_hooks.py:260] lr = 0.000120922894 (27.128 sec) I0716 04:40:00.334441 139795883440000 basic_session_run_hooks.py:692] global_step/sec: 3.85845 I0716 04:40:00.337281 139795883440000 basic_session_run_hooks.py:260] loss = 0.015168118, step = 8724 (25.915 sec) I0716 04:40:00.344412 139795883440000 basic_session_run_hooks.py:260] lr = 0.00011965554 (25.917 sec) I0716 04:40:03.324968 139795883440000 basic_session_run_hooks.py:606] Saving checkpoints for 8736 into ../model/elmo_bigru/model.ckpt. I0716 04:40:05.298134 139795883440000 estimator.py:1145] Calling model_fn. I0716 04:40:06.645496 139795883440000 saver.py:1499] Saver not created because there are no variables in the graph to restore I0716 04:40:07.228574 139795883440000 estimator.py:1147] Done calling model_fn. I0716 04:40:07.259759 139795883440000 evaluation.py:255] Starting evaluation at 2019-07-16T04:40:07Z I0716 04:40:07.483759 139795883440000 monitored_session.py:240] Graph was finalized. I0716 04:40:07.500017 139795883440000 saver.py:1280] Restoring parameters from ../model/elmo_bigru/model.ckpt-8736 I0716 04:40:08.392496 139795883440000 session_manager.py:500] Running local_init_op. I0716 04:40:08.454483 139795883440000 session_manager.py:502] Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
I0716 04:41:07.242574 139795883440000 evaluation.py:275] Finished evaluation at 2019-07-16-04:41:07 I0716 04:41:07.244690 139795883440000 estimator.py:2039] Saving dict for global step 8736: global_step = 8736, loss = 0.3608753 I0716 04:41:07.253153 139795883440000 estimator.py:2099] Saving 'checkpoint_path' summary for global step 8736: ../model/elmo_bigru/model.ckpt-8736 I0716 04:41:07.336138 139795883440000 estimator.py:368] Loss for final step: 0.0033090645.
Reading ../data/atis.test.w-intent.iob
I0716 04:41:07.933838 139795883440000 estimator.py:1145] Calling model_fn. I0716 04:41:09.617775 139795883440000 saver.py:1499] Saver not created because there are no variables in the graph to restore I0716 04:41:10.127541 139795883440000 estimator.py:1147] Done calling model_fn. I0716 04:41:10.352676 139795883440000 monitored_session.py:240] Graph was finalized. I0716 04:41:10.369623 139795883440000 saver.py:1280] Restoring parameters from ../model/elmo_bigru/model.ckpt-8736 I0716 04:41:11.262585 139795883440000 session_manager.py:500] Running local_init_op. I0716 04:41:11.308635 139795883440000 session_manager.py:502] Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.
'precision', 'predicted', average, warn_for)
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1439: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples.
'recall', 'true', average, warn_for)
I0716 04:42:09.205271 139795883440000 interactiveshell.py:2882]
precision recall f1-score support
atis_flight 0.984 0.992 0.988 632
atis_airfare 0.941 1.000 0.970 48
atis_ground_service 1.000 1.000 1.000 36
atis_airline 1.000 1.000 1.000 38
atis_abbreviation 0.943 1.000 0.971 33
atis_aircraft 1.000 0.889 0.941 9
atis_flight_time 0.500 1.000 0.667 1
atis_quantity 0.429 1.000 0.600 3
atis_flight#atis_airfare 0.833 0.417 0.556 12
atis_airport 1.000 1.000 1.000 18
atis_distance 1.000 1.000 1.000 10
atis_city 0.750 0.500 0.600 6
atis_ground_fare 1.000 0.857 0.923 7
atis_capacity 0.952 0.952 0.952 21
atis_flight_no 0.889 1.000 0.941 8
atis_meal 1.000 0.833 0.909 6
atis_restriction 0.000 0.000 0.000 0
atis_airline#atis_flight_no 0.000 0.000 0.000 0
atis_ground_service#atis_ground_fare 0.000 0.000 0.000 0
atis_airfare#atis_flight_time 0.000 0.000 0.000 0
atis_cheapest 0.000 0.000 0.000 0
atis_aircraft#atis_flight#atis_flight_no 0.000 0.000 0.000 0
micro avg 0.973 0.979 0.976 888
macro avg 0.646 0.656 0.637 888
weighted avg 0.975 0.979 0.975 888
I0716 04:42:09.242614 139795883440000 interactiveshell.py:2882]
precision recall f1-score support
O 0.000 0.000 0.000 0.0
B-toloc.city_name 0.975 0.993 0.984 716.0
B-fromloc.city_name 0.990 0.999 0.994 704.0
I-toloc.city_name 0.978 0.985 0.981 265.0
B-depart_date.day_name 0.991 0.991 0.991 212.0
B-airline_name 1.000 1.000 1.000 101.0
I-fromloc.city_name 0.973 1.000 0.986 177.0
B-depart_time.period_of_day 1.000 0.908 0.952 130.0
I-airline_name 1.000 1.000 1.000 65.0
B-depart_date.day_number 0.981 0.964 0.972 55.0
B-depart_date.month_name 0.982 0.964 0.973 56.0
B-depart_time.time 0.864 1.000 0.927 57.0
B-round_trip 1.000 1.000 1.000 73.0
B-cost_relative 0.973 0.973 0.973 37.0
I-round_trip 1.000 1.000 1.000 71.0
B-flight_mod 0.828 1.000 0.906 24.0
B-depart_time.time_relative 0.955 0.985 0.970 65.0
I-depart_time.time 0.912 1.000 0.954 52.0
B-stoploc.city_name 0.870 1.000 0.930 20.0
B-city_name 0.860 0.649 0.740 57.0
B-class_type 0.960 1.000 0.980 24.0
B-arrive_time.time 0.971 0.971 0.971 34.0
B-arrive_time.time_relative 0.906 0.935 0.921 31.0
I-class_type 1.000 1.000 1.000 17.0
B-flight_stop 1.000 1.000 1.000 21.0
I-arrive_time.time 1.000 0.971 0.986 35.0
B-airline_code 0.971 1.000 0.986 34.0
I-depart_date.day_number 1.000 0.933 0.966 15.0
I-fromloc.airport_name 0.441 1.000 0.612 15.0
B-fromloc.airport_name 0.444 1.000 0.615 12.0
B-arrive_date.day_name 0.786 1.000 0.880 11.0
B-toloc.state_code 1.000 1.000 1.000 18.0
B-depart_date.today_relative 1.000 0.889 0.941 9.0
B-flight_number 0.733 1.000 0.846 11.0
B-depart_date.date_relative 0.944 1.000 0.971 17.0
B-toloc.state_name 0.781 0.893 0.833 28.0
B-fare_basis_code 0.850 1.000 0.919 17.0
B-flight_time 1.000 1.000 1.000 1.0
B-or 1.000 1.000 1.000 3.0
B-arrive_time.period_of_day 0.750 1.000 0.857 6.0
B-meal_description 1.000 0.900 0.947 10.0
I-cost_relative 1.000 0.667 0.800 3.0
I-airport_name 0.846 0.379 0.524 29.0
B-fare_amount 1.000 1.000 1.000 2.0
I-fare_amount 1.000 1.000 1.000 2.0
I-city_name 1.000 0.567 0.723 30.0
I-toloc.airport_name 1.000 1.000 1.000 3.0
B-transport_type 1.000 1.000 1.000 10.0
B-arrive_date.month_name 0.714 0.833 0.769 6.0
B-arrive_date.day_number 0.714 0.833 0.769 6.0
I-stoploc.city_name 1.000 1.000 1.000 10.0
B-meal 1.000 1.000 1.000 16.0
B-fromloc.state_code 1.000 1.000 1.000 23.0
B-depart_time.period_mod 1.000 1.000 1.000 5.0
B-connect 1.000 1.000 1.000 6.0
B-flight_days 1.000 1.000 1.000 10.0
B-toloc.airport_name 1.000 1.000 1.000 3.0
B-fromloc.state_name 1.000 1.000 1.000 17.0
B-airport_name 0.750 0.286 0.414 21.0
B-economy 1.000 1.000 1.000 6.0
I-flight_time 1.000 1.000 1.000 1.0
B-aircraft_code 1.000 0.879 0.935 33.0
B-mod 1.000 0.500 0.667 2.0
B-airport_code 1.000 0.444 0.615 9.0
B-depart_time.start_time 1.000 0.667 0.800 3.0
B-depart_time.end_time 1.000 0.333 0.500 3.0
B-depart_date.year 1.000 1.000 1.000 3.0
I-transport_type 0.000 0.000 0.000 1.0
B-restriction_code 0.800 1.000 0.889 4.0
B-arrive_time.start_time 0.889 1.000 0.941 8.0
B-toloc.airport_code 1.000 0.750 0.857 4.0
B-arrive_time.end_time 0.889 1.000 0.941 8.0
I-arrive_time.end_time 0.889 1.000 0.941 8.0
I-depart_time.end_time 1.000 0.333 0.500 3.0
I-flight_stop 0.000 0.000 0.000 0.0
B-fromloc.airport_code 0.625 1.000 0.769 5.0
I-restriction_code 1.000 1.000 1.000 3.0
I-depart_time.start_time 1.000 1.000 1.000 1.0
I-toloc.state_name 1.000 1.000 1.000 1.0
I-depart_date.today_relative 0.000 0.000 0.000 0.0
B-arrive_date.date_relative 1.000 1.000 1.000 2.0
I-flight_mod 1.000 0.167 0.286 6.0
I-economy 0.000 0.000 0.000 0.0
B-return_date.date_relative 1.000 0.333 0.500 3.0
I-fromloc.state_name 1.000 1.000 1.000 1.0
B-state_code 1.000 1.000 1.000 1.0
I-arrive_time.start_time 1.000 1.000 1.000 1.0
I-arrive_date.day_number 0.000 0.000 0.000 0.0
B-meal_code 0.000 0.000 0.000 1.0
I-depart_time.period_of_day 1.000 1.000 1.000 1.0
B-day_name 1.000 0.500 0.667 2.0
B-period_of_day 1.000 0.750 0.857 4.0
B-stoploc.state_code 0.000 0.000 0.000 0.0
B-return_date.month_name 0.000 0.000 0.000 0.0
B-return_date.day_number 0.000 0.000 0.000 0.0
B-arrive_time.period_mod 0.000 0.000 0.000 0.0
I-meal_code 0.000 0.000 0.000 0.0
B-toloc.country_name 1.000 1.000 1.000 1.0
B-days_code 0.000 0.000 0.000 1.0
I-arrive_time.period_of_day 0.000 0.000 0.000 0.0
I-today_relative 0.000 0.000 0.000 0.0
B-return_time.period_of_day 0.000 0.000 0.000 0.0
B-time 0.000 0.000 0.000 0.0
I-fare_basis_code 0.000 0.000 0.000 0.0
I-arrive_time.time_relative 1.000 0.500 0.667 4.0
I-depart_time.time_relative 0.000 0.000 0.000 1.0
B-today_relative 0.000 0.000 0.000 0.0
B-state_name 0.000 0.000 0.000 9.0
B-arrive_date.today_relative 0.000 0.000 0.000 0.0
B-return_time.period_mod 0.000 0.000 0.000 0.0
B-month_name 0.000 0.000 0.000 0.0
B-day_number 0.000 0.000 0.000 0.0
I-return_date.date_relative 0.667 0.667 0.667 3.0
I-return_date.today_relative 0.000 0.000 0.000 0.0
B-stoploc.airport_name 0.000 0.000 0.000 0.0
B-time_relative 0.000 0.000 0.000 0.0
I-time 0.000 0.000 0.000 0.0
I-return_date.day_number 0.000 0.000 0.000 0.0
I-meal_description 0.000 0.000 0.000 0.0
B-return_date.today_relative 0.000 0.000 0.000 0.0
B-return_date.day_name 0.000 0.000 0.000 2.0
micro avg 0.957 0.958 0.958 3657.0
macro avg 0.681 0.647 0.648 3657.0
weighted avg 0.961 0.958 0.955 3657.0
I0716 04:42:09.264512 139795883440000 interactiveshell.py:2882] Best Slot F1: 0.960
I0716 04:42:09.265876 139795883440000 estimator_training.py:186] Not using Distribute Coordinator.
I0716 04:42:09.267675 139795883440000 training.py:612] Running training and evaluation locally (non-distributed).
I0716 04:42:09.269614 139795883440000 training.py:700] Start train and evaluate loop. The evaluate will happen after every checkpoint. Checkpoint frequency is determined based on RunConfig arguments: save_checkpoints_steps 312 or save_checkpoints_secs None.
I0716 04:42:09.341445 139795883440000 estimator.py:1145] Calling model_fn.
I0716 04:42:10.994044 139795883440000 saver.py:1499] Saver not created because there are no variables in the graph to restore
I0716 04:42:11.572091 139795883440000 <ipython-input-6-d6747b40360b>:55]
[<tf.Variable 'embedding:0' shape=(750, 300) dtype=float32_ref>,
<tf.Variable 'module/aggregation/weights:0' shape=(3,) dtype=float32>,
<tf.Variable 'module/aggregation/scaling:0' shape=() dtype=float32>,
<tf.Variable 'dense/kernel:0' shape=(1324, 300) dtype=float32_ref>,
<tf.Variable 'dense/bias:0' shape=(300,) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/fw/gru_cell/gates/kernel:0' shape=(600, 600) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/fw/gru_cell/gates/bias:0' shape=(600,) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/fw/gru_cell/candidate/kernel:0' shape=(600, 300) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/fw/gru_cell/candidate/bias:0' shape=(300,) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/bw/gru_cell/gates/kernel:0' shape=(600, 600) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/bw/gru_cell/gates/bias:0' shape=(600,) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/bw/gru_cell/candidate/kernel:0' shape=(600, 300) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/bw/gru_cell/candidate/bias:0' shape=(300,) dtype=float32_ref>,
<tf.Variable 'dense_1/kernel:0' shape=(600, 23) dtype=float32_ref>,
<tf.Variable 'dense_1/bias:0' shape=(23,) dtype=float32_ref>,
<tf.Variable 'dense_2/kernel:0' shape=(600, 122) dtype=float32_ref>,
<tf.Variable 'dense_2/bias:0' shape=(122,) dtype=float32_ref>]
I0716 04:42:12.658575 139795883440000 estimator.py:1147] Done calling model_fn.
I0716 04:42:12.662549 139795883440000 basic_session_run_hooks.py:541] Create CheckpointSaverHook.
I0716 04:42:12.995083 139795883440000 monitored_session.py:240] Graph was finalized.
I0716 04:42:13.012826 139795883440000 saver.py:1280] Restoring parameters from ../model/elmo_bigru/model.ckpt-8736
I0716 04:42:14.061174 139795883440000 session_manager.py:500] Running local_init_op.
I0716 04:42:14.132613 139795883440000 session_manager.py:502] Done running local_init_op.
I0716 04:42:17.146679 139795883440000 basic_session_run_hooks.py:606] Saving checkpoints for 8736 into ../model/elmo_bigru/model.ckpt.
Reading ../data/atis.train.w-intent.iob
I0716 04:42:22.121123 139795883440000 basic_session_run_hooks.py:262] loss = 0.0142422775, step = 8736 I0716 04:42:22.122906 139795883440000 basic_session_run_hooks.py:262] lr = 0.00011950435 I0716 04:42:49.735462 139795883440000 basic_session_run_hooks.py:692] global_step/sec: 3.62123 I0716 04:42:49.744070 139795883440000 basic_session_run_hooks.py:260] loss = 0.012751075, step = 8836 (27.623 sec) I0716 04:42:49.746460 139795883440000 basic_session_run_hooks.py:260] lr = 0.00011825185 (27.624 sec) I0716 04:43:15.778196 139795883440000 basic_session_run_hooks.py:692] global_step/sec: 3.83984 I0716 04:43:15.785495 139795883440000 basic_session_run_hooks.py:260] loss = 0.016511267, step = 8936 (26.041 sec) I0716 04:43:15.788418 139795883440000 basic_session_run_hooks.py:260] lr = 0.000117012474 (26.042 sec) I0716 04:43:42.846633 139795883440000 basic_session_run_hooks.py:692] global_step/sec: 3.69434 I0716 04:43:42.852755 139795883440000 basic_session_run_hooks.py:260] loss = 0.01867107, step = 9036 (27.067 sec) I0716 04:43:42.855791 139795883440000 basic_session_run_hooks.py:260] lr = 0.00011578611 (27.067 sec) I0716 04:43:45.541065 139795883440000 basic_session_run_hooks.py:606] Saving checkpoints for 9048 into ../model/elmo_bigru/model.ckpt. I0716 04:43:47.526752 139795883440000 estimator.py:1145] Calling model_fn. I0716 04:43:48.916526 139795883440000 saver.py:1499] Saver not created because there are no variables in the graph to restore I0716 04:43:49.481549 139795883440000 estimator.py:1147] Done calling model_fn. I0716 04:43:49.511429 139795883440000 evaluation.py:255] Starting evaluation at 2019-07-16T04:43:49Z I0716 04:43:49.733474 139795883440000 monitored_session.py:240] Graph was finalized. I0716 04:43:49.750476 139795883440000 saver.py:1280] Restoring parameters from ../model/elmo_bigru/model.ckpt-9048 I0716 04:43:50.664488 139795883440000 session_manager.py:500] Running local_init_op. I0716 04:43:50.723493 139795883440000 session_manager.py:502] Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
I0716 04:44:49.735225 139795883440000 evaluation.py:275] Finished evaluation at 2019-07-16-04:44:49 I0716 04:44:49.737258 139795883440000 estimator.py:2039] Saving dict for global step 9048: global_step = 9048, loss = 0.37418377 I0716 04:44:49.746189 139795883440000 estimator.py:2099] Saving 'checkpoint_path' summary for global step 9048: ../model/elmo_bigru/model.ckpt-9048 I0716 04:44:49.833694 139795883440000 estimator.py:368] Loss for final step: 0.0026269325.
Reading ../data/atis.test.w-intent.iob
I0716 04:44:50.443303 139795883440000 estimator.py:1145] Calling model_fn. I0716 04:44:52.121428 139795883440000 saver.py:1499] Saver not created because there are no variables in the graph to restore I0716 04:44:52.830277 139795883440000 estimator.py:1147] Done calling model_fn. I0716 04:44:53.050785 139795883440000 monitored_session.py:240] Graph was finalized. I0716 04:44:53.068392 139795883440000 saver.py:1280] Restoring parameters from ../model/elmo_bigru/model.ckpt-9048 I0716 04:44:53.968281 139795883440000 session_manager.py:500] Running local_init_op. I0716 04:44:54.016930 139795883440000 session_manager.py:502] Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.
'precision', 'predicted', average, warn_for)
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1439: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples.
'recall', 'true', average, warn_for)
I0716 04:45:52.206138 139795883440000 interactiveshell.py:2882]
precision recall f1-score support
atis_flight 0.984 0.992 0.988 632
atis_airfare 0.941 1.000 0.970 48
atis_ground_service 1.000 1.000 1.000 36
atis_airline 0.974 1.000 0.987 38
atis_abbreviation 0.943 1.000 0.971 33
atis_aircraft 1.000 0.889 0.941 9
atis_flight_time 0.500 1.000 0.667 1
atis_quantity 0.429 1.000 0.600 3
atis_flight#atis_airfare 0.833 0.417 0.556 12
atis_airport 1.000 0.944 0.971 18
atis_distance 1.000 1.000 1.000 10
atis_city 0.750 0.500 0.600 6
atis_ground_fare 1.000 0.857 0.923 7
atis_capacity 0.952 0.952 0.952 21
atis_flight_no 0.889 1.000 0.941 8
atis_meal 1.000 0.833 0.909 6
atis_restriction 0.000 0.000 0.000 0
atis_airline#atis_flight_no 0.000 0.000 0.000 0
atis_ground_service#atis_ground_fare 0.000 0.000 0.000 0
atis_airfare#atis_flight_time 0.000 0.000 0.000 0
atis_cheapest 0.000 0.000 0.000 0
atis_aircraft#atis_flight#atis_flight_no 0.000 0.000 0.000 0
micro avg 0.972 0.977 0.975 888
macro avg 0.645 0.654 0.635 888
weighted avg 0.974 0.977 0.974 888
I0716 04:45:52.241688 139795883440000 interactiveshell.py:2882]
precision recall f1-score support
O 0.000 0.000 0.000 0.0
B-toloc.city_name 0.977 0.993 0.985 716.0
B-fromloc.city_name 0.989 0.999 0.994 704.0
I-toloc.city_name 0.977 0.981 0.979 265.0
B-depart_date.day_name 0.986 0.991 0.988 212.0
B-airline_name 1.000 1.000 1.000 101.0
I-fromloc.city_name 0.962 1.000 0.981 177.0
B-depart_time.period_of_day 0.992 0.923 0.956 130.0
I-airline_name 1.000 1.000 1.000 65.0
B-depart_date.day_number 0.981 0.964 0.972 55.0
B-depart_date.month_name 0.982 0.964 0.973 56.0
B-depart_time.time 0.864 1.000 0.927 57.0
B-round_trip 1.000 1.000 1.000 73.0
B-cost_relative 0.973 0.973 0.973 37.0
I-round_trip 1.000 1.000 1.000 71.0
B-flight_mod 0.828 1.000 0.906 24.0
B-depart_time.time_relative 0.955 0.969 0.962 65.0
I-depart_time.time 0.945 1.000 0.972 52.0
B-stoploc.city_name 0.909 1.000 0.952 20.0
B-city_name 0.902 0.649 0.755 57.0
B-class_type 0.960 1.000 0.980 24.0
B-arrive_time.time 0.971 0.971 0.971 34.0
B-arrive_time.time_relative 0.967 0.935 0.951 31.0
I-class_type 1.000 1.000 1.000 17.0
B-flight_stop 1.000 1.000 1.000 21.0
I-arrive_time.time 1.000 0.971 0.986 35.0
B-airline_code 0.895 1.000 0.944 34.0
I-depart_date.day_number 1.000 1.000 1.000 15.0
I-fromloc.airport_name 0.441 1.000 0.612 15.0
B-fromloc.airport_name 0.444 1.000 0.615 12.0
B-arrive_date.day_name 0.769 0.909 0.833 11.0
B-toloc.state_code 1.000 1.000 1.000 18.0
B-depart_date.today_relative 1.000 1.000 1.000 9.0
B-flight_number 0.733 1.000 0.846 11.0
B-depart_date.date_relative 0.944 1.000 0.971 17.0
B-toloc.state_name 0.788 0.929 0.852 28.0
B-fare_basis_code 0.850 1.000 0.919 17.0
B-flight_time 1.000 1.000 1.000 1.0
B-or 1.000 1.000 1.000 3.0
B-arrive_time.period_of_day 0.750 1.000 0.857 6.0
B-meal_description 1.000 0.900 0.947 10.0
I-cost_relative 1.000 0.667 0.800 3.0
I-airport_name 0.846 0.379 0.524 29.0
B-fare_amount 1.000 1.000 1.000 2.0
I-fare_amount 1.000 1.000 1.000 2.0
I-city_name 1.000 0.533 0.696 30.0
I-toloc.airport_name 1.000 1.000 1.000 3.0
B-transport_type 1.000 1.000 1.000 10.0
B-arrive_date.month_name 0.714 0.833 0.769 6.0
B-arrive_date.day_number 0.714 0.833 0.769 6.0
I-stoploc.city_name 1.000 1.000 1.000 10.0
B-meal 0.941 1.000 0.970 16.0
B-fromloc.state_code 1.000 1.000 1.000 23.0
B-depart_time.period_mod 0.833 1.000 0.909 5.0
B-connect 1.000 1.000 1.000 6.0
B-flight_days 1.000 1.000 1.000 10.0
B-toloc.airport_name 1.000 1.000 1.000 3.0
B-fromloc.state_name 0.944 1.000 0.971 17.0
B-airport_name 0.778 0.333 0.467 21.0
B-economy 1.000 1.000 1.000 6.0
I-flight_time 1.000 1.000 1.000 1.0
B-aircraft_code 1.000 0.818 0.900 33.0
B-mod 1.000 0.500 0.667 2.0
B-airport_code 1.000 0.444 0.615 9.0
B-depart_time.start_time 0.750 1.000 0.857 3.0
B-depart_time.end_time 0.667 0.667 0.667 3.0
B-depart_date.year 1.000 1.000 1.000 3.0
I-transport_type 1.000 1.000 1.000 1.0
B-restriction_code 1.000 1.000 1.000 4.0
B-arrive_time.start_time 1.000 0.875 0.933 8.0
B-toloc.airport_code 1.000 0.750 0.857 4.0
B-arrive_time.end_time 1.000 0.875 0.933 8.0
I-arrive_time.end_time 1.000 1.000 1.000 8.0
I-depart_time.end_time 1.000 0.667 0.800 3.0
I-flight_stop 0.000 0.000 0.000 0.0
B-fromloc.airport_code 0.625 1.000 0.769 5.0
I-restriction_code 1.000 1.000 1.000 3.0
I-depart_time.start_time 1.000 1.000 1.000 1.0
I-toloc.state_name 1.000 1.000 1.000 1.0
I-depart_date.today_relative 0.000 0.000 0.000 0.0
B-arrive_date.date_relative 1.000 1.000 1.000 2.0
I-flight_mod 1.000 0.167 0.286 6.0
I-economy 0.000 0.000 0.000 0.0
B-return_date.date_relative 1.000 0.667 0.800 3.0
I-fromloc.state_name 1.000 1.000 1.000 1.0
B-state_code 1.000 1.000 1.000 1.0
I-arrive_time.start_time 1.000 1.000 1.000 1.0
I-arrive_date.day_number 0.000 0.000 0.000 0.0
B-meal_code 0.000 0.000 0.000 1.0
I-depart_time.period_of_day 0.000 0.000 0.000 1.0
B-day_name 1.000 0.500 0.667 2.0
B-period_of_day 1.000 0.750 0.857 4.0
B-stoploc.state_code 0.000 0.000 0.000 0.0
B-return_date.month_name 0.000 0.000 0.000 0.0
B-return_date.day_number 0.000 0.000 0.000 0.0
B-arrive_time.period_mod 0.000 0.000 0.000 0.0
I-meal_code 0.000 0.000 0.000 0.0
B-toloc.country_name 1.000 1.000 1.000 1.0
B-days_code 0.000 0.000 0.000 1.0
I-arrive_time.period_of_day 0.000 0.000 0.000 0.0
I-today_relative 0.000 0.000 0.000 0.0
B-return_time.period_of_day 0.000 0.000 0.000 0.0
B-time 0.000 0.000 0.000 0.0
I-fare_basis_code 0.000 0.000 0.000 0.0
I-arrive_time.time_relative 1.000 1.000 1.000 4.0
I-depart_time.time_relative 0.000 0.000 0.000 1.0
B-today_relative 0.000 0.000 0.000 0.0
B-state_name 0.000 0.000 0.000 9.0
B-arrive_date.today_relative 0.000 0.000 0.000 0.0
B-return_time.period_mod 0.000 0.000 0.000 0.0
B-month_name 0.000 0.000 0.000 0.0
B-day_number 0.000 0.000 0.000 0.0
I-return_date.date_relative 1.000 0.667 0.800 3.0
I-return_date.today_relative 0.000 0.000 0.000 0.0
B-stoploc.airport_name 0.000 0.000 0.000 0.0
B-time_relative 0.000 0.000 0.000 0.0
I-time 0.000 0.000 0.000 0.0
I-return_date.day_number 0.000 0.000 0.000 0.0
I-meal_description 0.000 0.000 0.000 0.0
B-return_date.today_relative 0.000 0.000 0.000 0.0
B-return_date.day_name 0.000 0.000 0.000 2.0
micro avg 0.958 0.960 0.959 3657.0
macro avg 0.682 0.661 0.660 3657.0
weighted avg 0.961 0.960 0.956 3657.0
I0716 04:45:52.263083 139795883440000 interactiveshell.py:2882] Best Slot F1: 0.960
I0716 04:45:52.264766 139795883440000 estimator_training.py:186] Not using Distribute Coordinator.
I0716 04:45:52.266667 139795883440000 training.py:612] Running training and evaluation locally (non-distributed).
I0716 04:45:52.268577 139795883440000 training.py:700] Start train and evaluate loop. The evaluate will happen after every checkpoint. Checkpoint frequency is determined based on RunConfig arguments: save_checkpoints_steps 312 or save_checkpoints_secs None.
I0716 04:45:52.337149 139795883440000 estimator.py:1145] Calling model_fn.
I0716 04:45:53.673756 139795883440000 saver.py:1499] Saver not created because there are no variables in the graph to restore
I0716 04:45:54.459498 139795883440000 <ipython-input-6-d6747b40360b>:55]
[<tf.Variable 'embedding:0' shape=(750, 300) dtype=float32_ref>,
<tf.Variable 'module/aggregation/weights:0' shape=(3,) dtype=float32>,
<tf.Variable 'module/aggregation/scaling:0' shape=() dtype=float32>,
<tf.Variable 'dense/kernel:0' shape=(1324, 300) dtype=float32_ref>,
<tf.Variable 'dense/bias:0' shape=(300,) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/fw/gru_cell/gates/kernel:0' shape=(600, 600) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/fw/gru_cell/gates/bias:0' shape=(600,) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/fw/gru_cell/candidate/kernel:0' shape=(600, 300) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/fw/gru_cell/candidate/bias:0' shape=(300,) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/bw/gru_cell/gates/kernel:0' shape=(600, 600) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/bw/gru_cell/gates/bias:0' shape=(600,) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/bw/gru_cell/candidate/kernel:0' shape=(600, 300) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/bw/gru_cell/candidate/bias:0' shape=(300,) dtype=float32_ref>,
<tf.Variable 'dense_1/kernel:0' shape=(600, 23) dtype=float32_ref>,
<tf.Variable 'dense_1/bias:0' shape=(23,) dtype=float32_ref>,
<tf.Variable 'dense_2/kernel:0' shape=(600, 122) dtype=float32_ref>,
<tf.Variable 'dense_2/bias:0' shape=(122,) dtype=float32_ref>]
I0716 04:45:55.540261 139795883440000 estimator.py:1147] Done calling model_fn.
I0716 04:45:55.543845 139795883440000 basic_session_run_hooks.py:541] Create CheckpointSaverHook.
I0716 04:45:55.876288 139795883440000 monitored_session.py:240] Graph was finalized.
I0716 04:45:55.897523 139795883440000 saver.py:1280] Restoring parameters from ../model/elmo_bigru/model.ckpt-9048
I0716 04:45:56.945858 139795883440000 session_manager.py:500] Running local_init_op.
I0716 04:45:57.013511 139795883440000 session_manager.py:502] Done running local_init_op.
I0716 04:46:00.010006 139795883440000 basic_session_run_hooks.py:606] Saving checkpoints for 9048 into ../model/elmo_bigru/model.ckpt.
Reading ../data/atis.train.w-intent.iob
I0716 04:46:04.935000 139795883440000 basic_session_run_hooks.py:262] loss = 0.010518936, step = 9048 I0716 04:46:04.937154 139795883440000 basic_session_run_hooks.py:262] lr = 0.00011563981 I0716 04:46:33.369325 139795883440000 basic_session_run_hooks.py:692] global_step/sec: 3.51678 I0716 04:46:33.371953 139795883440000 basic_session_run_hooks.py:260] loss = 0.012888418, step = 9148 (28.437 sec) I0716 04:46:33.378367 139795883440000 basic_session_run_hooks.py:260] lr = 0.00011442781 (28.441 sec) I0716 04:46:59.021768 139795883440000 basic_session_run_hooks.py:692] global_step/sec: 3.89827 I0716 04:46:59.028245 139795883440000 basic_session_run_hooks.py:260] loss = 0.0126974005, step = 9248 (25.656 sec) I0716 04:46:59.030674 139795883440000 basic_session_run_hooks.py:260] lr = 0.00011322854 (25.652 sec) I0716 04:47:25.371392 139795883440000 basic_session_run_hooks.py:692] global_step/sec: 3.79512 I0716 04:47:25.377985 139795883440000 basic_session_run_hooks.py:260] loss = 0.011571307, step = 9348 (26.350 sec) I0716 04:47:25.380642 139795883440000 basic_session_run_hooks.py:260] lr = 0.0001120418 (26.350 sec) I0716 04:47:27.977454 139795883440000 basic_session_run_hooks.py:606] Saving checkpoints for 9360 into ../model/elmo_bigru/model.ckpt. I0716 04:47:30.206533 139795883440000 estimator.py:1145] Calling model_fn. I0716 04:47:31.614375 139795883440000 saver.py:1499] Saver not created because there are no variables in the graph to restore I0716 04:47:32.210842 139795883440000 estimator.py:1147] Done calling model_fn. I0716 04:47:32.241384 139795883440000 evaluation.py:255] Starting evaluation at 2019-07-16T04:47:32Z I0716 04:47:32.468497 139795883440000 monitored_session.py:240] Graph was finalized. I0716 04:47:32.487444 139795883440000 saver.py:1280] Restoring parameters from ../model/elmo_bigru/model.ckpt-9360 I0716 04:47:33.419073 139795883440000 session_manager.py:500] Running local_init_op. I0716 04:47:33.478233 139795883440000 session_manager.py:502] Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
I0716 04:48:32.354332 139795883440000 evaluation.py:275] Finished evaluation at 2019-07-16-04:48:32 I0716 04:48:32.356114 139795883440000 estimator.py:2039] Saving dict for global step 9360: global_step = 9360, loss = 0.36635378 I0716 04:48:32.363076 139795883440000 estimator.py:2099] Saving 'checkpoint_path' summary for global step 9360: ../model/elmo_bigru/model.ckpt-9360 I0716 04:48:32.447439 139795883440000 estimator.py:368] Loss for final step: 0.002667467.
Reading ../data/atis.test.w-intent.iob
I0716 04:48:33.036143 139795883440000 estimator.py:1145] Calling model_fn. I0716 04:48:34.747920 139795883440000 saver.py:1499] Saver not created because there are no variables in the graph to restore I0716 04:48:35.261643 139795883440000 estimator.py:1147] Done calling model_fn. I0716 04:48:35.486868 139795883440000 monitored_session.py:240] Graph was finalized. I0716 04:48:35.502614 139795883440000 saver.py:1280] Restoring parameters from ../model/elmo_bigru/model.ckpt-9360 I0716 04:48:36.388677 139795883440000 session_manager.py:500] Running local_init_op. I0716 04:48:36.440654 139795883440000 session_manager.py:502] Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.
'precision', 'predicted', average, warn_for)
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1439: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples.
'recall', 'true', average, warn_for)
I0716 04:49:34.259678 139795883440000 interactiveshell.py:2882]
precision recall f1-score support
atis_flight 0.984 0.992 0.988 632
atis_airfare 0.941 1.000 0.970 48
atis_ground_service 1.000 1.000 1.000 36
atis_airline 0.974 1.000 0.987 38
atis_abbreviation 0.943 1.000 0.971 33
atis_aircraft 1.000 0.889 0.941 9
atis_flight_time 0.500 1.000 0.667 1
atis_quantity 0.429 1.000 0.600 3
atis_flight#atis_airfare 0.833 0.417 0.556 12
atis_airport 1.000 0.944 0.971 18
atis_distance 1.000 1.000 1.000 10
atis_city 0.600 0.500 0.545 6
atis_ground_fare 1.000 0.857 0.923 7
atis_capacity 0.952 0.952 0.952 21
atis_flight_no 0.889 1.000 0.941 8
atis_meal 1.000 0.667 0.800 6
atis_restriction 0.000 0.000 0.000 0
atis_airline#atis_flight_no 0.000 0.000 0.000 0
atis_ground_service#atis_ground_fare 0.000 0.000 0.000 0
atis_airfare#atis_flight_time 0.000 0.000 0.000 0
atis_cheapest 0.000 0.000 0.000 0
atis_aircraft#atis_flight#atis_flight_no 0.000 0.000 0.000 0
micro avg 0.971 0.976 0.974 888
macro avg 0.638 0.646 0.628 888
weighted avg 0.973 0.976 0.973 888
I0716 04:49:34.296104 139795883440000 interactiveshell.py:2882]
precision recall f1-score support
O 0.000 0.000 0.000 0.0
B-toloc.city_name 0.977 0.993 0.985 716.0
B-fromloc.city_name 0.992 0.999 0.995 704.0
I-toloc.city_name 0.981 0.981 0.981 265.0
B-depart_date.day_name 0.991 0.991 0.991 212.0
B-airline_name 1.000 1.000 1.000 101.0
I-fromloc.city_name 0.973 1.000 0.986 177.0
B-depart_time.period_of_day 1.000 0.923 0.960 130.0
I-airline_name 1.000 1.000 1.000 65.0
B-depart_date.day_number 0.981 0.927 0.953 55.0
B-depart_date.month_name 0.982 0.964 0.973 56.0
B-depart_time.time 0.864 1.000 0.927 57.0
B-round_trip 1.000 1.000 1.000 73.0
B-cost_relative 0.973 0.973 0.973 37.0
I-round_trip 1.000 1.000 1.000 71.0
B-flight_mod 0.828 1.000 0.906 24.0
B-depart_time.time_relative 0.955 0.985 0.970 65.0
I-depart_time.time 0.945 1.000 0.972 52.0
B-stoploc.city_name 0.833 1.000 0.909 20.0
B-city_name 0.844 0.667 0.745 57.0
B-class_type 0.889 1.000 0.941 24.0
B-arrive_time.time 0.971 0.971 0.971 34.0
B-arrive_time.time_relative 0.906 0.935 0.921 31.0
I-class_type 1.000 1.000 1.000 17.0
B-flight_stop 1.000 1.000 1.000 21.0
I-arrive_time.time 1.000 0.971 0.986 35.0
B-airline_code 0.895 1.000 0.944 34.0
I-depart_date.day_number 1.000 0.933 0.966 15.0
I-fromloc.airport_name 0.441 1.000 0.612 15.0
B-fromloc.airport_name 0.462 1.000 0.632 12.0
B-arrive_date.day_name 0.786 1.000 0.880 11.0
B-toloc.state_code 1.000 1.000 1.000 18.0
B-depart_date.today_relative 1.000 1.000 1.000 9.0
B-flight_number 0.733 1.000 0.846 11.0
B-depart_date.date_relative 0.944 1.000 0.971 17.0
B-toloc.state_name 0.806 0.893 0.847 28.0
B-fare_basis_code 0.850 1.000 0.919 17.0
B-flight_time 1.000 1.000 1.000 1.0
B-or 1.000 1.000 1.000 3.0
B-arrive_time.period_of_day 0.750 1.000 0.857 6.0
B-meal_description 1.000 0.900 0.947 10.0
I-cost_relative 1.000 0.667 0.800 3.0
I-airport_name 0.833 0.345 0.488 29.0
B-fare_amount 1.000 1.000 1.000 2.0
I-fare_amount 1.000 1.000 1.000 2.0
I-city_name 0.950 0.633 0.760 30.0
I-toloc.airport_name 1.000 1.000 1.000 3.0
B-transport_type 1.000 1.000 1.000 10.0
B-arrive_date.month_name 0.714 0.833 0.769 6.0
B-arrive_date.day_number 0.714 0.833 0.769 6.0
I-stoploc.city_name 0.909 1.000 0.952 10.0
B-meal 0.941 1.000 0.970 16.0
B-fromloc.state_code 1.000 1.000 1.000 23.0
B-depart_time.period_mod 1.000 1.000 1.000 5.0
B-connect 1.000 1.000 1.000 6.0
B-flight_days 1.000 1.000 1.000 10.0
B-toloc.airport_name 1.000 1.000 1.000 3.0
B-fromloc.state_name 1.000 1.000 1.000 17.0
B-airport_name 0.750 0.286 0.414 21.0
B-economy 1.000 1.000 1.000 6.0
I-flight_time 1.000 1.000 1.000 1.0
B-aircraft_code 1.000 0.818 0.900 33.0
B-mod 1.000 0.500 0.667 2.0
B-airport_code 1.000 0.444 0.615 9.0
B-depart_time.start_time 1.000 0.667 0.800 3.0
B-depart_time.end_time 1.000 0.333 0.500 3.0
B-depart_date.year 1.000 1.000 1.000 3.0
I-transport_type 1.000 1.000 1.000 1.0
B-restriction_code 1.000 1.000 1.000 4.0
B-arrive_time.start_time 0.889 1.000 0.941 8.0
B-toloc.airport_code 1.000 0.750 0.857 4.0
B-arrive_time.end_time 0.889 1.000 0.941 8.0
I-arrive_time.end_time 0.889 1.000 0.941 8.0
I-depart_time.end_time 1.000 0.333 0.500 3.0
I-flight_stop 0.000 0.000 0.000 0.0
B-fromloc.airport_code 0.625 1.000 0.769 5.0
I-restriction_code 1.000 1.000 1.000 3.0
I-depart_time.start_time 1.000 1.000 1.000 1.0
I-toloc.state_name 1.000 1.000 1.000 1.0
I-depart_date.today_relative 0.000 0.000 0.000 0.0
B-arrive_date.date_relative 1.000 1.000 1.000 2.0
I-flight_mod 1.000 0.167 0.286 6.0
I-economy 0.000 0.000 0.000 0.0
B-return_date.date_relative 1.000 0.333 0.500 3.0
I-fromloc.state_name 1.000 1.000 1.000 1.0
B-state_code 1.000 1.000 1.000 1.0
I-arrive_time.start_time 1.000 1.000 1.000 1.0
I-arrive_date.day_number 0.000 0.000 0.000 0.0
B-meal_code 0.000 0.000 0.000 1.0
I-depart_time.period_of_day 1.000 1.000 1.000 1.0
B-day_name 1.000 0.500 0.667 2.0
B-period_of_day 1.000 0.750 0.857 4.0
B-stoploc.state_code 0.000 0.000 0.000 0.0
B-return_date.month_name 0.000 0.000 0.000 0.0
B-return_date.day_number 0.000 0.000 0.000 0.0
B-arrive_time.period_mod 0.000 0.000 0.000 0.0
I-meal_code 0.000 0.000 0.000 0.0
B-toloc.country_name 1.000 1.000 1.000 1.0
B-days_code 0.000 0.000 0.000 1.0
I-arrive_time.period_of_day 0.000 0.000 0.000 0.0
I-today_relative 0.000 0.000 0.000 0.0
B-return_time.period_of_day 0.000 0.000 0.000 0.0
B-time 0.000 0.000 0.000 0.0
I-fare_basis_code 0.000 0.000 0.000 0.0
I-arrive_time.time_relative 1.000 0.500 0.667 4.0
I-depart_time.time_relative 0.000 0.000 0.000 1.0
B-today_relative 0.000 0.000 0.000 0.0
B-state_name 0.000 0.000 0.000 9.0
B-arrive_date.today_relative 0.000 0.000 0.000 0.0
B-return_time.period_mod 0.000 0.000 0.000 0.0
B-month_name 0.000 0.000 0.000 0.0
B-day_number 0.000 0.000 0.000 0.0
I-return_date.date_relative 0.667 0.667 0.667 3.0
I-return_date.today_relative 0.000 0.000 0.000 0.0
B-stoploc.airport_name 0.000 0.000 0.000 0.0
B-time_relative 0.000 0.000 0.000 0.0
I-time 0.000 0.000 0.000 0.0
I-return_date.day_number 0.000 0.000 0.000 0.0
I-meal_description 0.000 0.000 0.000 0.0
B-return_date.today_relative 0.000 0.000 0.000 0.0
B-return_date.day_name 0.000 0.000 0.000 2.0
micro avg 0.957 0.959 0.958 3657.0
macro avg 0.689 0.656 0.657 3657.0
weighted avg 0.960 0.959 0.955 3657.0
I0716 04:49:34.317782 139795883440000 interactiveshell.py:2882] Best Slot F1: 0.960
I0716 04:49:34.319455 139795883440000 estimator_training.py:186] Not using Distribute Coordinator.
I0716 04:49:34.322131 139795883440000 training.py:612] Running training and evaluation locally (non-distributed).
I0716 04:49:34.325804 139795883440000 training.py:700] Start train and evaluate loop. The evaluate will happen after every checkpoint. Checkpoint frequency is determined based on RunConfig arguments: save_checkpoints_steps 312 or save_checkpoints_secs None.
I0716 04:49:34.390808 139795883440000 estimator.py:1145] Calling model_fn.
I0716 04:49:35.879683 139795883440000 saver.py:1499] Saver not created because there are no variables in the graph to restore
I0716 04:49:36.453143 139795883440000 <ipython-input-6-d6747b40360b>:55]
[<tf.Variable 'embedding:0' shape=(750, 300) dtype=float32_ref>,
<tf.Variable 'module/aggregation/weights:0' shape=(3,) dtype=float32>,
<tf.Variable 'module/aggregation/scaling:0' shape=() dtype=float32>,
<tf.Variable 'dense/kernel:0' shape=(1324, 300) dtype=float32_ref>,
<tf.Variable 'dense/bias:0' shape=(300,) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/fw/gru_cell/gates/kernel:0' shape=(600, 600) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/fw/gru_cell/gates/bias:0' shape=(600,) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/fw/gru_cell/candidate/kernel:0' shape=(600, 300) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/fw/gru_cell/candidate/bias:0' shape=(300,) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/bw/gru_cell/gates/kernel:0' shape=(600, 600) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/bw/gru_cell/gates/bias:0' shape=(600,) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/bw/gru_cell/candidate/kernel:0' shape=(600, 300) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/bw/gru_cell/candidate/bias:0' shape=(300,) dtype=float32_ref>,
<tf.Variable 'dense_1/kernel:0' shape=(600, 23) dtype=float32_ref>,
<tf.Variable 'dense_1/bias:0' shape=(23,) dtype=float32_ref>,
<tf.Variable 'dense_2/kernel:0' shape=(600, 122) dtype=float32_ref>,
<tf.Variable 'dense_2/bias:0' shape=(122,) dtype=float32_ref>]
I0716 04:49:37.719774 139795883440000 estimator.py:1147] Done calling model_fn.
I0716 04:49:37.724008 139795883440000 basic_session_run_hooks.py:541] Create CheckpointSaverHook.
I0716 04:49:38.057800 139795883440000 monitored_session.py:240] Graph was finalized.
I0716 04:49:38.076627 139795883440000 saver.py:1280] Restoring parameters from ../model/elmo_bigru/model.ckpt-9360
I0716 04:49:39.114448 139795883440000 session_manager.py:500] Running local_init_op.
I0716 04:49:39.180683 139795883440000 session_manager.py:502] Done running local_init_op.
I0716 04:49:41.989519 139795883440000 basic_session_run_hooks.py:606] Saving checkpoints for 9360 into ../model/elmo_bigru/model.ckpt.
Reading ../data/atis.train.w-intent.iob
I0716 04:49:46.790730 139795883440000 basic_session_run_hooks.py:262] loss = 0.01551132, step = 9360 I0716 04:49:46.792497 139795883440000 basic_session_run_hooks.py:262] lr = 0.00011190023 I0716 04:50:14.630988 139795883440000 basic_session_run_hooks.py:692] global_step/sec: 3.59184 I0716 04:50:14.636307 139795883440000 basic_session_run_hooks.py:260] loss = 0.011015689, step = 9460 (27.846 sec) I0716 04:50:14.639242 139795883440000 basic_session_run_hooks.py:260] lr = 0.00011072745 (27.847 sec) I0716 04:50:41.028852 139795883440000 basic_session_run_hooks.py:692] global_step/sec: 3.78819 I0716 04:50:41.037099 139795883440000 basic_session_run_hooks.py:260] loss = 0.013117211, step = 9560 (26.401 sec) I0716 04:50:41.039311 139795883440000 basic_session_run_hooks.py:260] lr = 0.00010956693 (26.400 sec) I0716 04:51:06.975775 139795883440000 basic_session_run_hooks.py:692] global_step/sec: 3.85402 I0716 04:51:06.979199 139795883440000 basic_session_run_hooks.py:260] loss = 0.015385278, step = 9660 (25.942 sec) I0716 04:51:06.983596 139795883440000 basic_session_run_hooks.py:260] lr = 0.00010841859 (25.944 sec) I0716 04:51:09.733838 139795883440000 basic_session_run_hooks.py:606] Saving checkpoints for 9672 into ../model/elmo_bigru/model.ckpt. I0716 04:51:11.702174 139795883440000 estimator.py:1145] Calling model_fn. I0716 04:51:13.322754 139795883440000 saver.py:1499] Saver not created because there are no variables in the graph to restore I0716 04:51:13.908298 139795883440000 estimator.py:1147] Done calling model_fn. I0716 04:51:13.939200 139795883440000 evaluation.py:255] Starting evaluation at 2019-07-16T04:51:13Z I0716 04:51:14.166243 139795883440000 monitored_session.py:240] Graph was finalized. I0716 04:51:14.183458 139795883440000 saver.py:1280] Restoring parameters from ../model/elmo_bigru/model.ckpt-9672 I0716 04:51:15.079905 139795883440000 session_manager.py:500] Running local_init_op. I0716 04:51:15.133797 139795883440000 session_manager.py:502] Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
I0716 04:52:13.750108 139795883440000 evaluation.py:275] Finished evaluation at 2019-07-16-04:52:13 I0716 04:52:13.751840 139795883440000 estimator.py:2039] Saving dict for global step 9672: global_step = 9672, loss = 0.36763337 I0716 04:52:13.758711 139795883440000 estimator.py:2099] Saving 'checkpoint_path' summary for global step 9672: ../model/elmo_bigru/model.ckpt-9672 I0716 04:52:13.845093 139795883440000 estimator.py:368] Loss for final step: 0.00026629955.
Reading ../data/atis.test.w-intent.iob
I0716 04:52:14.444941 139795883440000 estimator.py:1145] Calling model_fn. I0716 04:52:15.972132 139795883440000 saver.py:1499] Saver not created because there are no variables in the graph to restore I0716 04:52:16.502604 139795883440000 estimator.py:1147] Done calling model_fn. I0716 04:52:16.716974 139795883440000 monitored_session.py:240] Graph was finalized. I0716 04:52:16.734670 139795883440000 saver.py:1280] Restoring parameters from ../model/elmo_bigru/model.ckpt-9672 I0716 04:52:17.638433 139795883440000 session_manager.py:500] Running local_init_op. I0716 04:52:17.689142 139795883440000 session_manager.py:502] Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.
'precision', 'predicted', average, warn_for)
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1439: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples.
'recall', 'true', average, warn_for)
I0716 04:53:15.658815 139795883440000 interactiveshell.py:2882]
precision recall f1-score support
atis_flight 0.984 0.991 0.987 632
atis_airfare 0.941 1.000 0.970 48
atis_ground_service 1.000 1.000 1.000 36
atis_airline 1.000 1.000 1.000 38
atis_abbreviation 0.943 1.000 0.971 33
atis_aircraft 1.000 0.889 0.941 9
atis_flight_time 0.500 1.000 0.667 1
atis_quantity 0.429 1.000 0.600 3
atis_flight#atis_airfare 0.833 0.417 0.556 12
atis_airport 0.947 1.000 0.973 18
atis_distance 1.000 1.000 1.000 10
atis_city 0.750 0.500 0.600 6
atis_ground_fare 1.000 0.857 0.923 7
atis_capacity 0.952 0.952 0.952 21
atis_flight_no 0.889 1.000 0.941 8
atis_meal 1.000 0.833 0.909 6
atis_restriction 0.000 0.000 0.000 0
atis_airline#atis_flight_no 0.000 0.000 0.000 0
atis_ground_service#atis_ground_fare 0.000 0.000 0.000 0
atis_airfare#atis_flight_time 0.000 0.000 0.000 0
atis_cheapest 0.000 0.000 0.000 0
atis_aircraft#atis_flight#atis_flight_no 0.000 0.000 0.000 0
micro avg 0.972 0.977 0.975 888
macro avg 0.644 0.656 0.636 888
weighted avg 0.974 0.977 0.974 888
I0716 04:53:15.698364 139795883440000 interactiveshell.py:2882]
precision recall f1-score support
O 0.000 0.000 0.000 0.0
B-toloc.city_name 0.979 0.993 0.986 716.0
B-fromloc.city_name 0.992 0.999 0.995 704.0
I-toloc.city_name 0.981 0.981 0.981 265.0
B-depart_date.day_name 0.991 0.991 0.991 212.0
B-airline_name 1.000 1.000 1.000 101.0
I-fromloc.city_name 0.973 1.000 0.986 177.0
B-depart_time.period_of_day 0.992 0.908 0.948 130.0
I-airline_name 1.000 1.000 1.000 65.0
B-depart_date.day_number 0.981 0.964 0.972 55.0
B-depart_date.month_name 0.982 0.964 0.973 56.0
B-depart_time.time 0.864 1.000 0.927 57.0
B-round_trip 1.000 1.000 1.000 73.0
B-cost_relative 0.973 0.973 0.973 37.0
I-round_trip 1.000 1.000 1.000 71.0
B-flight_mod 0.828 1.000 0.906 24.0
B-depart_time.time_relative 0.955 0.985 0.970 65.0
I-depart_time.time 0.912 1.000 0.954 52.0
B-stoploc.city_name 0.833 1.000 0.909 20.0
B-city_name 0.902 0.649 0.755 57.0
B-class_type 0.960 1.000 0.980 24.0
B-arrive_time.time 0.971 0.971 0.971 34.0
B-arrive_time.time_relative 0.967 0.935 0.951 31.0
I-class_type 1.000 1.000 1.000 17.0
B-flight_stop 1.000 1.000 1.000 21.0
I-arrive_time.time 1.000 0.971 0.986 35.0
B-airline_code 0.971 1.000 0.986 34.0
I-depart_date.day_number 1.000 0.933 0.966 15.0
I-fromloc.airport_name 0.441 1.000 0.612 15.0
B-fromloc.airport_name 0.462 1.000 0.632 12.0
B-arrive_date.day_name 0.786 1.000 0.880 11.0
B-toloc.state_code 1.000 1.000 1.000 18.0
B-depart_date.today_relative 1.000 0.889 0.941 9.0
B-flight_number 0.733 1.000 0.846 11.0
B-depart_date.date_relative 0.944 1.000 0.971 17.0
B-toloc.state_name 0.794 0.964 0.871 28.0
B-fare_basis_code 0.895 1.000 0.944 17.0
B-flight_time 1.000 1.000 1.000 1.0
B-or 1.000 1.000 1.000 3.0
B-arrive_time.period_of_day 0.750 1.000 0.857 6.0
B-meal_description 1.000 0.900 0.947 10.0
I-cost_relative 1.000 0.667 0.800 3.0
I-airport_name 0.846 0.379 0.524 29.0
B-fare_amount 1.000 1.000 1.000 2.0
I-fare_amount 1.000 1.000 1.000 2.0
I-city_name 1.000 0.633 0.776 30.0
I-toloc.airport_name 1.000 1.000 1.000 3.0
B-transport_type 1.000 1.000 1.000 10.0
B-arrive_date.month_name 0.714 0.833 0.769 6.0
B-arrive_date.day_number 0.714 0.833 0.769 6.0
I-stoploc.city_name 0.909 1.000 0.952 10.0
B-meal 0.941 1.000 0.970 16.0
B-fromloc.state_code 1.000 1.000 1.000 23.0
B-depart_time.period_mod 1.000 1.000 1.000 5.0
B-connect 1.000 1.000 1.000 6.0
B-flight_days 1.000 1.000 1.000 10.0
B-toloc.airport_name 1.000 1.000 1.000 3.0
B-fromloc.state_name 1.000 1.000 1.000 17.0
B-airport_name 0.778 0.333 0.467 21.0
B-economy 1.000 1.000 1.000 6.0
I-flight_time 1.000 1.000 1.000 1.0
B-aircraft_code 1.000 0.879 0.935 33.0
B-mod 1.000 0.500 0.667 2.0
B-airport_code 1.000 0.444 0.615 9.0
B-depart_time.start_time 1.000 0.667 0.800 3.0
B-depart_time.end_time 1.000 0.333 0.500 3.0
B-depart_date.year 1.000 1.000 1.000 3.0
I-transport_type 0.000 0.000 0.000 1.0
B-restriction_code 0.800 1.000 0.889 4.0
B-arrive_time.start_time 0.889 1.000 0.941 8.0
B-toloc.airport_code 1.000 1.000 1.000 4.0
B-arrive_time.end_time 0.889 1.000 0.941 8.0
I-arrive_time.end_time 0.889 1.000 0.941 8.0
I-depart_time.end_time 1.000 0.333 0.500 3.0
I-flight_stop 0.000 0.000 0.000 0.0
B-fromloc.airport_code 0.556 1.000 0.714 5.0
I-restriction_code 1.000 1.000 1.000 3.0
I-depart_time.start_time 1.000 1.000 1.000 1.0
I-toloc.state_name 1.000 1.000 1.000 1.0
I-depart_date.today_relative 0.000 0.000 0.000 0.0
B-arrive_date.date_relative 1.000 1.000 1.000 2.0
I-flight_mod 1.000 0.167 0.286 6.0
I-economy 0.000 0.000 0.000 0.0
B-return_date.date_relative 1.000 0.333 0.500 3.0
I-fromloc.state_name 1.000 1.000 1.000 1.0
B-state_code 1.000 1.000 1.000 1.0
I-arrive_time.start_time 1.000 1.000 1.000 1.0
I-arrive_date.day_number 0.000 0.000 0.000 0.0
B-meal_code 0.000 0.000 0.000 1.0
I-depart_time.period_of_day 1.000 1.000 1.000 1.0
B-day_name 1.000 0.500 0.667 2.0
B-period_of_day 1.000 0.750 0.857 4.0
B-stoploc.state_code 0.000 0.000 0.000 0.0
B-return_date.month_name 0.000 0.000 0.000 0.0
B-return_date.day_number 0.000 0.000 0.000 0.0
B-arrive_time.period_mod 0.000 0.000 0.000 0.0
I-meal_code 0.000 0.000 0.000 0.0
B-toloc.country_name 1.000 1.000 1.000 1.0
B-days_code 0.000 0.000 0.000 1.0
I-arrive_time.period_of_day 0.000 0.000 0.000 0.0
I-today_relative 0.000 0.000 0.000 0.0
B-return_time.period_of_day 0.000 0.000 0.000 0.0
B-time 0.000 0.000 0.000 0.0
I-fare_basis_code 0.000 0.000 0.000 0.0
I-arrive_time.time_relative 1.000 1.000 1.000 4.0
I-depart_time.time_relative 0.000 0.000 0.000 1.0
B-today_relative 0.000 0.000 0.000 0.0
B-state_name 0.000 0.000 0.000 9.0
B-arrive_date.today_relative 0.000 0.000 0.000 0.0
B-return_time.period_mod 0.000 0.000 0.000 0.0
B-month_name 0.000 0.000 0.000 0.0
B-day_number 0.000 0.000 0.000 0.0
I-return_date.date_relative 0.667 0.667 0.667 3.0
I-return_date.today_relative 0.000 0.000 0.000 0.0
B-stoploc.airport_name 0.000 0.000 0.000 0.0
B-time_relative 0.000 0.000 0.000 0.0
I-time 0.000 0.000 0.000 0.0
I-return_date.day_number 0.000 0.000 0.000 0.0
I-meal_description 0.000 0.000 0.000 0.0
B-return_date.today_relative 0.000 0.000 0.000 0.0
B-return_date.day_name 0.000 0.000 0.000 2.0
micro avg 0.959 0.960 0.960 3657.0
macro avg 0.681 0.655 0.653 3657.0
weighted avg 0.963 0.960 0.957 3657.0
I0716 04:53:15.721315 139795883440000 interactiveshell.py:2882] Best Slot F1: 0.960
I0716 04:53:15.723052 139795883440000 estimator_training.py:186] Not using Distribute Coordinator.
I0716 04:53:15.724583 139795883440000 training.py:612] Running training and evaluation locally (non-distributed).
I0716 04:53:15.726242 139795883440000 training.py:700] Start train and evaluate loop. The evaluate will happen after every checkpoint. Checkpoint frequency is determined based on RunConfig arguments: save_checkpoints_steps 312 or save_checkpoints_secs None.
I0716 04:53:15.794797 139795883440000 estimator.py:1145] Calling model_fn.
I0716 04:53:17.253643 139795883440000 saver.py:1499] Saver not created because there are no variables in the graph to restore
I0716 04:53:17.837841 139795883440000 <ipython-input-6-d6747b40360b>:55]
[<tf.Variable 'embedding:0' shape=(750, 300) dtype=float32_ref>,
<tf.Variable 'module/aggregation/weights:0' shape=(3,) dtype=float32>,
<tf.Variable 'module/aggregation/scaling:0' shape=() dtype=float32>,
<tf.Variable 'dense/kernel:0' shape=(1324, 300) dtype=float32_ref>,
<tf.Variable 'dense/bias:0' shape=(300,) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/fw/gru_cell/gates/kernel:0' shape=(600, 600) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/fw/gru_cell/gates/bias:0' shape=(600,) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/fw/gru_cell/candidate/kernel:0' shape=(600, 300) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/fw/gru_cell/candidate/bias:0' shape=(300,) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/bw/gru_cell/gates/kernel:0' shape=(600, 600) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/bw/gru_cell/gates/bias:0' shape=(600,) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/bw/gru_cell/candidate/kernel:0' shape=(600, 300) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/bw/gru_cell/candidate/bias:0' shape=(300,) dtype=float32_ref>,
<tf.Variable 'dense_1/kernel:0' shape=(600, 23) dtype=float32_ref>,
<tf.Variable 'dense_1/bias:0' shape=(23,) dtype=float32_ref>,
<tf.Variable 'dense_2/kernel:0' shape=(600, 122) dtype=float32_ref>,
<tf.Variable 'dense_2/bias:0' shape=(122,) dtype=float32_ref>]
I0716 04:53:19.109848 139795883440000 estimator.py:1147] Done calling model_fn.
I0716 04:53:19.113912 139795883440000 basic_session_run_hooks.py:541] Create CheckpointSaverHook.
I0716 04:53:19.430395 139795883440000 monitored_session.py:240] Graph was finalized.
I0716 04:53:19.450618 139795883440000 saver.py:1280] Restoring parameters from ../model/elmo_bigru/model.ckpt-9672
I0716 04:53:20.499954 139795883440000 session_manager.py:500] Running local_init_op.
I0716 04:53:20.569100 139795883440000 session_manager.py:502] Done running local_init_op.
I0716 04:53:23.373302 139795883440000 basic_session_run_hooks.py:606] Saving checkpoints for 9672 into ../model/elmo_bigru/model.ckpt.
Reading ../data/atis.train.w-intent.iob
I0716 04:53:28.404636 139795883440000 basic_session_run_hooks.py:262] loss = 0.020569747, step = 9672 I0716 04:53:28.406489 139795883440000 basic_session_run_hooks.py:262] lr = 0.000108281594 I0716 04:53:55.312132 139795883440000 basic_session_run_hooks.py:692] global_step/sec: 3.71635 I0716 04:53:55.315697 139795883440000 basic_session_run_hooks.py:260] loss = 0.012076699, step = 9772 (26.911 sec) I0716 04:53:55.319453 139795883440000 basic_session_run_hooks.py:260] lr = 0.000107146734 (26.913 sec) I0716 04:54:21.683307 139795883440000 basic_session_run_hooks.py:692] global_step/sec: 3.79202 I0716 04:54:21.689350 139795883440000 basic_session_run_hooks.py:260] loss = 0.013083725, step = 9872 (26.374 sec) I0716 04:54:21.691671 139795883440000 basic_session_run_hooks.py:260] lr = 0.00010602375 (26.372 sec) I0716 04:54:48.945347 139795883440000 basic_session_run_hooks.py:692] global_step/sec: 3.66811 I0716 04:54:48.953207 139795883440000 basic_session_run_hooks.py:260] loss = 0.013616568, step = 9972 (27.264 sec) I0716 04:54:48.955022 139795883440000 basic_session_run_hooks.py:260] lr = 0.00010491254 (27.263 sec) I0716 04:54:51.518794 139795883440000 basic_session_run_hooks.py:606] Saving checkpoints for 9984 into ../model/elmo_bigru/model.ckpt. I0716 04:54:53.406455 139795883440000 estimator.py:1145] Calling model_fn. I0716 04:54:55.017331 139795883440000 saver.py:1499] Saver not created because there are no variables in the graph to restore I0716 04:54:55.584659 139795883440000 estimator.py:1147] Done calling model_fn. I0716 04:54:55.615095 139795883440000 evaluation.py:255] Starting evaluation at 2019-07-16T04:54:55Z I0716 04:54:55.838979 139795883440000 monitored_session.py:240] Graph was finalized. I0716 04:54:55.857573 139795883440000 saver.py:1280] Restoring parameters from ../model/elmo_bigru/model.ckpt-9984 I0716 04:54:56.831674 139795883440000 session_manager.py:500] Running local_init_op. I0716 04:54:56.906322 139795883440000 session_manager.py:502] Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
I0716 04:55:56.055133 139795883440000 evaluation.py:275] Finished evaluation at 2019-07-16-04:55:56 I0716 04:55:56.056856 139795883440000 estimator.py:2039] Saving dict for global step 9984: global_step = 9984, loss = 0.38307405 I0716 04:55:56.065994 139795883440000 estimator.py:2099] Saving 'checkpoint_path' summary for global step 9984: ../model/elmo_bigru/model.ckpt-9984 I0716 04:55:56.149209 139795883440000 estimator.py:368] Loss for final step: 0.0013525934.
Reading ../data/atis.test.w-intent.iob
I0716 04:55:56.767248 139795883440000 estimator.py:1145] Calling model_fn. I0716 04:55:58.302370 139795883440000 saver.py:1499] Saver not created because there are no variables in the graph to restore I0716 04:55:58.829959 139795883440000 estimator.py:1147] Done calling model_fn. I0716 04:55:59.043296 139795883440000 monitored_session.py:240] Graph was finalized. I0716 04:55:59.062215 139795883440000 saver.py:1280] Restoring parameters from ../model/elmo_bigru/model.ckpt-9984 I0716 04:55:59.979383 139795883440000 session_manager.py:500] Running local_init_op. I0716 04:56:00.027868 139795883440000 session_manager.py:502] Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.
'precision', 'predicted', average, warn_for)
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1439: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples.
'recall', 'true', average, warn_for)
I0716 04:56:57.841996 139795883440000 interactiveshell.py:2882]
precision recall f1-score support
atis_flight 0.986 0.991 0.988 632
atis_airfare 0.941 1.000 0.970 48
atis_ground_service 1.000 1.000 1.000 36
atis_airline 0.974 1.000 0.987 38
atis_abbreviation 0.943 1.000 0.971 33
atis_aircraft 1.000 0.889 0.941 9
atis_flight_time 0.500 1.000 0.667 1
atis_quantity 0.429 1.000 0.600 3
atis_flight#atis_airfare 0.833 0.417 0.556 12
atis_airport 0.944 0.944 0.944 18
atis_distance 1.000 1.000 1.000 10
atis_city 0.800 0.667 0.727 6
atis_ground_fare 1.000 0.857 0.923 7
atis_capacity 0.952 0.952 0.952 21
atis_flight_no 0.889 1.000 0.941 8
atis_meal 1.000 0.833 0.909 6
atis_restriction 0.000 0.000 0.000 0
atis_airline#atis_flight_no 0.000 0.000 0.000 0
atis_ground_service#atis_ground_fare 0.000 0.000 0.000 0
atis_airfare#atis_flight_time 0.000 0.000 0.000 0
atis_cheapest 0.000 0.000 0.000 0
atis_aircraft#atis_flight#atis_flight_no 0.000 0.000 0.000 0
micro avg 0.972 0.977 0.975 888
macro avg 0.645 0.661 0.640 888
weighted avg 0.974 0.977 0.974 888
I0716 04:56:57.888020 139795883440000 interactiveshell.py:2882]
precision recall f1-score support
O 0.000 0.000 0.000 0.0
B-toloc.city_name 0.975 0.994 0.985 716.0
B-fromloc.city_name 0.990 0.999 0.994 704.0
I-toloc.city_name 0.978 0.985 0.981 265.0
B-depart_date.day_name 0.986 0.991 0.988 212.0
B-airline_name 1.000 1.000 1.000 101.0
I-fromloc.city_name 0.973 1.000 0.986 177.0
B-depart_time.period_of_day 0.984 0.923 0.952 130.0
I-airline_name 1.000 1.000 1.000 65.0
B-depart_date.day_number 0.981 0.964 0.972 55.0
B-depart_date.month_name 0.982 0.964 0.973 56.0
B-depart_time.time 0.864 1.000 0.927 57.0
B-round_trip 1.000 1.000 1.000 73.0
B-cost_relative 0.973 0.973 0.973 37.0
I-round_trip 1.000 1.000 1.000 71.0
B-flight_mod 0.828 1.000 0.906 24.0
B-depart_time.time_relative 0.955 0.985 0.970 65.0
I-depart_time.time 0.945 1.000 0.972 52.0
B-stoploc.city_name 0.909 1.000 0.952 20.0
B-city_name 0.881 0.649 0.747 57.0
B-class_type 0.960 1.000 0.980 24.0
B-arrive_time.time 0.971 0.971 0.971 34.0
B-arrive_time.time_relative 0.967 0.935 0.951 31.0
I-class_type 1.000 1.000 1.000 17.0
B-flight_stop 1.000 1.000 1.000 21.0
I-arrive_time.time 1.000 0.971 0.986 35.0
B-airline_code 0.895 1.000 0.944 34.0
I-depart_date.day_number 1.000 1.000 1.000 15.0
I-fromloc.airport_name 0.441 1.000 0.612 15.0
B-fromloc.airport_name 0.462 1.000 0.632 12.0
B-arrive_date.day_name 0.769 0.909 0.833 11.0
B-toloc.state_code 1.000 1.000 1.000 18.0
B-depart_date.today_relative 1.000 0.889 0.941 9.0
B-flight_number 0.733 1.000 0.846 11.0
B-depart_date.date_relative 0.944 1.000 0.971 17.0
B-toloc.state_name 0.833 0.893 0.862 28.0
B-fare_basis_code 0.850 1.000 0.919 17.0
B-flight_time 1.000 1.000 1.000 1.0
B-or 1.000 1.000 1.000 3.0
B-arrive_time.period_of_day 0.750 1.000 0.857 6.0
B-meal_description 1.000 0.900 0.947 10.0
I-cost_relative 1.000 0.667 0.800 3.0
I-airport_name 0.846 0.379 0.524 29.0
B-fare_amount 1.000 1.000 1.000 2.0
I-fare_amount 1.000 1.000 1.000 2.0
I-city_name 1.000 0.600 0.750 30.0
I-toloc.airport_name 1.000 1.000 1.000 3.0
B-transport_type 1.000 1.000 1.000 10.0
B-arrive_date.month_name 0.714 0.833 0.769 6.0
B-arrive_date.day_number 0.714 0.833 0.769 6.0
I-stoploc.city_name 1.000 1.000 1.000 10.0
B-meal 1.000 1.000 1.000 16.0
B-fromloc.state_code 1.000 1.000 1.000 23.0
B-depart_time.period_mod 1.000 1.000 1.000 5.0
B-connect 1.000 1.000 1.000 6.0
B-flight_days 1.000 1.000 1.000 10.0
B-toloc.airport_name 1.000 1.000 1.000 3.0
B-fromloc.state_name 1.000 1.000 1.000 17.0
B-airport_name 0.636 0.333 0.437 21.0
B-economy 1.000 1.000 1.000 6.0
I-flight_time 1.000 1.000 1.000 1.0
B-aircraft_code 1.000 0.818 0.900 33.0
B-mod 1.000 0.500 0.667 2.0
B-airport_code 1.000 0.444 0.615 9.0
B-depart_time.start_time 1.000 1.000 1.000 3.0
B-depart_time.end_time 1.000 0.667 0.800 3.0
B-depart_date.year 1.000 1.000 1.000 3.0
I-transport_type 0.000 0.000 0.000 1.0
B-restriction_code 1.000 1.000 1.000 4.0
B-arrive_time.start_time 1.000 1.000 1.000 8.0
B-toloc.airport_code 1.000 0.750 0.857 4.0
B-arrive_time.end_time 1.000 1.000 1.000 8.0
I-arrive_time.end_time 1.000 1.000 1.000 8.0
I-depart_time.end_time 1.000 0.667 0.800 3.0
I-flight_stop 0.000 0.000 0.000 0.0
B-fromloc.airport_code 0.556 1.000 0.714 5.0
I-restriction_code 1.000 1.000 1.000 3.0
I-depart_time.start_time 1.000 1.000 1.000 1.0
I-toloc.state_name 1.000 1.000 1.000 1.0
I-depart_date.today_relative 0.000 0.000 0.000 0.0
B-arrive_date.date_relative 1.000 1.000 1.000 2.0
I-flight_mod 1.000 0.167 0.286 6.0
I-economy 0.000 0.000 0.000 0.0
B-return_date.date_relative 1.000 0.667 0.800 3.0
I-fromloc.state_name 1.000 1.000 1.000 1.0
B-state_code 1.000 1.000 1.000 1.0
I-arrive_time.start_time 1.000 1.000 1.000 1.0
I-arrive_date.day_number 0.000 0.000 0.000 0.0
B-meal_code 0.000 0.000 0.000 1.0
I-depart_time.period_of_day 0.000 0.000 0.000 1.0
B-day_name 1.000 0.500 0.667 2.0
B-period_of_day 1.000 0.750 0.857 4.0
B-stoploc.state_code 0.000 0.000 0.000 0.0
B-return_date.month_name 0.000 0.000 0.000 0.0
B-return_date.day_number 0.000 0.000 0.000 0.0
B-arrive_time.period_mod 0.000 0.000 0.000 0.0
I-meal_code 0.000 0.000 0.000 0.0
B-toloc.country_name 1.000 1.000 1.000 1.0
B-days_code 0.000 0.000 0.000 1.0
I-arrive_time.period_of_day 0.000 0.000 0.000 0.0
I-today_relative 0.000 0.000 0.000 0.0
B-return_time.period_of_day 0.000 0.000 0.000 0.0
B-time 0.000 0.000 0.000 0.0
I-fare_basis_code 0.000 0.000 0.000 0.0
I-arrive_time.time_relative 1.000 1.000 1.000 4.0
I-depart_time.time_relative 0.000 0.000 0.000 1.0
B-today_relative 0.000 0.000 0.000 0.0
B-state_name 0.000 0.000 0.000 9.0
B-arrive_date.today_relative 0.000 0.000 0.000 0.0
B-return_time.period_mod 0.000 0.000 0.000 0.0
B-month_name 0.000 0.000 0.000 0.0
B-day_number 0.000 0.000 0.000 0.0
I-return_date.date_relative 1.000 0.667 0.800 3.0
I-return_date.today_relative 0.000 0.000 0.000 0.0
B-stoploc.airport_name 0.000 0.000 0.000 0.0
B-time_relative 0.000 0.000 0.000 0.0
I-time 0.000 0.000 0.000 0.0
I-return_date.day_number 0.000 0.000 0.000 0.0
I-meal_description 0.000 0.000 0.000 0.0
B-return_date.today_relative 0.000 0.000 0.000 0.0
B-return_date.day_name 0.000 0.000 0.000 2.0
micro avg 0.959 0.961 0.960 3657.0
macro avg 0.680 0.654 0.656 3657.0
weighted avg 0.962 0.961 0.957 3657.0
I0716 04:56:57.910744 139795883440000 interactiveshell.py:2882] Best Slot F1: 0.960
I0716 04:56:57.912216 139795883440000 estimator_training.py:186] Not using Distribute Coordinator.
I0716 04:56:57.915825 139795883440000 training.py:612] Running training and evaluation locally (non-distributed).
I0716 04:56:57.918539 139795883440000 training.py:700] Start train and evaluate loop. The evaluate will happen after every checkpoint. Checkpoint frequency is determined based on RunConfig arguments: save_checkpoints_steps 312 or save_checkpoints_secs None.
I0716 04:56:57.983797 139795883440000 estimator.py:1145] Calling model_fn.
I0716 04:56:59.463993 139795883440000 saver.py:1499] Saver not created because there are no variables in the graph to restore
I0716 04:57:00.242153 139795883440000 <ipython-input-6-d6747b40360b>:55]
[<tf.Variable 'embedding:0' shape=(750, 300) dtype=float32_ref>,
<tf.Variable 'module/aggregation/weights:0' shape=(3,) dtype=float32>,
<tf.Variable 'module/aggregation/scaling:0' shape=() dtype=float32>,
<tf.Variable 'dense/kernel:0' shape=(1324, 300) dtype=float32_ref>,
<tf.Variable 'dense/bias:0' shape=(300,) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/fw/gru_cell/gates/kernel:0' shape=(600, 600) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/fw/gru_cell/gates/bias:0' shape=(600,) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/fw/gru_cell/candidate/kernel:0' shape=(600, 300) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/fw/gru_cell/candidate/bias:0' shape=(300,) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/bw/gru_cell/gates/kernel:0' shape=(600, 600) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/bw/gru_cell/gates/bias:0' shape=(600,) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/bw/gru_cell/candidate/kernel:0' shape=(600, 300) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/bw/gru_cell/candidate/bias:0' shape=(300,) dtype=float32_ref>,
<tf.Variable 'dense_1/kernel:0' shape=(600, 23) dtype=float32_ref>,
<tf.Variable 'dense_1/bias:0' shape=(23,) dtype=float32_ref>,
<tf.Variable 'dense_2/kernel:0' shape=(600, 122) dtype=float32_ref>,
<tf.Variable 'dense_2/bias:0' shape=(122,) dtype=float32_ref>]
I0716 04:57:01.325490 139795883440000 estimator.py:1147] Done calling model_fn.
I0716 04:57:01.329560 139795883440000 basic_session_run_hooks.py:541] Create CheckpointSaverHook.
I0716 04:57:01.651513 139795883440000 monitored_session.py:240] Graph was finalized.
I0716 04:57:01.670848 139795883440000 saver.py:1280] Restoring parameters from ../model/elmo_bigru/model.ckpt-9984
I0716 04:57:02.726634 139795883440000 session_manager.py:500] Running local_init_op.
I0716 04:57:02.803383 139795883440000 session_manager.py:502] Done running local_init_op.
I0716 04:57:05.796596 139795883440000 basic_session_run_hooks.py:606] Saving checkpoints for 9984 into ../model/elmo_bigru/model.ckpt.
Reading ../data/atis.train.w-intent.iob
I0716 04:57:10.651218 139795883440000 basic_session_run_hooks.py:262] loss = 0.016779646, step = 9984 I0716 04:57:10.653019 139795883440000 basic_session_run_hooks.py:262] lr = 0.00010477998 I0716 04:57:37.472784 139795883440000 basic_session_run_hooks.py:692] global_step/sec: 3.72826 I0716 04:57:37.478230 139795883440000 basic_session_run_hooks.py:260] loss = 0.012659789, step = 10084 (26.827 sec) I0716 04:57:37.483809 139795883440000 basic_session_run_hooks.py:260] lr = 0.000103681814 (26.831 sec) I0716 04:58:03.187721 139795883440000 basic_session_run_hooks.py:692] global_step/sec: 3.88878 I0716 04:58:03.193797 139795883440000 basic_session_run_hooks.py:260] loss = 0.00961412, step = 10184 (25.716 sec) I0716 04:58:03.197678 139795883440000 basic_session_run_hooks.py:260] lr = 0.000102595164 (25.714 sec) I0716 04:58:31.202835 139795883440000 basic_session_run_hooks.py:692] global_step/sec: 3.56951 I0716 04:58:31.212342 139795883440000 basic_session_run_hooks.py:260] loss = 0.016829388, step = 10284 (28.019 sec) I0716 04:58:31.213922 139795883440000 basic_session_run_hooks.py:260] lr = 0.00010151988 (28.016 sec) I0716 04:58:33.982954 139795883440000 basic_session_run_hooks.py:606] Saving checkpoints for 10296 into ../model/elmo_bigru/model.ckpt. I0716 04:58:35.982879 139795883440000 estimator.py:1145] Calling model_fn. I0716 04:58:37.380511 139795883440000 saver.py:1499] Saver not created because there are no variables in the graph to restore I0716 04:58:37.954942 139795883440000 estimator.py:1147] Done calling model_fn. I0716 04:58:37.984667 139795883440000 evaluation.py:255] Starting evaluation at 2019-07-16T04:58:37Z I0716 04:58:38.220102 139795883440000 monitored_session.py:240] Graph was finalized. I0716 04:58:38.235623 139795883440000 saver.py:1280] Restoring parameters from ../model/elmo_bigru/model.ckpt-10296 I0716 04:58:39.129740 139795883440000 session_manager.py:500] Running local_init_op. I0716 04:58:39.201298 139795883440000 session_manager.py:502] Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
I0716 04:59:38.152055 139795883440000 evaluation.py:275] Finished evaluation at 2019-07-16-04:59:38 I0716 04:59:38.153831 139795883440000 estimator.py:2039] Saving dict for global step 10296: global_step = 10296, loss = 0.37506437 I0716 04:59:38.162429 139795883440000 estimator.py:2099] Saving 'checkpoint_path' summary for global step 10296: ../model/elmo_bigru/model.ckpt-10296 I0716 04:59:38.247560 139795883440000 estimator.py:368] Loss for final step: 0.01134271.
Reading ../data/atis.test.w-intent.iob
I0716 04:59:38.860542 139795883440000 estimator.py:1145] Calling model_fn. I0716 04:59:40.533974 139795883440000 saver.py:1499] Saver not created because there are no variables in the graph to restore I0716 04:59:41.061809 139795883440000 estimator.py:1147] Done calling model_fn. I0716 04:59:41.465726 139795883440000 monitored_session.py:240] Graph was finalized. I0716 04:59:41.481816 139795883440000 saver.py:1280] Restoring parameters from ../model/elmo_bigru/model.ckpt-10296 I0716 04:59:42.392911 139795883440000 session_manager.py:500] Running local_init_op. I0716 04:59:42.440152 139795883440000 session_manager.py:502] Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.
'precision', 'predicted', average, warn_for)
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1439: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples.
'recall', 'true', average, warn_for)
I0716 05:00:40.164161 139795883440000 interactiveshell.py:2882]
precision recall f1-score support
atis_flight 0.984 0.992 0.988 632
atis_airfare 0.941 1.000 0.970 48
atis_ground_service 1.000 1.000 1.000 36
atis_airline 1.000 1.000 1.000 38
atis_abbreviation 0.943 1.000 0.971 33
atis_aircraft 1.000 0.889 0.941 9
atis_flight_time 0.500 1.000 0.667 1
atis_quantity 0.429 1.000 0.600 3
atis_flight#atis_airfare 0.833 0.417 0.556 12
atis_airport 1.000 1.000 1.000 18
atis_distance 1.000 1.000 1.000 10
atis_city 0.600 0.500 0.545 6
atis_ground_fare 1.000 0.857 0.923 7
atis_capacity 0.952 0.952 0.952 21
atis_flight_no 0.889 1.000 0.941 8
atis_meal 1.000 0.667 0.800 6
atis_restriction 0.000 0.000 0.000 0
atis_airline#atis_flight_no 0.000 0.000 0.000 0
atis_ground_service#atis_ground_fare 0.000 0.000 0.000 0
atis_airfare#atis_flight_time 0.000 0.000 0.000 0
atis_cheapest 0.000 0.000 0.000 0
atis_aircraft#atis_flight#atis_flight_no 0.000 0.000 0.000 0
micro avg 0.972 0.977 0.975 888
macro avg 0.640 0.649 0.630 888
weighted avg 0.974 0.977 0.974 888
I0716 05:00:40.202049 139795883440000 interactiveshell.py:2882]
precision recall f1-score support
O 0.000 0.000 0.000 0.0
B-toloc.city_name 0.975 0.993 0.984 716.0
B-fromloc.city_name 0.989 0.999 0.994 704.0
I-toloc.city_name 0.978 0.985 0.981 265.0
B-depart_date.day_name 0.991 0.991 0.991 212.0
B-airline_name 1.000 1.000 1.000 101.0
I-fromloc.city_name 0.967 1.000 0.983 177.0
B-depart_time.period_of_day 0.992 0.923 0.956 130.0
I-airline_name 1.000 1.000 1.000 65.0
B-depart_date.day_number 0.981 0.964 0.972 55.0
B-depart_date.month_name 0.982 0.964 0.973 56.0
B-depart_time.time 0.864 1.000 0.927 57.0
B-round_trip 1.000 1.000 1.000 73.0
B-cost_relative 0.973 0.973 0.973 37.0
I-round_trip 1.000 1.000 1.000 71.0
B-flight_mod 0.828 1.000 0.906 24.0
B-depart_time.time_relative 0.955 0.985 0.970 65.0
I-depart_time.time 0.945 1.000 0.972 52.0
B-stoploc.city_name 0.870 1.000 0.930 20.0
B-city_name 0.949 0.649 0.771 57.0
B-class_type 0.960 1.000 0.980 24.0
B-arrive_time.time 0.971 0.971 0.971 34.0
B-arrive_time.time_relative 0.967 0.935 0.951 31.0
I-class_type 1.000 1.000 1.000 17.0
B-flight_stop 1.000 1.000 1.000 21.0
I-arrive_time.time 1.000 0.971 0.986 35.0
B-airline_code 0.944 1.000 0.971 34.0
I-depart_date.day_number 1.000 0.933 0.966 15.0
I-fromloc.airport_name 0.441 1.000 0.612 15.0
B-fromloc.airport_name 0.462 1.000 0.632 12.0
B-arrive_date.day_name 0.786 1.000 0.880 11.0
B-toloc.state_code 1.000 1.000 1.000 18.0
B-depart_date.today_relative 1.000 0.889 0.941 9.0
B-flight_number 0.733 1.000 0.846 11.0
B-depart_date.date_relative 0.944 1.000 0.971 17.0
B-toloc.state_name 0.833 0.893 0.862 28.0
B-fare_basis_code 0.895 1.000 0.944 17.0
B-flight_time 1.000 1.000 1.000 1.0
B-or 1.000 1.000 1.000 3.0
B-arrive_time.period_of_day 0.750 1.000 0.857 6.0
B-meal_description 1.000 0.900 0.947 10.0
I-cost_relative 1.000 0.667 0.800 3.0
I-airport_name 0.846 0.379 0.524 29.0
B-fare_amount 1.000 1.000 1.000 2.0
I-fare_amount 1.000 1.000 1.000 2.0
I-city_name 1.000 0.567 0.723 30.0
I-toloc.airport_name 1.000 1.000 1.000 3.0
B-transport_type 1.000 1.000 1.000 10.0
B-arrive_date.month_name 0.714 0.833 0.769 6.0
B-arrive_date.day_number 0.714 0.833 0.769 6.0
I-stoploc.city_name 1.000 1.000 1.000 10.0
B-meal 1.000 1.000 1.000 16.0
B-fromloc.state_code 1.000 1.000 1.000 23.0
B-depart_time.period_mod 1.000 1.000 1.000 5.0
B-connect 1.000 1.000 1.000 6.0
B-flight_days 1.000 1.000 1.000 10.0
B-toloc.airport_name 1.000 1.000 1.000 3.0
B-fromloc.state_name 0.944 1.000 0.971 17.0
B-airport_name 0.667 0.381 0.485 21.0
B-economy 1.000 1.000 1.000 6.0
I-flight_time 1.000 1.000 1.000 1.0
B-aircraft_code 1.000 0.879 0.935 33.0
B-mod 1.000 0.500 0.667 2.0
B-airport_code 1.000 0.444 0.615 9.0
B-depart_time.start_time 1.000 1.000 1.000 3.0
B-depart_time.end_time 1.000 0.667 0.800 3.0
B-depart_date.year 1.000 1.000 1.000 3.0
I-transport_type 0.000 0.000 0.000 1.0
B-restriction_code 1.000 1.000 1.000 4.0
B-arrive_time.start_time 1.000 1.000 1.000 8.0
B-toloc.airport_code 1.000 0.750 0.857 4.0
B-arrive_time.end_time 1.000 1.000 1.000 8.0
I-arrive_time.end_time 0.889 1.000 0.941 8.0
I-depart_time.end_time 1.000 0.333 0.500 3.0
I-flight_stop 0.000 0.000 0.000 0.0
B-fromloc.airport_code 0.556 1.000 0.714 5.0
I-restriction_code 1.000 1.000 1.000 3.0
I-depart_time.start_time 1.000 1.000 1.000 1.0
I-toloc.state_name 1.000 1.000 1.000 1.0
I-depart_date.today_relative 0.000 0.000 0.000 0.0
B-arrive_date.date_relative 1.000 1.000 1.000 2.0
I-flight_mod 1.000 0.167 0.286 6.0
I-economy 0.000 0.000 0.000 0.0
B-return_date.date_relative 1.000 0.667 0.800 3.0
I-fromloc.state_name 1.000 1.000 1.000 1.0
B-state_code 1.000 1.000 1.000 1.0
I-arrive_time.start_time 1.000 1.000 1.000 1.0
I-arrive_date.day_number 0.000 0.000 0.000 0.0
B-meal_code 0.000 0.000 0.000 1.0
I-depart_time.period_of_day 1.000 1.000 1.000 1.0
B-day_name 1.000 0.500 0.667 2.0
B-period_of_day 1.000 0.750 0.857 4.0
B-stoploc.state_code 0.000 0.000 0.000 0.0
B-return_date.month_name 0.000 0.000 0.000 0.0
B-return_date.day_number 0.000 0.000 0.000 0.0
B-arrive_time.period_mod 0.000 0.000 0.000 0.0
I-meal_code 0.000 0.000 0.000 0.0
B-toloc.country_name 1.000 1.000 1.000 1.0
B-days_code 1.000 1.000 1.000 1.0
I-arrive_time.period_of_day 0.000 0.000 0.000 0.0
I-today_relative 0.000 0.000 0.000 0.0
B-return_time.period_of_day 0.000 0.000 0.000 0.0
B-time 0.000 0.000 0.000 0.0
I-fare_basis_code 0.000 0.000 0.000 0.0
I-arrive_time.time_relative 1.000 1.000 1.000 4.0
I-depart_time.time_relative 0.000 0.000 0.000 1.0
B-today_relative 0.000 0.000 0.000 0.0
B-state_name 0.000 0.000 0.000 9.0
B-arrive_date.today_relative 0.000 0.000 0.000 0.0
B-return_time.period_mod 0.000 0.000 0.000 0.0
B-month_name 0.000 0.000 0.000 0.0
B-day_number 0.000 0.000 0.000 0.0
I-return_date.date_relative 1.000 0.667 0.800 3.0
I-return_date.today_relative 0.000 0.000 0.000 0.0
B-stoploc.airport_name 0.000 0.000 0.000 0.0
B-time_relative 0.000 0.000 0.000 0.0
I-time 0.000 0.000 0.000 0.0
I-return_date.day_number 0.000 0.000 0.000 0.0
I-meal_description 0.000 0.000 0.000 0.0
B-return_date.today_relative 0.000 0.000 0.000 0.0
B-return_date.day_name 0.000 0.000 0.000 2.0
micro avg 0.960 0.961 0.960 3657.0
macro avg 0.696 0.669 0.670 3657.0
weighted avg 0.963 0.961 0.958 3657.0
I0716 05:00:40.224560 139795883440000 interactiveshell.py:2882] Best Slot F1: 0.960
I0716 05:00:40.226076 139795883440000 estimator_training.py:186] Not using Distribute Coordinator.
I0716 05:00:40.228124 139795883440000 training.py:612] Running training and evaluation locally (non-distributed).
I0716 05:00:40.230159 139795883440000 training.py:700] Start train and evaluate loop. The evaluate will happen after every checkpoint. Checkpoint frequency is determined based on RunConfig arguments: save_checkpoints_steps 312 or save_checkpoints_secs None.
I0716 05:00:40.298190 139795883440000 estimator.py:1145] Calling model_fn.
I0716 05:00:41.621490 139795883440000 saver.py:1499] Saver not created because there are no variables in the graph to restore
I0716 05:00:42.228225 139795883440000 <ipython-input-6-d6747b40360b>:55]
[<tf.Variable 'embedding:0' shape=(750, 300) dtype=float32_ref>,
<tf.Variable 'module/aggregation/weights:0' shape=(3,) dtype=float32>,
<tf.Variable 'module/aggregation/scaling:0' shape=() dtype=float32>,
<tf.Variable 'dense/kernel:0' shape=(1324, 300) dtype=float32_ref>,
<tf.Variable 'dense/bias:0' shape=(300,) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/fw/gru_cell/gates/kernel:0' shape=(600, 600) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/fw/gru_cell/gates/bias:0' shape=(600,) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/fw/gru_cell/candidate/kernel:0' shape=(600, 300) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/fw/gru_cell/candidate/bias:0' shape=(300,) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/bw/gru_cell/gates/kernel:0' shape=(600, 600) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/bw/gru_cell/gates/bias:0' shape=(600,) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/bw/gru_cell/candidate/kernel:0' shape=(600, 300) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/bw/gru_cell/candidate/bias:0' shape=(300,) dtype=float32_ref>,
<tf.Variable 'dense_1/kernel:0' shape=(600, 23) dtype=float32_ref>,
<tf.Variable 'dense_1/bias:0' shape=(23,) dtype=float32_ref>,
<tf.Variable 'dense_2/kernel:0' shape=(600, 122) dtype=float32_ref>,
<tf.Variable 'dense_2/bias:0' shape=(122,) dtype=float32_ref>]
I0716 05:00:43.507210 139795883440000 estimator.py:1147] Done calling model_fn.
I0716 05:00:43.511086 139795883440000 basic_session_run_hooks.py:541] Create CheckpointSaverHook.
I0716 05:00:43.832129 139795883440000 monitored_session.py:240] Graph was finalized.
I0716 05:00:43.852826 139795883440000 saver.py:1280] Restoring parameters from ../model/elmo_bigru/model.ckpt-10296
I0716 05:00:44.897862 139795883440000 session_manager.py:500] Running local_init_op.
I0716 05:00:44.963641 139795883440000 session_manager.py:502] Done running local_init_op.
I0716 05:00:47.794221 139795883440000 basic_session_run_hooks.py:606] Saving checkpoints for 10296 into ../model/elmo_bigru/model.ckpt.
Reading ../data/atis.train.w-intent.iob
I0716 05:00:52.672652 139795883440000 basic_session_run_hooks.py:262] loss = 0.01384804, step = 10296 I0716 05:00:52.675022 139795883440000 basic_session_run_hooks.py:262] lr = 0.00010139161 I0716 05:01:19.914961 139795883440000 basic_session_run_hooks.py:692] global_step/sec: 3.67065 I0716 05:01:19.922856 139795883440000 basic_session_run_hooks.py:260] loss = 0.011046371, step = 10396 (27.250 sec) I0716 05:01:19.925280 139795883440000 basic_session_run_hooks.py:260] lr = 0.000100328936 (27.250 sec) I0716 05:01:46.527574 139795883440000 basic_session_run_hooks.py:692] global_step/sec: 3.75762 I0716 05:01:46.532375 139795883440000 basic_session_run_hooks.py:260] loss = 0.011879209, step = 10496 (26.610 sec) I0716 05:01:46.536274 139795883440000 basic_session_run_hooks.py:260] lr = 9.927743e-05 (26.611 sec) I0716 05:02:13.149505 139795883440000 basic_session_run_hooks.py:692] global_step/sec: 3.75629 I0716 05:02:13.151699 139795883440000 basic_session_run_hooks.py:260] loss = 0.01011859, step = 10596 (26.619 sec) I0716 05:02:13.154321 139795883440000 basic_session_run_hooks.py:260] lr = 9.823692e-05 (26.618 sec) I0716 05:02:15.924441 139795883440000 basic_session_run_hooks.py:606] Saving checkpoints for 10608 into ../model/elmo_bigru/model.ckpt. I0716 05:02:18.027540 139795883440000 estimator.py:1145] Calling model_fn. I0716 05:02:19.667282 139795883440000 saver.py:1499] Saver not created because there are no variables in the graph to restore I0716 05:02:20.244174 139795883440000 estimator.py:1147] Done calling model_fn. I0716 05:02:20.274494 139795883440000 evaluation.py:255] Starting evaluation at 2019-07-16T05:02:20Z I0716 05:02:20.502266 139795883440000 monitored_session.py:240] Graph was finalized. I0716 05:02:20.528296 139795883440000 saver.py:1280] Restoring parameters from ../model/elmo_bigru/model.ckpt-10608 I0716 05:02:21.418348 139795883440000 session_manager.py:500] Running local_init_op. I0716 05:02:21.476039 139795883440000 session_manager.py:502] Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
I0716 05:03:20.553541 139795883440000 evaluation.py:275] Finished evaluation at 2019-07-16-05:03:20 I0716 05:03:20.555404 139795883440000 estimator.py:2039] Saving dict for global step 10608: global_step = 10608, loss = 0.36588457 I0716 05:03:20.564051 139795883440000 estimator.py:2099] Saving 'checkpoint_path' summary for global step 10608: ../model/elmo_bigru/model.ckpt-10608 I0716 05:03:20.651795 139795883440000 estimator.py:368] Loss for final step: 0.0058385767.
Reading ../data/atis.test.w-intent.iob
I0716 05:03:21.270508 139795883440000 estimator.py:1145] Calling model_fn. I0716 05:03:22.821979 139795883440000 saver.py:1499] Saver not created because there are no variables in the graph to restore I0716 05:03:23.353141 139795883440000 estimator.py:1147] Done calling model_fn. I0716 05:03:23.571113 139795883440000 monitored_session.py:240] Graph was finalized. I0716 05:03:23.588160 139795883440000 saver.py:1280] Restoring parameters from ../model/elmo_bigru/model.ckpt-10608 I0716 05:03:24.530436 139795883440000 session_manager.py:500] Running local_init_op. I0716 05:03:24.580191 139795883440000 session_manager.py:502] Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.
'precision', 'predicted', average, warn_for)
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1439: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples.
'recall', 'true', average, warn_for)
I0716 05:04:22.721977 139795883440000 interactiveshell.py:2882]
precision recall f1-score support
atis_flight 0.984 0.992 0.988 632
atis_airfare 0.941 1.000 0.970 48
atis_ground_service 1.000 1.000 1.000 36
atis_airline 1.000 1.000 1.000 38
atis_abbreviation 0.943 1.000 0.971 33
atis_aircraft 1.000 0.889 0.941 9
atis_flight_time 0.500 1.000 0.667 1
atis_quantity 0.429 1.000 0.600 3
atis_flight#atis_airfare 0.833 0.417 0.556 12
atis_airport 1.000 1.000 1.000 18
atis_distance 1.000 1.000 1.000 10
atis_city 0.600 0.500 0.545 6
atis_ground_fare 1.000 0.857 0.923 7
atis_capacity 0.952 0.952 0.952 21
atis_flight_no 0.889 1.000 0.941 8
atis_meal 1.000 0.667 0.800 6
atis_restriction 0.000 0.000 0.000 0
atis_airline#atis_flight_no 0.000 0.000 0.000 0
atis_ground_service#atis_ground_fare 0.000 0.000 0.000 0
atis_airfare#atis_flight_time 0.000 0.000 0.000 0
atis_cheapest 0.000 0.000 0.000 0
atis_aircraft#atis_flight#atis_flight_no 0.000 0.000 0.000 0
micro avg 0.972 0.977 0.975 888
macro avg 0.640 0.649 0.630 888
weighted avg 0.974 0.977 0.974 888
I0716 05:04:22.761313 139795883440000 interactiveshell.py:2882]
precision recall f1-score support
O 0.000 0.000 0.000 0.0
B-toloc.city_name 0.977 0.993 0.985 716.0
B-fromloc.city_name 0.990 0.999 0.994 704.0
I-toloc.city_name 0.977 0.981 0.979 265.0
B-depart_date.day_name 0.991 0.991 0.991 212.0
B-airline_name 1.000 1.000 1.000 101.0
I-fromloc.city_name 0.973 1.000 0.986 177.0
B-depart_time.period_of_day 0.992 0.923 0.956 130.0
I-airline_name 1.000 1.000 1.000 65.0
B-depart_date.day_number 0.981 0.964 0.972 55.0
B-depart_date.month_name 0.982 0.964 0.973 56.0
B-depart_time.time 0.864 1.000 0.927 57.0
B-round_trip 1.000 1.000 1.000 73.0
B-cost_relative 0.973 0.973 0.973 37.0
I-round_trip 1.000 1.000 1.000 71.0
B-flight_mod 0.828 1.000 0.906 24.0
B-depart_time.time_relative 0.955 0.985 0.970 65.0
I-depart_time.time 0.945 1.000 0.972 52.0
B-stoploc.city_name 0.870 1.000 0.930 20.0
B-city_name 0.902 0.649 0.755 57.0
B-class_type 0.960 1.000 0.980 24.0
B-arrive_time.time 0.971 0.971 0.971 34.0
B-arrive_time.time_relative 0.967 0.935 0.951 31.0
I-class_type 1.000 1.000 1.000 17.0
B-flight_stop 1.000 1.000 1.000 21.0
I-arrive_time.time 1.000 0.971 0.986 35.0
B-airline_code 0.944 1.000 0.971 34.0
I-depart_date.day_number 1.000 1.000 1.000 15.0
I-fromloc.airport_name 0.441 1.000 0.612 15.0
B-fromloc.airport_name 0.462 1.000 0.632 12.0
B-arrive_date.day_name 0.786 1.000 0.880 11.0
B-toloc.state_code 1.000 1.000 1.000 18.0
B-depart_date.today_relative 1.000 0.889 0.941 9.0
B-flight_number 0.733 1.000 0.846 11.0
B-depart_date.date_relative 0.944 1.000 0.971 17.0
B-toloc.state_name 0.788 0.929 0.852 28.0
B-fare_basis_code 0.895 1.000 0.944 17.0
B-flight_time 1.000 1.000 1.000 1.0
B-or 1.000 1.000 1.000 3.0
B-arrive_time.period_of_day 0.750 1.000 0.857 6.0
B-meal_description 1.000 0.900 0.947 10.0
I-cost_relative 1.000 0.667 0.800 3.0
I-airport_name 0.846 0.379 0.524 29.0
B-fare_amount 1.000 1.000 1.000 2.0
I-fare_amount 1.000 1.000 1.000 2.0
I-city_name 1.000 0.600 0.750 30.0
I-toloc.airport_name 1.000 1.000 1.000 3.0
B-transport_type 1.000 1.000 1.000 10.0
B-arrive_date.month_name 0.714 0.833 0.769 6.0
B-arrive_date.day_number 0.714 0.833 0.769 6.0
I-stoploc.city_name 1.000 1.000 1.000 10.0
B-meal 1.000 1.000 1.000 16.0
B-fromloc.state_code 1.000 1.000 1.000 23.0
B-depart_time.period_mod 1.000 1.000 1.000 5.0
B-connect 1.000 1.000 1.000 6.0
B-flight_days 1.000 1.000 1.000 10.0
B-toloc.airport_name 1.000 1.000 1.000 3.0
B-fromloc.state_name 0.944 1.000 0.971 17.0
B-airport_name 0.778 0.333 0.467 21.0
B-economy 1.000 1.000 1.000 6.0
I-flight_time 1.000 1.000 1.000 1.0
B-aircraft_code 1.000 0.879 0.935 33.0
B-mod 1.000 0.500 0.667 2.0
B-airport_code 1.000 0.444 0.615 9.0
B-depart_time.start_time 1.000 1.000 1.000 3.0
B-depart_time.end_time 1.000 0.667 0.800 3.0
B-depart_date.year 1.000 1.000 1.000 3.0
I-transport_type 0.000 0.000 0.000 1.0
B-restriction_code 1.000 1.000 1.000 4.0
B-arrive_time.start_time 1.000 1.000 1.000 8.0
B-toloc.airport_code 1.000 0.750 0.857 4.0
B-arrive_time.end_time 1.000 1.000 1.000 8.0
I-arrive_time.end_time 0.889 1.000 0.941 8.0
I-depart_time.end_time 1.000 0.333 0.500 3.0
I-flight_stop 0.000 0.000 0.000 0.0
B-fromloc.airport_code 0.556 1.000 0.714 5.0
I-restriction_code 1.000 1.000 1.000 3.0
I-depart_time.start_time 1.000 1.000 1.000 1.0
I-toloc.state_name 1.000 1.000 1.000 1.0
I-depart_date.today_relative 0.000 0.000 0.000 0.0
B-arrive_date.date_relative 1.000 1.000 1.000 2.0
I-flight_mod 1.000 0.167 0.286 6.0
I-economy 0.000 0.000 0.000 0.0
B-return_date.date_relative 1.000 0.333 0.500 3.0
I-fromloc.state_name 1.000 1.000 1.000 1.0
B-state_code 1.000 1.000 1.000 1.0
I-arrive_time.start_time 1.000 1.000 1.000 1.0
I-arrive_date.day_number 0.000 0.000 0.000 0.0
B-meal_code 0.000 0.000 0.000 1.0
I-depart_time.period_of_day 1.000 1.000 1.000 1.0
B-day_name 1.000 0.500 0.667 2.0
B-period_of_day 1.000 0.750 0.857 4.0
B-stoploc.state_code 0.000 0.000 0.000 0.0
B-return_date.month_name 0.000 0.000 0.000 0.0
B-return_date.day_number 0.000 0.000 0.000 0.0
B-arrive_time.period_mod 0.000 0.000 0.000 0.0
I-meal_code 0.000 0.000 0.000 0.0
B-toloc.country_name 1.000 1.000 1.000 1.0
B-days_code 1.000 1.000 1.000 1.0
I-arrive_time.period_of_day 0.000 0.000 0.000 0.0
I-today_relative 0.000 0.000 0.000 0.0
B-return_time.period_of_day 0.000 0.000 0.000 0.0
B-time 0.000 0.000 0.000 0.0
I-fare_basis_code 0.000 0.000 0.000 0.0
I-arrive_time.time_relative 1.000 1.000 1.000 4.0
I-depart_time.time_relative 0.000 0.000 0.000 1.0
B-today_relative 0.000 0.000 0.000 0.0
B-state_name 0.000 0.000 0.000 9.0
B-arrive_date.today_relative 0.000 0.000 0.000 0.0
B-return_time.period_mod 0.000 0.000 0.000 0.0
B-month_name 0.000 0.000 0.000 0.0
B-day_number 0.000 0.000 0.000 0.0
I-return_date.date_relative 0.667 0.667 0.667 3.0
I-return_date.today_relative 0.000 0.000 0.000 0.0
B-stoploc.airport_name 0.000 0.000 0.000 0.0
B-time_relative 0.000 0.000 0.000 0.0
I-time 0.000 0.000 0.000 0.0
I-return_date.day_number 0.000 0.000 0.000 0.0
I-meal_description 0.000 0.000 0.000 0.0
B-return_date.today_relative 0.000 0.000 0.000 0.0
B-return_date.day_name 0.000 0.000 0.000 2.0
micro avg 0.960 0.961 0.960 3657.0
macro avg 0.694 0.667 0.667 3657.0
weighted avg 0.964 0.961 0.958 3657.0
I0716 05:04:22.784583 139795883440000 interactiveshell.py:2882] Best Slot F1: 0.960
I0716 05:04:22.786408 139795883440000 estimator_training.py:186] Not using Distribute Coordinator.
I0716 05:04:22.788550 139795883440000 training.py:612] Running training and evaluation locally (non-distributed).
I0716 05:04:22.790685 139795883440000 training.py:700] Start train and evaluate loop. The evaluate will happen after every checkpoint. Checkpoint frequency is determined based on RunConfig arguments: save_checkpoints_steps 312 or save_checkpoints_secs None.
I0716 05:04:22.860918 139795883440000 estimator.py:1145] Calling model_fn.
I0716 05:04:24.386466 139795883440000 saver.py:1499] Saver not created because there are no variables in the graph to restore
I0716 05:04:25.001788 139795883440000 <ipython-input-6-d6747b40360b>:55]
[<tf.Variable 'embedding:0' shape=(750, 300) dtype=float32_ref>,
<tf.Variable 'module/aggregation/weights:0' shape=(3,) dtype=float32>,
<tf.Variable 'module/aggregation/scaling:0' shape=() dtype=float32>,
<tf.Variable 'dense/kernel:0' shape=(1324, 300) dtype=float32_ref>,
<tf.Variable 'dense/bias:0' shape=(300,) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/fw/gru_cell/gates/kernel:0' shape=(600, 600) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/fw/gru_cell/gates/bias:0' shape=(600,) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/fw/gru_cell/candidate/kernel:0' shape=(600, 300) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/fw/gru_cell/candidate/bias:0' shape=(300,) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/bw/gru_cell/gates/kernel:0' shape=(600, 600) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/bw/gru_cell/gates/bias:0' shape=(600,) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/bw/gru_cell/candidate/kernel:0' shape=(600, 300) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/bw/gru_cell/candidate/bias:0' shape=(300,) dtype=float32_ref>,
<tf.Variable 'dense_1/kernel:0' shape=(600, 23) dtype=float32_ref>,
<tf.Variable 'dense_1/bias:0' shape=(23,) dtype=float32_ref>,
<tf.Variable 'dense_2/kernel:0' shape=(600, 122) dtype=float32_ref>,
<tf.Variable 'dense_2/bias:0' shape=(122,) dtype=float32_ref>]
I0716 05:04:26.290348 139795883440000 estimator.py:1147] Done calling model_fn.
I0716 05:04:26.294201 139795883440000 basic_session_run_hooks.py:541] Create CheckpointSaverHook.
I0716 05:04:26.632357 139795883440000 monitored_session.py:240] Graph was finalized.
I0716 05:04:26.651122 139795883440000 saver.py:1280] Restoring parameters from ../model/elmo_bigru/model.ckpt-10608
I0716 05:04:27.718711 139795883440000 session_manager.py:500] Running local_init_op.
I0716 05:04:27.792746 139795883440000 session_manager.py:502] Done running local_init_op.
I0716 05:04:30.619942 139795883440000 basic_session_run_hooks.py:606] Saving checkpoints for 10608 into ../model/elmo_bigru/model.ckpt.
Reading ../data/atis.train.w-intent.iob
I0716 05:04:35.721859 139795883440000 basic_session_run_hooks.py:262] loss = 0.0103135025, step = 10608 I0716 05:04:35.723819 139795883440000 basic_session_run_hooks.py:262] lr = 9.8112796e-05 I0716 05:05:03.612728 139795883440000 basic_session_run_hooks.py:692] global_step/sec: 3.58534 I0716 05:05:03.621054 139795883440000 basic_session_run_hooks.py:260] loss = 0.0095020505, step = 10708 (27.899 sec) I0716 05:05:03.624384 139795883440000 basic_session_run_hooks.py:260] lr = 9.708451e-05 (27.901 sec) I0716 05:05:30.273659 139795883440000 basic_session_run_hooks.py:692] global_step/sec: 3.7508 I0716 05:05:30.280654 139795883440000 basic_session_run_hooks.py:260] loss = 0.011504768, step = 10808 (26.659 sec) I0716 05:05:30.284494 139795883440000 basic_session_run_hooks.py:260] lr = 9.6066986e-05 (26.660 sec) I0716 05:05:56.251807 139795883440000 basic_session_run_hooks.py:692] global_step/sec: 3.84939 I0716 05:05:56.258651 139795883440000 basic_session_run_hooks.py:260] loss = 0.009077469, step = 10908 (25.978 sec) I0716 05:05:56.261323 139795883440000 basic_session_run_hooks.py:260] lr = 9.506013e-05 (25.977 sec) I0716 05:05:59.056719 139795883440000 basic_session_run_hooks.py:606] Saving checkpoints for 10920 into ../model/elmo_bigru/model.ckpt. I0716 05:06:00.985701 139795883440000 estimator.py:1145] Calling model_fn. I0716 05:06:02.611641 139795883440000 saver.py:1499] Saver not created because there are no variables in the graph to restore I0716 05:06:03.195277 139795883440000 estimator.py:1147] Done calling model_fn. I0716 05:06:03.226158 139795883440000 evaluation.py:255] Starting evaluation at 2019-07-16T05:06:03Z I0716 05:06:03.456297 139795883440000 monitored_session.py:240] Graph was finalized. I0716 05:06:03.473925 139795883440000 saver.py:1280] Restoring parameters from ../model/elmo_bigru/model.ckpt-10920 I0716 05:06:04.373917 139795883440000 session_manager.py:500] Running local_init_op. I0716 05:06:04.427072 139795883440000 session_manager.py:502] Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
I0716 05:07:03.300335 139795883440000 evaluation.py:275] Finished evaluation at 2019-07-16-05:07:03 I0716 05:07:03.302104 139795883440000 estimator.py:2039] Saving dict for global step 10920: global_step = 10920, loss = 0.36773965 I0716 05:07:03.312190 139795883440000 estimator.py:2099] Saving 'checkpoint_path' summary for global step 10920: ../model/elmo_bigru/model.ckpt-10920 I0716 05:07:03.396445 139795883440000 estimator.py:368] Loss for final step: 0.004308995.
Reading ../data/atis.test.w-intent.iob
I0716 05:07:04.000622 139795883440000 estimator.py:1145] Calling model_fn. I0716 05:07:05.549351 139795883440000 saver.py:1499] Saver not created because there are no variables in the graph to restore I0716 05:07:06.064721 139795883440000 estimator.py:1147] Done calling model_fn. I0716 05:07:06.476282 139795883440000 monitored_session.py:240] Graph was finalized. I0716 05:07:06.488260 139795883440000 saver.py:1280] Restoring parameters from ../model/elmo_bigru/model.ckpt-10920 I0716 05:07:07.387382 139795883440000 session_manager.py:500] Running local_init_op. I0716 05:07:07.434978 139795883440000 session_manager.py:502] Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.
'precision', 'predicted', average, warn_for)
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1439: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples.
'recall', 'true', average, warn_for)
I0716 05:08:05.408970 139795883440000 interactiveshell.py:2882]
precision recall f1-score support
atis_flight 0.986 0.991 0.988 632
atis_airfare 0.941 1.000 0.970 48
atis_ground_service 1.000 1.000 1.000 36
atis_airline 1.000 1.000 1.000 38
atis_abbreviation 0.943 1.000 0.971 33
atis_aircraft 1.000 0.889 0.941 9
atis_flight_time 0.500 1.000 0.667 1
atis_quantity 0.429 1.000 0.600 3
atis_flight#atis_airfare 0.833 0.417 0.556 12
atis_airport 0.947 1.000 0.973 18
atis_distance 1.000 1.000 1.000 10
atis_city 0.667 0.667 0.667 6
atis_ground_fare 1.000 0.857 0.923 7
atis_capacity 0.952 0.952 0.952 21
atis_flight_no 0.889 1.000 0.941 8
atis_meal 1.000 0.667 0.800 6
atis_restriction 0.000 0.000 0.000 0
atis_airline#atis_flight_no 0.000 0.000 0.000 0
atis_ground_service#atis_ground_fare 0.000 0.000 0.000 0
atis_airfare#atis_flight_time 0.000 0.000 0.000 0
atis_cheapest 0.000 0.000 0.000 0
atis_aircraft#atis_flight#atis_flight_no 0.000 0.000 0.000 0
micro avg 0.972 0.977 0.975 888
macro avg 0.640 0.656 0.634 888
weighted avg 0.974 0.977 0.974 888
I0716 05:08:05.447309 139795883440000 interactiveshell.py:2882]
precision recall f1-score support
O 0.000 0.000 0.000 0.0
B-toloc.city_name 0.974 0.994 0.984 716.0
B-fromloc.city_name 0.992 0.999 0.995 704.0
I-toloc.city_name 0.978 0.992 0.985 265.0
B-depart_date.day_name 0.991 0.991 0.991 212.0
B-airline_name 1.000 1.000 1.000 101.0
I-fromloc.city_name 0.983 1.000 0.992 177.0
B-depart_time.period_of_day 0.992 0.923 0.956 130.0
I-airline_name 1.000 1.000 1.000 65.0
B-depart_date.day_number 0.981 0.964 0.972 55.0
B-depart_date.month_name 0.981 0.946 0.964 56.0
B-depart_time.time 0.864 1.000 0.927 57.0
B-round_trip 1.000 1.000 1.000 73.0
B-cost_relative 0.973 0.973 0.973 37.0
I-round_trip 1.000 1.000 1.000 71.0
B-flight_mod 0.828 1.000 0.906 24.0
B-depart_time.time_relative 0.955 0.985 0.970 65.0
I-depart_time.time 0.945 1.000 0.972 52.0
B-stoploc.city_name 0.870 1.000 0.930 20.0
B-city_name 0.900 0.632 0.742 57.0
B-class_type 0.960 1.000 0.980 24.0
B-arrive_time.time 0.971 0.971 0.971 34.0
B-arrive_time.time_relative 0.967 0.935 0.951 31.0
I-class_type 1.000 1.000 1.000 17.0
B-flight_stop 1.000 1.000 1.000 21.0
I-arrive_time.time 1.000 0.971 0.986 35.0
B-airline_code 0.944 1.000 0.971 34.0
I-depart_date.day_number 1.000 0.933 0.966 15.0
I-fromloc.airport_name 0.441 1.000 0.612 15.0
B-fromloc.airport_name 0.462 1.000 0.632 12.0
B-arrive_date.day_name 0.786 1.000 0.880 11.0
B-toloc.state_code 1.000 1.000 1.000 18.0
B-depart_date.today_relative 1.000 0.889 0.941 9.0
B-flight_number 0.733 1.000 0.846 11.0
B-depart_date.date_relative 0.944 1.000 0.971 17.0
B-toloc.state_name 0.781 0.893 0.833 28.0
B-fare_basis_code 0.850 1.000 0.919 17.0
B-flight_time 1.000 1.000 1.000 1.0
B-or 1.000 1.000 1.000 3.0
B-arrive_time.period_of_day 0.750 1.000 0.857 6.0
B-meal_description 1.000 0.900 0.947 10.0
I-cost_relative 1.000 0.667 0.800 3.0
I-airport_name 0.846 0.379 0.524 29.0
B-fare_amount 1.000 1.000 1.000 2.0
I-fare_amount 1.000 1.000 1.000 2.0
I-city_name 1.000 0.600 0.750 30.0
I-toloc.airport_name 1.000 1.000 1.000 3.0
B-transport_type 1.000 1.000 1.000 10.0
B-arrive_date.month_name 0.625 0.833 0.714 6.0
B-arrive_date.day_number 0.714 0.833 0.769 6.0
I-stoploc.city_name 0.909 1.000 0.952 10.0
B-meal 1.000 1.000 1.000 16.0
B-fromloc.state_code 1.000 1.000 1.000 23.0
B-depart_time.period_mod 1.000 1.000 1.000 5.0
B-connect 1.000 1.000 1.000 6.0
B-flight_days 1.000 1.000 1.000 10.0
B-toloc.airport_name 1.000 1.000 1.000 3.0
B-fromloc.state_name 1.000 1.000 1.000 17.0
B-airport_name 0.778 0.333 0.467 21.0
B-economy 1.000 1.000 1.000 6.0
I-flight_time 1.000 1.000 1.000 1.0
B-aircraft_code 1.000 0.879 0.935 33.0
B-mod 1.000 0.500 0.667 2.0
B-airport_code 1.000 0.444 0.615 9.0
B-depart_time.start_time 1.000 0.667 0.800 3.0
B-depart_time.end_time 1.000 0.333 0.500 3.0
B-depart_date.year 1.000 1.000 1.000 3.0
I-transport_type 0.000 0.000 0.000 1.0
B-restriction_code 1.000 1.000 1.000 4.0
B-arrive_time.start_time 0.889 1.000 0.941 8.0
B-toloc.airport_code 1.000 0.750 0.857 4.0
B-arrive_time.end_time 0.889 1.000 0.941 8.0
I-arrive_time.end_time 0.889 1.000 0.941 8.0
I-depart_time.end_time 1.000 0.333 0.500 3.0
I-flight_stop 0.000 0.000 0.000 0.0
B-fromloc.airport_code 0.556 1.000 0.714 5.0
I-restriction_code 1.000 1.000 1.000 3.0
I-depart_time.start_time 1.000 1.000 1.000 1.0
I-toloc.state_name 1.000 1.000 1.000 1.0
I-depart_date.today_relative 0.000 0.000 0.000 0.0
B-arrive_date.date_relative 1.000 1.000 1.000 2.0
I-flight_mod 1.000 0.167 0.286 6.0
I-economy 0.000 0.000 0.000 0.0
B-return_date.date_relative 1.000 0.333 0.500 3.0
I-fromloc.state_name 1.000 1.000 1.000 1.0
B-state_code 1.000 1.000 1.000 1.0
I-arrive_time.start_time 1.000 1.000 1.000 1.0
I-arrive_date.day_number 0.000 0.000 0.000 0.0
B-meal_code 0.000 0.000 0.000 1.0
I-depart_time.period_of_day 1.000 1.000 1.000 1.0
B-day_name 1.000 0.500 0.667 2.0
B-period_of_day 1.000 0.750 0.857 4.0
B-stoploc.state_code 0.000 0.000 0.000 0.0
B-return_date.month_name 0.000 0.000 0.000 0.0
B-return_date.day_number 0.000 0.000 0.000 0.0
B-arrive_time.period_mod 0.000 0.000 0.000 0.0
I-meal_code 0.000 0.000 0.000 0.0
B-toloc.country_name 1.000 1.000 1.000 1.0
B-days_code 0.000 0.000 0.000 1.0
I-arrive_time.period_of_day 0.000 0.000 0.000 0.0
I-today_relative 0.000 0.000 0.000 0.0
B-return_time.period_of_day 0.000 0.000 0.000 0.0
B-time 0.000 0.000 0.000 0.0
I-fare_basis_code 0.000 0.000 0.000 0.0
I-arrive_time.time_relative 1.000 1.000 1.000 4.0
I-depart_time.time_relative 0.000 0.000 0.000 1.0
B-today_relative 0.000 0.000 0.000 0.0
B-state_name 0.000 0.000 0.000 9.0
B-arrive_date.today_relative 0.000 0.000 0.000 0.0
B-return_time.period_mod 0.000 0.000 0.000 0.0
B-month_name 0.000 0.000 0.000 0.0
B-day_number 0.000 0.000 0.000 0.0
I-return_date.date_relative 0.667 0.667 0.667 3.0
I-return_date.today_relative 0.000 0.000 0.000 0.0
B-stoploc.airport_name 0.000 0.000 0.000 0.0
B-time_relative 0.000 0.000 0.000 0.0
I-time 0.000 0.000 0.000 0.0
I-return_date.day_number 0.000 0.000 0.000 0.0
I-meal_description 0.000 0.000 0.000 0.0
B-return_date.today_relative 0.000 0.000 0.000 0.0
B-return_date.day_name 0.000 0.000 0.000 2.0
micro avg 0.959 0.960 0.960 3657.0
macro avg 0.682 0.652 0.652 3657.0
weighted avg 0.963 0.960 0.957 3657.0
I0716 05:08:05.473785 139795883440000 interactiveshell.py:2882] Best Slot F1: 0.960
I0716 05:08:05.481377 139795883440000 estimator_training.py:186] Not using Distribute Coordinator.
I0716 05:08:05.482621 139795883440000 training.py:612] Running training and evaluation locally (non-distributed).
I0716 05:08:05.485518 139795883440000 training.py:700] Start train and evaluate loop. The evaluate will happen after every checkpoint. Checkpoint frequency is determined based on RunConfig arguments: save_checkpoints_steps 312 or save_checkpoints_secs None.
I0716 05:08:05.558955 139795883440000 estimator.py:1145] Calling model_fn.
I0716 05:08:06.878779 139795883440000 saver.py:1499] Saver not created because there are no variables in the graph to restore
I0716 05:08:07.455803 139795883440000 <ipython-input-6-d6747b40360b>:55]
[<tf.Variable 'embedding:0' shape=(750, 300) dtype=float32_ref>,
<tf.Variable 'module/aggregation/weights:0' shape=(3,) dtype=float32>,
<tf.Variable 'module/aggregation/scaling:0' shape=() dtype=float32>,
<tf.Variable 'dense/kernel:0' shape=(1324, 300) dtype=float32_ref>,
<tf.Variable 'dense/bias:0' shape=(300,) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/fw/gru_cell/gates/kernel:0' shape=(600, 600) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/fw/gru_cell/gates/bias:0' shape=(600,) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/fw/gru_cell/candidate/kernel:0' shape=(600, 300) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/fw/gru_cell/candidate/bias:0' shape=(300,) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/bw/gru_cell/gates/kernel:0' shape=(600, 600) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/bw/gru_cell/gates/bias:0' shape=(600,) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/bw/gru_cell/candidate/kernel:0' shape=(600, 300) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/bw/gru_cell/candidate/bias:0' shape=(300,) dtype=float32_ref>,
<tf.Variable 'dense_1/kernel:0' shape=(600, 23) dtype=float32_ref>,
<tf.Variable 'dense_1/bias:0' shape=(23,) dtype=float32_ref>,
<tf.Variable 'dense_2/kernel:0' shape=(600, 122) dtype=float32_ref>,
<tf.Variable 'dense_2/bias:0' shape=(122,) dtype=float32_ref>]
I0716 05:08:08.773962 139795883440000 estimator.py:1147] Done calling model_fn.
I0716 05:08:08.777846 139795883440000 basic_session_run_hooks.py:541] Create CheckpointSaverHook.
I0716 05:08:09.101681 139795883440000 monitored_session.py:240] Graph was finalized.
I0716 05:08:09.123294 139795883440000 saver.py:1280] Restoring parameters from ../model/elmo_bigru/model.ckpt-10920
I0716 05:08:10.187270 139795883440000 session_manager.py:500] Running local_init_op.
I0716 05:08:10.259480 139795883440000 session_manager.py:502] Done running local_init_op.
I0716 05:08:13.139483 139795883440000 basic_session_run_hooks.py:606] Saving checkpoints for 10920 into ../model/elmo_bigru/model.ckpt.
Reading ../data/atis.train.w-intent.iob
I0716 05:08:18.349324 139795883440000 basic_session_run_hooks.py:262] loss = 0.017390745, step = 10920 I0716 05:08:18.351059 139795883440000 basic_session_run_hooks.py:262] lr = 9.494002e-05 I0716 05:08:46.093603 139795883440000 basic_session_run_hooks.py:692] global_step/sec: 3.60425 I0716 05:08:46.101212 139795883440000 basic_session_run_hooks.py:260] loss = 0.012770555, step = 11020 (27.752 sec) I0716 05:08:46.103015 139795883440000 basic_session_run_hooks.py:260] lr = 9.394498e-05 (27.752 sec) I0716 05:09:12.375688 139795883440000 basic_session_run_hooks.py:692] global_step/sec: 3.80488 I0716 05:09:12.383651 139795883440000 basic_session_run_hooks.py:260] loss = 0.014521266, step = 11120 (26.282 sec) I0716 05:09:12.385815 139795883440000 basic_session_run_hooks.py:260] lr = 9.296037e-05 (26.283 sec) I0716 05:09:38.238720 139795883440000 basic_session_run_hooks.py:692] global_step/sec: 3.86651 I0716 05:09:38.245971 139795883440000 basic_session_run_hooks.py:260] loss = 0.014142913, step = 11220 (25.862 sec) I0716 05:09:38.248379 139795883440000 basic_session_run_hooks.py:260] lr = 9.198607e-05 (25.863 sec) I0716 05:09:41.196772 139795883440000 basic_session_run_hooks.py:606] Saving checkpoints for 11232 into ../model/elmo_bigru/model.ckpt. I0716 05:09:43.256902 139795883440000 estimator.py:1145] Calling model_fn. I0716 05:09:44.903710 139795883440000 saver.py:1499] Saver not created because there are no variables in the graph to restore I0716 05:09:45.473230 139795883440000 estimator.py:1147] Done calling model_fn. I0716 05:09:45.503853 139795883440000 evaluation.py:255] Starting evaluation at 2019-07-16T05:09:45Z I0716 05:09:45.734034 139795883440000 monitored_session.py:240] Graph was finalized. I0716 05:09:45.750732 139795883440000 saver.py:1280] Restoring parameters from ../model/elmo_bigru/model.ckpt-11232 I0716 05:09:46.685195 139795883440000 session_manager.py:500] Running local_init_op. I0716 05:09:46.751442 139795883440000 session_manager.py:502] Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
I0716 05:10:45.692678 139795883440000 evaluation.py:275] Finished evaluation at 2019-07-16-05:10:45 I0716 05:10:45.694537 139795883440000 estimator.py:2039] Saving dict for global step 11232: global_step = 11232, loss = 0.383035 I0716 05:10:45.701206 139795883440000 estimator.py:2099] Saving 'checkpoint_path' summary for global step 11232: ../model/elmo_bigru/model.ckpt-11232 I0716 05:10:45.789470 139795883440000 estimator.py:368] Loss for final step: 0.0029675646.
Reading ../data/atis.test.w-intent.iob
I0716 05:10:46.391677 139795883440000 estimator.py:1145] Calling model_fn. I0716 05:10:47.934705 139795883440000 saver.py:1499] Saver not created because there are no variables in the graph to restore I0716 05:10:48.634458 139795883440000 estimator.py:1147] Done calling model_fn. I0716 05:10:48.851438 139795883440000 monitored_session.py:240] Graph was finalized. I0716 05:10:48.867738 139795883440000 saver.py:1280] Restoring parameters from ../model/elmo_bigru/model.ckpt-11232 I0716 05:10:49.769472 139795883440000 session_manager.py:500] Running local_init_op. I0716 05:10:49.824869 139795883440000 session_manager.py:502] Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.
'precision', 'predicted', average, warn_for)
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1439: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples.
'recall', 'true', average, warn_for)
I0716 05:11:47.888665 139795883440000 interactiveshell.py:2882]
precision recall f1-score support
atis_flight 0.984 0.992 0.988 632
atis_airfare 0.941 1.000 0.970 48
atis_ground_service 1.000 1.000 1.000 36
atis_airline 1.000 1.000 1.000 38
atis_abbreviation 0.943 1.000 0.971 33
atis_aircraft 1.000 0.889 0.941 9
atis_flight_time 0.500 1.000 0.667 1
atis_quantity 0.429 1.000 0.600 3
atis_flight#atis_airfare 0.833 0.417 0.556 12
atis_airport 1.000 1.000 1.000 18
atis_distance 1.000 1.000 1.000 10
atis_city 0.600 0.500 0.545 6
atis_ground_fare 1.000 0.857 0.923 7
atis_capacity 0.952 0.952 0.952 21
atis_flight_no 0.889 1.000 0.941 8
atis_meal 1.000 0.667 0.800 6
atis_restriction 0.000 0.000 0.000 0
atis_airline#atis_flight_no 0.000 0.000 0.000 0
atis_ground_service#atis_ground_fare 0.000 0.000 0.000 0
atis_airfare#atis_flight_time 0.000 0.000 0.000 0
atis_cheapest 0.000 0.000 0.000 0
atis_aircraft#atis_flight#atis_flight_no 0.000 0.000 0.000 0
micro avg 0.972 0.977 0.975 888
macro avg 0.640 0.649 0.630 888
weighted avg 0.974 0.977 0.974 888
I0716 05:11:47.926136 139795883440000 interactiveshell.py:2882]
precision recall f1-score support
O 0.000 0.000 0.000 0.0
B-toloc.city_name 0.977 0.992 0.984 716.0
B-fromloc.city_name 0.987 0.999 0.993 704.0
I-toloc.city_name 0.978 0.985 0.981 265.0
B-depart_date.day_name 0.991 0.991 0.991 212.0
B-airline_name 1.000 1.000 1.000 101.0
I-fromloc.city_name 0.973 1.000 0.986 177.0
B-depart_time.period_of_day 1.000 0.923 0.960 130.0
I-airline_name 1.000 1.000 1.000 65.0
B-depart_date.day_number 0.981 0.964 0.972 55.0
B-depart_date.month_name 0.981 0.946 0.964 56.0
B-depart_time.time 0.864 1.000 0.927 57.0
B-round_trip 1.000 0.986 0.993 73.0
B-cost_relative 0.973 0.973 0.973 37.0
I-round_trip 1.000 1.000 1.000 71.0
B-flight_mod 0.828 1.000 0.906 24.0
B-depart_time.time_relative 0.955 0.985 0.970 65.0
I-depart_time.time 0.945 1.000 0.972 52.0
B-stoploc.city_name 0.909 1.000 0.952 20.0
B-city_name 0.902 0.649 0.755 57.0
B-class_type 0.960 1.000 0.980 24.0
B-arrive_time.time 0.971 0.971 0.971 34.0
B-arrive_time.time_relative 0.935 0.935 0.935 31.0
I-class_type 1.000 1.000 1.000 17.0
B-flight_stop 1.000 1.000 1.000 21.0
I-arrive_time.time 1.000 0.971 0.986 35.0
B-airline_code 0.944 1.000 0.971 34.0
I-depart_date.day_number 1.000 0.933 0.966 15.0
I-fromloc.airport_name 0.441 1.000 0.612 15.0
B-fromloc.airport_name 0.462 1.000 0.632 12.0
B-arrive_date.day_name 0.786 1.000 0.880 11.0
B-toloc.state_code 1.000 1.000 1.000 18.0
B-depart_date.today_relative 1.000 0.889 0.941 9.0
B-flight_number 0.733 1.000 0.846 11.0
B-depart_date.date_relative 0.944 1.000 0.971 17.0
B-toloc.state_name 0.788 0.929 0.852 28.0
B-fare_basis_code 0.850 1.000 0.919 17.0
B-flight_time 1.000 1.000 1.000 1.0
B-or 1.000 1.000 1.000 3.0
B-arrive_time.period_of_day 0.750 1.000 0.857 6.0
B-meal_description 1.000 0.900 0.947 10.0
I-cost_relative 1.000 0.667 0.800 3.0
I-airport_name 0.846 0.379 0.524 29.0
B-fare_amount 1.000 1.000 1.000 2.0
I-fare_amount 1.000 1.000 1.000 2.0
I-city_name 1.000 0.600 0.750 30.0
I-toloc.airport_name 1.000 1.000 1.000 3.0
B-transport_type 1.000 1.000 1.000 10.0
B-arrive_date.month_name 0.625 0.833 0.714 6.0
B-arrive_date.day_number 0.714 0.833 0.769 6.0
I-stoploc.city_name 1.000 1.000 1.000 10.0
B-meal 1.000 1.000 1.000 16.0
B-fromloc.state_code 1.000 1.000 1.000 23.0
B-depart_time.period_mod 1.000 1.000 1.000 5.0
B-connect 1.000 1.000 1.000 6.0
B-flight_days 1.000 1.000 1.000 10.0
B-toloc.airport_name 1.000 1.000 1.000 3.0
B-fromloc.state_name 0.944 1.000 0.971 17.0
B-airport_name 0.778 0.333 0.467 21.0
B-economy 1.000 1.000 1.000 6.0
I-flight_time 1.000 1.000 1.000 1.0
B-aircraft_code 1.000 0.879 0.935 33.0
B-mod 1.000 0.500 0.667 2.0
B-airport_code 1.000 0.444 0.615 9.0
B-depart_time.start_time 1.000 0.667 0.800 3.0
B-depart_time.end_time 1.000 0.333 0.500 3.0
B-depart_date.year 1.000 1.000 1.000 3.0
I-transport_type 0.000 0.000 0.000 1.0
B-restriction_code 1.000 1.000 1.000 4.0
B-arrive_time.start_time 0.889 1.000 0.941 8.0
B-toloc.airport_code 1.000 0.750 0.857 4.0
B-arrive_time.end_time 0.889 1.000 0.941 8.0
I-arrive_time.end_time 0.889 1.000 0.941 8.0
I-depart_time.end_time 1.000 0.333 0.500 3.0
I-flight_stop 0.000 0.000 0.000 0.0
B-fromloc.airport_code 0.556 1.000 0.714 5.0
I-restriction_code 1.000 1.000 1.000 3.0
I-depart_time.start_time 1.000 1.000 1.000 1.0
I-toloc.state_name 1.000 1.000 1.000 1.0
I-depart_date.today_relative 0.000 0.000 0.000 0.0
B-arrive_date.date_relative 1.000 1.000 1.000 2.0
I-flight_mod 1.000 0.167 0.286 6.0
I-economy 0.000 0.000 0.000 0.0
B-return_date.date_relative 1.000 0.333 0.500 3.0
I-fromloc.state_name 1.000 1.000 1.000 1.0
B-state_code 1.000 1.000 1.000 1.0
I-arrive_time.start_time 1.000 1.000 1.000 1.0
I-arrive_date.day_number 0.000 0.000 0.000 0.0
B-meal_code 0.000 0.000 0.000 1.0
I-depart_time.period_of_day 1.000 1.000 1.000 1.0
B-day_name 1.000 0.500 0.667 2.0
B-period_of_day 1.000 0.750 0.857 4.0
B-stoploc.state_code 0.000 0.000 0.000 0.0
B-return_date.month_name 0.000 0.000 0.000 0.0
B-return_date.day_number 0.000 0.000 0.000 0.0
B-arrive_time.period_mod 0.000 0.000 0.000 0.0
I-meal_code 0.000 0.000 0.000 0.0
B-toloc.country_name 1.000 1.000 1.000 1.0
B-days_code 0.000 0.000 0.000 1.0
I-arrive_time.period_of_day 0.000 0.000 0.000 0.0
I-today_relative 0.000 0.000 0.000 0.0
B-return_time.period_of_day 0.000 0.000 0.000 0.0
B-time 0.000 0.000 0.000 0.0
I-fare_basis_code 0.000 0.000 0.000 0.0
I-arrive_time.time_relative 1.000 0.750 0.857 4.0
I-depart_time.time_relative 0.000 0.000 0.000 1.0
B-today_relative 0.000 0.000 0.000 0.0
B-state_name 0.000 0.000 0.000 9.0
B-arrive_date.today_relative 0.000 0.000 0.000 0.0
B-return_time.period_mod 0.000 0.000 0.000 0.0
B-month_name 0.000 0.000 0.000 0.0
B-day_number 0.000 0.000 0.000 0.0
I-return_date.date_relative 0.667 0.667 0.667 3.0
I-return_date.today_relative 0.000 0.000 0.000 0.0
B-stoploc.airport_name 0.000 0.000 0.000 0.0
B-time_relative 0.000 0.000 0.000 0.0
I-time 0.000 0.000 0.000 0.0
I-return_date.day_number 0.000 0.000 0.000 0.0
I-meal_description 0.000 0.000 0.000 0.0
B-return_date.today_relative 0.000 0.000 0.000 0.0
B-return_date.day_name 0.000 0.000 0.000 2.0
micro avg 0.958 0.959 0.958 3657.0
macro avg 0.683 0.650 0.651 3657.0
weighted avg 0.962 0.959 0.956 3657.0
I0716 05:11:47.950561 139795883440000 interactiveshell.py:2882] Best Slot F1: 0.960
I0716 05:11:47.952192 139795883440000 estimator_training.py:186] Not using Distribute Coordinator.
I0716 05:11:47.954119 139795883440000 training.py:612] Running training and evaluation locally (non-distributed).
I0716 05:11:47.956347 139795883440000 training.py:700] Start train and evaluate loop. The evaluate will happen after every checkpoint. Checkpoint frequency is determined based on RunConfig arguments: save_checkpoints_steps 312 or save_checkpoints_secs None.
I0716 05:11:48.025987 139795883440000 estimator.py:1145] Calling model_fn.
I0716 05:11:49.363261 139795883440000 saver.py:1499] Saver not created because there are no variables in the graph to restore
I0716 05:11:50.152054 139795883440000 <ipython-input-6-d6747b40360b>:55]
[<tf.Variable 'embedding:0' shape=(750, 300) dtype=float32_ref>,
<tf.Variable 'module/aggregation/weights:0' shape=(3,) dtype=float32>,
<tf.Variable 'module/aggregation/scaling:0' shape=() dtype=float32>,
<tf.Variable 'dense/kernel:0' shape=(1324, 300) dtype=float32_ref>,
<tf.Variable 'dense/bias:0' shape=(300,) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/fw/gru_cell/gates/kernel:0' shape=(600, 600) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/fw/gru_cell/gates/bias:0' shape=(600,) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/fw/gru_cell/candidate/kernel:0' shape=(600, 300) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/fw/gru_cell/candidate/bias:0' shape=(300,) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/bw/gru_cell/gates/kernel:0' shape=(600, 600) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/bw/gru_cell/gates/bias:0' shape=(600,) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/bw/gru_cell/candidate/kernel:0' shape=(600, 300) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/bw/gru_cell/candidate/bias:0' shape=(300,) dtype=float32_ref>,
<tf.Variable 'dense_1/kernel:0' shape=(600, 23) dtype=float32_ref>,
<tf.Variable 'dense_1/bias:0' shape=(23,) dtype=float32_ref>,
<tf.Variable 'dense_2/kernel:0' shape=(600, 122) dtype=float32_ref>,
<tf.Variable 'dense_2/bias:0' shape=(122,) dtype=float32_ref>]
I0716 05:11:51.244380 139795883440000 estimator.py:1147] Done calling model_fn.
I0716 05:11:51.248389 139795883440000 basic_session_run_hooks.py:541] Create CheckpointSaverHook.
I0716 05:11:51.570209 139795883440000 monitored_session.py:240] Graph was finalized.
I0716 05:11:51.590957 139795883440000 saver.py:1280] Restoring parameters from ../model/elmo_bigru/model.ckpt-11232
I0716 05:11:52.627373 139795883440000 session_manager.py:500] Running local_init_op.
I0716 05:11:52.704813 139795883440000 session_manager.py:502] Done running local_init_op.
I0716 05:11:55.754058 139795883440000 basic_session_run_hooks.py:606] Saving checkpoints for 11232 into ../model/elmo_bigru/model.ckpt.
Reading ../data/atis.train.w-intent.iob
I0716 05:12:00.553711 139795883440000 basic_session_run_hooks.py:262] loss = 0.008925883, step = 11232 I0716 05:12:00.555503 139795883440000 basic_session_run_hooks.py:262] lr = 9.186985e-05 I0716 05:12:27.661089 139795883440000 basic_session_run_hooks.py:692] global_step/sec: 3.68895 I0716 05:12:27.664352 139795883440000 basic_session_run_hooks.py:260] loss = 0.013533504, step = 11332 (27.111 sec) I0716 05:12:27.669419 139795883440000 basic_session_run_hooks.py:260] lr = 9.090697e-05 (27.114 sec) I0716 05:12:54.545852 139795883440000 basic_session_run_hooks.py:692] global_step/sec: 3.71959 I0716 05:12:54.552693 139795883440000 basic_session_run_hooks.py:260] loss = 0.0067935092, step = 11432 (26.888 sec) I0716 05:12:54.556052 139795883440000 basic_session_run_hooks.py:260] lr = 8.9954214e-05 (26.887 sec) I0716 05:13:21.051222 139795883440000 basic_session_run_hooks.py:692] global_step/sec: 3.77281 I0716 05:13:21.059669 139795883440000 basic_session_run_hooks.py:260] loss = 0.008703114, step = 11532 (26.507 sec) I0716 05:13:21.062155 139795883440000 basic_session_run_hooks.py:260] lr = 8.9011424e-05 (26.506 sec) I0716 05:13:24.135333 139795883440000 basic_session_run_hooks.py:606] Saving checkpoints for 11544 into ../model/elmo_bigru/model.ckpt. I0716 05:13:26.381620 139795883440000 estimator.py:1145] Calling model_fn. I0716 05:13:27.781203 139795883440000 saver.py:1499] Saver not created because there are no variables in the graph to restore I0716 05:13:28.389799 139795883440000 estimator.py:1147] Done calling model_fn. I0716 05:13:28.420340 139795883440000 evaluation.py:255] Starting evaluation at 2019-07-16T05:13:28Z I0716 05:13:28.651257 139795883440000 monitored_session.py:240] Graph was finalized. I0716 05:13:28.673435 139795883440000 saver.py:1280] Restoring parameters from ../model/elmo_bigru/model.ckpt-11544 I0716 05:13:29.603673 139795883440000 session_manager.py:500] Running local_init_op. I0716 05:13:29.663183 139795883440000 session_manager.py:502] Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
I0716 05:14:28.770346 139795883440000 evaluation.py:275] Finished evaluation at 2019-07-16-05:14:28 I0716 05:14:28.772348 139795883440000 estimator.py:2039] Saving dict for global step 11544: global_step = 11544, loss = 0.38771486 I0716 05:14:28.782020 139795883440000 estimator.py:2099] Saving 'checkpoint_path' summary for global step 11544: ../model/elmo_bigru/model.ckpt-11544 I0716 05:14:28.863938 139795883440000 estimator.py:368] Loss for final step: 0.0041209282.
Reading ../data/atis.test.w-intent.iob
I0716 05:14:29.477226 139795883440000 estimator.py:1145] Calling model_fn. I0716 05:14:31.202950 139795883440000 saver.py:1499] Saver not created because there are no variables in the graph to restore I0716 05:14:31.746677 139795883440000 estimator.py:1147] Done calling model_fn. I0716 05:14:31.969483 139795883440000 monitored_session.py:240] Graph was finalized. I0716 05:14:31.987271 139795883440000 saver.py:1280] Restoring parameters from ../model/elmo_bigru/model.ckpt-11544 I0716 05:14:32.905600 139795883440000 session_manager.py:500] Running local_init_op. I0716 05:14:32.958049 139795883440000 session_manager.py:502] Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.
'precision', 'predicted', average, warn_for)
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1439: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples.
'recall', 'true', average, warn_for)
I0716 05:15:31.219921 139795883440000 interactiveshell.py:2882]
precision recall f1-score support
atis_flight 0.984 0.992 0.988 632
atis_airfare 0.941 1.000 0.970 48
atis_ground_service 1.000 1.000 1.000 36
atis_airline 0.974 1.000 0.987 38
atis_abbreviation 0.943 1.000 0.971 33
atis_aircraft 1.000 0.889 0.941 9
atis_flight_time 0.500 1.000 0.667 1
atis_quantity 0.429 1.000 0.600 3
atis_flight#atis_airfare 0.833 0.417 0.556 12
atis_airport 1.000 0.944 0.971 18
atis_distance 1.000 1.000 1.000 10
atis_city 0.750 0.500 0.600 6
atis_ground_fare 1.000 0.857 0.923 7
atis_capacity 0.952 0.952 0.952 21
atis_flight_no 0.889 1.000 0.941 8
atis_meal 1.000 0.833 0.909 6
atis_restriction 0.000 0.000 0.000 0
atis_airline#atis_flight_no 0.000 0.000 0.000 0
atis_ground_service#atis_ground_fare 0.000 0.000 0.000 0
atis_airfare#atis_flight_time 0.000 0.000 0.000 0
atis_cheapest 0.000 0.000 0.000 0
atis_aircraft#atis_flight#atis_flight_no 0.000 0.000 0.000 0
micro avg 0.972 0.977 0.975 888
macro avg 0.645 0.654 0.635 888
weighted avg 0.974 0.977 0.974 888
I0716 05:15:31.258357 139795883440000 interactiveshell.py:2882]
precision recall f1-score support
O 0.000 0.000 0.000 0.0
B-toloc.city_name 0.973 0.994 0.983 716.0
B-fromloc.city_name 0.992 0.999 0.995 704.0
I-toloc.city_name 0.978 0.989 0.983 265.0
B-depart_date.day_name 0.991 0.991 0.991 212.0
B-airline_name 1.000 1.000 1.000 101.0
I-fromloc.city_name 0.973 1.000 0.986 177.0
B-depart_time.period_of_day 1.000 0.923 0.960 130.0
I-airline_name 1.000 1.000 1.000 65.0
B-depart_date.day_number 0.981 0.964 0.972 55.0
B-depart_date.month_name 0.981 0.946 0.964 56.0
B-depart_time.time 0.864 1.000 0.927 57.0
B-round_trip 1.000 1.000 1.000 73.0
B-cost_relative 0.973 0.973 0.973 37.0
I-round_trip 1.000 1.000 1.000 71.0
B-flight_mod 0.828 1.000 0.906 24.0
B-depart_time.time_relative 0.955 0.985 0.970 65.0
I-depart_time.time 0.945 1.000 0.972 52.0
B-stoploc.city_name 0.870 1.000 0.930 20.0
B-city_name 0.923 0.632 0.750 57.0
B-class_type 0.960 1.000 0.980 24.0
B-arrive_time.time 0.971 0.971 0.971 34.0
B-arrive_time.time_relative 0.967 0.935 0.951 31.0
I-class_type 1.000 1.000 1.000 17.0
B-flight_stop 1.000 1.000 1.000 21.0
I-arrive_time.time 1.000 0.971 0.986 35.0
B-airline_code 0.944 1.000 0.971 34.0
I-depart_date.day_number 1.000 0.933 0.966 15.0
I-fromloc.airport_name 0.441 1.000 0.612 15.0
B-fromloc.airport_name 0.462 1.000 0.632 12.0
B-arrive_date.day_name 0.786 1.000 0.880 11.0
B-toloc.state_code 1.000 1.000 1.000 18.0
B-depart_date.today_relative 1.000 0.889 0.941 9.0
B-flight_number 0.733 1.000 0.846 11.0
B-depart_date.date_relative 0.944 1.000 0.971 17.0
B-toloc.state_name 0.828 0.857 0.842 28.0
B-fare_basis_code 0.850 1.000 0.919 17.0
B-flight_time 1.000 1.000 1.000 1.0
B-or 1.000 1.000 1.000 3.0
B-arrive_time.period_of_day 0.750 1.000 0.857 6.0
B-meal_description 1.000 0.900 0.947 10.0
I-cost_relative 1.000 0.667 0.800 3.0
I-airport_name 0.846 0.379 0.524 29.0
B-fare_amount 1.000 1.000 1.000 2.0
I-fare_amount 1.000 1.000 1.000 2.0
I-city_name 1.000 0.533 0.696 30.0
I-toloc.airport_name 1.000 1.000 1.000 3.0
B-transport_type 1.000 1.000 1.000 10.0
B-arrive_date.month_name 0.625 0.833 0.714 6.0
B-arrive_date.day_number 0.714 0.833 0.769 6.0
I-stoploc.city_name 1.000 1.000 1.000 10.0
B-meal 1.000 1.000 1.000 16.0
B-fromloc.state_code 1.000 1.000 1.000 23.0
B-depart_time.period_mod 1.000 1.000 1.000 5.0
B-connect 1.000 1.000 1.000 6.0
B-flight_days 1.000 1.000 1.000 10.0
B-toloc.airport_name 1.000 1.000 1.000 3.0
B-fromloc.state_name 0.944 1.000 0.971 17.0
B-airport_name 0.667 0.381 0.485 21.0
B-economy 1.000 1.000 1.000 6.0
I-flight_time 1.000 1.000 1.000 1.0
B-aircraft_code 1.000 0.879 0.935 33.0
B-mod 1.000 0.500 0.667 2.0
B-airport_code 1.000 0.444 0.615 9.0
B-depart_time.start_time 1.000 0.667 0.800 3.0
B-depart_time.end_time 1.000 0.333 0.500 3.0
B-depart_date.year 1.000 1.000 1.000 3.0
I-transport_type 0.000 0.000 0.000 1.0
B-restriction_code 1.000 1.000 1.000 4.0
B-arrive_time.start_time 0.889 1.000 0.941 8.0
B-toloc.airport_code 1.000 0.750 0.857 4.0
B-arrive_time.end_time 0.889 1.000 0.941 8.0
I-arrive_time.end_time 0.889 1.000 0.941 8.0
I-depart_time.end_time 1.000 0.333 0.500 3.0
I-flight_stop 0.000 0.000 0.000 0.0
B-fromloc.airport_code 0.556 1.000 0.714 5.0
I-restriction_code 1.000 1.000 1.000 3.0
I-depart_time.start_time 1.000 1.000 1.000 1.0
I-toloc.state_name 1.000 1.000 1.000 1.0
I-depart_date.today_relative 0.000 0.000 0.000 0.0
B-arrive_date.date_relative 1.000 1.000 1.000 2.0
I-flight_mod 1.000 0.167 0.286 6.0
I-economy 0.000 0.000 0.000 0.0
B-return_date.date_relative 1.000 0.667 0.800 3.0
I-fromloc.state_name 1.000 1.000 1.000 1.0
B-state_code 1.000 1.000 1.000 1.0
I-arrive_time.start_time 1.000 1.000 1.000 1.0
I-arrive_date.day_number 0.000 0.000 0.000 0.0
B-meal_code 0.000 0.000 0.000 1.0
I-depart_time.period_of_day 1.000 1.000 1.000 1.0
B-day_name 1.000 0.500 0.667 2.0
B-period_of_day 1.000 0.750 0.857 4.0
B-stoploc.state_code 0.000 0.000 0.000 0.0
B-return_date.month_name 0.000 0.000 0.000 0.0
B-return_date.day_number 0.000 0.000 0.000 0.0
B-arrive_time.period_mod 0.000 0.000 0.000 0.0
I-meal_code 0.000 0.000 0.000 0.0
B-toloc.country_name 1.000 1.000 1.000 1.0
B-days_code 0.000 0.000 0.000 1.0
I-arrive_time.period_of_day 0.000 0.000 0.000 0.0
I-today_relative 0.000 0.000 0.000 0.0
B-return_time.period_of_day 0.000 0.000 0.000 0.0
B-time 0.000 0.000 0.000 0.0
I-fare_basis_code 0.000 0.000 0.000 0.0
I-arrive_time.time_relative 1.000 1.000 1.000 4.0
I-depart_time.time_relative 0.000 0.000 0.000 1.0
B-today_relative 0.000 0.000 0.000 0.0
B-state_name 0.000 0.000 0.000 9.0
B-arrive_date.today_relative 0.000 0.000 0.000 0.0
B-return_time.period_mod 0.000 0.000 0.000 0.0
B-month_name 0.000 0.000 0.000 0.0
B-day_number 0.000 0.000 0.000 0.0
I-return_date.date_relative 1.000 0.667 0.800 3.0
I-return_date.today_relative 0.000 0.000 0.000 0.0
B-stoploc.airport_name 0.000 0.000 0.000 0.0
B-time_relative 0.000 0.000 0.000 0.0
I-time 0.000 0.000 0.000 0.0
I-return_date.day_number 0.000 0.000 0.000 0.0
I-meal_description 0.000 0.000 0.000 0.0
B-return_date.today_relative 0.000 0.000 0.000 0.0
B-return_date.day_name 0.000 0.000 0.000 2.0
micro avg 0.958 0.960 0.959 3657.0
macro avg 0.685 0.654 0.655 3657.0
weighted avg 0.963 0.960 0.957 3657.0
I0716 05:15:31.287286 139795883440000 interactiveshell.py:2882] Best Slot F1: 0.960
I0716 05:15:31.288828 139795883440000 estimator_training.py:186] Not using Distribute Coordinator.
I0716 05:15:31.290701 139795883440000 training.py:612] Running training and evaluation locally (non-distributed).
I0716 05:15:31.292778 139795883440000 training.py:700] Start train and evaluate loop. The evaluate will happen after every checkpoint. Checkpoint frequency is determined based on RunConfig arguments: save_checkpoints_steps 312 or save_checkpoints_secs None.
I0716 05:15:31.363937 139795883440000 estimator.py:1145] Calling model_fn.
I0716 05:15:32.893659 139795883440000 saver.py:1499] Saver not created because there are no variables in the graph to restore
I0716 05:15:33.484046 139795883440000 <ipython-input-6-d6747b40360b>:55]
[<tf.Variable 'embedding:0' shape=(750, 300) dtype=float32_ref>,
<tf.Variable 'module/aggregation/weights:0' shape=(3,) dtype=float32>,
<tf.Variable 'module/aggregation/scaling:0' shape=() dtype=float32>,
<tf.Variable 'dense/kernel:0' shape=(1324, 300) dtype=float32_ref>,
<tf.Variable 'dense/bias:0' shape=(300,) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/fw/gru_cell/gates/kernel:0' shape=(600, 600) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/fw/gru_cell/gates/bias:0' shape=(600,) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/fw/gru_cell/candidate/kernel:0' shape=(600, 300) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/fw/gru_cell/candidate/bias:0' shape=(300,) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/bw/gru_cell/gates/kernel:0' shape=(600, 600) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/bw/gru_cell/gates/bias:0' shape=(600,) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/bw/gru_cell/candidate/kernel:0' shape=(600, 300) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/bw/gru_cell/candidate/bias:0' shape=(300,) dtype=float32_ref>,
<tf.Variable 'dense_1/kernel:0' shape=(600, 23) dtype=float32_ref>,
<tf.Variable 'dense_1/bias:0' shape=(23,) dtype=float32_ref>,
<tf.Variable 'dense_2/kernel:0' shape=(600, 122) dtype=float32_ref>,
<tf.Variable 'dense_2/bias:0' shape=(122,) dtype=float32_ref>]
I0716 05:15:34.796094 139795883440000 estimator.py:1147] Done calling model_fn.
I0716 05:15:34.800277 139795883440000 basic_session_run_hooks.py:541] Create CheckpointSaverHook.
I0716 05:15:35.143608 139795883440000 monitored_session.py:240] Graph was finalized.
I0716 05:15:35.165673 139795883440000 saver.py:1280] Restoring parameters from ../model/elmo_bigru/model.ckpt-11544
I0716 05:15:36.244131 139795883440000 session_manager.py:500] Running local_init_op.
I0716 05:15:36.319446 139795883440000 session_manager.py:502] Done running local_init_op.
I0716 05:15:39.263292 139795883440000 basic_session_run_hooks.py:606] Saving checkpoints for 11544 into ../model/elmo_bigru/model.ckpt.
Reading ../data/atis.train.w-intent.iob
I0716 05:15:44.056493 139795883440000 basic_session_run_hooks.py:262] loss = 0.007963607, step = 11544 I0716 05:15:44.058551 139795883440000 basic_session_run_hooks.py:262] lr = 8.889895e-05 I0716 05:16:11.216505 139795883440000 basic_session_run_hooks.py:692] global_step/sec: 3.68181 I0716 05:16:11.223562 139795883440000 basic_session_run_hooks.py:260] loss = 0.00936635, step = 11644 (27.167 sec) I0716 05:16:11.227022 139795883440000 basic_session_run_hooks.py:260] lr = 8.7967215e-05 (27.168 sec) I0716 05:16:37.509724 139795883440000 basic_session_run_hooks.py:692] global_step/sec: 3.80326 I0716 05:16:37.513396 139795883440000 basic_session_run_hooks.py:260] loss = 0.01121197, step = 11744 (26.290 sec) I0716 05:16:37.519301 139795883440000 basic_session_run_hooks.py:260] lr = 8.704526e-05 (26.291 sec) I0716 05:17:04.397915 139795883440000 basic_session_run_hooks.py:692] global_step/sec: 3.71911 I0716 05:17:04.406521 139795883440000 basic_session_run_hooks.py:260] loss = 0.014993166, step = 11844 (26.893 sec) I0716 05:17:04.408484 139795883440000 basic_session_run_hooks.py:260] lr = 8.613297e-05 (26.890 sec) I0716 05:17:07.297354 139795883440000 basic_session_run_hooks.py:606] Saving checkpoints for 11856 into ../model/elmo_bigru/model.ckpt. I0716 05:17:09.314856 139795883440000 estimator.py:1145] Calling model_fn. I0716 05:17:10.967811 139795883440000 saver.py:1499] Saver not created because there are no variables in the graph to restore I0716 05:17:11.550818 139795883440000 estimator.py:1147] Done calling model_fn. I0716 05:17:11.581848 139795883440000 evaluation.py:255] Starting evaluation at 2019-07-16T05:17:11Z I0716 05:17:11.808008 139795883440000 monitored_session.py:240] Graph was finalized. I0716 05:17:11.826797 139795883440000 saver.py:1280] Restoring parameters from ../model/elmo_bigru/model.ckpt-11856 I0716 05:17:12.744184 139795883440000 session_manager.py:500] Running local_init_op. I0716 05:17:12.809362 139795883440000 session_manager.py:502] Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
I0716 05:18:11.775803 139795883440000 evaluation.py:275] Finished evaluation at 2019-07-16-05:18:11 I0716 05:18:11.777727 139795883440000 estimator.py:2039] Saving dict for global step 11856: global_step = 11856, loss = 0.39094597 I0716 05:18:11.788371 139795883440000 estimator.py:2099] Saving 'checkpoint_path' summary for global step 11856: ../model/elmo_bigru/model.ckpt-11856 I0716 05:18:11.875290 139795883440000 estimator.py:368] Loss for final step: 0.0013453001.
Reading ../data/atis.test.w-intent.iob
I0716 05:18:12.474029 139795883440000 estimator.py:1145] Calling model_fn. I0716 05:18:14.052869 139795883440000 saver.py:1499] Saver not created because there are no variables in the graph to restore I0716 05:18:14.570518 139795883440000 estimator.py:1147] Done calling model_fn. I0716 05:18:14.795010 139795883440000 monitored_session.py:240] Graph was finalized. I0716 05:18:14.810380 139795883440000 saver.py:1280] Restoring parameters from ../model/elmo_bigru/model.ckpt-11856 I0716 05:18:15.705967 139795883440000 session_manager.py:500] Running local_init_op. I0716 05:18:15.757150 139795883440000 session_manager.py:502] Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.
'precision', 'predicted', average, warn_for)
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1439: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples.
'recall', 'true', average, warn_for)
I0716 05:19:13.985109 139795883440000 interactiveshell.py:2882]
precision recall f1-score support
atis_flight 0.984 0.992 0.988 632
atis_airfare 0.960 1.000 0.980 48
atis_ground_service 1.000 1.000 1.000 36
atis_airline 1.000 1.000 1.000 38
atis_abbreviation 0.943 1.000 0.971 33
atis_aircraft 1.000 0.889 0.941 9
atis_flight_time 0.500 1.000 0.667 1
atis_quantity 0.375 1.000 0.545 3
atis_flight#atis_airfare 0.833 0.417 0.556 12
atis_airport 1.000 1.000 1.000 18
atis_distance 1.000 1.000 1.000 10
atis_city 0.750 0.500 0.600 6
atis_ground_fare 1.000 0.857 0.923 7
atis_capacity 0.952 0.952 0.952 21
atis_flight_no 0.889 1.000 0.941 8
atis_meal 1.000 0.833 0.909 6
atis_restriction 0.000 0.000 0.000 0
atis_airline#atis_flight_no 0.000 0.000 0.000 0
atis_ground_service#atis_ground_fare 0.000 0.000 0.000 0
atis_airfare#atis_flight_time 0.000 0.000 0.000 0
atis_cheapest 0.000 0.000 0.000 0
atis_aircraft#atis_flight#atis_flight_no 0.000 0.000 0.000 0
micro avg 0.973 0.979 0.976 888
macro avg 0.645 0.656 0.635 888
weighted avg 0.976 0.979 0.975 888
I0716 05:19:14.023535 139795883440000 interactiveshell.py:2882]
precision recall f1-score support
O 0.000 0.000 0.000 0.0
B-toloc.city_name 0.979 0.993 0.986 716.0
B-fromloc.city_name 0.989 0.999 0.994 704.0
I-toloc.city_name 0.978 0.989 0.983 265.0
B-depart_date.day_name 0.991 0.991 0.991 212.0
B-airline_name 1.000 1.000 1.000 101.0
I-fromloc.city_name 0.973 1.000 0.986 177.0
B-depart_time.period_of_day 0.992 0.923 0.956 130.0
I-airline_name 1.000 1.000 1.000 65.0
B-depart_date.day_number 0.981 0.964 0.972 55.0
B-depart_date.month_name 0.982 0.964 0.973 56.0
B-depart_time.time 0.864 1.000 0.927 57.0
B-round_trip 1.000 1.000 1.000 73.0
B-cost_relative 0.973 0.973 0.973 37.0
I-round_trip 1.000 1.000 1.000 71.0
B-flight_mod 0.828 1.000 0.906 24.0
B-depart_time.time_relative 0.955 0.985 0.970 65.0
I-depart_time.time 0.945 1.000 0.972 52.0
B-stoploc.city_name 0.909 1.000 0.952 20.0
B-city_name 0.923 0.632 0.750 57.0
B-class_type 0.960 1.000 0.980 24.0
B-arrive_time.time 0.971 0.971 0.971 34.0
B-arrive_time.time_relative 0.967 0.935 0.951 31.0
I-class_type 1.000 1.000 1.000 17.0
B-flight_stop 1.000 1.000 1.000 21.0
I-arrive_time.time 1.000 0.971 0.986 35.0
B-airline_code 0.971 1.000 0.986 34.0
I-depart_date.day_number 1.000 0.933 0.966 15.0
I-fromloc.airport_name 0.441 1.000 0.612 15.0
B-fromloc.airport_name 0.462 1.000 0.632 12.0
B-arrive_date.day_name 0.786 1.000 0.880 11.0
B-toloc.state_code 1.000 1.000 1.000 18.0
B-depart_date.today_relative 1.000 0.889 0.941 9.0
B-flight_number 0.733 1.000 0.846 11.0
B-depart_date.date_relative 0.944 1.000 0.971 17.0
B-toloc.state_name 0.800 1.000 0.889 28.0
B-fare_basis_code 0.850 1.000 0.919 17.0
B-flight_time 1.000 1.000 1.000 1.0
B-or 1.000 1.000 1.000 3.0
B-arrive_time.period_of_day 0.750 1.000 0.857 6.0
B-meal_description 1.000 0.900 0.947 10.0
I-cost_relative 1.000 0.667 0.800 3.0
I-airport_name 0.846 0.379 0.524 29.0
B-fare_amount 1.000 1.000 1.000 2.0
I-fare_amount 1.000 1.000 1.000 2.0
I-city_name 1.000 0.533 0.696 30.0
I-toloc.airport_name 1.000 1.000 1.000 3.0
B-transport_type 1.000 1.000 1.000 10.0
B-arrive_date.month_name 0.714 0.833 0.769 6.0
B-arrive_date.day_number 0.714 0.833 0.769 6.0
I-stoploc.city_name 1.000 1.000 1.000 10.0
B-meal 1.000 1.000 1.000 16.0
B-fromloc.state_code 1.000 1.000 1.000 23.0
B-depart_time.period_mod 1.000 1.000 1.000 5.0
B-connect 1.000 1.000 1.000 6.0
B-flight_days 1.000 1.000 1.000 10.0
B-toloc.airport_name 1.000 1.000 1.000 3.0
B-fromloc.state_name 0.944 1.000 0.971 17.0
B-airport_name 0.800 0.381 0.516 21.0
B-economy 1.000 1.000 1.000 6.0
I-flight_time 1.000 1.000 1.000 1.0
B-aircraft_code 1.000 0.879 0.935 33.0
B-mod 1.000 0.500 0.667 2.0
B-airport_code 1.000 0.444 0.615 9.0
B-depart_time.start_time 1.000 0.667 0.800 3.0
B-depart_time.end_time 1.000 0.333 0.500 3.0
B-depart_date.year 1.000 1.000 1.000 3.0
I-transport_type 0.000 0.000 0.000 1.0
B-restriction_code 0.800 1.000 0.889 4.0
B-arrive_time.start_time 0.889 1.000 0.941 8.0
B-toloc.airport_code 1.000 1.000 1.000 4.0
B-arrive_time.end_time 0.889 1.000 0.941 8.0
I-arrive_time.end_time 0.889 1.000 0.941 8.0
I-depart_time.end_time 1.000 0.333 0.500 3.0
I-flight_stop 0.000 0.000 0.000 0.0
B-fromloc.airport_code 0.556 1.000 0.714 5.0
I-restriction_code 1.000 1.000 1.000 3.0
I-depart_time.start_time 1.000 1.000 1.000 1.0
I-toloc.state_name 1.000 1.000 1.000 1.0
I-depart_date.today_relative 0.000 0.000 0.000 0.0
B-arrive_date.date_relative 1.000 1.000 1.000 2.0
I-flight_mod 1.000 0.167 0.286 6.0
I-economy 0.000 0.000 0.000 0.0
B-return_date.date_relative 1.000 0.333 0.500 3.0
I-fromloc.state_name 1.000 1.000 1.000 1.0
B-state_code 1.000 1.000 1.000 1.0
I-arrive_time.start_time 1.000 1.000 1.000 1.0
I-arrive_date.day_number 0.000 0.000 0.000 0.0
B-meal_code 0.000 0.000 0.000 1.0
I-depart_time.period_of_day 1.000 1.000 1.000 1.0
B-day_name 1.000 0.500 0.667 2.0
B-period_of_day 1.000 0.750 0.857 4.0
B-stoploc.state_code 0.000 0.000 0.000 0.0
B-return_date.month_name 0.000 0.000 0.000 0.0
B-return_date.day_number 0.000 0.000 0.000 0.0
B-arrive_time.period_mod 0.000 0.000 0.000 0.0
I-meal_code 0.000 0.000 0.000 0.0
B-toloc.country_name 1.000 1.000 1.000 1.0
B-days_code 0.000 0.000 0.000 1.0
I-arrive_time.period_of_day 0.000 0.000 0.000 0.0
I-today_relative 0.000 0.000 0.000 0.0
B-return_time.period_of_day 0.000 0.000 0.000 0.0
B-time 0.000 0.000 0.000 0.0
I-fare_basis_code 0.000 0.000 0.000 0.0
I-arrive_time.time_relative 1.000 1.000 1.000 4.0
I-depart_time.time_relative 0.000 0.000 0.000 1.0
B-today_relative 0.000 0.000 0.000 0.0
B-state_name 0.000 0.000 0.000 9.0
B-arrive_date.today_relative 0.000 0.000 0.000 0.0
B-return_time.period_mod 0.000 0.000 0.000 0.0
B-month_name 0.000 0.000 0.000 0.0
B-day_number 0.000 0.000 0.000 0.0
I-return_date.date_relative 0.667 0.667 0.667 3.0
I-return_date.today_relative 0.000 0.000 0.000 0.0
B-stoploc.airport_name 0.000 0.000 0.000 0.0
B-time_relative 0.000 0.000 0.000 0.0
I-time 0.000 0.000 0.000 0.0
I-return_date.day_number 0.000 0.000 0.000 0.0
I-meal_description 0.000 0.000 0.000 0.0
B-return_date.today_relative 0.000 0.000 0.000 0.0
B-return_date.day_name 0.000 0.000 0.000 2.0
micro avg 0.959 0.961 0.960 3657.0
macro avg 0.683 0.655 0.654 3657.0
weighted avg 0.964 0.961 0.957 3657.0
I0716 05:19:14.047439 139795883440000 interactiveshell.py:2882] Best Slot F1: 0.960
I0716 05:19:14.049042 139795883440000 estimator_training.py:186] Not using Distribute Coordinator.
I0716 05:19:14.054128 139795883440000 training.py:612] Running training and evaluation locally (non-distributed).
I0716 05:19:14.056412 139795883440000 training.py:700] Start train and evaluate loop. The evaluate will happen after every checkpoint. Checkpoint frequency is determined based on RunConfig arguments: save_checkpoints_steps 312 or save_checkpoints_secs None.
I0716 05:19:14.123416 139795883440000 estimator.py:1145] Calling model_fn.
I0716 05:19:15.632527 139795883440000 saver.py:1499] Saver not created because there are no variables in the graph to restore
I0716 05:19:16.231479 139795883440000 <ipython-input-6-d6747b40360b>:55]
[<tf.Variable 'embedding:0' shape=(750, 300) dtype=float32_ref>,
<tf.Variable 'module/aggregation/weights:0' shape=(3,) dtype=float32>,
<tf.Variable 'module/aggregation/scaling:0' shape=() dtype=float32>,
<tf.Variable 'dense/kernel:0' shape=(1324, 300) dtype=float32_ref>,
<tf.Variable 'dense/bias:0' shape=(300,) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/fw/gru_cell/gates/kernel:0' shape=(600, 600) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/fw/gru_cell/gates/bias:0' shape=(600,) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/fw/gru_cell/candidate/kernel:0' shape=(600, 300) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/fw/gru_cell/candidate/bias:0' shape=(300,) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/bw/gru_cell/gates/kernel:0' shape=(600, 600) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/bw/gru_cell/gates/bias:0' shape=(600,) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/bw/gru_cell/candidate/kernel:0' shape=(600, 300) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/bw/gru_cell/candidate/bias:0' shape=(300,) dtype=float32_ref>,
<tf.Variable 'dense_1/kernel:0' shape=(600, 23) dtype=float32_ref>,
<tf.Variable 'dense_1/bias:0' shape=(23,) dtype=float32_ref>,
<tf.Variable 'dense_2/kernel:0' shape=(600, 122) dtype=float32_ref>,
<tf.Variable 'dense_2/bias:0' shape=(122,) dtype=float32_ref>]
I0716 05:19:17.529751 139795883440000 estimator.py:1147] Done calling model_fn.
I0716 05:19:17.533969 139795883440000 basic_session_run_hooks.py:541] Create CheckpointSaverHook.
I0716 05:19:17.854725 139795883440000 monitored_session.py:240] Graph was finalized.
I0716 05:19:17.874505 139795883440000 saver.py:1280] Restoring parameters from ../model/elmo_bigru/model.ckpt-11856
I0716 05:19:18.951483 139795883440000 session_manager.py:500] Running local_init_op.
I0716 05:19:19.021305 139795883440000 session_manager.py:502] Done running local_init_op.
I0716 05:19:21.874391 139795883440000 basic_session_run_hooks.py:606] Saving checkpoints for 11856 into ../model/elmo_bigru/model.ckpt.
Reading ../data/atis.train.w-intent.iob
I0716 05:19:26.661842 139795883440000 basic_session_run_hooks.py:262] loss = 0.010607904, step = 11856 I0716 05:19:26.663809 139795883440000 basic_session_run_hooks.py:262] lr = 8.6024134e-05 I0716 05:19:54.552931 139795883440000 basic_session_run_hooks.py:692] global_step/sec: 3.58531 I0716 05:19:54.556187 139795883440000 basic_session_run_hooks.py:260] loss = 0.011345856, step = 11956 (27.894 sec) I0716 05:19:54.558486 139795883440000 basic_session_run_hooks.py:260] lr = 8.5122534e-05 (27.895 sec) I0716 05:20:20.401957 139795883440000 basic_session_run_hooks.py:692] global_step/sec: 3.86861 I0716 05:20:20.410420 139795883440000 basic_session_run_hooks.py:260] loss = 0.01699819, step = 12056 (25.854 sec) I0716 05:20:20.412324 139795883440000 basic_session_run_hooks.py:260] lr = 8.4230385e-05 (25.854 sec) I0716 05:20:47.246008 139795883440000 basic_session_run_hooks.py:692] global_step/sec: 3.72522 I0716 05:20:47.254420 139795883440000 basic_session_run_hooks.py:260] loss = 0.008648333, step = 12156 (26.844 sec) I0716 05:20:47.256663 139795883440000 basic_session_run_hooks.py:260] lr = 8.334759e-05 (26.844 sec) I0716 05:20:50.029184 139795883440000 basic_session_run_hooks.py:606] Saving checkpoints for 12168 into ../model/elmo_bigru/model.ckpt. I0716 05:20:52.050083 139795883440000 estimator.py:1145] Calling model_fn. I0716 05:20:53.702356 139795883440000 saver.py:1499] Saver not created because there are no variables in the graph to restore I0716 05:20:54.303798 139795883440000 estimator.py:1147] Done calling model_fn. I0716 05:20:54.333423 139795883440000 evaluation.py:255] Starting evaluation at 2019-07-16T05:20:54Z I0716 05:20:54.558538 139795883440000 monitored_session.py:240] Graph was finalized. I0716 05:20:54.576385 139795883440000 saver.py:1280] Restoring parameters from ../model/elmo_bigru/model.ckpt-12168 I0716 05:20:55.496074 139795883440000 session_manager.py:500] Running local_init_op. I0716 05:20:55.558856 139795883440000 session_manager.py:502] Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
I0716 05:21:54.515105 139795883440000 evaluation.py:275] Finished evaluation at 2019-07-16-05:21:54 I0716 05:21:54.517024 139795883440000 estimator.py:2039] Saving dict for global step 12168: global_step = 12168, loss = 0.38762906 I0716 05:21:54.524751 139795883440000 estimator.py:2099] Saving 'checkpoint_path' summary for global step 12168: ../model/elmo_bigru/model.ckpt-12168 I0716 05:21:54.619015 139795883440000 estimator.py:368] Loss for final step: 0.0029825827.
Reading ../data/atis.test.w-intent.iob
I0716 05:21:55.235663 139795883440000 estimator.py:1145] Calling model_fn. I0716 05:21:56.801426 139795883440000 saver.py:1499] Saver not created because there are no variables in the graph to restore I0716 05:21:57.311077 139795883440000 estimator.py:1147] Done calling model_fn. I0716 05:21:57.526308 139795883440000 monitored_session.py:240] Graph was finalized. I0716 05:21:57.553237 139795883440000 saver.py:1280] Restoring parameters from ../model/elmo_bigru/model.ckpt-12168 I0716 05:21:58.451761 139795883440000 session_manager.py:500] Running local_init_op. I0716 05:21:58.505653 139795883440000 session_manager.py:502] Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.
'precision', 'predicted', average, warn_for)
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1439: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples.
'recall', 'true', average, warn_for)
I0716 05:22:56.483352 139795883440000 interactiveshell.py:2882]
precision recall f1-score support
atis_flight 0.984 0.992 0.988 632
atis_airfare 0.960 1.000 0.980 48
atis_ground_service 1.000 1.000 1.000 36
atis_airline 0.974 1.000 0.987 38
atis_abbreviation 0.943 1.000 0.971 33
atis_aircraft 1.000 0.889 0.941 9
atis_flight_time 0.500 1.000 0.667 1
atis_quantity 0.375 1.000 0.545 3
atis_flight#atis_airfare 0.833 0.417 0.556 12
atis_airport 1.000 0.944 0.971 18
atis_distance 1.000 1.000 1.000 10
atis_city 0.750 0.500 0.600 6
atis_ground_fare 1.000 0.857 0.923 7
atis_capacity 0.952 0.952 0.952 21
atis_flight_no 0.889 1.000 0.941 8
atis_meal 1.000 0.833 0.909 6
atis_restriction 0.000 0.000 0.000 0
atis_airline#atis_flight_no 0.000 0.000 0.000 0
atis_ground_service#atis_ground_fare 0.000 0.000 0.000 0
atis_airfare#atis_flight_time 0.000 0.000 0.000 0
atis_cheapest 0.000 0.000 0.000 0
atis_aircraft#atis_flight#atis_flight_no 0.000 0.000 0.000 0
micro avg 0.972 0.977 0.975 888
macro avg 0.644 0.654 0.633 888
weighted avg 0.975 0.977 0.974 888
I0716 05:22:56.522810 139795883440000 interactiveshell.py:2882]
precision recall f1-score support
O 0.000 0.000 0.000 0.0
B-toloc.city_name 0.982 0.992 0.987 716.0
B-fromloc.city_name 0.987 0.999 0.993 704.0
I-toloc.city_name 0.977 0.981 0.979 265.0
B-depart_date.day_name 0.991 0.991 0.991 212.0
B-airline_name 1.000 1.000 1.000 101.0
I-fromloc.city_name 0.967 1.000 0.983 177.0
B-depart_time.period_of_day 1.000 0.923 0.960 130.0
I-airline_name 1.000 1.000 1.000 65.0
B-depart_date.day_number 0.981 0.964 0.972 55.0
B-depart_date.month_name 0.982 0.964 0.973 56.0
B-depart_time.time 0.864 1.000 0.927 57.0
B-round_trip 1.000 1.000 1.000 73.0
B-cost_relative 0.973 0.973 0.973 37.0
I-round_trip 1.000 1.000 1.000 71.0
B-flight_mod 0.828 1.000 0.906 24.0
B-depart_time.time_relative 0.970 0.985 0.977 65.0
I-depart_time.time 0.945 1.000 0.972 52.0
B-stoploc.city_name 0.909 1.000 0.952 20.0
B-city_name 0.927 0.667 0.776 57.0
B-class_type 0.960 1.000 0.980 24.0
B-arrive_time.time 0.971 0.971 0.971 34.0
B-arrive_time.time_relative 0.967 0.935 0.951 31.0
I-class_type 1.000 1.000 1.000 17.0
B-flight_stop 1.000 1.000 1.000 21.0
I-arrive_time.time 1.000 0.971 0.986 35.0
B-airline_code 0.971 1.000 0.986 34.0
I-depart_date.day_number 1.000 0.933 0.966 15.0
I-fromloc.airport_name 0.441 1.000 0.612 15.0
B-fromloc.airport_name 0.462 1.000 0.632 12.0
B-arrive_date.day_name 0.786 1.000 0.880 11.0
B-toloc.state_code 1.000 1.000 1.000 18.0
B-depart_date.today_relative 1.000 0.889 0.941 9.0
B-flight_number 0.733 1.000 0.846 11.0
B-depart_date.date_relative 0.944 1.000 0.971 17.0
B-toloc.state_name 0.800 1.000 0.889 28.0
B-fare_basis_code 0.850 1.000 0.919 17.0
B-flight_time 1.000 1.000 1.000 1.0
B-or 1.000 1.000 1.000 3.0
B-arrive_time.period_of_day 0.750 1.000 0.857 6.0
B-meal_description 1.000 0.900 0.947 10.0
I-cost_relative 1.000 0.667 0.800 3.0
I-airport_name 0.846 0.379 0.524 29.0
B-fare_amount 1.000 1.000 1.000 2.0
I-fare_amount 1.000 1.000 1.000 2.0
I-city_name 1.000 0.533 0.696 30.0
I-toloc.airport_name 1.000 1.000 1.000 3.0
B-transport_type 1.000 1.000 1.000 10.0
B-arrive_date.month_name 0.714 0.833 0.769 6.0
B-arrive_date.day_number 0.714 0.833 0.769 6.0
I-stoploc.city_name 1.000 1.000 1.000 10.0
B-meal 0.941 1.000 0.970 16.0
B-fromloc.state_code 1.000 1.000 1.000 23.0
B-depart_time.period_mod 1.000 1.000 1.000 5.0
B-connect 1.000 1.000 1.000 6.0
B-flight_days 1.000 1.000 1.000 10.0
B-toloc.airport_name 1.000 1.000 1.000 3.0
B-fromloc.state_name 0.944 1.000 0.971 17.0
B-airport_name 0.800 0.381 0.516 21.0
B-economy 1.000 1.000 1.000 6.0
I-flight_time 1.000 1.000 1.000 1.0
B-aircraft_code 1.000 0.879 0.935 33.0
B-mod 1.000 0.500 0.667 2.0
B-airport_code 1.000 0.444 0.615 9.0
B-depart_time.start_time 1.000 0.667 0.800 3.0
B-depart_time.end_time 1.000 0.333 0.500 3.0
B-depart_date.year 1.000 1.000 1.000 3.0
I-transport_type 0.000 0.000 0.000 1.0
B-restriction_code 0.800 1.000 0.889 4.0
B-arrive_time.start_time 0.889 1.000 0.941 8.0
B-toloc.airport_code 1.000 1.000 1.000 4.0
B-arrive_time.end_time 0.889 1.000 0.941 8.0
I-arrive_time.end_time 0.889 1.000 0.941 8.0
I-depart_time.end_time 1.000 0.333 0.500 3.0
I-flight_stop 0.000 0.000 0.000 0.0
B-fromloc.airport_code 0.556 1.000 0.714 5.0
I-restriction_code 1.000 1.000 1.000 3.0
I-depart_time.start_time 1.000 1.000 1.000 1.0
I-toloc.state_name 1.000 1.000 1.000 1.0
I-depart_date.today_relative 0.000 0.000 0.000 0.0
B-arrive_date.date_relative 1.000 1.000 1.000 2.0
I-flight_mod 1.000 0.167 0.286 6.0
I-economy 0.000 0.000 0.000 0.0
B-return_date.date_relative 1.000 0.333 0.500 3.0
I-fromloc.state_name 1.000 1.000 1.000 1.0
B-state_code 1.000 1.000 1.000 1.0
I-arrive_time.start_time 1.000 1.000 1.000 1.0
I-arrive_date.day_number 0.000 0.000 0.000 0.0
B-meal_code 0.000 0.000 0.000 1.0
I-depart_time.period_of_day 1.000 1.000 1.000 1.0
B-day_name 1.000 0.500 0.667 2.0
B-period_of_day 1.000 0.750 0.857 4.0
B-stoploc.state_code 0.000 0.000 0.000 0.0
B-return_date.month_name 0.000 0.000 0.000 0.0
B-return_date.day_number 0.000 0.000 0.000 0.0
B-arrive_time.period_mod 0.000 0.000 0.000 0.0
I-meal_code 0.000 0.000 0.000 0.0
B-toloc.country_name 1.000 1.000 1.000 1.0
B-days_code 0.000 0.000 0.000 1.0
I-arrive_time.period_of_day 0.000 0.000 0.000 0.0
I-today_relative 0.000 0.000 0.000 0.0
B-return_time.period_of_day 0.000 0.000 0.000 0.0
B-time 0.000 0.000 0.000 0.0
I-fare_basis_code 0.000 0.000 0.000 0.0
I-arrive_time.time_relative 1.000 1.000 1.000 4.0
I-depart_time.time_relative 0.000 0.000 0.000 1.0
B-today_relative 0.000 0.000 0.000 0.0
B-state_name 0.000 0.000 0.000 9.0
B-arrive_date.today_relative 0.000 0.000 0.000 0.0
B-return_time.period_mod 0.000 0.000 0.000 0.0
B-month_name 0.000 0.000 0.000 0.0
B-day_number 0.000 0.000 0.000 0.0
I-return_date.date_relative 0.667 0.667 0.667 3.0
I-return_date.today_relative 0.000 0.000 0.000 0.0
B-stoploc.airport_name 0.000 0.000 0.000 0.0
B-time_relative 0.000 0.000 0.000 0.0
I-time 0.000 0.000 0.000 0.0
I-return_date.day_number 0.000 0.000 0.000 0.0
I-meal_description 0.000 0.000 0.000 0.0
B-return_date.today_relative 0.000 0.000 0.000 0.0
B-return_date.day_name 0.000 0.000 0.000 2.0
micro avg 0.959 0.961 0.960 3657.0
macro avg 0.682 0.655 0.654 3657.0
weighted avg 0.964 0.961 0.957 3657.0
I0716 05:22:56.546641 139795883440000 interactiveshell.py:2882] Best Slot F1: 0.960
I0716 05:22:56.548236 139795883440000 estimator_training.py:186] Not using Distribute Coordinator.
I0716 05:22:56.550099 139795883440000 training.py:612] Running training and evaluation locally (non-distributed).
I0716 05:22:56.553230 139795883440000 training.py:700] Start train and evaluate loop. The evaluate will happen after every checkpoint. Checkpoint frequency is determined based on RunConfig arguments: save_checkpoints_steps 312 or save_checkpoints_secs None.
I0716 05:22:56.622224 139795883440000 estimator.py:1145] Calling model_fn.
I0716 05:22:58.115841 139795883440000 saver.py:1499] Saver not created because there are no variables in the graph to restore
I0716 05:22:58.702072 139795883440000 <ipython-input-6-d6747b40360b>:55]
[<tf.Variable 'embedding:0' shape=(750, 300) dtype=float32_ref>,
<tf.Variable 'module/aggregation/weights:0' shape=(3,) dtype=float32>,
<tf.Variable 'module/aggregation/scaling:0' shape=() dtype=float32>,
<tf.Variable 'dense/kernel:0' shape=(1324, 300) dtype=float32_ref>,
<tf.Variable 'dense/bias:0' shape=(300,) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/fw/gru_cell/gates/kernel:0' shape=(600, 600) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/fw/gru_cell/gates/bias:0' shape=(600,) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/fw/gru_cell/candidate/kernel:0' shape=(600, 300) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/fw/gru_cell/candidate/bias:0' shape=(300,) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/bw/gru_cell/gates/kernel:0' shape=(600, 600) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/bw/gru_cell/gates/bias:0' shape=(600,) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/bw/gru_cell/candidate/kernel:0' shape=(600, 300) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/bw/gru_cell/candidate/bias:0' shape=(300,) dtype=float32_ref>,
<tf.Variable 'dense_1/kernel:0' shape=(600, 23) dtype=float32_ref>,
<tf.Variable 'dense_1/bias:0' shape=(23,) dtype=float32_ref>,
<tf.Variable 'dense_2/kernel:0' shape=(600, 122) dtype=float32_ref>,
<tf.Variable 'dense_2/bias:0' shape=(122,) dtype=float32_ref>]
I0716 05:22:59.997102 139795883440000 estimator.py:1147] Done calling model_fn.
I0716 05:23:00.000872 139795883440000 basic_session_run_hooks.py:541] Create CheckpointSaverHook.
I0716 05:23:00.332017 139795883440000 monitored_session.py:240] Graph was finalized.
I0716 05:23:00.355240 139795883440000 saver.py:1280] Restoring parameters from ../model/elmo_bigru/model.ckpt-12168
I0716 05:23:01.405848 139795883440000 session_manager.py:500] Running local_init_op.
I0716 05:23:01.476660 139795883440000 session_manager.py:502] Done running local_init_op.
I0716 05:23:04.277058 139795883440000 basic_session_run_hooks.py:606] Saving checkpoints for 12168 into ../model/elmo_bigru/model.ckpt.
Reading ../data/atis.train.w-intent.iob
I0716 05:23:08.941863 139795883440000 basic_session_run_hooks.py:262] loss = 0.009021113, step = 12168 I0716 05:23:08.943840 139795883440000 basic_session_run_hooks.py:262] lr = 8.324228e-05 I0716 05:23:36.479554 139795883440000 basic_session_run_hooks.py:692] global_step/sec: 3.63132 I0716 05:23:36.483439 139795883440000 basic_session_run_hooks.py:260] loss = 0.008717981, step = 12268 (27.542 sec) I0716 05:23:36.488025 139795883440000 basic_session_run_hooks.py:260] lr = 8.2369836e-05 (27.544 sec) I0716 05:24:02.972101 139795883440000 basic_session_run_hooks.py:692] global_step/sec: 3.77464 I0716 05:24:02.974956 139795883440000 basic_session_run_hooks.py:260] loss = 0.012833698, step = 12368 (26.492 sec) I0716 05:24:02.982555 139795883440000 basic_session_run_hooks.py:260] lr = 8.150654e-05 (26.495 sec) I0716 05:24:29.164611 139795883440000 basic_session_run_hooks.py:692] global_step/sec: 3.81788 I0716 05:24:29.168652 139795883440000 basic_session_run_hooks.py:260] loss = 0.013801475, step = 12468 (26.194 sec) I0716 05:24:29.172936 139795883440000 basic_session_run_hooks.py:260] lr = 8.0652295e-05 (26.190 sec) I0716 05:24:32.146581 139795883440000 basic_session_run_hooks.py:606] Saving checkpoints for 12480 into ../model/elmo_bigru/model.ckpt. I0716 05:24:34.259977 139795883440000 estimator.py:1145] Calling model_fn. I0716 05:24:35.882333 139795883440000 saver.py:1499] Saver not created because there are no variables in the graph to restore I0716 05:24:36.459211 139795883440000 estimator.py:1147] Done calling model_fn. I0716 05:24:36.488796 139795883440000 evaluation.py:255] Starting evaluation at 2019-07-16T05:24:36Z I0716 05:24:36.715009 139795883440000 monitored_session.py:240] Graph was finalized. I0716 05:24:36.728446 139795883440000 saver.py:1280] Restoring parameters from ../model/elmo_bigru/model.ckpt-12480 I0716 05:24:37.638752 139795883440000 session_manager.py:500] Running local_init_op. I0716 05:24:37.698906 139795883440000 session_manager.py:502] Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
I0716 05:25:36.709956 139795883440000 evaluation.py:275] Finished evaluation at 2019-07-16-05:25:36 I0716 05:25:36.711731 139795883440000 estimator.py:2039] Saving dict for global step 12480: global_step = 12480, loss = 0.3937179 I0716 05:25:36.718591 139795883440000 estimator.py:2099] Saving 'checkpoint_path' summary for global step 12480: ../model/elmo_bigru/model.ckpt-12480 I0716 05:25:36.809600 139795883440000 estimator.py:368] Loss for final step: 0.0027113266.
Reading ../data/atis.test.w-intent.iob
I0716 05:25:37.415125 139795883440000 estimator.py:1145] Calling model_fn. I0716 05:25:38.997050 139795883440000 saver.py:1499] Saver not created because there are no variables in the graph to restore I0716 05:25:39.525370 139795883440000 estimator.py:1147] Done calling model_fn. I0716 05:25:39.755532 139795883440000 monitored_session.py:240] Graph was finalized. I0716 05:25:39.773118 139795883440000 saver.py:1280] Restoring parameters from ../model/elmo_bigru/model.ckpt-12480 I0716 05:25:40.659859 139795883440000 session_manager.py:500] Running local_init_op. I0716 05:25:40.732056 139795883440000 session_manager.py:502] Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.
'precision', 'predicted', average, warn_for)
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1439: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples.
'recall', 'true', average, warn_for)
I0716 05:26:38.619279 139795883440000 interactiveshell.py:2882]
precision recall f1-score support
atis_flight 0.984 0.992 0.988 632
atis_airfare 0.941 1.000 0.970 48
atis_ground_service 1.000 1.000 1.000 36
atis_airline 0.974 1.000 0.987 38
atis_abbreviation 0.943 1.000 0.971 33
atis_aircraft 1.000 0.889 0.941 9
atis_flight_time 0.500 1.000 0.667 1
atis_quantity 0.429 1.000 0.600 3
atis_flight#atis_airfare 0.833 0.417 0.556 12
atis_airport 1.000 0.944 0.971 18
atis_distance 1.000 1.000 1.000 10
atis_city 0.750 0.500 0.600 6
atis_ground_fare 1.000 0.857 0.923 7
atis_capacity 0.952 0.952 0.952 21
atis_flight_no 0.889 1.000 0.941 8
atis_meal 1.000 0.833 0.909 6
atis_restriction 0.000 0.000 0.000 0
atis_airline#atis_flight_no 0.000 0.000 0.000 0
atis_ground_service#atis_ground_fare 0.000 0.000 0.000 0
atis_airfare#atis_flight_time 0.000 0.000 0.000 0
atis_cheapest 0.000 0.000 0.000 0
atis_aircraft#atis_flight#atis_flight_no 0.000 0.000 0.000 0
micro avg 0.972 0.977 0.975 888
macro avg 0.645 0.654 0.635 888
weighted avg 0.974 0.977 0.974 888
I0716 05:26:38.657084 139795883440000 interactiveshell.py:2882]
precision recall f1-score support
O 0.000 0.000 0.000 0.0
B-toloc.city_name 0.977 0.994 0.985 716.0
B-fromloc.city_name 0.990 0.999 0.994 704.0
I-toloc.city_name 0.978 0.989 0.983 265.0
B-depart_date.day_name 0.991 0.991 0.991 212.0
B-airline_name 1.000 1.000 1.000 101.0
I-fromloc.city_name 0.973 1.000 0.986 177.0
B-depart_time.period_of_day 1.000 0.923 0.960 130.0
I-airline_name 1.000 1.000 1.000 65.0
B-depart_date.day_number 0.981 0.964 0.972 55.0
B-depart_date.month_name 0.981 0.946 0.964 56.0
B-depart_time.time 0.864 1.000 0.927 57.0
B-round_trip 1.000 1.000 1.000 73.0
B-cost_relative 0.973 0.973 0.973 37.0
I-round_trip 1.000 1.000 1.000 71.0
B-flight_mod 0.828 1.000 0.906 24.0
B-depart_time.time_relative 0.955 0.985 0.970 65.0
I-depart_time.time 0.945 1.000 0.972 52.0
B-stoploc.city_name 0.909 1.000 0.952 20.0
B-city_name 0.925 0.649 0.763 57.0
B-class_type 0.960 1.000 0.980 24.0
B-arrive_time.time 0.971 0.971 0.971 34.0
B-arrive_time.time_relative 0.967 0.935 0.951 31.0
I-class_type 1.000 1.000 1.000 17.0
B-flight_stop 1.000 1.000 1.000 21.0
I-arrive_time.time 1.000 0.971 0.986 35.0
B-airline_code 0.971 1.000 0.986 34.0
I-depart_date.day_number 1.000 0.933 0.966 15.0
I-fromloc.airport_name 0.441 1.000 0.612 15.0
B-fromloc.airport_name 0.462 1.000 0.632 12.0
B-arrive_date.day_name 0.786 1.000 0.880 11.0
B-toloc.state_code 1.000 1.000 1.000 18.0
B-depart_date.today_relative 1.000 0.889 0.941 9.0
B-flight_number 0.733 1.000 0.846 11.0
B-depart_date.date_relative 0.944 1.000 0.971 17.0
B-toloc.state_name 0.788 0.929 0.852 28.0
B-fare_basis_code 0.944 1.000 0.971 17.0
B-flight_time 1.000 1.000 1.000 1.0
B-or 1.000 1.000 1.000 3.0
B-arrive_time.period_of_day 0.750 1.000 0.857 6.0
B-meal_description 1.000 0.900 0.947 10.0
I-cost_relative 1.000 0.667 0.800 3.0
I-airport_name 0.846 0.379 0.524 29.0
B-fare_amount 1.000 1.000 1.000 2.0
I-fare_amount 1.000 1.000 1.000 2.0
I-city_name 1.000 0.567 0.723 30.0
I-toloc.airport_name 1.000 1.000 1.000 3.0
B-transport_type 1.000 1.000 1.000 10.0
B-arrive_date.month_name 0.625 0.833 0.714 6.0
B-arrive_date.day_number 0.714 0.833 0.769 6.0
I-stoploc.city_name 1.000 1.000 1.000 10.0
B-meal 1.000 1.000 1.000 16.0
B-fromloc.state_code 1.000 1.000 1.000 23.0
B-depart_time.period_mod 1.000 1.000 1.000 5.0
B-connect 1.000 1.000 1.000 6.0
B-flight_days 1.000 1.000 1.000 10.0
B-toloc.airport_name 1.000 1.000 1.000 3.0
B-fromloc.state_name 0.944 1.000 0.971 17.0
B-airport_name 0.800 0.381 0.516 21.0
B-economy 1.000 1.000 1.000 6.0
I-flight_time 1.000 1.000 1.000 1.0
B-aircraft_code 1.000 0.879 0.935 33.0
B-mod 1.000 0.500 0.667 2.0
B-airport_code 1.000 0.444 0.615 9.0
B-depart_time.start_time 1.000 0.667 0.800 3.0
B-depart_time.end_time 1.000 0.333 0.500 3.0
B-depart_date.year 1.000 1.000 1.000 3.0
I-transport_type 0.000 0.000 0.000 1.0
B-restriction_code 0.800 1.000 0.889 4.0
B-arrive_time.start_time 0.889 1.000 0.941 8.0
B-toloc.airport_code 1.000 0.750 0.857 4.0
B-arrive_time.end_time 0.889 1.000 0.941 8.0
I-arrive_time.end_time 0.889 1.000 0.941 8.0
I-depart_time.end_time 1.000 0.333 0.500 3.0
I-flight_stop 0.000 0.000 0.000 0.0
B-fromloc.airport_code 0.556 1.000 0.714 5.0
I-restriction_code 1.000 1.000 1.000 3.0
I-depart_time.start_time 1.000 1.000 1.000 1.0
I-toloc.state_name 1.000 1.000 1.000 1.0
I-depart_date.today_relative 0.000 0.000 0.000 0.0
B-arrive_date.date_relative 1.000 1.000 1.000 2.0
I-flight_mod 1.000 0.167 0.286 6.0
I-economy 0.000 0.000 0.000 0.0
B-return_date.date_relative 1.000 0.333 0.500 3.0
I-fromloc.state_name 1.000 1.000 1.000 1.0
B-state_code 1.000 1.000 1.000 1.0
I-arrive_time.start_time 1.000 1.000 1.000 1.0
I-arrive_date.day_number 0.000 0.000 0.000 0.0
B-meal_code 0.000 0.000 0.000 1.0
I-depart_time.period_of_day 1.000 1.000 1.000 1.0
B-day_name 1.000 0.500 0.667 2.0
B-period_of_day 1.000 0.750 0.857 4.0
B-stoploc.state_code 0.000 0.000 0.000 0.0
B-return_date.month_name 0.000 0.000 0.000 0.0
B-return_date.day_number 0.000 0.000 0.000 0.0
B-arrive_time.period_mod 0.000 0.000 0.000 0.0
I-meal_code 0.000 0.000 0.000 0.0
B-toloc.country_name 1.000 1.000 1.000 1.0
B-days_code 1.000 1.000 1.000 1.0
I-arrive_time.period_of_day 0.000 0.000 0.000 0.0
I-today_relative 0.000 0.000 0.000 0.0
B-return_time.period_of_day 0.000 0.000 0.000 0.0
B-time 0.000 0.000 0.000 0.0
I-fare_basis_code 0.000 0.000 0.000 0.0
I-arrive_time.time_relative 1.000 1.000 1.000 4.0
I-depart_time.time_relative 0.000 0.000 0.000 1.0
B-today_relative 0.000 0.000 0.000 0.0
B-state_name 0.000 0.000 0.000 9.0
B-arrive_date.today_relative 0.000 0.000 0.000 0.0
B-return_time.period_mod 0.000 0.000 0.000 0.0
B-month_name 0.000 0.000 0.000 0.0
B-day_number 0.000 0.000 0.000 0.0
I-return_date.date_relative 0.750 1.000 0.857 3.0
I-return_date.today_relative 0.000 0.000 0.000 0.0
B-stoploc.airport_name 0.000 0.000 0.000 0.0
B-time_relative 0.000 0.000 0.000 0.0
I-time 0.000 0.000 0.000 0.0
I-return_date.day_number 0.000 0.000 0.000 0.0
I-meal_description 0.000 0.000 0.000 0.0
B-return_date.today_relative 0.000 0.000 0.000 0.0
B-return_date.day_name 0.000 0.000 0.000 2.0
micro avg 0.960 0.961 0.960 3657.0
macro avg 0.692 0.663 0.662 3657.0
weighted avg 0.964 0.961 0.958 3657.0
I0716 05:26:38.680395 139795883440000 interactiveshell.py:2882] Best Slot F1: 0.960
I0716 05:26:38.682062 139795883440000 estimator_training.py:186] Not using Distribute Coordinator.
I0716 05:26:38.684017 139795883440000 training.py:612] Running training and evaluation locally (non-distributed).
I0716 05:26:38.685970 139795883440000 training.py:700] Start train and evaluate loop. The evaluate will happen after every checkpoint. Checkpoint frequency is determined based on RunConfig arguments: save_checkpoints_steps 312 or save_checkpoints_secs None.
I0716 05:26:38.756810 139795883440000 estimator.py:1145] Calling model_fn.
I0716 05:26:40.255664 139795883440000 saver.py:1499] Saver not created because there are no variables in the graph to restore
I0716 05:26:40.830694 139795883440000 <ipython-input-6-d6747b40360b>:55]
[<tf.Variable 'embedding:0' shape=(750, 300) dtype=float32_ref>,
<tf.Variable 'module/aggregation/weights:0' shape=(3,) dtype=float32>,
<tf.Variable 'module/aggregation/scaling:0' shape=() dtype=float32>,
<tf.Variable 'dense/kernel:0' shape=(1324, 300) dtype=float32_ref>,
<tf.Variable 'dense/bias:0' shape=(300,) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/fw/gru_cell/gates/kernel:0' shape=(600, 600) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/fw/gru_cell/gates/bias:0' shape=(600,) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/fw/gru_cell/candidate/kernel:0' shape=(600, 300) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/fw/gru_cell/candidate/bias:0' shape=(300,) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/bw/gru_cell/gates/kernel:0' shape=(600, 600) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/bw/gru_cell/gates/bias:0' shape=(600,) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/bw/gru_cell/candidate/kernel:0' shape=(600, 300) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/bw/gru_cell/candidate/bias:0' shape=(300,) dtype=float32_ref>,
<tf.Variable 'dense_1/kernel:0' shape=(600, 23) dtype=float32_ref>,
<tf.Variable 'dense_1/bias:0' shape=(23,) dtype=float32_ref>,
<tf.Variable 'dense_2/kernel:0' shape=(600, 122) dtype=float32_ref>,
<tf.Variable 'dense_2/bias:0' shape=(122,) dtype=float32_ref>]
I0716 05:26:42.128607 139795883440000 estimator.py:1147] Done calling model_fn.
I0716 05:26:42.132558 139795883440000 basic_session_run_hooks.py:541] Create CheckpointSaverHook.
I0716 05:26:42.452786 139795883440000 monitored_session.py:240] Graph was finalized.
I0716 05:26:42.471553 139795883440000 saver.py:1280] Restoring parameters from ../model/elmo_bigru/model.ckpt-12480
I0716 05:26:43.507312 139795883440000 session_manager.py:500] Running local_init_op.
I0716 05:26:43.579066 139795883440000 session_manager.py:502] Done running local_init_op.
I0716 05:26:46.398946 139795883440000 basic_session_run_hooks.py:606] Saving checkpoints for 12480 into ../model/elmo_bigru/model.ckpt.
Reading ../data/atis.train.w-intent.iob
I0716 05:26:51.132528 139795883440000 basic_session_run_hooks.py:262] loss = 0.013520321, step = 12480 I0716 05:26:51.134556 139795883440000 basic_session_run_hooks.py:262] lr = 8.055038e-05 I0716 05:27:19.365086 139795883440000 basic_session_run_hooks.py:692] global_step/sec: 3.54194 I0716 05:27:19.370056 139795883440000 basic_session_run_hooks.py:260] loss = 0.01258297, step = 12580 (28.238 sec) I0716 05:27:19.375082 139795883440000 basic_session_run_hooks.py:260] lr = 7.970615e-05 (28.241 sec) I0716 05:27:44.703674 139795883440000 basic_session_run_hooks.py:692] global_step/sec: 3.94662 I0716 05:27:44.711696 139795883440000 basic_session_run_hooks.py:260] loss = 0.005740271, step = 12680 (25.342 sec) I0716 05:27:44.713462 139795883440000 basic_session_run_hooks.py:260] lr = 7.887078e-05 (25.338 sec) I0716 05:28:10.674861 139795883440000 basic_session_run_hooks.py:692] global_step/sec: 3.85037 I0716 05:28:10.683306 139795883440000 basic_session_run_hooks.py:260] loss = 0.008855683, step = 12780 (25.972 sec) I0716 05:28:10.684962 139795883440000 basic_session_run_hooks.py:260] lr = 7.804416e-05 (25.971 sec) I0716 05:28:13.473150 139795883440000 basic_session_run_hooks.py:606] Saving checkpoints for 12792 into ../model/elmo_bigru/model.ckpt. I0716 05:28:15.436150 139795883440000 estimator.py:1145] Calling model_fn. I0716 05:28:17.078930 139795883440000 saver.py:1499] Saver not created because there are no variables in the graph to restore I0716 05:28:17.668273 139795883440000 estimator.py:1147] Done calling model_fn. I0716 05:28:17.698833 139795883440000 evaluation.py:255] Starting evaluation at 2019-07-16T05:28:17Z I0716 05:28:17.923661 139795883440000 monitored_session.py:240] Graph was finalized. I0716 05:28:17.941342 139795883440000 saver.py:1280] Restoring parameters from ../model/elmo_bigru/model.ckpt-12792 I0716 05:28:18.848675 139795883440000 session_manager.py:500] Running local_init_op. I0716 05:28:18.905096 139795883440000 session_manager.py:502] Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
I0716 05:29:17.960201 139795883440000 evaluation.py:275] Finished evaluation at 2019-07-16-05:29:17 I0716 05:29:17.961978 139795883440000 estimator.py:2039] Saving dict for global step 12792: global_step = 12792, loss = 0.400475 I0716 05:29:17.968168 139795883440000 estimator.py:2099] Saving 'checkpoint_path' summary for global step 12792: ../model/elmo_bigru/model.ckpt-12792 I0716 05:29:18.057105 139795883440000 estimator.py:368] Loss for final step: 0.0023489797.
Reading ../data/atis.test.w-intent.iob
I0716 05:29:18.660959 139795883440000 estimator.py:1145] Calling model_fn. I0716 05:29:20.220807 139795883440000 saver.py:1499] Saver not created because there are no variables in the graph to restore I0716 05:29:20.743836 139795883440000 estimator.py:1147] Done calling model_fn. I0716 05:29:20.963388 139795883440000 monitored_session.py:240] Graph was finalized. I0716 05:29:20.982475 139795883440000 saver.py:1280] Restoring parameters from ../model/elmo_bigru/model.ckpt-12792 I0716 05:29:21.911297 139795883440000 session_manager.py:500] Running local_init_op. I0716 05:29:21.957988 139795883440000 session_manager.py:502] Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.
'precision', 'predicted', average, warn_for)
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1439: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples.
'recall', 'true', average, warn_for)
I0716 05:30:19.955024 139795883440000 interactiveshell.py:2882]
precision recall f1-score support
atis_flight 0.984 0.992 0.988 632
atis_airfare 0.941 1.000 0.970 48
atis_ground_service 1.000 1.000 1.000 36
atis_airline 0.974 1.000 0.987 38
atis_abbreviation 0.943 1.000 0.971 33
atis_aircraft 1.000 0.889 0.941 9
atis_flight_time 0.500 1.000 0.667 1
atis_quantity 0.429 1.000 0.600 3
atis_flight#atis_airfare 0.833 0.417 0.556 12
atis_airport 1.000 0.944 0.971 18
atis_distance 1.000 1.000 1.000 10
atis_city 0.750 0.500 0.600 6
atis_ground_fare 1.000 0.857 0.923 7
atis_capacity 0.952 0.952 0.952 21
atis_flight_no 0.889 1.000 0.941 8
atis_meal 1.000 0.833 0.909 6
atis_restriction 0.000 0.000 0.000 0
atis_airline#atis_flight_no 0.000 0.000 0.000 0
atis_ground_service#atis_ground_fare 0.000 0.000 0.000 0
atis_airfare#atis_flight_time 0.000 0.000 0.000 0
atis_cheapest 0.000 0.000 0.000 0
atis_aircraft#atis_flight#atis_flight_no 0.000 0.000 0.000 0
micro avg 0.972 0.977 0.975 888
macro avg 0.645 0.654 0.635 888
weighted avg 0.974 0.977 0.974 888
I0716 05:30:19.991873 139795883440000 interactiveshell.py:2882]
precision recall f1-score support
O 0.000 0.000 0.000 0.0
B-toloc.city_name 0.978 0.992 0.985 716.0
B-fromloc.city_name 0.987 0.999 0.993 704.0
I-toloc.city_name 0.978 0.985 0.981 265.0
B-depart_date.day_name 0.991 0.991 0.991 212.0
B-airline_name 1.000 1.000 1.000 101.0
I-fromloc.city_name 0.967 1.000 0.983 177.0
B-depart_time.period_of_day 1.000 0.923 0.960 130.0
I-airline_name 1.000 1.000 1.000 65.0
B-depart_date.day_number 0.981 0.964 0.972 55.0
B-depart_date.month_name 0.982 0.964 0.973 56.0
B-depart_time.time 0.864 1.000 0.927 57.0
B-round_trip 1.000 1.000 1.000 73.0
B-cost_relative 0.973 0.973 0.973 37.0
I-round_trip 1.000 1.000 1.000 71.0
B-flight_mod 0.828 1.000 0.906 24.0
B-depart_time.time_relative 0.955 0.985 0.970 65.0
I-depart_time.time 0.945 1.000 0.972 52.0
B-stoploc.city_name 0.909 1.000 0.952 20.0
B-city_name 0.923 0.632 0.750 57.0
B-class_type 0.960 1.000 0.980 24.0
B-arrive_time.time 0.971 0.971 0.971 34.0
B-arrive_time.time_relative 0.967 0.935 0.951 31.0
I-class_type 1.000 1.000 1.000 17.0
B-flight_stop 1.000 1.000 1.000 21.0
I-arrive_time.time 1.000 0.971 0.986 35.0
B-airline_code 0.971 1.000 0.986 34.0
I-depart_date.day_number 1.000 0.933 0.966 15.0
I-fromloc.airport_name 0.441 1.000 0.612 15.0
B-fromloc.airport_name 0.462 1.000 0.632 12.0
B-arrive_date.day_name 0.786 1.000 0.880 11.0
B-toloc.state_code 1.000 1.000 1.000 18.0
B-depart_date.today_relative 1.000 0.889 0.941 9.0
B-flight_number 0.733 1.000 0.846 11.0
B-depart_date.date_relative 0.944 1.000 0.971 17.0
B-toloc.state_name 0.794 0.964 0.871 28.0
B-fare_basis_code 0.850 1.000 0.919 17.0
B-flight_time 1.000 1.000 1.000 1.0
B-or 1.000 1.000 1.000 3.0
B-arrive_time.period_of_day 0.750 1.000 0.857 6.0
B-meal_description 1.000 0.900 0.947 10.0
I-cost_relative 1.000 0.667 0.800 3.0
I-airport_name 0.846 0.379 0.524 29.0
B-fare_amount 1.000 1.000 1.000 2.0
I-fare_amount 1.000 1.000 1.000 2.0
I-city_name 1.000 0.567 0.723 30.0
I-toloc.airport_name 1.000 1.000 1.000 3.0
B-transport_type 1.000 1.000 1.000 10.0
B-arrive_date.month_name 0.714 0.833 0.769 6.0
B-arrive_date.day_number 0.714 0.833 0.769 6.0
I-stoploc.city_name 1.000 1.000 1.000 10.0
B-meal 1.000 1.000 1.000 16.0
B-fromloc.state_code 1.000 1.000 1.000 23.0
B-depart_time.period_mod 1.000 1.000 1.000 5.0
B-connect 1.000 1.000 1.000 6.0
B-flight_days 1.000 1.000 1.000 10.0
B-toloc.airport_name 1.000 1.000 1.000 3.0
B-fromloc.state_name 0.944 1.000 0.971 17.0
B-airport_name 0.800 0.381 0.516 21.0
B-economy 1.000 1.000 1.000 6.0
I-flight_time 1.000 1.000 1.000 1.0
B-aircraft_code 1.000 0.879 0.935 33.0
B-mod 1.000 0.500 0.667 2.0
B-airport_code 1.000 0.444 0.615 9.0
B-depart_time.start_time 1.000 0.667 0.800 3.0
B-depart_time.end_time 1.000 0.333 0.500 3.0
B-depart_date.year 1.000 1.000 1.000 3.0
I-transport_type 0.000 0.000 0.000 1.0
B-restriction_code 0.800 1.000 0.889 4.0
B-arrive_time.start_time 0.889 1.000 0.941 8.0
B-toloc.airport_code 1.000 1.000 1.000 4.0
B-arrive_time.end_time 0.889 1.000 0.941 8.0
I-arrive_time.end_time 0.889 1.000 0.941 8.0
I-depart_time.end_time 1.000 0.333 0.500 3.0
I-flight_stop 0.000 0.000 0.000 0.0
B-fromloc.airport_code 0.556 1.000 0.714 5.0
I-restriction_code 1.000 1.000 1.000 3.0
I-depart_time.start_time 1.000 1.000 1.000 1.0
I-toloc.state_name 1.000 1.000 1.000 1.0
I-depart_date.today_relative 0.000 0.000 0.000 0.0
B-arrive_date.date_relative 1.000 1.000 1.000 2.0
I-flight_mod 1.000 0.167 0.286 6.0
I-economy 0.000 0.000 0.000 0.0
B-return_date.date_relative 1.000 0.667 0.800 3.0
I-fromloc.state_name 1.000 1.000 1.000 1.0
B-state_code 1.000 1.000 1.000 1.0
I-arrive_time.start_time 1.000 1.000 1.000 1.0
I-arrive_date.day_number 0.000 0.000 0.000 0.0
B-meal_code 0.000 0.000 0.000 1.0
I-depart_time.period_of_day 1.000 1.000 1.000 1.0
B-day_name 1.000 0.500 0.667 2.0
B-period_of_day 1.000 0.750 0.857 4.0
B-stoploc.state_code 0.000 0.000 0.000 0.0
B-return_date.month_name 0.000 0.000 0.000 0.0
B-return_date.day_number 0.000 0.000 0.000 0.0
B-arrive_time.period_mod 0.000 0.000 0.000 0.0
I-meal_code 0.000 0.000 0.000 0.0
B-toloc.country_name 1.000 1.000 1.000 1.0
B-days_code 0.000 0.000 0.000 1.0
I-arrive_time.period_of_day 0.000 0.000 0.000 0.0
I-today_relative 0.000 0.000 0.000 0.0
B-return_time.period_of_day 0.000 0.000 0.000 0.0
B-time 0.000 0.000 0.000 0.0
I-fare_basis_code 0.000 0.000 0.000 0.0
I-arrive_time.time_relative 1.000 1.000 1.000 4.0
I-depart_time.time_relative 0.000 0.000 0.000 1.0
B-today_relative 0.000 0.000 0.000 0.0
B-state_name 0.000 0.000 0.000 9.0
B-arrive_date.today_relative 0.000 0.000 0.000 0.0
B-return_time.period_mod 0.000 0.000 0.000 0.0
B-month_name 0.000 0.000 0.000 0.0
B-day_number 0.000 0.000 0.000 0.0
I-return_date.date_relative 1.000 0.667 0.800 3.0
I-return_date.today_relative 0.000 0.000 0.000 0.0
B-stoploc.airport_name 0.000 0.000 0.000 0.0
B-time_relative 0.000 0.000 0.000 0.0
I-time 0.000 0.000 0.000 0.0
I-return_date.day_number 0.000 0.000 0.000 0.0
I-meal_description 0.000 0.000 0.000 0.0
B-return_date.today_relative 0.000 0.000 0.000 0.0
B-return_date.day_name 0.000 0.000 0.000 2.0
micro avg 0.959 0.961 0.960 3657.0
macro avg 0.685 0.657 0.657 3657.0
weighted avg 0.963 0.961 0.957 3657.0
I0716 05:30:20.019577 139795883440000 interactiveshell.py:2882] Best Slot F1: 0.960
I0716 05:30:20.023614 139795883440000 estimator_training.py:186] Not using Distribute Coordinator.
I0716 05:30:20.025929 139795883440000 training.py:612] Running training and evaluation locally (non-distributed).
I0716 05:30:20.027685 139795883440000 training.py:700] Start train and evaluate loop. The evaluate will happen after every checkpoint. Checkpoint frequency is determined based on RunConfig arguments: save_checkpoints_steps 312 or save_checkpoints_secs None.
I0716 05:30:20.102035 139795883440000 estimator.py:1145] Calling model_fn.
I0716 05:30:21.588777 139795883440000 saver.py:1499] Saver not created because there are no variables in the graph to restore
I0716 05:30:22.181025 139795883440000 <ipython-input-6-d6747b40360b>:55]
[<tf.Variable 'embedding:0' shape=(750, 300) dtype=float32_ref>,
<tf.Variable 'module/aggregation/weights:0' shape=(3,) dtype=float32>,
<tf.Variable 'module/aggregation/scaling:0' shape=() dtype=float32>,
<tf.Variable 'dense/kernel:0' shape=(1324, 300) dtype=float32_ref>,
<tf.Variable 'dense/bias:0' shape=(300,) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/fw/gru_cell/gates/kernel:0' shape=(600, 600) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/fw/gru_cell/gates/bias:0' shape=(600,) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/fw/gru_cell/candidate/kernel:0' shape=(600, 300) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/fw/gru_cell/candidate/bias:0' shape=(300,) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/bw/gru_cell/gates/kernel:0' shape=(600, 600) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/bw/gru_cell/gates/bias:0' shape=(600,) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/bw/gru_cell/candidate/kernel:0' shape=(600, 300) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/bw/gru_cell/candidate/bias:0' shape=(300,) dtype=float32_ref>,
<tf.Variable 'dense_1/kernel:0' shape=(600, 23) dtype=float32_ref>,
<tf.Variable 'dense_1/bias:0' shape=(23,) dtype=float32_ref>,
<tf.Variable 'dense_2/kernel:0' shape=(600, 122) dtype=float32_ref>,
<tf.Variable 'dense_2/bias:0' shape=(122,) dtype=float32_ref>]
I0716 05:30:23.469295 139795883440000 estimator.py:1147] Done calling model_fn.
I0716 05:30:23.473536 139795883440000 basic_session_run_hooks.py:541] Create CheckpointSaverHook.
I0716 05:30:23.794810 139795883440000 monitored_session.py:240] Graph was finalized.
I0716 05:30:23.816685 139795883440000 saver.py:1280] Restoring parameters from ../model/elmo_bigru/model.ckpt-12792
I0716 05:30:24.864028 139795883440000 session_manager.py:500] Running local_init_op.
I0716 05:30:24.939088 139795883440000 session_manager.py:502] Done running local_init_op.
I0716 05:30:27.726537 139795883440000 basic_session_run_hooks.py:606] Saving checkpoints for 12792 into ../model/elmo_bigru/model.ckpt.
Reading ../data/atis.train.w-intent.iob
I0716 05:30:32.478336 139795883440000 basic_session_run_hooks.py:262] loss = 0.010260903, step = 12792 I0716 05:30:32.480191 139795883440000 basic_session_run_hooks.py:262] lr = 7.7945544e-05 I0716 05:31:00.489519 139795883440000 basic_session_run_hooks.py:692] global_step/sec: 3.56993 I0716 05:31:00.497004 139795883440000 basic_session_run_hooks.py:260] loss = 0.009084362, step = 12892 (28.019 sec) I0716 05:31:00.499659 139795883440000 basic_session_run_hooks.py:260] lr = 7.712862e-05 (28.019 sec) I0716 05:31:26.556809 139795883440000 basic_session_run_hooks.py:692] global_step/sec: 3.83623 I0716 05:31:26.560199 139795883440000 basic_session_run_hooks.py:260] loss = 0.008441718, step = 12992 (26.063 sec) I0716 05:31:26.567608 139795883440000 basic_session_run_hooks.py:260] lr = 7.632024e-05 (26.068 sec) I0716 05:31:53.002555 139795883440000 basic_session_run_hooks.py:692] global_step/sec: 3.78133 I0716 05:31:53.007514 139795883440000 basic_session_run_hooks.py:260] loss = 0.012072688, step = 13092 (26.447 sec) I0716 05:31:53.012902 139795883440000 basic_session_run_hooks.py:260] lr = 7.552035e-05 (26.445 sec) I0716 05:31:55.986191 139795883440000 basic_session_run_hooks.py:606] Saving checkpoints for 13104 into ../model/elmo_bigru/model.ckpt. I0716 05:31:58.048022 139795883440000 estimator.py:1145] Calling model_fn. I0716 05:31:59.683301 139795883440000 saver.py:1499] Saver not created because there are no variables in the graph to restore I0716 05:32:00.265349 139795883440000 estimator.py:1147] Done calling model_fn. I0716 05:32:00.295489 139795883440000 evaluation.py:255] Starting evaluation at 2019-07-16T05:32:00Z I0716 05:32:00.528330 139795883440000 monitored_session.py:240] Graph was finalized. I0716 05:32:00.546332 139795883440000 saver.py:1280] Restoring parameters from ../model/elmo_bigru/model.ckpt-13104 I0716 05:32:01.450866 139795883440000 session_manager.py:500] Running local_init_op. I0716 05:32:01.516671 139795883440000 session_manager.py:502] Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
I0716 05:33:00.227576 139795883440000 evaluation.py:275] Finished evaluation at 2019-07-16-05:33:00 I0716 05:33:00.229390 139795883440000 estimator.py:2039] Saving dict for global step 13104: global_step = 13104, loss = 0.40517667 I0716 05:33:00.236587 139795883440000 estimator.py:2099] Saving 'checkpoint_path' summary for global step 13104: ../model/elmo_bigru/model.ckpt-13104 I0716 05:33:00.323032 139795883440000 estimator.py:368] Loss for final step: 0.0011228888.
Reading ../data/atis.test.w-intent.iob
I0716 05:33:00.930331 139795883440000 estimator.py:1145] Calling model_fn. I0716 05:33:02.460177 139795883440000 saver.py:1499] Saver not created because there are no variables in the graph to restore I0716 05:33:02.986480 139795883440000 estimator.py:1147] Done calling model_fn. I0716 05:33:03.196828 139795883440000 monitored_session.py:240] Graph was finalized. I0716 05:33:03.215934 139795883440000 saver.py:1280] Restoring parameters from ../model/elmo_bigru/model.ckpt-13104 I0716 05:33:04.122166 139795883440000 session_manager.py:500] Running local_init_op. I0716 05:33:04.172596 139795883440000 session_manager.py:502] Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.
'precision', 'predicted', average, warn_for)
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1439: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples.
'recall', 'true', average, warn_for)
I0716 05:34:02.142135 139795883440000 interactiveshell.py:2882]
precision recall f1-score support
atis_flight 0.984 0.992 0.988 632
atis_airfare 0.960 1.000 0.980 48
atis_ground_service 1.000 1.000 1.000 36
atis_airline 0.974 1.000 0.987 38
atis_abbreviation 0.943 1.000 0.971 33
atis_aircraft 1.000 0.889 0.941 9
atis_flight_time 0.500 1.000 0.667 1
atis_quantity 0.375 1.000 0.545 3
atis_flight#atis_airfare 0.833 0.417 0.556 12
atis_airport 1.000 0.944 0.971 18
atis_distance 1.000 1.000 1.000 10
atis_city 0.750 0.500 0.600 6
atis_ground_fare 1.000 0.857 0.923 7
atis_capacity 0.952 0.952 0.952 21
atis_flight_no 0.889 1.000 0.941 8
atis_meal 1.000 0.833 0.909 6
atis_restriction 0.000 0.000 0.000 0
atis_airline#atis_flight_no 0.000 0.000 0.000 0
atis_ground_service#atis_ground_fare 0.000 0.000 0.000 0
atis_airfare#atis_flight_time 0.000 0.000 0.000 0
atis_cheapest 0.000 0.000 0.000 0
atis_aircraft#atis_flight#atis_flight_no 0.000 0.000 0.000 0
micro avg 0.972 0.977 0.975 888
macro avg 0.644 0.654 0.633 888
weighted avg 0.975 0.977 0.974 888
I0716 05:34:02.187218 139795883440000 interactiveshell.py:2882]
precision recall f1-score support
O 0.000 0.000 0.000 0.0
B-toloc.city_name 0.974 0.994 0.984 716.0
B-fromloc.city_name 0.990 0.999 0.994 704.0
I-toloc.city_name 0.978 0.989 0.983 265.0
B-depart_date.day_name 0.991 0.991 0.991 212.0
B-airline_name 1.000 1.000 1.000 101.0
I-fromloc.city_name 0.973 1.000 0.986 177.0
B-depart_time.period_of_day 1.000 0.915 0.956 130.0
I-airline_name 1.000 1.000 1.000 65.0
B-depart_date.day_number 0.981 0.964 0.972 55.0
B-depart_date.month_name 0.982 0.964 0.973 56.0
B-depart_time.time 0.864 1.000 0.927 57.0
B-round_trip 1.000 1.000 1.000 73.0
B-cost_relative 0.973 0.973 0.973 37.0
I-round_trip 1.000 1.000 1.000 71.0
B-flight_mod 0.828 1.000 0.906 24.0
B-depart_time.time_relative 0.970 0.985 0.977 65.0
I-depart_time.time 0.929 1.000 0.963 52.0
B-stoploc.city_name 0.909 1.000 0.952 20.0
B-city_name 0.923 0.632 0.750 57.0
B-class_type 0.960 1.000 0.980 24.0
B-arrive_time.time 0.971 0.971 0.971 34.0
B-arrive_time.time_relative 0.967 0.935 0.951 31.0
I-class_type 1.000 1.000 1.000 17.0
B-flight_stop 1.000 1.000 1.000 21.0
I-arrive_time.time 1.000 0.971 0.986 35.0
B-airline_code 0.944 1.000 0.971 34.0
I-depart_date.day_number 1.000 0.933 0.966 15.0
I-fromloc.airport_name 0.441 1.000 0.612 15.0
B-fromloc.airport_name 0.462 1.000 0.632 12.0
B-arrive_date.day_name 0.786 1.000 0.880 11.0
B-toloc.state_code 1.000 1.000 1.000 18.0
B-depart_date.today_relative 1.000 0.889 0.941 9.0
B-flight_number 0.733 1.000 0.846 11.0
B-depart_date.date_relative 0.944 1.000 0.971 17.0
B-toloc.state_name 0.833 0.893 0.862 28.0
B-fare_basis_code 0.850 1.000 0.919 17.0
B-flight_time 1.000 1.000 1.000 1.0
B-or 1.000 1.000 1.000 3.0
B-arrive_time.period_of_day 0.750 1.000 0.857 6.0
B-meal_description 1.000 0.900 0.947 10.0
I-cost_relative 1.000 0.667 0.800 3.0
I-airport_name 0.846 0.379 0.524 29.0
B-fare_amount 1.000 1.000 1.000 2.0
I-fare_amount 1.000 1.000 1.000 2.0
I-city_name 1.000 0.533 0.696 30.0
I-toloc.airport_name 1.000 1.000 1.000 3.0
B-transport_type 1.000 1.000 1.000 10.0
B-arrive_date.month_name 0.714 0.833 0.769 6.0
B-arrive_date.day_number 0.714 0.833 0.769 6.0
I-stoploc.city_name 1.000 1.000 1.000 10.0
B-meal 1.000 1.000 1.000 16.0
B-fromloc.state_code 1.000 1.000 1.000 23.0
B-depart_time.period_mod 1.000 1.000 1.000 5.0
B-connect 1.000 1.000 1.000 6.0
B-flight_days 1.000 1.000 1.000 10.0
B-toloc.airport_name 1.000 1.000 1.000 3.0
B-fromloc.state_name 0.944 1.000 0.971 17.0
B-airport_name 0.667 0.381 0.485 21.0
B-economy 1.000 1.000 1.000 6.0
I-flight_time 1.000 1.000 1.000 1.0
B-aircraft_code 1.000 0.879 0.935 33.0
B-mod 1.000 0.500 0.667 2.0
B-airport_code 1.000 0.444 0.615 9.0
B-depart_time.start_time 1.000 0.667 0.800 3.0
B-depart_time.end_time 1.000 0.333 0.500 3.0
B-depart_date.year 1.000 1.000 1.000 3.0
I-transport_type 0.000 0.000 0.000 1.0
B-restriction_code 1.000 1.000 1.000 4.0
B-arrive_time.start_time 0.889 1.000 0.941 8.0
B-toloc.airport_code 1.000 0.750 0.857 4.0
B-arrive_time.end_time 0.889 1.000 0.941 8.0
I-arrive_time.end_time 0.889 1.000 0.941 8.0
I-depart_time.end_time 1.000 0.333 0.500 3.0
I-flight_stop 0.000 0.000 0.000 0.0
B-fromloc.airport_code 0.556 1.000 0.714 5.0
I-restriction_code 1.000 1.000 1.000 3.0
I-depart_time.start_time 1.000 1.000 1.000 1.0
I-toloc.state_name 1.000 1.000 1.000 1.0
I-depart_date.today_relative 0.000 0.000 0.000 0.0
B-arrive_date.date_relative 1.000 1.000 1.000 2.0
I-flight_mod 1.000 0.167 0.286 6.0
I-economy 0.000 0.000 0.000 0.0
B-return_date.date_relative 1.000 0.333 0.500 3.0
I-fromloc.state_name 1.000 1.000 1.000 1.0
B-state_code 1.000 1.000 1.000 1.0
I-arrive_time.start_time 1.000 1.000 1.000 1.0
I-arrive_date.day_number 0.000 0.000 0.000 0.0
B-meal_code 0.000 0.000 0.000 1.0
I-depart_time.period_of_day 1.000 1.000 1.000 1.0
B-day_name 1.000 0.500 0.667 2.0
B-period_of_day 1.000 0.750 0.857 4.0
B-stoploc.state_code 0.000 0.000 0.000 0.0
B-return_date.month_name 0.000 0.000 0.000 0.0
B-return_date.day_number 0.000 0.000 0.000 0.0
B-arrive_time.period_mod 0.000 0.000 0.000 0.0
I-meal_code 0.000 0.000 0.000 0.0
B-toloc.country_name 1.000 1.000 1.000 1.0
B-days_code 0.000 0.000 0.000 1.0
I-arrive_time.period_of_day 0.000 0.000 0.000 0.0
I-today_relative 0.000 0.000 0.000 0.0
B-return_time.period_of_day 0.000 0.000 0.000 0.0
B-time 0.000 0.000 0.000 0.0
I-fare_basis_code 0.000 0.000 0.000 0.0
I-arrive_time.time_relative 1.000 1.000 1.000 4.0
I-depart_time.time_relative 0.000 0.000 0.000 1.0
B-today_relative 0.000 0.000 0.000 0.0
B-state_name 0.000 0.000 0.000 9.0
B-arrive_date.today_relative 0.000 0.000 0.000 0.0
B-return_time.period_mod 0.000 0.000 0.000 0.0
B-month_name 0.000 0.000 0.000 0.0
B-day_number 0.000 0.000 0.000 0.0
I-return_date.date_relative 0.667 0.667 0.667 3.0
I-return_date.today_relative 0.000 0.000 0.000 0.0
B-stoploc.airport_name 0.000 0.000 0.000 0.0
B-time_relative 0.000 0.000 0.000 0.0
I-time 0.000 0.000 0.000 0.0
I-return_date.day_number 0.000 0.000 0.000 0.0
I-meal_description 0.000 0.000 0.000 0.0
B-return_date.today_relative 0.000 0.000 0.000 0.0
B-return_date.day_name 0.000 0.000 0.000 2.0
micro avg 0.958 0.960 0.959 3657.0
macro avg 0.683 0.652 0.653 3657.0
weighted avg 0.963 0.960 0.957 3657.0
I0716 05:34:02.211748 139795883440000 interactiveshell.py:2882] Best Slot F1: 0.960
I0716 05:34:02.213319 139795883440000 estimator_training.py:186] Not using Distribute Coordinator.
I0716 05:34:02.216927 139795883440000 training.py:612] Running training and evaluation locally (non-distributed).
I0716 05:34:02.219686 139795883440000 training.py:700] Start train and evaluate loop. The evaluate will happen after every checkpoint. Checkpoint frequency is determined based on RunConfig arguments: save_checkpoints_steps 312 or save_checkpoints_secs None.
I0716 05:34:02.287338 139795883440000 estimator.py:1145] Calling model_fn.
I0716 05:34:03.786828 139795883440000 saver.py:1499] Saver not created because there are no variables in the graph to restore
I0716 05:34:04.382152 139795883440000 <ipython-input-6-d6747b40360b>:55]
[<tf.Variable 'embedding:0' shape=(750, 300) dtype=float32_ref>,
<tf.Variable 'module/aggregation/weights:0' shape=(3,) dtype=float32>,
<tf.Variable 'module/aggregation/scaling:0' shape=() dtype=float32>,
<tf.Variable 'dense/kernel:0' shape=(1324, 300) dtype=float32_ref>,
<tf.Variable 'dense/bias:0' shape=(300,) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/fw/gru_cell/gates/kernel:0' shape=(600, 600) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/fw/gru_cell/gates/bias:0' shape=(600,) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/fw/gru_cell/candidate/kernel:0' shape=(600, 300) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/fw/gru_cell/candidate/bias:0' shape=(300,) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/bw/gru_cell/gates/kernel:0' shape=(600, 600) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/bw/gru_cell/gates/bias:0' shape=(600,) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/bw/gru_cell/candidate/kernel:0' shape=(600, 300) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/bw/gru_cell/candidate/bias:0' shape=(300,) dtype=float32_ref>,
<tf.Variable 'dense_1/kernel:0' shape=(600, 23) dtype=float32_ref>,
<tf.Variable 'dense_1/bias:0' shape=(23,) dtype=float32_ref>,
<tf.Variable 'dense_2/kernel:0' shape=(600, 122) dtype=float32_ref>,
<tf.Variable 'dense_2/bias:0' shape=(122,) dtype=float32_ref>]
I0716 05:34:05.677148 139795883440000 estimator.py:1147] Done calling model_fn.
I0716 05:34:05.680834 139795883440000 basic_session_run_hooks.py:541] Create CheckpointSaverHook.
I0716 05:34:06.006551 139795883440000 monitored_session.py:240] Graph was finalized.
I0716 05:34:06.029321 139795883440000 saver.py:1280] Restoring parameters from ../model/elmo_bigru/model.ckpt-13104
I0716 05:34:07.111840 139795883440000 session_manager.py:500] Running local_init_op.
I0716 05:34:07.186209 139795883440000 session_manager.py:502] Done running local_init_op.
I0716 05:34:10.109262 139795883440000 basic_session_run_hooks.py:606] Saving checkpoints for 13104 into ../model/elmo_bigru/model.ckpt.
Reading ../data/atis.train.w-intent.iob
I0716 05:34:15.083790 139795883440000 basic_session_run_hooks.py:262] loss = 0.008863335, step = 13104 I0716 05:34:15.085566 139795883440000 basic_session_run_hooks.py:262] lr = 7.542493e-05 I0716 05:34:42.543205 139795883440000 basic_session_run_hooks.py:692] global_step/sec: 3.64162 I0716 05:34:42.549776 139795883440000 basic_session_run_hooks.py:260] loss = 0.008364289, step = 13204 (27.466 sec) I0716 05:34:42.552909 139795883440000 basic_session_run_hooks.py:260] lr = 7.463443e-05 (27.467 sec) I0716 05:35:09.690642 139795883440000 basic_session_run_hooks.py:692] global_step/sec: 3.68359 I0716 05:35:09.699984 139795883440000 basic_session_run_hooks.py:260] loss = 0.011803856, step = 13304 (27.150 sec) I0716 05:35:09.701951 139795883440000 basic_session_run_hooks.py:260] lr = 7.385219e-05 (27.149 sec) I0716 05:35:35.376586 139795883440000 basic_session_run_hooks.py:692] global_step/sec: 3.89318 I0716 05:35:35.386376 139795883440000 basic_session_run_hooks.py:260] loss = 0.012592914, step = 13404 (25.686 sec) I0716 05:35:35.388249 139795883440000 basic_session_run_hooks.py:260] lr = 7.3078176e-05 (25.686 sec) I0716 05:35:38.137051 139795883440000 basic_session_run_hooks.py:606] Saving checkpoints for 13416 into ../model/elmo_bigru/model.ckpt. I0716 05:35:40.099873 139795883440000 estimator.py:1145] Calling model_fn. I0716 05:35:41.769351 139795883440000 saver.py:1499] Saver not created because there are no variables in the graph to restore I0716 05:35:42.353270 139795883440000 estimator.py:1147] Done calling model_fn. I0716 05:35:42.386612 139795883440000 evaluation.py:255] Starting evaluation at 2019-07-16T05:35:42Z I0716 05:35:42.613936 139795883440000 monitored_session.py:240] Graph was finalized. I0716 05:35:42.632848 139795883440000 saver.py:1280] Restoring parameters from ../model/elmo_bigru/model.ckpt-13416 I0716 05:35:43.552337 139795883440000 session_manager.py:500] Running local_init_op. I0716 05:35:43.637071 139795883440000 session_manager.py:502] Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
I0716 05:36:42.807984 139795883440000 evaluation.py:275] Finished evaluation at 2019-07-16-05:36:42 I0716 05:36:42.809957 139795883440000 estimator.py:2039] Saving dict for global step 13416: global_step = 13416, loss = 0.41098255 I0716 05:36:42.819127 139795883440000 estimator.py:2099] Saving 'checkpoint_path' summary for global step 13416: ../model/elmo_bigru/model.ckpt-13416 I0716 05:36:42.902808 139795883440000 estimator.py:368] Loss for final step: 0.013912404.
Reading ../data/atis.test.w-intent.iob
I0716 05:36:43.498341 139795883440000 estimator.py:1145] Calling model_fn. I0716 05:36:45.058119 139795883440000 saver.py:1499] Saver not created because there are no variables in the graph to restore I0716 05:36:45.578801 139795883440000 estimator.py:1147] Done calling model_fn. I0716 05:36:45.795963 139795883440000 monitored_session.py:240] Graph was finalized. I0716 05:36:45.814599 139795883440000 saver.py:1280] Restoring parameters from ../model/elmo_bigru/model.ckpt-13416 I0716 05:36:46.731653 139795883440000 session_manager.py:500] Running local_init_op. I0716 05:36:46.780838 139795883440000 session_manager.py:502] Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.
'precision', 'predicted', average, warn_for)
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1439: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples.
'recall', 'true', average, warn_for)
I0716 05:37:44.828861 139795883440000 interactiveshell.py:2882]
precision recall f1-score support
atis_flight 0.986 0.992 0.989 632
atis_airfare 0.923 1.000 0.960 48
atis_ground_service 1.000 1.000 1.000 36
atis_airline 0.974 1.000 0.987 38
atis_abbreviation 0.943 1.000 0.971 33
atis_aircraft 1.000 0.889 0.941 9
atis_flight_time 0.500 1.000 0.667 1
atis_quantity 0.429 1.000 0.600 3
atis_flight#atis_airfare 0.833 0.417 0.556 12
atis_airport 1.000 0.944 0.971 18
atis_distance 1.000 1.000 1.000 10
atis_city 0.750 0.500 0.600 6
atis_ground_fare 1.000 0.857 0.923 7
atis_capacity 0.952 0.952 0.952 21
atis_flight_no 0.889 1.000 0.941 8
atis_meal 1.000 0.833 0.909 6
atis_restriction 0.000 0.000 0.000 0
atis_airline#atis_flight_no 0.000 0.000 0.000 0
atis_ground_service#atis_ground_fare 0.000 0.000 0.000 0
atis_airfare#atis_flight_time 0.000 0.000 0.000 0
atis_cheapest 0.000 0.000 0.000 0
atis_aircraft#atis_flight#atis_flight_no 0.000 0.000 0.000 0
micro avg 0.972 0.977 0.975 888
macro avg 0.645 0.654 0.635 888
weighted avg 0.974 0.977 0.974 888
I0716 05:37:44.868015 139795883440000 interactiveshell.py:2882]
precision recall f1-score support
O 0.000 0.000 0.000 0.0
B-toloc.city_name 0.973 0.996 0.984 716.0
B-fromloc.city_name 0.992 0.999 0.995 704.0
I-toloc.city_name 0.978 0.992 0.985 265.0
B-depart_date.day_name 0.991 0.991 0.991 212.0
B-airline_name 1.000 1.000 1.000 101.0
I-fromloc.city_name 0.983 1.000 0.992 177.0
B-depart_time.period_of_day 0.992 0.923 0.956 130.0
I-airline_name 1.000 1.000 1.000 65.0
B-depart_date.day_number 0.981 0.964 0.972 55.0
B-depart_date.month_name 0.982 0.964 0.973 56.0
B-depart_time.time 0.864 1.000 0.927 57.0
B-round_trip 1.000 0.986 0.993 73.0
B-cost_relative 0.973 0.973 0.973 37.0
I-round_trip 1.000 1.000 1.000 71.0
B-flight_mod 0.828 1.000 0.906 24.0
B-depart_time.time_relative 0.970 0.985 0.977 65.0
I-depart_time.time 0.945 1.000 0.972 52.0
B-stoploc.city_name 0.909 1.000 0.952 20.0
B-city_name 0.897 0.614 0.729 57.0
B-class_type 0.960 1.000 0.980 24.0
B-arrive_time.time 0.971 0.971 0.971 34.0
B-arrive_time.time_relative 0.906 0.935 0.921 31.0
I-class_type 1.000 1.000 1.000 17.0
B-flight_stop 1.000 1.000 1.000 21.0
I-arrive_time.time 1.000 0.971 0.986 35.0
B-airline_code 0.944 1.000 0.971 34.0
I-depart_date.day_number 1.000 0.933 0.966 15.0
I-fromloc.airport_name 0.441 1.000 0.612 15.0
B-fromloc.airport_name 0.462 1.000 0.632 12.0
B-arrive_date.day_name 0.786 1.000 0.880 11.0
B-toloc.state_code 1.000 1.000 1.000 18.0
B-depart_date.today_relative 1.000 0.889 0.941 9.0
B-flight_number 0.733 1.000 0.846 11.0
B-depart_date.date_relative 0.944 1.000 0.971 17.0
B-toloc.state_name 0.833 0.893 0.862 28.0
B-fare_basis_code 0.895 1.000 0.944 17.0
B-flight_time 1.000 1.000 1.000 1.0
B-or 1.000 1.000 1.000 3.0
B-arrive_time.period_of_day 0.750 1.000 0.857 6.0
B-meal_description 1.000 0.900 0.947 10.0
I-cost_relative 1.000 0.667 0.800 3.0
I-airport_name 0.846 0.379 0.524 29.0
B-fare_amount 1.000 1.000 1.000 2.0
I-fare_amount 1.000 1.000 1.000 2.0
I-city_name 1.000 0.567 0.723 30.0
I-toloc.airport_name 1.000 1.000 1.000 3.0
B-transport_type 1.000 1.000 1.000 10.0
B-arrive_date.month_name 0.714 0.833 0.769 6.0
B-arrive_date.day_number 0.714 0.833 0.769 6.0
I-stoploc.city_name 1.000 1.000 1.000 10.0
B-meal 1.000 1.000 1.000 16.0
B-fromloc.state_code 1.000 1.000 1.000 23.0
B-depart_time.period_mod 1.000 1.000 1.000 5.0
B-connect 1.000 1.000 1.000 6.0
B-flight_days 1.000 1.000 1.000 10.0
B-toloc.airport_name 1.000 1.000 1.000 3.0
B-fromloc.state_name 0.944 1.000 0.971 17.0
B-airport_name 0.636 0.333 0.437 21.0
B-economy 1.000 1.000 1.000 6.0
I-flight_time 1.000 1.000 1.000 1.0
B-aircraft_code 1.000 0.879 0.935 33.0
B-mod 1.000 0.500 0.667 2.0
B-airport_code 1.000 0.444 0.615 9.0
B-depart_time.start_time 1.000 0.667 0.800 3.0
B-depart_time.end_time 1.000 0.333 0.500 3.0
B-depart_date.year 1.000 1.000 1.000 3.0
I-transport_type 0.000 0.000 0.000 1.0
B-restriction_code 1.000 1.000 1.000 4.0
B-arrive_time.start_time 0.889 1.000 0.941 8.0
B-toloc.airport_code 1.000 0.750 0.857 4.0
B-arrive_time.end_time 0.889 1.000 0.941 8.0
I-arrive_time.end_time 0.889 1.000 0.941 8.0
I-depart_time.end_time 1.000 0.333 0.500 3.0
I-flight_stop 0.000 0.000 0.000 0.0
B-fromloc.airport_code 0.556 1.000 0.714 5.0
I-restriction_code 1.000 1.000 1.000 3.0
I-depart_time.start_time 1.000 1.000 1.000 1.0
I-toloc.state_name 1.000 1.000 1.000 1.0
I-depart_date.today_relative 0.000 0.000 0.000 0.0
B-arrive_date.date_relative 1.000 1.000 1.000 2.0
I-flight_mod 1.000 0.167 0.286 6.0
I-economy 0.000 0.000 0.000 0.0
B-return_date.date_relative 1.000 0.333 0.500 3.0
I-fromloc.state_name 1.000 1.000 1.000 1.0
B-state_code 1.000 1.000 1.000 1.0
I-arrive_time.start_time 1.000 1.000 1.000 1.0
I-arrive_date.day_number 0.000 0.000 0.000 0.0
B-meal_code 0.000 0.000 0.000 1.0
I-depart_time.period_of_day 1.000 1.000 1.000 1.0
B-day_name 1.000 0.500 0.667 2.0
B-period_of_day 1.000 0.750 0.857 4.0
B-stoploc.state_code 0.000 0.000 0.000 0.0
B-return_date.month_name 0.000 0.000 0.000 0.0
B-return_date.day_number 0.000 0.000 0.000 0.0
B-arrive_time.period_mod 0.000 0.000 0.000 0.0
I-meal_code 0.000 0.000 0.000 0.0
B-toloc.country_name 1.000 1.000 1.000 1.0
B-days_code 1.000 1.000 1.000 1.0
I-arrive_time.period_of_day 0.000 0.000 0.000 0.0
I-today_relative 0.000 0.000 0.000 0.0
B-return_time.period_of_day 0.000 0.000 0.000 0.0
B-time 0.000 0.000 0.000 0.0
I-fare_basis_code 0.000 0.000 0.000 0.0
I-arrive_time.time_relative 1.000 0.500 0.667 4.0
I-depart_time.time_relative 0.000 0.000 0.000 1.0
B-today_relative 0.000 0.000 0.000 0.0
B-state_name 0.000 0.000 0.000 9.0
B-arrive_date.today_relative 0.000 0.000 0.000 0.0
B-return_time.period_mod 0.000 0.000 0.000 0.0
B-month_name 0.000 0.000 0.000 0.0
B-day_number 0.000 0.000 0.000 0.0
I-return_date.date_relative 0.667 0.667 0.667 3.0
I-return_date.today_relative 0.000 0.000 0.000 0.0
B-stoploc.airport_name 0.000 0.000 0.000 0.0
B-time_relative 0.000 0.000 0.000 0.0
I-time 0.000 0.000 0.000 0.0
I-return_date.day_number 0.000 0.000 0.000 0.0
I-meal_description 0.000 0.000 0.000 0.0
B-return_date.today_relative 0.000 0.000 0.000 0.0
B-return_date.day_name 0.000 0.000 0.000 2.0
micro avg 0.958 0.960 0.959 3657.0
macro avg 0.691 0.655 0.658 3657.0
weighted avg 0.963 0.960 0.956 3657.0
I0716 05:37:44.891681 139795883440000 interactiveshell.py:2882] Best Slot F1: 0.960
I0716 05:37:44.893238 139795883440000 estimator_training.py:186] Not using Distribute Coordinator.
I0716 05:37:44.895620 139795883440000 training.py:612] Running training and evaluation locally (non-distributed).
I0716 05:37:44.898410 139795883440000 training.py:700] Start train and evaluate loop. The evaluate will happen after every checkpoint. Checkpoint frequency is determined based on RunConfig arguments: save_checkpoints_steps 312 or save_checkpoints_secs None.
I0716 05:37:44.966067 139795883440000 estimator.py:1145] Calling model_fn.
I0716 05:37:46.474880 139795883440000 saver.py:1499] Saver not created because there are no variables in the graph to restore
I0716 05:37:47.060688 139795883440000 <ipython-input-6-d6747b40360b>:55]
[<tf.Variable 'embedding:0' shape=(750, 300) dtype=float32_ref>,
<tf.Variable 'module/aggregation/weights:0' shape=(3,) dtype=float32>,
<tf.Variable 'module/aggregation/scaling:0' shape=() dtype=float32>,
<tf.Variable 'dense/kernel:0' shape=(1324, 300) dtype=float32_ref>,
<tf.Variable 'dense/bias:0' shape=(300,) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/fw/gru_cell/gates/kernel:0' shape=(600, 600) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/fw/gru_cell/gates/bias:0' shape=(600,) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/fw/gru_cell/candidate/kernel:0' shape=(600, 300) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/fw/gru_cell/candidate/bias:0' shape=(300,) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/bw/gru_cell/gates/kernel:0' shape=(600, 600) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/bw/gru_cell/gates/bias:0' shape=(600,) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/bw/gru_cell/candidate/kernel:0' shape=(600, 300) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/bw/gru_cell/candidate/bias:0' shape=(300,) dtype=float32_ref>,
<tf.Variable 'dense_1/kernel:0' shape=(600, 23) dtype=float32_ref>,
<tf.Variable 'dense_1/bias:0' shape=(23,) dtype=float32_ref>,
<tf.Variable 'dense_2/kernel:0' shape=(600, 122) dtype=float32_ref>,
<tf.Variable 'dense_2/bias:0' shape=(122,) dtype=float32_ref>]
I0716 05:37:48.348798 139795883440000 estimator.py:1147] Done calling model_fn.
I0716 05:37:48.357215 139795883440000 basic_session_run_hooks.py:541] Create CheckpointSaverHook.
I0716 05:37:48.674329 139795883440000 monitored_session.py:240] Graph was finalized.
I0716 05:37:48.693415 139795883440000 saver.py:1280] Restoring parameters from ../model/elmo_bigru/model.ckpt-13416
I0716 05:37:49.772269 139795883440000 session_manager.py:500] Running local_init_op.
I0716 05:37:49.842147 139795883440000 session_manager.py:502] Done running local_init_op.
I0716 05:37:52.684488 139795883440000 basic_session_run_hooks.py:606] Saving checkpoints for 13416 into ../model/elmo_bigru/model.ckpt.
Reading ../data/atis.train.w-intent.iob
I0716 05:37:57.513536 139795883440000 basic_session_run_hooks.py:262] loss = 0.013887277, step = 13416 I0716 05:37:57.515413 139795883440000 basic_session_run_hooks.py:262] lr = 7.298584e-05 I0716 05:38:24.983728 139795883440000 basic_session_run_hooks.py:692] global_step/sec: 3.64024 I0716 05:38:24.987360 139795883440000 basic_session_run_hooks.py:260] loss = 0.012135924, step = 13516 (27.474 sec) I0716 05:38:24.992357 139795883440000 basic_session_run_hooks.py:260] lr = 7.2220886e-05 (27.477 sec) I0716 05:38:51.416158 139795883440000 basic_session_run_hooks.py:692] global_step/sec: 3.78322 I0716 05:38:51.424153 139795883440000 basic_session_run_hooks.py:260] loss = 0.01026399, step = 13616 (26.437 sec) I0716 05:38:51.426458 139795883440000 basic_session_run_hooks.py:260] lr = 7.146397e-05 (26.434 sec) I0716 05:39:18.190498 139795883440000 basic_session_run_hooks.py:692] global_step/sec: 3.73492 I0716 05:39:18.199103 139795883440000 basic_session_run_hooks.py:260] loss = 0.0123380385, step = 13716 (26.775 sec) I0716 05:39:18.201380 139795883440000 basic_session_run_hooks.py:260] lr = 7.071497e-05 (26.775 sec) I0716 05:39:20.919991 139795883440000 basic_session_run_hooks.py:606] Saving checkpoints for 13728 into ../model/elmo_bigru/model.ckpt. I0716 05:39:23.175500 139795883440000 estimator.py:1145] Calling model_fn. I0716 05:39:24.846983 139795883440000 saver.py:1499] Saver not created because there are no variables in the graph to restore I0716 05:39:25.436039 139795883440000 estimator.py:1147] Done calling model_fn. I0716 05:39:25.467047 139795883440000 evaluation.py:255] Starting evaluation at 2019-07-16T05:39:25Z I0716 05:39:25.705051 139795883440000 monitored_session.py:240] Graph was finalized. I0716 05:39:25.720287 139795883440000 saver.py:1280] Restoring parameters from ../model/elmo_bigru/model.ckpt-13728 I0716 05:39:26.682774 139795883440000 session_manager.py:500] Running local_init_op. I0716 05:39:26.746172 139795883440000 session_manager.py:502] Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
I0716 05:40:25.841186 139795883440000 evaluation.py:275] Finished evaluation at 2019-07-16-05:40:25 I0716 05:40:25.843400 139795883440000 estimator.py:2039] Saving dict for global step 13728: global_step = 13728, loss = 0.40032935 I0716 05:40:25.851620 139795883440000 estimator.py:2099] Saving 'checkpoint_path' summary for global step 13728: ../model/elmo_bigru/model.ckpt-13728 I0716 05:40:25.939419 139795883440000 estimator.py:368] Loss for final step: 0.012668098.
Reading ../data/atis.test.w-intent.iob
I0716 05:40:26.538111 139795883440000 estimator.py:1145] Calling model_fn. I0716 05:40:28.083170 139795883440000 saver.py:1499] Saver not created because there are no variables in the graph to restore I0716 05:40:28.598452 139795883440000 estimator.py:1147] Done calling model_fn. I0716 05:40:28.815289 139795883440000 monitored_session.py:240] Graph was finalized. I0716 05:40:28.834128 139795883440000 saver.py:1280] Restoring parameters from ../model/elmo_bigru/model.ckpt-13728 I0716 05:40:29.743818 139795883440000 session_manager.py:500] Running local_init_op. I0716 05:40:29.792791 139795883440000 session_manager.py:502] Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.
'precision', 'predicted', average, warn_for)
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1439: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples.
'recall', 'true', average, warn_for)
I0716 05:41:27.823527 139795883440000 interactiveshell.py:2882]
precision recall f1-score support
atis_flight 0.987 0.991 0.989 632
atis_airfare 0.960 1.000 0.980 48
atis_ground_service 1.000 1.000 1.000 36
atis_airline 0.974 1.000 0.987 38
atis_abbreviation 0.943 1.000 0.971 33
atis_aircraft 1.000 0.889 0.941 9
atis_flight_time 0.333 1.000 0.500 1
atis_quantity 0.375 1.000 0.545 3
atis_flight#atis_airfare 0.833 0.417 0.556 12
atis_airport 0.944 0.944 0.944 18
atis_distance 1.000 1.000 1.000 10
atis_city 0.800 0.667 0.727 6
atis_ground_fare 1.000 0.857 0.923 7
atis_capacity 0.952 0.952 0.952 21
atis_flight_no 0.889 1.000 0.941 8
atis_meal 1.000 0.833 0.909 6
atis_restriction 0.000 0.000 0.000 0
atis_airline#atis_flight_no 0.000 0.000 0.000 0
atis_ground_service#atis_ground_fare 0.000 0.000 0.000 0
atis_airfare#atis_flight_time 0.000 0.000 0.000 0
atis_cheapest 0.000 0.000 0.000 0
atis_aircraft#atis_flight#atis_flight_no 0.000 0.000 0.000 0
micro avg 0.972 0.977 0.975 888
macro avg 0.636 0.661 0.630 888
weighted avg 0.976 0.977 0.975 888
I0716 05:41:27.862074 139795883440000 interactiveshell.py:2882]
precision recall f1-score support
O 0.000 0.000 0.000 0.0
B-toloc.city_name 0.974 0.992 0.983 716.0
B-fromloc.city_name 0.985 1.000 0.992 704.0
I-toloc.city_name 0.978 0.985 0.981 265.0
B-depart_date.day_name 0.991 0.991 0.991 212.0
B-airline_name 1.000 1.000 1.000 101.0
I-fromloc.city_name 0.967 1.000 0.983 177.0
B-depart_time.period_of_day 0.992 0.923 0.956 130.0
I-airline_name 0.985 1.000 0.992 65.0
B-depart_date.day_number 0.981 0.964 0.972 55.0
B-depart_date.month_name 0.982 0.964 0.973 56.0
B-depart_time.time 0.864 1.000 0.927 57.0
B-round_trip 1.000 1.000 1.000 73.0
B-cost_relative 0.973 0.973 0.973 37.0
I-round_trip 1.000 1.000 1.000 71.0
B-flight_mod 0.828 1.000 0.906 24.0
B-depart_time.time_relative 0.970 0.985 0.977 65.0
I-depart_time.time 0.945 1.000 0.972 52.0
B-stoploc.city_name 0.952 1.000 0.976 20.0
B-city_name 0.946 0.614 0.745 57.0
B-class_type 0.960 1.000 0.980 24.0
B-arrive_time.time 0.971 0.971 0.971 34.0
B-arrive_time.time_relative 0.967 0.935 0.951 31.0
I-class_type 1.000 1.000 1.000 17.0
B-flight_stop 1.000 1.000 1.000 21.0
I-arrive_time.time 1.000 0.971 0.986 35.0
B-airline_code 0.944 1.000 0.971 34.0
I-depart_date.day_number 1.000 0.933 0.966 15.0
I-fromloc.airport_name 0.441 1.000 0.612 15.0
B-fromloc.airport_name 0.462 1.000 0.632 12.0
B-arrive_date.day_name 0.786 1.000 0.880 11.0
B-toloc.state_code 1.000 1.000 1.000 18.0
B-depart_date.today_relative 1.000 0.889 0.941 9.0
B-flight_number 0.733 1.000 0.846 11.0
B-depart_date.date_relative 0.944 1.000 0.971 17.0
B-toloc.state_name 0.862 0.893 0.877 28.0
B-fare_basis_code 0.850 1.000 0.919 17.0
B-flight_time 1.000 1.000 1.000 1.0
B-or 1.000 1.000 1.000 3.0
B-arrive_time.period_of_day 0.750 1.000 0.857 6.0
B-meal_description 1.000 0.900 0.947 10.0
I-cost_relative 1.000 0.667 0.800 3.0
I-airport_name 0.846 0.379 0.524 29.0
B-fare_amount 1.000 1.000 1.000 2.0
I-fare_amount 1.000 1.000 1.000 2.0
I-city_name 1.000 0.533 0.696 30.0
I-toloc.airport_name 1.000 1.000 1.000 3.0
B-transport_type 1.000 1.000 1.000 10.0
B-arrive_date.month_name 0.714 0.833 0.769 6.0
B-arrive_date.day_number 0.714 0.833 0.769 6.0
I-stoploc.city_name 1.000 1.000 1.000 10.0
B-meal 1.000 1.000 1.000 16.0
B-fromloc.state_code 1.000 1.000 1.000 23.0
B-depart_time.period_mod 1.000 1.000 1.000 5.0
B-connect 1.000 1.000 1.000 6.0
B-flight_days 1.000 1.000 1.000 10.0
B-toloc.airport_name 1.000 1.000 1.000 3.0
B-fromloc.state_name 0.895 1.000 0.944 17.0
B-airport_name 0.667 0.381 0.485 21.0
B-economy 1.000 1.000 1.000 6.0
I-flight_time 1.000 1.000 1.000 1.0
B-aircraft_code 1.000 0.879 0.935 33.0
B-mod 1.000 0.500 0.667 2.0
B-airport_code 1.000 0.444 0.615 9.0
B-depart_time.start_time 1.000 0.667 0.800 3.0
B-depart_time.end_time 1.000 0.333 0.500 3.0
B-depart_date.year 1.000 1.000 1.000 3.0
I-transport_type 0.000 0.000 0.000 1.0
B-restriction_code 1.000 1.000 1.000 4.0
B-arrive_time.start_time 0.889 1.000 0.941 8.0
B-toloc.airport_code 1.000 0.750 0.857 4.0
B-arrive_time.end_time 0.889 1.000 0.941 8.0
I-arrive_time.end_time 0.889 1.000 0.941 8.0
I-depart_time.end_time 1.000 0.333 0.500 3.0
I-flight_stop 0.000 0.000 0.000 0.0
B-fromloc.airport_code 0.556 1.000 0.714 5.0
I-restriction_code 1.000 1.000 1.000 3.0
I-depart_time.start_time 1.000 1.000 1.000 1.0
I-toloc.state_name 1.000 1.000 1.000 1.0
I-depart_date.today_relative 0.000 0.000 0.000 0.0
B-arrive_date.date_relative 1.000 1.000 1.000 2.0
I-flight_mod 1.000 0.167 0.286 6.0
I-economy 0.000 0.000 0.000 0.0
B-return_date.date_relative 1.000 0.333 0.500 3.0
I-fromloc.state_name 1.000 1.000 1.000 1.0
B-state_code 1.000 1.000 1.000 1.0
I-arrive_time.start_time 1.000 1.000 1.000 1.0
I-arrive_date.day_number 0.000 0.000 0.000 0.0
B-meal_code 0.000 0.000 0.000 1.0
I-depart_time.period_of_day 1.000 1.000 1.000 1.0
B-day_name 1.000 0.500 0.667 2.0
B-period_of_day 1.000 0.750 0.857 4.0
B-stoploc.state_code 0.000 0.000 0.000 0.0
B-return_date.month_name 0.000 0.000 0.000 0.0
B-return_date.day_number 0.000 0.000 0.000 0.0
B-arrive_time.period_mod 0.000 0.000 0.000 0.0
I-meal_code 0.000 0.000 0.000 0.0
B-toloc.country_name 0.000 0.000 0.000 1.0
B-days_code 0.000 0.000 0.000 1.0
I-arrive_time.period_of_day 0.000 0.000 0.000 0.0
I-today_relative 0.000 0.000 0.000 0.0
B-return_time.period_of_day 0.000 0.000 0.000 0.0
B-time 0.000 0.000 0.000 0.0
I-fare_basis_code 0.000 0.000 0.000 0.0
I-arrive_time.time_relative 1.000 1.000 1.000 4.0
I-depart_time.time_relative 0.000 0.000 0.000 1.0
B-today_relative 0.000 0.000 0.000 0.0
B-state_name 0.000 0.000 0.000 9.0
B-arrive_date.today_relative 0.000 0.000 0.000 0.0
B-return_time.period_mod 0.000 0.000 0.000 0.0
B-month_name 0.000 0.000 0.000 0.0
B-day_number 0.000 0.000 0.000 0.0
I-return_date.date_relative 0.667 0.667 0.667 3.0
I-return_date.today_relative 0.000 0.000 0.000 0.0
B-stoploc.airport_name 0.000 0.000 0.000 0.0
B-time_relative 0.000 0.000 0.000 0.0
I-time 0.000 0.000 0.000 0.0
I-return_date.day_number 0.000 0.000 0.000 0.0
I-meal_description 0.000 0.000 0.000 0.0
B-return_date.today_relative 0.000 0.000 0.000 0.0
B-return_date.day_name 0.000 0.000 0.000 2.0
micro avg 0.957 0.959 0.958 3657.0
macro avg 0.675 0.643 0.644 3657.0
weighted avg 0.961 0.959 0.955 3657.0
I0716 05:41:27.884293 139795883440000 interactiveshell.py:2882] Best Slot F1: 0.960
I0716 05:41:27.885876 139795883440000 estimator_training.py:186] Not using Distribute Coordinator.
I0716 05:41:27.887674 139795883440000 training.py:612] Running training and evaluation locally (non-distributed).
I0716 05:41:27.889172 139795883440000 training.py:700] Start train and evaluate loop. The evaluate will happen after every checkpoint. Checkpoint frequency is determined based on RunConfig arguments: save_checkpoints_steps 312 or save_checkpoints_secs None.
I0716 05:41:27.958157 139795883440000 estimator.py:1145] Calling model_fn.
I0716 05:41:29.460845 139795883440000 saver.py:1499] Saver not created because there are no variables in the graph to restore
I0716 05:41:30.036725 139795883440000 <ipython-input-6-d6747b40360b>:55]
[<tf.Variable 'embedding:0' shape=(750, 300) dtype=float32_ref>,
<tf.Variable 'module/aggregation/weights:0' shape=(3,) dtype=float32>,
<tf.Variable 'module/aggregation/scaling:0' shape=() dtype=float32>,
<tf.Variable 'dense/kernel:0' shape=(1324, 300) dtype=float32_ref>,
<tf.Variable 'dense/bias:0' shape=(300,) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/fw/gru_cell/gates/kernel:0' shape=(600, 600) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/fw/gru_cell/gates/bias:0' shape=(600,) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/fw/gru_cell/candidate/kernel:0' shape=(600, 300) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/fw/gru_cell/candidate/bias:0' shape=(300,) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/bw/gru_cell/gates/kernel:0' shape=(600, 600) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/bw/gru_cell/gates/bias:0' shape=(600,) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/bw/gru_cell/candidate/kernel:0' shape=(600, 300) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/bw/gru_cell/candidate/bias:0' shape=(300,) dtype=float32_ref>,
<tf.Variable 'dense_1/kernel:0' shape=(600, 23) dtype=float32_ref>,
<tf.Variable 'dense_1/bias:0' shape=(23,) dtype=float32_ref>,
<tf.Variable 'dense_2/kernel:0' shape=(600, 122) dtype=float32_ref>,
<tf.Variable 'dense_2/bias:0' shape=(122,) dtype=float32_ref>]
I0716 05:41:31.329411 139795883440000 estimator.py:1147] Done calling model_fn.
I0716 05:41:31.334978 139795883440000 basic_session_run_hooks.py:541] Create CheckpointSaverHook.
I0716 05:41:31.666571 139795883440000 monitored_session.py:240] Graph was finalized.
I0716 05:41:31.685984 139795883440000 saver.py:1280] Restoring parameters from ../model/elmo_bigru/model.ckpt-13728
I0716 05:41:32.754310 139795883440000 session_manager.py:500] Running local_init_op.
I0716 05:41:32.839430 139795883440000 session_manager.py:502] Done running local_init_op.
I0716 05:41:35.610599 139795883440000 basic_session_run_hooks.py:606] Saving checkpoints for 13728 into ../model/elmo_bigru/model.ckpt.
Reading ../data/atis.train.w-intent.iob
I0716 05:41:40.410586 139795883440000 basic_session_run_hooks.py:262] loss = 0.010003799, step = 13728 I0716 05:41:40.412563 139795883440000 basic_session_run_hooks.py:262] lr = 7.062562e-05 I0716 05:42:08.743339 139795883440000 basic_session_run_hooks.py:692] global_step/sec: 3.52939 I0716 05:42:08.751814 139795883440000 basic_session_run_hooks.py:260] loss = 0.009100836, step = 13828 (28.341 sec) I0716 05:42:08.753760 139795883440000 basic_session_run_hooks.py:260] lr = 6.98854e-05 (28.341 sec) I0716 05:42:35.140239 139795883440000 basic_session_run_hooks.py:692] global_step/sec: 3.78833 I0716 05:42:35.145344 139795883440000 basic_session_run_hooks.py:260] loss = 0.012343187, step = 13928 (26.394 sec) I0716 05:42:35.150030 139795883440000 basic_session_run_hooks.py:260] lr = 6.915296e-05 (26.396 sec) I0716 05:43:00.936875 139795883440000 basic_session_run_hooks.py:692] global_step/sec: 3.87648 I0716 05:43:00.940275 139795883440000 basic_session_run_hooks.py:260] loss = 0.014199471, step = 14028 (25.795 sec) I0716 05:43:00.944169 139795883440000 basic_session_run_hooks.py:260] lr = 6.842818e-05 (25.794 sec) I0716 05:43:03.559301 139795883440000 basic_session_run_hooks.py:606] Saving checkpoints for 14040 into ../model/elmo_bigru/model.ckpt. I0716 05:43:05.664705 139795883440000 estimator.py:1145] Calling model_fn. I0716 05:43:07.298515 139795883440000 saver.py:1499] Saver not created because there are no variables in the graph to restore I0716 05:43:07.874066 139795883440000 estimator.py:1147] Done calling model_fn. I0716 05:43:07.904097 139795883440000 evaluation.py:255] Starting evaluation at 2019-07-16T05:43:07Z I0716 05:43:08.125681 139795883440000 monitored_session.py:240] Graph was finalized. I0716 05:43:08.143678 139795883440000 saver.py:1280] Restoring parameters from ../model/elmo_bigru/model.ckpt-14040 I0716 05:43:09.050469 139795883440000 session_manager.py:500] Running local_init_op. I0716 05:43:09.107095 139795883440000 session_manager.py:502] Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
I0716 05:44:08.006983 139795883440000 evaluation.py:275] Finished evaluation at 2019-07-16-05:44:08 I0716 05:44:08.008811 139795883440000 estimator.py:2039] Saving dict for global step 14040: global_step = 14040, loss = 0.40883175 I0716 05:44:08.016630 139795883440000 estimator.py:2099] Saving 'checkpoint_path' summary for global step 14040: ../model/elmo_bigru/model.ckpt-14040 I0716 05:44:08.101225 139795883440000 estimator.py:368] Loss for final step: 0.0031542224.
Reading ../data/atis.test.w-intent.iob
I0716 05:44:08.701605 139795883440000 estimator.py:1145] Calling model_fn. I0716 05:44:10.260324 139795883440000 saver.py:1499] Saver not created because there are no variables in the graph to restore I0716 05:44:10.775668 139795883440000 estimator.py:1147] Done calling model_fn. I0716 05:44:11.004148 139795883440000 monitored_session.py:240] Graph was finalized. I0716 05:44:11.021503 139795883440000 saver.py:1280] Restoring parameters from ../model/elmo_bigru/model.ckpt-14040 I0716 05:44:11.922996 139795883440000 session_manager.py:500] Running local_init_op. I0716 05:44:11.984531 139795883440000 session_manager.py:502] Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.
'precision', 'predicted', average, warn_for)
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1439: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples.
'recall', 'true', average, warn_for)
I0716 05:45:10.302753 139795883440000 interactiveshell.py:2882]
precision recall f1-score support
atis_flight 0.983 0.992 0.987 632
atis_airfare 0.941 1.000 0.970 48
atis_ground_service 1.000 1.000 1.000 36
atis_airline 1.000 1.000 1.000 38
atis_abbreviation 0.943 1.000 0.971 33
atis_aircraft 1.000 0.889 0.941 9
atis_flight_time 0.500 1.000 0.667 1
atis_quantity 0.429 1.000 0.600 3
atis_flight#atis_airfare 0.833 0.417 0.556 12
atis_airport 1.000 1.000 1.000 18
atis_distance 1.000 1.000 1.000 10
atis_city 1.000 0.500 0.667 6
atis_ground_fare 1.000 0.857 0.923 7
atis_capacity 0.952 0.952 0.952 21
atis_flight_no 0.889 1.000 0.941 8
atis_meal 1.000 0.833 0.909 6
atis_restriction 0.000 0.000 0.000 0
atis_airline#atis_flight_no 0.000 0.000 0.000 0
atis_ground_service#atis_ground_fare 0.000 0.000 0.000 0
atis_airfare#atis_flight_time 0.000 0.000 0.000 0
atis_cheapest 0.000 0.000 0.000 0
atis_aircraft#atis_flight#atis_flight_no 0.000 0.000 0.000 0
micro avg 0.973 0.979 0.976 888
macro avg 0.658 0.656 0.640 888
weighted avg 0.976 0.979 0.975 888
I0716 05:45:10.341657 139795883440000 interactiveshell.py:2882]
precision recall f1-score support
O 0.000 0.000 0.000 0.0
B-toloc.city_name 0.975 0.994 0.985 716.0
B-fromloc.city_name 0.990 0.999 0.994 704.0
I-toloc.city_name 0.978 0.989 0.983 265.0
B-depart_date.day_name 0.991 0.991 0.991 212.0
B-airline_name 1.000 1.000 1.000 101.0
I-fromloc.city_name 0.978 1.000 0.989 177.0
B-depart_time.period_of_day 1.000 0.923 0.960 130.0
I-airline_name 1.000 1.000 1.000 65.0
B-depart_date.day_number 0.981 0.964 0.972 55.0
B-depart_date.month_name 0.982 0.964 0.973 56.0
B-depart_time.time 0.864 1.000 0.927 57.0
B-round_trip 1.000 1.000 1.000 73.0
B-cost_relative 0.973 0.973 0.973 37.0
I-round_trip 1.000 1.000 1.000 71.0
B-flight_mod 0.828 1.000 0.906 24.0
B-depart_time.time_relative 0.970 0.985 0.977 65.0
I-depart_time.time 0.945 1.000 0.972 52.0
B-stoploc.city_name 0.909 1.000 0.952 20.0
B-city_name 0.923 0.632 0.750 57.0
B-class_type 0.960 1.000 0.980 24.0
B-arrive_time.time 0.971 0.971 0.971 34.0
B-arrive_time.time_relative 0.967 0.935 0.951 31.0
I-class_type 1.000 1.000 1.000 17.0
B-flight_stop 1.000 1.000 1.000 21.0
I-arrive_time.time 1.000 0.971 0.986 35.0
B-airline_code 0.971 1.000 0.986 34.0
I-depart_date.day_number 1.000 0.933 0.966 15.0
I-fromloc.airport_name 0.441 1.000 0.612 15.0
B-fromloc.airport_name 0.462 1.000 0.632 12.0
B-arrive_date.day_name 0.786 1.000 0.880 11.0
B-toloc.state_code 1.000 1.000 1.000 18.0
B-depart_date.today_relative 1.000 0.889 0.941 9.0
B-flight_number 0.733 1.000 0.846 11.0
B-depart_date.date_relative 0.944 1.000 0.971 17.0
B-toloc.state_name 0.788 0.929 0.852 28.0
B-fare_basis_code 0.850 1.000 0.919 17.0
B-flight_time 1.000 1.000 1.000 1.0
B-or 1.000 1.000 1.000 3.0
B-arrive_time.period_of_day 0.750 1.000 0.857 6.0
B-meal_description 1.000 0.900 0.947 10.0
I-cost_relative 1.000 0.667 0.800 3.0
I-airport_name 0.846 0.379 0.524 29.0
B-fare_amount 1.000 1.000 1.000 2.0
I-fare_amount 1.000 1.000 1.000 2.0
I-city_name 1.000 0.567 0.723 30.0
I-toloc.airport_name 1.000 1.000 1.000 3.0
B-transport_type 1.000 1.000 1.000 10.0
B-arrive_date.month_name 0.714 0.833 0.769 6.0
B-arrive_date.day_number 0.714 0.833 0.769 6.0
I-stoploc.city_name 1.000 1.000 1.000 10.0
B-meal 1.000 1.000 1.000 16.0
B-fromloc.state_code 1.000 1.000 1.000 23.0
B-depart_time.period_mod 1.000 1.000 1.000 5.0
B-connect 1.000 1.000 1.000 6.0
B-flight_days 1.000 1.000 1.000 10.0
B-toloc.airport_name 1.000 1.000 1.000 3.0
B-fromloc.state_name 0.944 1.000 0.971 17.0
B-airport_name 0.800 0.381 0.516 21.0
B-economy 1.000 1.000 1.000 6.0
I-flight_time 1.000 1.000 1.000 1.0
B-aircraft_code 1.000 0.879 0.935 33.0
B-mod 1.000 0.500 0.667 2.0
B-airport_code 1.000 0.444 0.615 9.0
B-depart_time.start_time 1.000 0.667 0.800 3.0
B-depart_time.end_time 1.000 0.333 0.500 3.0
B-depart_date.year 1.000 1.000 1.000 3.0
I-transport_type 0.000 0.000 0.000 1.0
B-restriction_code 0.800 1.000 0.889 4.0
B-arrive_time.start_time 0.889 1.000 0.941 8.0
B-toloc.airport_code 1.000 0.750 0.857 4.0
B-arrive_time.end_time 0.889 1.000 0.941 8.0
I-arrive_time.end_time 0.889 1.000 0.941 8.0
I-depart_time.end_time 1.000 0.333 0.500 3.0
I-flight_stop 0.000 0.000 0.000 0.0
B-fromloc.airport_code 0.556 1.000 0.714 5.0
I-restriction_code 1.000 1.000 1.000 3.0
I-depart_time.start_time 1.000 1.000 1.000 1.0
I-toloc.state_name 1.000 1.000 1.000 1.0
I-depart_date.today_relative 0.000 0.000 0.000 0.0
B-arrive_date.date_relative 1.000 1.000 1.000 2.0
I-flight_mod 1.000 0.167 0.286 6.0
I-economy 0.000 0.000 0.000 0.0
B-return_date.date_relative 1.000 0.333 0.500 3.0
I-fromloc.state_name 1.000 1.000 1.000 1.0
B-state_code 1.000 1.000 1.000 1.0
I-arrive_time.start_time 1.000 1.000 1.000 1.0
I-arrive_date.day_number 0.000 0.000 0.000 0.0
B-meal_code 0.000 0.000 0.000 1.0
I-depart_time.period_of_day 1.000 1.000 1.000 1.0
B-day_name 1.000 0.500 0.667 2.0
B-period_of_day 1.000 0.750 0.857 4.0
B-stoploc.state_code 0.000 0.000 0.000 0.0
B-return_date.month_name 0.000 0.000 0.000 0.0
B-return_date.day_number 0.000 0.000 0.000 0.0
B-arrive_time.period_mod 0.000 0.000 0.000 0.0
I-meal_code 0.000 0.000 0.000 0.0
B-toloc.country_name 1.000 1.000 1.000 1.0
B-days_code 0.000 0.000 0.000 1.0
I-arrive_time.period_of_day 0.000 0.000 0.000 0.0
I-today_relative 0.000 0.000 0.000 0.0
B-return_time.period_of_day 0.000 0.000 0.000 0.0
B-time 0.000 0.000 0.000 0.0
I-fare_basis_code 0.000 0.000 0.000 0.0
I-arrive_time.time_relative 1.000 1.000 1.000 4.0
I-depart_time.time_relative 0.000 0.000 0.000 1.0
B-today_relative 0.000 0.000 0.000 0.0
B-state_name 0.000 0.000 0.000 9.0
B-arrive_date.today_relative 0.000 0.000 0.000 0.0
B-return_time.period_mod 0.000 0.000 0.000 0.0
B-month_name 0.000 0.000 0.000 0.0
B-day_number 0.000 0.000 0.000 0.0
I-return_date.date_relative 0.667 0.667 0.667 3.0
I-return_date.today_relative 0.000 0.000 0.000 0.0
B-stoploc.airport_name 0.000 0.000 0.000 0.0
B-time_relative 0.000 0.000 0.000 0.0
I-time 0.000 0.000 0.000 0.0
I-return_date.day_number 0.000 0.000 0.000 0.0
I-meal_description 0.000 0.000 0.000 0.0
B-return_date.today_relative 0.000 0.000 0.000 0.0
B-return_date.day_name 0.000 0.000 0.000 2.0
micro avg 0.959 0.961 0.960 3657.0
macro avg 0.683 0.652 0.652 3657.0
weighted avg 0.964 0.961 0.957 3657.0
I0716 05:45:10.364182 139795883440000 interactiveshell.py:2882] Best Slot F1: 0.960
I0716 05:45:10.365823 139795883440000 estimator_training.py:186] Not using Distribute Coordinator.
I0716 05:45:10.367828 139795883440000 training.py:612] Running training and evaluation locally (non-distributed).
I0716 05:45:10.370361 139795883440000 training.py:700] Start train and evaluate loop. The evaluate will happen after every checkpoint. Checkpoint frequency is determined based on RunConfig arguments: save_checkpoints_steps 312 or save_checkpoints_secs None.
I0716 05:45:10.443487 139795883440000 estimator.py:1145] Calling model_fn.
I0716 05:45:12.025765 139795883440000 saver.py:1499] Saver not created because there are no variables in the graph to restore
I0716 05:45:12.837126 139795883440000 <ipython-input-6-d6747b40360b>:55]
[<tf.Variable 'embedding:0' shape=(750, 300) dtype=float32_ref>,
<tf.Variable 'module/aggregation/weights:0' shape=(3,) dtype=float32>,
<tf.Variable 'module/aggregation/scaling:0' shape=() dtype=float32>,
<tf.Variable 'dense/kernel:0' shape=(1324, 300) dtype=float32_ref>,
<tf.Variable 'dense/bias:0' shape=(300,) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/fw/gru_cell/gates/kernel:0' shape=(600, 600) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/fw/gru_cell/gates/bias:0' shape=(600,) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/fw/gru_cell/candidate/kernel:0' shape=(600, 300) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/fw/gru_cell/candidate/bias:0' shape=(300,) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/bw/gru_cell/gates/kernel:0' shape=(600, 600) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/bw/gru_cell/gates/bias:0' shape=(600,) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/bw/gru_cell/candidate/kernel:0' shape=(600, 300) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/bw/gru_cell/candidate/bias:0' shape=(300,) dtype=float32_ref>,
<tf.Variable 'dense_1/kernel:0' shape=(600, 23) dtype=float32_ref>,
<tf.Variable 'dense_1/bias:0' shape=(23,) dtype=float32_ref>,
<tf.Variable 'dense_2/kernel:0' shape=(600, 122) dtype=float32_ref>,
<tf.Variable 'dense_2/bias:0' shape=(122,) dtype=float32_ref>]
I0716 05:45:13.959752 139795883440000 estimator.py:1147] Done calling model_fn.
I0716 05:45:13.963981 139795883440000 basic_session_run_hooks.py:541] Create CheckpointSaverHook.
I0716 05:45:14.293300 139795883440000 monitored_session.py:240] Graph was finalized.
I0716 05:45:14.313684 139795883440000 saver.py:1280] Restoring parameters from ../model/elmo_bigru/model.ckpt-14040
I0716 05:45:15.402346 139795883440000 session_manager.py:500] Running local_init_op.
I0716 05:45:15.498242 139795883440000 session_manager.py:502] Done running local_init_op.
I0716 05:45:18.427085 139795883440000 basic_session_run_hooks.py:606] Saving checkpoints for 14040 into ../model/elmo_bigru/model.ckpt.
Reading ../data/atis.train.w-intent.iob
I0716 05:45:23.187589 139795883440000 basic_session_run_hooks.py:262] loss = 0.010440143, step = 14040 I0716 05:45:23.189447 139795883440000 basic_session_run_hooks.py:262] lr = 6.834172e-05 I0716 05:45:51.074040 139795883440000 basic_session_run_hooks.py:692] global_step/sec: 3.58589 I0716 05:45:51.079168 139795883440000 basic_session_run_hooks.py:260] loss = 0.010018, step = 14140 (27.892 sec) I0716 05:45:51.081705 139795883440000 basic_session_run_hooks.py:260] lr = 6.762545e-05 (27.892 sec) I0716 05:46:17.054210 139795883440000 basic_session_run_hooks.py:692] global_step/sec: 3.84908 I0716 05:46:17.059381 139795883440000 basic_session_run_hooks.py:260] loss = 0.010949598, step = 14240 (25.980 sec) I0716 05:46:17.065196 139795883440000 basic_session_run_hooks.py:260] lr = 6.6916684e-05 (25.983 sec) I0716 05:46:44.069175 139795883440000 basic_session_run_hooks.py:692] global_step/sec: 3.70165 I0716 05:46:44.074118 139795883440000 basic_session_run_hooks.py:260] loss = 0.008432481, step = 14340 (27.015 sec) I0716 05:46:44.078580 139795883440000 basic_session_run_hooks.py:260] lr = 6.621535e-05 (27.013 sec) I0716 05:46:46.845710 139795883440000 basic_session_run_hooks.py:606] Saving checkpoints for 14352 into ../model/elmo_bigru/model.ckpt. I0716 05:46:49.039951 139795883440000 estimator.py:1145] Calling model_fn. I0716 05:46:50.422496 139795883440000 saver.py:1499] Saver not created because there are no variables in the graph to restore I0716 05:46:51.007021 139795883440000 estimator.py:1147] Done calling model_fn. I0716 05:46:51.037232 139795883440000 evaluation.py:255] Starting evaluation at 2019-07-16T05:46:51Z I0716 05:46:51.265338 139795883440000 monitored_session.py:240] Graph was finalized. I0716 05:46:51.284544 139795883440000 saver.py:1280] Restoring parameters from ../model/elmo_bigru/model.ckpt-14352 I0716 05:46:52.211852 139795883440000 session_manager.py:500] Running local_init_op. I0716 05:46:52.275646 139795883440000 session_manager.py:502] Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
I0716 05:47:51.278919 139795883440000 evaluation.py:275] Finished evaluation at 2019-07-16-05:47:51 I0716 05:47:51.280749 139795883440000 estimator.py:2039] Saving dict for global step 14352: global_step = 14352, loss = 0.4107694 I0716 05:47:51.289415 139795883440000 estimator.py:2099] Saving 'checkpoint_path' summary for global step 14352: ../model/elmo_bigru/model.ckpt-14352 I0716 05:47:51.372225 139795883440000 estimator.py:368] Loss for final step: 0.0029536623.
Reading ../data/atis.test.w-intent.iob
I0716 05:47:51.982342 139795883440000 estimator.py:1145] Calling model_fn. I0716 05:47:53.540017 139795883440000 saver.py:1499] Saver not created because there are no variables in the graph to restore I0716 05:47:54.252370 139795883440000 estimator.py:1147] Done calling model_fn. I0716 05:47:54.467233 139795883440000 monitored_session.py:240] Graph was finalized. I0716 05:47:54.485304 139795883440000 saver.py:1280] Restoring parameters from ../model/elmo_bigru/model.ckpt-14352 I0716 05:47:55.402657 139795883440000 session_manager.py:500] Running local_init_op. I0716 05:47:55.448067 139795883440000 session_manager.py:502] Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.
'precision', 'predicted', average, warn_for)
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1439: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples.
'recall', 'true', average, warn_for)
I0716 05:48:53.400110 139795883440000 interactiveshell.py:2882]
precision recall f1-score support
atis_flight 0.984 0.992 0.988 632
atis_airfare 0.941 1.000 0.970 48
atis_ground_service 1.000 1.000 1.000 36
atis_airline 1.000 1.000 1.000 38
atis_abbreviation 0.943 1.000 0.971 33
atis_aircraft 1.000 0.889 0.941 9
atis_flight_time 0.500 1.000 0.667 1
atis_quantity 0.429 1.000 0.600 3
atis_flight#atis_airfare 0.833 0.417 0.556 12
atis_airport 1.000 1.000 1.000 18
atis_distance 1.000 1.000 1.000 10
atis_city 0.750 0.500 0.600 6
atis_ground_fare 1.000 0.857 0.923 7
atis_capacity 0.952 0.952 0.952 21
atis_flight_no 0.889 1.000 0.941 8
atis_meal 1.000 0.833 0.909 6
atis_restriction 0.000 0.000 0.000 0
atis_airline#atis_flight_no 0.000 0.000 0.000 0
atis_ground_service#atis_ground_fare 0.000 0.000 0.000 0
atis_airfare#atis_flight_time 0.000 0.000 0.000 0
atis_cheapest 0.000 0.000 0.000 0
atis_aircraft#atis_flight#atis_flight_no 0.000 0.000 0.000 0
micro avg 0.973 0.979 0.976 888
macro avg 0.646 0.656 0.637 888
weighted avg 0.975 0.979 0.975 888
I0716 05:48:53.439727 139795883440000 interactiveshell.py:2882]
precision recall f1-score support
O 0.000 0.000 0.000 0.0
B-toloc.city_name 0.975 0.994 0.985 716.0
B-fromloc.city_name 0.990 0.999 0.994 704.0
I-toloc.city_name 0.978 0.989 0.983 265.0
B-depart_date.day_name 0.991 0.991 0.991 212.0
B-airline_name 1.000 1.000 1.000 101.0
I-fromloc.city_name 0.973 1.000 0.986 177.0
B-depart_time.period_of_day 1.000 0.923 0.960 130.0
I-airline_name 1.000 1.000 1.000 65.0
B-depart_date.day_number 0.981 0.964 0.972 55.0
B-depart_date.month_name 0.982 0.964 0.973 56.0
B-depart_time.time 0.864 1.000 0.927 57.0
B-round_trip 1.000 0.986 0.993 73.0
B-cost_relative 0.973 0.973 0.973 37.0
I-round_trip 1.000 1.000 1.000 71.0
B-flight_mod 0.828 1.000 0.906 24.0
B-depart_time.time_relative 0.970 0.985 0.977 65.0
I-depart_time.time 0.945 1.000 0.972 52.0
B-stoploc.city_name 0.909 1.000 0.952 20.0
B-city_name 0.923 0.632 0.750 57.0
B-class_type 0.960 1.000 0.980 24.0
B-arrive_time.time 0.971 0.971 0.971 34.0
B-arrive_time.time_relative 0.967 0.935 0.951 31.0
I-class_type 1.000 1.000 1.000 17.0
B-flight_stop 1.000 1.000 1.000 21.0
I-arrive_time.time 1.000 0.971 0.986 35.0
B-airline_code 0.971 1.000 0.986 34.0
I-depart_date.day_number 1.000 0.933 0.966 15.0
I-fromloc.airport_name 0.441 1.000 0.612 15.0
B-fromloc.airport_name 0.462 1.000 0.632 12.0
B-arrive_date.day_name 0.786 1.000 0.880 11.0
B-toloc.state_code 1.000 1.000 1.000 18.0
B-depart_date.today_relative 1.000 0.889 0.941 9.0
B-flight_number 0.733 1.000 0.846 11.0
B-depart_date.date_relative 0.944 1.000 0.971 17.0
B-toloc.state_name 0.788 0.929 0.852 28.0
B-fare_basis_code 0.850 1.000 0.919 17.0
B-flight_time 1.000 1.000 1.000 1.0
B-or 1.000 1.000 1.000 3.0
B-arrive_time.period_of_day 0.750 1.000 0.857 6.0
B-meal_description 1.000 0.900 0.947 10.0
I-cost_relative 1.000 0.667 0.800 3.0
I-airport_name 0.846 0.379 0.524 29.0
B-fare_amount 1.000 1.000 1.000 2.0
I-fare_amount 1.000 1.000 1.000 2.0
I-city_name 1.000 0.533 0.696 30.0
I-toloc.airport_name 1.000 1.000 1.000 3.0
B-transport_type 1.000 1.000 1.000 10.0
B-arrive_date.month_name 0.714 0.833 0.769 6.0
B-arrive_date.day_number 0.714 0.833 0.769 6.0
I-stoploc.city_name 1.000 1.000 1.000 10.0
B-meal 1.000 1.000 1.000 16.0
B-fromloc.state_code 1.000 1.000 1.000 23.0
B-depart_time.period_mod 1.000 1.000 1.000 5.0
B-connect 1.000 1.000 1.000 6.0
B-flight_days 1.000 1.000 1.000 10.0
B-toloc.airport_name 1.000 1.000 1.000 3.0
B-fromloc.state_name 0.944 1.000 0.971 17.0
B-airport_name 0.800 0.381 0.516 21.0
B-economy 1.000 1.000 1.000 6.0
I-flight_time 1.000 1.000 1.000 1.0
B-aircraft_code 1.000 0.879 0.935 33.0
B-mod 1.000 0.500 0.667 2.0
B-airport_code 1.000 0.444 0.615 9.0
B-depart_time.start_time 1.000 0.667 0.800 3.0
B-depart_time.end_time 1.000 0.333 0.500 3.0
B-depart_date.year 1.000 1.000 1.000 3.0
I-transport_type 0.000 0.000 0.000 1.0
B-restriction_code 0.800 1.000 0.889 4.0
B-arrive_time.start_time 0.889 1.000 0.941 8.0
B-toloc.airport_code 1.000 0.750 0.857 4.0
B-arrive_time.end_time 0.889 1.000 0.941 8.0
I-arrive_time.end_time 0.889 1.000 0.941 8.0
I-depart_time.end_time 1.000 0.333 0.500 3.0
I-flight_stop 0.000 0.000 0.000 0.0
B-fromloc.airport_code 0.556 1.000 0.714 5.0
I-restriction_code 1.000 1.000 1.000 3.0
I-depart_time.start_time 1.000 1.000 1.000 1.0
I-toloc.state_name 1.000 1.000 1.000 1.0
I-depart_date.today_relative 0.000 0.000 0.000 0.0
B-arrive_date.date_relative 1.000 1.000 1.000 2.0
I-flight_mod 1.000 0.167 0.286 6.0
I-economy 0.000 0.000 0.000 0.0
B-return_date.date_relative 1.000 0.333 0.500 3.0
I-fromloc.state_name 1.000 1.000 1.000 1.0
B-state_code 1.000 1.000 1.000 1.0
I-arrive_time.start_time 1.000 1.000 1.000 1.0
I-arrive_date.day_number 0.000 0.000 0.000 0.0
B-meal_code 0.000 0.000 0.000 1.0
I-depart_time.period_of_day 1.000 1.000 1.000 1.0
B-day_name 1.000 0.500 0.667 2.0
B-period_of_day 1.000 0.750 0.857 4.0
B-stoploc.state_code 0.000 0.000 0.000 0.0
B-return_date.month_name 0.000 0.000 0.000 0.0
B-return_date.day_number 0.000 0.000 0.000 0.0
B-arrive_time.period_mod 0.000 0.000 0.000 0.0
I-meal_code 0.000 0.000 0.000 0.0
B-toloc.country_name 1.000 1.000 1.000 1.0
B-days_code 0.000 0.000 0.000 1.0
I-arrive_time.period_of_day 0.000 0.000 0.000 0.0
I-today_relative 0.000 0.000 0.000 0.0
B-return_time.period_of_day 0.000 0.000 0.000 0.0
B-time 0.000 0.000 0.000 0.0
I-fare_basis_code 0.000 0.000 0.000 0.0
I-arrive_time.time_relative 1.000 1.000 1.000 4.0
I-depart_time.time_relative 0.000 0.000 0.000 1.0
B-today_relative 0.000 0.000 0.000 0.0
B-state_name 0.000 0.000 0.000 9.0
B-arrive_date.today_relative 0.000 0.000 0.000 0.0
B-return_time.period_mod 0.000 0.000 0.000 0.0
B-month_name 0.000 0.000 0.000 0.0
B-day_number 0.000 0.000 0.000 0.0
I-return_date.date_relative 0.667 0.667 0.667 3.0
I-return_date.today_relative 0.000 0.000 0.000 0.0
B-stoploc.airport_name 0.000 0.000 0.000 0.0
B-time_relative 0.000 0.000 0.000 0.0
I-time 0.000 0.000 0.000 0.0
I-return_date.day_number 0.000 0.000 0.000 0.0
I-meal_description 0.000 0.000 0.000 0.0
B-return_date.today_relative 0.000 0.000 0.000 0.0
B-return_date.day_name 0.000 0.000 0.000 2.0
micro avg 0.959 0.960 0.959 3657.0
macro avg 0.683 0.652 0.652 3657.0
weighted avg 0.964 0.960 0.957 3657.0
I0716 05:48:53.476049 139795883440000 interactiveshell.py:2882] Best Slot F1: 0.960
I0716 05:48:53.477536 139795883440000 estimator_training.py:186] Not using Distribute Coordinator.
I0716 05:48:53.483242 139795883440000 training.py:612] Running training and evaluation locally (non-distributed).
I0716 05:48:53.487344 139795883440000 training.py:700] Start train and evaluate loop. The evaluate will happen after every checkpoint. Checkpoint frequency is determined based on RunConfig arguments: save_checkpoints_steps 312 or save_checkpoints_secs None.
I0716 05:48:53.553193 139795883440000 estimator.py:1145] Calling model_fn.
I0716 05:48:54.885581 139795883440000 saver.py:1499] Saver not created because there are no variables in the graph to restore
I0716 05:48:55.670599 139795883440000 <ipython-input-6-d6747b40360b>:55]
[<tf.Variable 'embedding:0' shape=(750, 300) dtype=float32_ref>,
<tf.Variable 'module/aggregation/weights:0' shape=(3,) dtype=float32>,
<tf.Variable 'module/aggregation/scaling:0' shape=() dtype=float32>,
<tf.Variable 'dense/kernel:0' shape=(1324, 300) dtype=float32_ref>,
<tf.Variable 'dense/bias:0' shape=(300,) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/fw/gru_cell/gates/kernel:0' shape=(600, 600) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/fw/gru_cell/gates/bias:0' shape=(600,) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/fw/gru_cell/candidate/kernel:0' shape=(600, 300) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/fw/gru_cell/candidate/bias:0' shape=(300,) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/bw/gru_cell/gates/kernel:0' shape=(600, 600) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/bw/gru_cell/gates/bias:0' shape=(600,) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/bw/gru_cell/candidate/kernel:0' shape=(600, 300) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/bw/gru_cell/candidate/bias:0' shape=(300,) dtype=float32_ref>,
<tf.Variable 'dense_1/kernel:0' shape=(600, 23) dtype=float32_ref>,
<tf.Variable 'dense_1/bias:0' shape=(23,) dtype=float32_ref>,
<tf.Variable 'dense_2/kernel:0' shape=(600, 122) dtype=float32_ref>,
<tf.Variable 'dense_2/bias:0' shape=(122,) dtype=float32_ref>]
I0716 05:48:56.756968 139795883440000 estimator.py:1147] Done calling model_fn.
I0716 05:48:56.761645 139795883440000 basic_session_run_hooks.py:541] Create CheckpointSaverHook.
I0716 05:48:57.091637 139795883440000 monitored_session.py:240] Graph was finalized.
I0716 05:48:57.112784 139795883440000 saver.py:1280] Restoring parameters from ../model/elmo_bigru/model.ckpt-14352
I0716 05:48:58.171647 139795883440000 session_manager.py:500] Running local_init_op.
I0716 05:48:58.240034 139795883440000 session_manager.py:502] Done running local_init_op.
I0716 05:49:01.294649 139795883440000 basic_session_run_hooks.py:606] Saving checkpoints for 14352 into ../model/elmo_bigru/model.ckpt.
Reading ../data/atis.train.w-intent.iob
I0716 05:49:06.017440 139795883440000 basic_session_run_hooks.py:262] loss = 0.005521031, step = 14352 I0716 05:49:06.019191 139795883440000 basic_session_run_hooks.py:262] lr = 6.613168e-05 I0716 05:49:33.814277 139795883440000 basic_session_run_hooks.py:692] global_step/sec: 3.59743 I0716 05:49:33.818360 139795883440000 basic_session_run_hooks.py:260] loss = 0.0083391145, step = 14452 (27.801 sec) I0716 05:49:33.824078 139795883440000 basic_session_run_hooks.py:260] lr = 6.543857e-05 (27.805 sec) I0716 05:49:59.538691 139795883440000 basic_session_run_hooks.py:692] global_step/sec: 3.88735 I0716 05:49:59.544354 139795883440000 basic_session_run_hooks.py:260] loss = 0.00884945, step = 14552 (25.726 sec) I0716 05:49:59.547071 139795883440000 basic_session_run_hooks.py:260] lr = 6.4752734e-05 (25.723 sec) I0716 05:50:26.581260 139795883440000 basic_session_run_hooks.py:692] global_step/sec: 3.69788 I0716 05:50:26.590217 139795883440000 basic_session_run_hooks.py:260] loss = 0.011286377, step = 14652 (27.046 sec) I0716 05:50:26.592428 139795883440000 basic_session_run_hooks.py:260] lr = 6.407407e-05 (27.045 sec) I0716 05:50:29.580576 139795883440000 basic_session_run_hooks.py:606] Saving checkpoints for 14664 into ../model/elmo_bigru/model.ckpt. I0716 05:50:31.657208 139795883440000 estimator.py:1145] Calling model_fn. I0716 05:50:33.072262 139795883440000 saver.py:1499] Saver not created because there are no variables in the graph to restore I0716 05:50:33.644398 139795883440000 estimator.py:1147] Done calling model_fn. I0716 05:50:33.675320 139795883440000 evaluation.py:255] Starting evaluation at 2019-07-16T05:50:33Z I0716 05:50:33.899181 139795883440000 monitored_session.py:240] Graph was finalized. I0716 05:50:33.917637 139795883440000 saver.py:1280] Restoring parameters from ../model/elmo_bigru/model.ckpt-14664 I0716 05:50:34.822535 139795883440000 session_manager.py:500] Running local_init_op. I0716 05:50:34.882165 139795883440000 session_manager.py:502] Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
I0716 05:51:33.744959 139795883440000 evaluation.py:275] Finished evaluation at 2019-07-16-05:51:33 I0716 05:51:33.746666 139795883440000 estimator.py:2039] Saving dict for global step 14664: global_step = 14664, loss = 0.41299483 I0716 05:51:33.756397 139795883440000 estimator.py:2099] Saving 'checkpoint_path' summary for global step 14664: ../model/elmo_bigru/model.ckpt-14664 I0716 05:51:33.845106 139795883440000 estimator.py:368] Loss for final step: 0.0035273344.
Reading ../data/atis.test.w-intent.iob
I0716 05:51:34.460048 139795883440000 estimator.py:1145] Calling model_fn. I0716 05:51:36.192692 139795883440000 saver.py:1499] Saver not created because there are no variables in the graph to restore I0716 05:51:36.715398 139795883440000 estimator.py:1147] Done calling model_fn. I0716 05:51:36.933637 139795883440000 monitored_session.py:240] Graph was finalized. I0716 05:51:36.958074 139795883440000 saver.py:1280] Restoring parameters from ../model/elmo_bigru/model.ckpt-14664 I0716 05:51:37.884623 139795883440000 session_manager.py:500] Running local_init_op. I0716 05:51:37.932129 139795883440000 session_manager.py:502] Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.
'precision', 'predicted', average, warn_for)
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1439: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples.
'recall', 'true', average, warn_for)
I0716 05:52:35.795669 139795883440000 interactiveshell.py:2882]
precision recall f1-score support
atis_flight 0.984 0.992 0.988 632
atis_airfare 0.941 1.000 0.970 48
atis_ground_service 1.000 1.000 1.000 36
atis_airline 1.000 1.000 1.000 38
atis_abbreviation 0.943 1.000 0.971 33
atis_aircraft 1.000 0.889 0.941 9
atis_flight_time 0.500 1.000 0.667 1
atis_quantity 0.429 1.000 0.600 3
atis_flight#atis_airfare 0.833 0.417 0.556 12
atis_airport 1.000 1.000 1.000 18
atis_distance 1.000 1.000 1.000 10
atis_city 0.750 0.500 0.600 6
atis_ground_fare 1.000 0.857 0.923 7
atis_capacity 0.952 0.952 0.952 21
atis_flight_no 0.889 1.000 0.941 8
atis_meal 1.000 0.833 0.909 6
atis_restriction 0.000 0.000 0.000 0
atis_airline#atis_flight_no 0.000 0.000 0.000 0
atis_ground_service#atis_ground_fare 0.000 0.000 0.000 0
atis_airfare#atis_flight_time 0.000 0.000 0.000 0
atis_cheapest 0.000 0.000 0.000 0
atis_aircraft#atis_flight#atis_flight_no 0.000 0.000 0.000 0
micro avg 0.973 0.979 0.976 888
macro avg 0.646 0.656 0.637 888
weighted avg 0.975 0.979 0.975 888
I0716 05:52:35.841406 139795883440000 interactiveshell.py:2882]
precision recall f1-score support
O 0.000 0.000 0.000 0.0
B-toloc.city_name 0.977 0.994 0.985 716.0
B-fromloc.city_name 0.990 0.999 0.994 704.0
I-toloc.city_name 0.978 0.989 0.983 265.0
B-depart_date.day_name 0.991 0.991 0.991 212.0
B-airline_name 1.000 1.000 1.000 101.0
I-fromloc.city_name 0.978 1.000 0.989 177.0
B-depart_time.period_of_day 1.000 0.923 0.960 130.0
I-airline_name 1.000 1.000 1.000 65.0
B-depart_date.day_number 0.981 0.964 0.972 55.0
B-depart_date.month_name 0.982 0.964 0.973 56.0
B-depart_time.time 0.864 1.000 0.927 57.0
B-round_trip 1.000 1.000 1.000 73.0
B-cost_relative 0.973 0.973 0.973 37.0
I-round_trip 1.000 1.000 1.000 71.0
B-flight_mod 0.828 1.000 0.906 24.0
B-depart_time.time_relative 0.955 0.985 0.970 65.0
I-depart_time.time 0.945 1.000 0.972 52.0
B-stoploc.city_name 0.909 1.000 0.952 20.0
B-city_name 0.923 0.632 0.750 57.0
B-class_type 0.960 1.000 0.980 24.0
B-arrive_time.time 0.971 0.971 0.971 34.0
B-arrive_time.time_relative 0.967 0.935 0.951 31.0
I-class_type 1.000 1.000 1.000 17.0
B-flight_stop 1.000 1.000 1.000 21.0
I-arrive_time.time 1.000 0.971 0.986 35.0
B-airline_code 0.971 1.000 0.986 34.0
I-depart_date.day_number 1.000 0.933 0.966 15.0
I-fromloc.airport_name 0.455 1.000 0.625 15.0
B-fromloc.airport_name 0.462 1.000 0.632 12.0
B-arrive_date.day_name 0.786 1.000 0.880 11.0
B-toloc.state_code 1.000 1.000 1.000 18.0
B-depart_date.today_relative 1.000 0.889 0.941 9.0
B-flight_number 0.733 1.000 0.846 11.0
B-depart_date.date_relative 0.944 1.000 0.971 17.0
B-toloc.state_name 0.788 0.929 0.852 28.0
B-fare_basis_code 0.895 1.000 0.944 17.0
B-flight_time 1.000 1.000 1.000 1.0
B-or 1.000 1.000 1.000 3.0
B-arrive_time.period_of_day 0.750 1.000 0.857 6.0
B-meal_description 1.000 0.900 0.947 10.0
I-cost_relative 1.000 0.667 0.800 3.0
I-airport_name 0.857 0.414 0.558 29.0
B-fare_amount 1.000 1.000 1.000 2.0
I-fare_amount 1.000 1.000 1.000 2.0
I-city_name 1.000 0.567 0.723 30.0
I-toloc.airport_name 1.000 1.000 1.000 3.0
B-transport_type 1.000 1.000 1.000 10.0
B-arrive_date.month_name 0.714 0.833 0.769 6.0
B-arrive_date.day_number 0.714 0.833 0.769 6.0
I-stoploc.city_name 1.000 1.000 1.000 10.0
B-meal 1.000 1.000 1.000 16.0
B-fromloc.state_code 1.000 1.000 1.000 23.0
B-depart_time.period_mod 1.000 1.000 1.000 5.0
B-connect 1.000 1.000 1.000 6.0
B-flight_days 1.000 1.000 1.000 10.0
B-toloc.airport_name 1.000 1.000 1.000 3.0
B-fromloc.state_name 0.944 1.000 0.971 17.0
B-airport_name 0.800 0.381 0.516 21.0
B-economy 1.000 1.000 1.000 6.0
I-flight_time 1.000 1.000 1.000 1.0
B-aircraft_code 1.000 0.879 0.935 33.0
B-mod 1.000 0.500 0.667 2.0
B-airport_code 1.000 0.444 0.615 9.0
B-depart_time.start_time 1.000 0.667 0.800 3.0
B-depart_time.end_time 1.000 0.333 0.500 3.0
B-depart_date.year 1.000 1.000 1.000 3.0
I-transport_type 0.000 0.000 0.000 1.0
B-restriction_code 0.800 1.000 0.889 4.0
B-arrive_time.start_time 0.889 1.000 0.941 8.0
B-toloc.airport_code 1.000 1.000 1.000 4.0
B-arrive_time.end_time 0.889 1.000 0.941 8.0
I-arrive_time.end_time 0.889 1.000 0.941 8.0
I-depart_time.end_time 1.000 0.333 0.500 3.0
I-flight_stop 0.000 0.000 0.000 0.0
B-fromloc.airport_code 0.556 1.000 0.714 5.0
I-restriction_code 1.000 1.000 1.000 3.0
I-depart_time.start_time 1.000 1.000 1.000 1.0
I-toloc.state_name 1.000 1.000 1.000 1.0
I-depart_date.today_relative 0.000 0.000 0.000 0.0
B-arrive_date.date_relative 1.000 1.000 1.000 2.0
I-flight_mod 1.000 0.167 0.286 6.0
I-economy 0.000 0.000 0.000 0.0
B-return_date.date_relative 1.000 0.333 0.500 3.0
I-fromloc.state_name 1.000 1.000 1.000 1.0
B-state_code 1.000 1.000 1.000 1.0
I-arrive_time.start_time 1.000 1.000 1.000 1.0
I-arrive_date.day_number 0.000 0.000 0.000 0.0
B-meal_code 0.000 0.000 0.000 1.0
I-depart_time.period_of_day 1.000 1.000 1.000 1.0
B-day_name 1.000 0.500 0.667 2.0
B-period_of_day 1.000 0.750 0.857 4.0
B-stoploc.state_code 0.000 0.000 0.000 0.0
B-return_date.month_name 0.000 0.000 0.000 0.0
B-return_date.day_number 0.000 0.000 0.000 0.0
B-arrive_time.period_mod 0.000 0.000 0.000 0.0
I-meal_code 0.000 0.000 0.000 0.0
B-toloc.country_name 1.000 1.000 1.000 1.0
B-days_code 1.000 1.000 1.000 1.0
I-arrive_time.period_of_day 0.000 0.000 0.000 0.0
I-today_relative 0.000 0.000 0.000 0.0
B-return_time.period_of_day 0.000 0.000 0.000 0.0
B-time 0.000 0.000 0.000 0.0
I-fare_basis_code 0.000 0.000 0.000 0.0
I-arrive_time.time_relative 1.000 1.000 1.000 4.0
I-depart_time.time_relative 0.000 0.000 0.000 1.0
B-today_relative 0.000 0.000 0.000 0.0
B-state_name 0.000 0.000 0.000 9.0
B-arrive_date.today_relative 0.000 0.000 0.000 0.0
B-return_time.period_mod 0.000 0.000 0.000 0.0
B-month_name 0.000 0.000 0.000 0.0
B-day_number 0.000 0.000 0.000 0.0
I-return_date.date_relative 0.667 0.667 0.667 3.0
I-return_date.today_relative 0.000 0.000 0.000 0.0
B-stoploc.airport_name 0.000 0.000 0.000 0.0
B-time_relative 0.000 0.000 0.000 0.0
I-time 0.000 0.000 0.000 0.0
I-return_date.day_number 0.000 0.000 0.000 0.0
I-meal_description 0.000 0.000 0.000 0.0
B-return_date.today_relative 0.000 0.000 0.000 0.0
B-return_date.day_name 0.000 0.000 0.000 2.0
micro avg 0.960 0.961 0.961 3657.0
macro avg 0.692 0.663 0.662 3657.0
weighted avg 0.964 0.961 0.958 3657.0
I0716 05:52:35.869594 139795883440000 interactiveshell.py:2882] Best Slot F1: 0.961
I0716 05:52:35.871094 139795883440000 estimator_training.py:186] Not using Distribute Coordinator.
I0716 05:52:35.872925 139795883440000 training.py:612] Running training and evaluation locally (non-distributed).
I0716 05:52:35.876721 139795883440000 training.py:700] Start train and evaluate loop. The evaluate will happen after every checkpoint. Checkpoint frequency is determined based on RunConfig arguments: save_checkpoints_steps 312 or save_checkpoints_secs None.
I0716 05:52:35.945642 139795883440000 estimator.py:1145] Calling model_fn.
I0716 05:52:37.465623 139795883440000 saver.py:1499] Saver not created because there are no variables in the graph to restore
I0716 05:52:38.056463 139795883440000 <ipython-input-6-d6747b40360b>:55]
[<tf.Variable 'embedding:0' shape=(750, 300) dtype=float32_ref>,
<tf.Variable 'module/aggregation/weights:0' shape=(3,) dtype=float32>,
<tf.Variable 'module/aggregation/scaling:0' shape=() dtype=float32>,
<tf.Variable 'dense/kernel:0' shape=(1324, 300) dtype=float32_ref>,
<tf.Variable 'dense/bias:0' shape=(300,) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/fw/gru_cell/gates/kernel:0' shape=(600, 600) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/fw/gru_cell/gates/bias:0' shape=(600,) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/fw/gru_cell/candidate/kernel:0' shape=(600, 300) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/fw/gru_cell/candidate/bias:0' shape=(300,) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/bw/gru_cell/gates/kernel:0' shape=(600, 600) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/bw/gru_cell/gates/bias:0' shape=(600,) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/bw/gru_cell/candidate/kernel:0' shape=(600, 300) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/bw/gru_cell/candidate/bias:0' shape=(300,) dtype=float32_ref>,
<tf.Variable 'dense_1/kernel:0' shape=(600, 23) dtype=float32_ref>,
<tf.Variable 'dense_1/bias:0' shape=(23,) dtype=float32_ref>,
<tf.Variable 'dense_2/kernel:0' shape=(600, 122) dtype=float32_ref>,
<tf.Variable 'dense_2/bias:0' shape=(122,) dtype=float32_ref>]
I0716 05:52:39.361430 139795883440000 estimator.py:1147] Done calling model_fn.
I0716 05:52:39.365440 139795883440000 basic_session_run_hooks.py:541] Create CheckpointSaverHook.
I0716 05:52:39.690876 139795883440000 monitored_session.py:240] Graph was finalized.
I0716 05:52:39.711251 139795883440000 saver.py:1280] Restoring parameters from ../model/elmo_bigru/model.ckpt-14664
I0716 05:52:40.747165 139795883440000 session_manager.py:500] Running local_init_op.
I0716 05:52:40.822594 139795883440000 session_manager.py:502] Done running local_init_op.
I0716 05:52:43.647684 139795883440000 basic_session_run_hooks.py:606] Saving checkpoints for 14664 into ../model/elmo_bigru/model.ckpt.
Reading ../data/atis.train.w-intent.iob
I0716 05:52:48.409260 139795883440000 basic_session_run_hooks.py:262] loss = 0.009573579, step = 14664 I0716 05:52:48.410746 139795883440000 basic_session_run_hooks.py:262] lr = 6.399311e-05 I0716 05:53:17.095580 139795883440000 basic_session_run_hooks.py:692] global_step/sec: 3.48591 I0716 05:53:17.100201 139795883440000 basic_session_run_hooks.py:260] loss = 0.010442889, step = 14764 (28.691 sec) I0716 05:53:17.104258 139795883440000 basic_session_run_hooks.py:260] lr = 6.332242e-05 (28.693 sec) I0716 05:53:42.701854 139795883440000 basic_session_run_hooks.py:692] global_step/sec: 3.90529 I0716 05:53:42.711499 139795883440000 basic_session_run_hooks.py:260] loss = 0.009410427, step = 14864 (25.611 sec) I0716 05:53:42.713646 139795883440000 basic_session_run_hooks.py:260] lr = 6.265875e-05 (25.609 sec) I0716 05:54:08.258992 139795883440000 basic_session_run_hooks.py:692] global_step/sec: 3.91281 I0716 05:54:08.267603 139795883440000 basic_session_run_hooks.py:260] loss = 0.006956376, step = 14964 (25.556 sec) I0716 05:54:08.270033 139795883440000 basic_session_run_hooks.py:260] lr = 6.2002044e-05 (25.556 sec) I0716 05:54:11.027354 139795883440000 basic_session_run_hooks.py:606] Saving checkpoints for 14976 into ../model/elmo_bigru/model.ckpt. I0716 05:54:13.016072 139795883440000 estimator.py:1145] Calling model_fn. I0716 05:54:14.394398 139795883440000 saver.py:1499] Saver not created because there are no variables in the graph to restore I0716 05:54:15.255473 139795883440000 estimator.py:1147] Done calling model_fn. I0716 05:54:15.287425 139795883440000 evaluation.py:255] Starting evaluation at 2019-07-16T05:54:15Z I0716 05:54:15.506654 139795883440000 monitored_session.py:240] Graph was finalized. I0716 05:54:15.527868 139795883440000 saver.py:1280] Restoring parameters from ../model/elmo_bigru/model.ckpt-14976 I0716 05:54:16.437136 139795883440000 session_manager.py:500] Running local_init_op. I0716 05:54:16.495558 139795883440000 session_manager.py:502] Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
I0716 05:55:15.609049 139795883440000 evaluation.py:275] Finished evaluation at 2019-07-16-05:55:15 I0716 05:55:15.610839 139795883440000 estimator.py:2039] Saving dict for global step 14976: global_step = 14976, loss = 0.41449836 I0716 05:55:15.617541 139795883440000 estimator.py:2099] Saving 'checkpoint_path' summary for global step 14976: ../model/elmo_bigru/model.ckpt-14976 I0716 05:55:15.705172 139795883440000 estimator.py:368] Loss for final step: 0.0009599638.
Reading ../data/atis.test.w-intent.iob
I0716 05:55:16.311159 139795883440000 estimator.py:1145] Calling model_fn. I0716 05:55:17.373431 139795883440000 saver.py:1499] Saver not created because there are no variables in the graph to restore I0716 05:55:18.466601 139795883440000 estimator.py:1147] Done calling model_fn. I0716 05:55:18.704450 139795883440000 monitored_session.py:240] Graph was finalized. I0716 05:55:18.721217 139795883440000 saver.py:1280] Restoring parameters from ../model/elmo_bigru/model.ckpt-14976 I0716 05:55:19.648371 139795883440000 session_manager.py:500] Running local_init_op. I0716 05:55:19.720381 139795883440000 session_manager.py:502] Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.
'precision', 'predicted', average, warn_for)
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1439: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples.
'recall', 'true', average, warn_for)
I0716 05:56:18.118442 139795883440000 interactiveshell.py:2882]
precision recall f1-score support
atis_flight 0.984 0.992 0.988 632
atis_airfare 0.941 1.000 0.970 48
atis_ground_service 1.000 1.000 1.000 36
atis_airline 1.000 1.000 1.000 38
atis_abbreviation 0.943 1.000 0.971 33
atis_aircraft 1.000 0.889 0.941 9
atis_flight_time 0.500 1.000 0.667 1
atis_quantity 0.429 1.000 0.600 3
atis_flight#atis_airfare 0.833 0.417 0.556 12
atis_airport 1.000 1.000 1.000 18
atis_distance 1.000 1.000 1.000 10
atis_city 0.750 0.500 0.600 6
atis_ground_fare 1.000 0.857 0.923 7
atis_capacity 0.952 0.952 0.952 21
atis_flight_no 0.889 1.000 0.941 8
atis_meal 1.000 0.833 0.909 6
atis_restriction 0.000 0.000 0.000 0
atis_airline#atis_flight_no 0.000 0.000 0.000 0
atis_ground_service#atis_ground_fare 0.000 0.000 0.000 0
atis_airfare#atis_flight_time 0.000 0.000 0.000 0
atis_cheapest 0.000 0.000 0.000 0
atis_aircraft#atis_flight#atis_flight_no 0.000 0.000 0.000 0
micro avg 0.973 0.979 0.976 888
macro avg 0.646 0.656 0.637 888
weighted avg 0.975 0.979 0.975 888
I0716 05:56:18.155181 139795883440000 interactiveshell.py:2882]
precision recall f1-score support
O 0.000 0.000 0.000 0.0
B-toloc.city_name 0.977 0.994 0.985 716.0
B-fromloc.city_name 0.992 0.999 0.995 704.0
I-toloc.city_name 0.978 0.989 0.983 265.0
B-depart_date.day_name 0.991 0.991 0.991 212.0
B-airline_name 1.000 1.000 1.000 101.0
I-fromloc.city_name 0.978 1.000 0.989 177.0
B-depart_time.period_of_day 1.000 0.923 0.960 130.0
I-airline_name 1.000 1.000 1.000 65.0
B-depart_date.day_number 0.981 0.964 0.972 55.0
B-depart_date.month_name 0.982 0.964 0.973 56.0
B-depart_time.time 0.864 1.000 0.927 57.0
B-round_trip 1.000 1.000 1.000 73.0
B-cost_relative 0.973 0.973 0.973 37.0
I-round_trip 1.000 1.000 1.000 71.0
B-flight_mod 0.828 1.000 0.906 24.0
B-depart_time.time_relative 0.955 0.985 0.970 65.0
I-depart_time.time 0.945 1.000 0.972 52.0
B-stoploc.city_name 0.870 1.000 0.930 20.0
B-city_name 0.925 0.649 0.763 57.0
B-class_type 0.960 1.000 0.980 24.0
B-arrive_time.time 0.971 0.971 0.971 34.0
B-arrive_time.time_relative 0.967 0.935 0.951 31.0
I-class_type 1.000 1.000 1.000 17.0
B-flight_stop 1.000 1.000 1.000 21.0
I-arrive_time.time 1.000 0.971 0.986 35.0
B-airline_code 0.971 1.000 0.986 34.0
I-depart_date.day_number 1.000 0.933 0.966 15.0
I-fromloc.airport_name 0.441 1.000 0.612 15.0
B-fromloc.airport_name 0.462 1.000 0.632 12.0
B-arrive_date.day_name 0.786 1.000 0.880 11.0
B-toloc.state_code 1.000 1.000 1.000 18.0
B-depart_date.today_relative 1.000 0.889 0.941 9.0
B-flight_number 0.733 1.000 0.846 11.0
B-depart_date.date_relative 0.944 1.000 0.971 17.0
B-toloc.state_name 0.833 0.893 0.862 28.0
B-fare_basis_code 0.850 1.000 0.919 17.0
B-flight_time 1.000 1.000 1.000 1.0
B-or 1.000 1.000 1.000 3.0
B-arrive_time.period_of_day 0.750 1.000 0.857 6.0
B-meal_description 1.000 0.900 0.947 10.0
I-cost_relative 1.000 0.667 0.800 3.0
I-airport_name 0.846 0.379 0.524 29.0
B-fare_amount 1.000 1.000 1.000 2.0
I-fare_amount 1.000 1.000 1.000 2.0
I-city_name 1.000 0.600 0.750 30.0
I-toloc.airport_name 1.000 1.000 1.000 3.0
B-transport_type 1.000 1.000 1.000 10.0
B-arrive_date.month_name 0.714 0.833 0.769 6.0
B-arrive_date.day_number 0.714 0.833 0.769 6.0
I-stoploc.city_name 1.000 1.000 1.000 10.0
B-meal 0.941 1.000 0.970 16.0
B-fromloc.state_code 1.000 1.000 1.000 23.0
B-depart_time.period_mod 1.000 1.000 1.000 5.0
B-connect 1.000 1.000 1.000 6.0
B-flight_days 1.000 1.000 1.000 10.0
B-toloc.airport_name 1.000 1.000 1.000 3.0
B-fromloc.state_name 0.944 1.000 0.971 17.0
B-airport_name 0.667 0.381 0.485 21.0
B-economy 1.000 1.000 1.000 6.0
I-flight_time 1.000 1.000 1.000 1.0
B-aircraft_code 1.000 0.879 0.935 33.0
B-mod 1.000 0.500 0.667 2.0
B-airport_code 1.000 0.444 0.615 9.0
B-depart_time.start_time 1.000 0.667 0.800 3.0
B-depart_time.end_time 1.000 0.333 0.500 3.0
B-depart_date.year 1.000 1.000 1.000 3.0
I-transport_type 0.000 0.000 0.000 1.0
B-restriction_code 0.800 1.000 0.889 4.0
B-arrive_time.start_time 0.889 1.000 0.941 8.0
B-toloc.airport_code 1.000 1.000 1.000 4.0
B-arrive_time.end_time 0.889 1.000 0.941 8.0
I-arrive_time.end_time 0.889 1.000 0.941 8.0
I-depart_time.end_time 1.000 0.333 0.500 3.0
I-flight_stop 0.000 0.000 0.000 0.0
B-fromloc.airport_code 0.556 1.000 0.714 5.0
I-restriction_code 1.000 1.000 1.000 3.0
I-depart_time.start_time 1.000 1.000 1.000 1.0
I-toloc.state_name 1.000 1.000 1.000 1.0
I-depart_date.today_relative 0.000 0.000 0.000 0.0
B-arrive_date.date_relative 1.000 1.000 1.000 2.0
I-flight_mod 1.000 0.167 0.286 6.0
I-economy 0.000 0.000 0.000 0.0
B-return_date.date_relative 1.000 0.333 0.500 3.0
I-fromloc.state_name 1.000 1.000 1.000 1.0
B-state_code 1.000 1.000 1.000 1.0
I-arrive_time.start_time 1.000 1.000 1.000 1.0
I-arrive_date.day_number 0.000 0.000 0.000 0.0
B-meal_code 0.000 0.000 0.000 1.0
I-depart_time.period_of_day 1.000 1.000 1.000 1.0
B-day_name 1.000 0.500 0.667 2.0
B-period_of_day 1.000 0.750 0.857 4.0
B-stoploc.state_code 0.000 0.000 0.000 0.0
B-return_date.month_name 0.000 0.000 0.000 0.0
B-return_date.day_number 0.000 0.000 0.000 0.0
B-arrive_time.period_mod 0.000 0.000 0.000 0.0
I-meal_code 0.000 0.000 0.000 0.0
B-toloc.country_name 1.000 1.000 1.000 1.0
B-days_code 0.000 0.000 0.000 1.0
I-arrive_time.period_of_day 0.000 0.000 0.000 0.0
I-today_relative 0.000 0.000 0.000 0.0
B-return_time.period_of_day 0.000 0.000 0.000 0.0
B-time 0.000 0.000 0.000 0.0
I-fare_basis_code 0.000 0.000 0.000 0.0
I-arrive_time.time_relative 1.000 1.000 1.000 4.0
I-depart_time.time_relative 0.000 0.000 0.000 1.0
B-today_relative 0.000 0.000 0.000 0.0
B-state_name 0.000 0.000 0.000 9.0
B-arrive_date.today_relative 0.000 0.000 0.000 0.0
B-return_time.period_mod 0.000 0.000 0.000 0.0
B-month_name 0.000 0.000 0.000 0.0
B-day_number 0.000 0.000 0.000 0.0
I-return_date.date_relative 0.667 0.667 0.667 3.0
I-return_date.today_relative 0.000 0.000 0.000 0.0
B-stoploc.airport_name 0.000 0.000 0.000 0.0
B-time_relative 0.000 0.000 0.000 0.0
I-time 0.000 0.000 0.000 0.0
I-return_date.day_number 0.000 0.000 0.000 0.0
I-meal_description 0.000 0.000 0.000 0.0
B-return_date.today_relative 0.000 0.000 0.000 0.0
B-return_date.day_name 0.000 0.000 0.000 2.0
micro avg 0.960 0.961 0.960 3657.0
macro avg 0.681 0.654 0.653 3657.0
weighted avg 0.963 0.961 0.958 3657.0
I0716 05:56:18.178083 139795883440000 interactiveshell.py:2882] Best Slot F1: 0.961
I0716 05:56:18.179729 139795883440000 estimator_training.py:186] Not using Distribute Coordinator.
I0716 05:56:18.181779 139795883440000 training.py:612] Running training and evaluation locally (non-distributed).
I0716 05:56:18.183770 139795883440000 training.py:700] Start train and evaluate loop. The evaluate will happen after every checkpoint. Checkpoint frequency is determined based on RunConfig arguments: save_checkpoints_steps 312 or save_checkpoints_secs None.
I0716 05:56:18.252672 139795883440000 estimator.py:1145] Calling model_fn.
I0716 05:56:19.590207 139795883440000 saver.py:1499] Saver not created because there are no variables in the graph to restore
I0716 05:56:20.392826 139795883440000 <ipython-input-6-d6747b40360b>:55]
[<tf.Variable 'embedding:0' shape=(750, 300) dtype=float32_ref>,
<tf.Variable 'module/aggregation/weights:0' shape=(3,) dtype=float32>,
<tf.Variable 'module/aggregation/scaling:0' shape=() dtype=float32>,
<tf.Variable 'dense/kernel:0' shape=(1324, 300) dtype=float32_ref>,
<tf.Variable 'dense/bias:0' shape=(300,) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/fw/gru_cell/gates/kernel:0' shape=(600, 600) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/fw/gru_cell/gates/bias:0' shape=(600,) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/fw/gru_cell/candidate/kernel:0' shape=(600, 300) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/fw/gru_cell/candidate/bias:0' shape=(300,) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/bw/gru_cell/gates/kernel:0' shape=(600, 600) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/bw/gru_cell/gates/bias:0' shape=(600,) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/bw/gru_cell/candidate/kernel:0' shape=(600, 300) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/bw/gru_cell/candidate/bias:0' shape=(300,) dtype=float32_ref>,
<tf.Variable 'dense_1/kernel:0' shape=(600, 23) dtype=float32_ref>,
<tf.Variable 'dense_1/bias:0' shape=(23,) dtype=float32_ref>,
<tf.Variable 'dense_2/kernel:0' shape=(600, 122) dtype=float32_ref>,
<tf.Variable 'dense_2/bias:0' shape=(122,) dtype=float32_ref>]
I0716 05:56:21.472462 139795883440000 estimator.py:1147] Done calling model_fn.
I0716 05:56:21.476660 139795883440000 basic_session_run_hooks.py:541] Create CheckpointSaverHook.
I0716 05:56:21.795012 139795883440000 monitored_session.py:240] Graph was finalized.
I0716 05:56:21.818181 139795883440000 saver.py:1280] Restoring parameters from ../model/elmo_bigru/model.ckpt-14976
I0716 05:56:22.884191 139795883440000 session_manager.py:500] Running local_init_op.
I0716 05:56:22.954678 139795883440000 session_manager.py:502] Done running local_init_op.
I0716 05:56:25.807198 139795883440000 basic_session_run_hooks.py:606] Saving checkpoints for 14976 into ../model/elmo_bigru/model.ckpt.
Reading ../data/atis.train.w-intent.iob
I0716 05:56:30.610659 139795883440000 basic_session_run_hooks.py:262] loss = 0.008995927, step = 14976 I0716 05:56:30.612615 139795883440000 basic_session_run_hooks.py:262] lr = 6.19237e-05 I0716 05:56:57.552406 139795883440000 basic_session_run_hooks.py:692] global_step/sec: 3.71164 I0716 05:56:57.555257 139795883440000 basic_session_run_hooks.py:260] loss = 0.013111635, step = 15076 (26.945 sec) I0716 05:56:57.557056 139795883440000 basic_session_run_hooks.py:260] lr = 6.127469e-05 (26.944 sec) I0716 05:57:24.568656 139795883440000 basic_session_run_hooks.py:692] global_step/sec: 3.70148 I0716 05:57:24.571431 139795883440000 basic_session_run_hooks.py:260] loss = 0.011711226, step = 15176 (27.016 sec) I0716 05:57:24.576102 139795883440000 basic_session_run_hooks.py:260] lr = 6.063249e-05 (27.019 sec) I0716 05:57:51.199509 139795883440000 basic_session_run_hooks.py:692] global_step/sec: 3.75505 I0716 05:57:51.207821 139795883440000 basic_session_run_hooks.py:260] loss = 0.0053373585, step = 15276 (26.636 sec) I0716 05:57:51.210559 139795883440000 basic_session_run_hooks.py:260] lr = 5.9997008e-05 (26.634 sec) I0716 05:57:53.953844 139795883440000 basic_session_run_hooks.py:606] Saving checkpoints for 15288 into ../model/elmo_bigru/model.ckpt. I0716 05:57:55.954508 139795883440000 estimator.py:1145] Calling model_fn. I0716 05:57:57.582303 139795883440000 saver.py:1499] Saver not created because there are no variables in the graph to restore I0716 05:57:58.166869 139795883440000 estimator.py:1147] Done calling model_fn. I0716 05:57:58.198586 139795883440000 evaluation.py:255] Starting evaluation at 2019-07-16T05:57:58Z I0716 05:57:58.439807 139795883440000 monitored_session.py:240] Graph was finalized. I0716 05:57:58.462229 139795883440000 saver.py:1280] Restoring parameters from ../model/elmo_bigru/model.ckpt-15288 I0716 05:57:59.353767 139795883440000 session_manager.py:500] Running local_init_op. I0716 05:57:59.408347 139795883440000 session_manager.py:502] Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
I0716 05:58:58.318442 139795883440000 evaluation.py:275] Finished evaluation at 2019-07-16-05:58:58 I0716 05:58:58.320344 139795883440000 estimator.py:2039] Saving dict for global step 15288: global_step = 15288, loss = 0.42268425 I0716 05:58:58.328033 139795883440000 estimator.py:2099] Saving 'checkpoint_path' summary for global step 15288: ../model/elmo_bigru/model.ckpt-15288 I0716 05:58:58.413577 139795883440000 estimator.py:368] Loss for final step: 0.0028340353.
Reading ../data/atis.test.w-intent.iob
I0716 05:58:59.028976 139795883440000 estimator.py:1145] Calling model_fn. I0716 05:59:00.581136 139795883440000 saver.py:1499] Saver not created because there are no variables in the graph to restore I0716 05:59:01.108357 139795883440000 estimator.py:1147] Done calling model_fn. I0716 05:59:01.517195 139795883440000 monitored_session.py:240] Graph was finalized. I0716 05:59:01.536671 139795883440000 saver.py:1280] Restoring parameters from ../model/elmo_bigru/model.ckpt-15288 I0716 05:59:02.462764 139795883440000 session_manager.py:500] Running local_init_op. I0716 05:59:02.510772 139795883440000 session_manager.py:502] Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.
'precision', 'predicted', average, warn_for)
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1439: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples.
'recall', 'true', average, warn_for)
I0716 06:00:00.528573 139795883440000 interactiveshell.py:2882]
precision recall f1-score support
atis_flight 0.983 0.992 0.987 632
atis_airfare 0.941 1.000 0.970 48
atis_ground_service 1.000 1.000 1.000 36
atis_airline 1.000 1.000 1.000 38
atis_abbreviation 0.943 1.000 0.971 33
atis_aircraft 1.000 0.889 0.941 9
atis_flight_time 0.500 1.000 0.667 1
atis_quantity 0.429 1.000 0.600 3
atis_flight#atis_airfare 0.833 0.417 0.556 12
atis_airport 1.000 1.000 1.000 18
atis_distance 1.000 1.000 1.000 10
atis_city 1.000 0.500 0.667 6
atis_ground_fare 1.000 0.857 0.923 7
atis_capacity 0.952 0.952 0.952 21
atis_flight_no 0.889 1.000 0.941 8
atis_meal 1.000 0.833 0.909 6
atis_restriction 0.000 0.000 0.000 0
atis_airline#atis_flight_no 0.000 0.000 0.000 0
atis_ground_service#atis_ground_fare 0.000 0.000 0.000 0
atis_airfare#atis_flight_time 0.000 0.000 0.000 0
atis_cheapest 0.000 0.000 0.000 0
atis_aircraft#atis_flight#atis_flight_no 0.000 0.000 0.000 0
micro avg 0.973 0.979 0.976 888
macro avg 0.658 0.656 0.640 888
weighted avg 0.976 0.979 0.975 888
I0716 06:00:00.569924 139795883440000 interactiveshell.py:2882]
precision recall f1-score support
O 0.000 0.000 0.000 0.0
B-toloc.city_name 0.977 0.994 0.985 716.0
B-fromloc.city_name 0.990 0.999 0.994 704.0
I-toloc.city_name 0.978 0.989 0.983 265.0
B-depart_date.day_name 0.991 0.991 0.991 212.0
B-airline_name 1.000 1.000 1.000 101.0
I-fromloc.city_name 0.978 1.000 0.989 177.0
B-depart_time.period_of_day 1.000 0.923 0.960 130.0
I-airline_name 1.000 1.000 1.000 65.0
B-depart_date.day_number 0.981 0.964 0.972 55.0
B-depart_date.month_name 0.982 0.964 0.973 56.0
B-depart_time.time 0.864 1.000 0.927 57.0
B-round_trip 1.000 0.986 0.993 73.0
B-cost_relative 0.973 0.973 0.973 37.0
I-round_trip 1.000 1.000 1.000 71.0
B-flight_mod 0.828 1.000 0.906 24.0
B-depart_time.time_relative 0.955 0.985 0.970 65.0
I-depart_time.time 0.945 1.000 0.972 52.0
B-stoploc.city_name 0.909 1.000 0.952 20.0
B-city_name 0.923 0.632 0.750 57.0
B-class_type 0.960 1.000 0.980 24.0
B-arrive_time.time 0.971 0.971 0.971 34.0
B-arrive_time.time_relative 0.967 0.935 0.951 31.0
I-class_type 1.000 1.000 1.000 17.0
B-flight_stop 1.000 1.000 1.000 21.0
I-arrive_time.time 1.000 0.971 0.986 35.0
B-airline_code 0.971 1.000 0.986 34.0
I-depart_date.day_number 1.000 0.933 0.966 15.0
I-fromloc.airport_name 0.441 1.000 0.612 15.0
B-fromloc.airport_name 0.462 1.000 0.632 12.0
B-arrive_date.day_name 0.786 1.000 0.880 11.0
B-toloc.state_code 1.000 1.000 1.000 18.0
B-depart_date.today_relative 1.000 1.000 1.000 9.0
B-flight_number 0.733 1.000 0.846 11.0
B-depart_date.date_relative 0.944 1.000 0.971 17.0
B-toloc.state_name 0.788 0.929 0.852 28.0
B-fare_basis_code 0.850 1.000 0.919 17.0
B-flight_time 1.000 1.000 1.000 1.0
B-or 1.000 1.000 1.000 3.0
B-arrive_time.period_of_day 0.750 1.000 0.857 6.0
B-meal_description 1.000 0.900 0.947 10.0
I-cost_relative 1.000 0.667 0.800 3.0
I-airport_name 0.846 0.379 0.524 29.0
B-fare_amount 1.000 1.000 1.000 2.0
I-fare_amount 1.000 1.000 1.000 2.0
I-city_name 1.000 0.567 0.723 30.0
I-toloc.airport_name 1.000 1.000 1.000 3.0
B-transport_type 1.000 1.000 1.000 10.0
B-arrive_date.month_name 0.714 0.833 0.769 6.0
B-arrive_date.day_number 0.714 0.833 0.769 6.0
I-stoploc.city_name 1.000 1.000 1.000 10.0
B-meal 1.000 1.000 1.000 16.0
B-fromloc.state_code 1.000 1.000 1.000 23.0
B-depart_time.period_mod 1.000 1.000 1.000 5.0
B-connect 1.000 1.000 1.000 6.0
B-flight_days 1.000 1.000 1.000 10.0
B-toloc.airport_name 1.000 1.000 1.000 3.0
B-fromloc.state_name 0.944 1.000 0.971 17.0
B-airport_name 0.800 0.381 0.516 21.0
B-economy 1.000 1.000 1.000 6.0
I-flight_time 1.000 1.000 1.000 1.0
B-aircraft_code 1.000 0.879 0.935 33.0
B-mod 1.000 0.500 0.667 2.0
B-airport_code 1.000 0.444 0.615 9.0
B-depart_time.start_time 1.000 0.667 0.800 3.0
B-depart_time.end_time 1.000 0.333 0.500 3.0
B-depart_date.year 1.000 1.000 1.000 3.0
I-transport_type 0.000 0.000 0.000 1.0
B-restriction_code 0.800 1.000 0.889 4.0
B-arrive_time.start_time 0.889 1.000 0.941 8.0
B-toloc.airport_code 1.000 1.000 1.000 4.0
B-arrive_time.end_time 0.889 1.000 0.941 8.0
I-arrive_time.end_time 0.889 1.000 0.941 8.0
I-depart_time.end_time 1.000 0.333 0.500 3.0
I-flight_stop 0.000 0.000 0.000 0.0
B-fromloc.airport_code 0.556 1.000 0.714 5.0
I-restriction_code 1.000 1.000 1.000 3.0
I-depart_time.start_time 1.000 1.000 1.000 1.0
I-toloc.state_name 1.000 1.000 1.000 1.0
I-depart_date.today_relative 0.000 0.000 0.000 0.0
B-arrive_date.date_relative 1.000 1.000 1.000 2.0
I-flight_mod 1.000 0.167 0.286 6.0
I-economy 0.000 0.000 0.000 0.0
B-return_date.date_relative 1.000 0.333 0.500 3.0
I-fromloc.state_name 1.000 1.000 1.000 1.0
B-state_code 1.000 1.000 1.000 1.0
I-arrive_time.start_time 1.000 1.000 1.000 1.0
I-arrive_date.day_number 0.000 0.000 0.000 0.0
B-meal_code 0.000 0.000 0.000 1.0
I-depart_time.period_of_day 1.000 1.000 1.000 1.0
B-day_name 1.000 0.500 0.667 2.0
B-period_of_day 1.000 0.750 0.857 4.0
B-stoploc.state_code 0.000 0.000 0.000 0.0
B-return_date.month_name 0.000 0.000 0.000 0.0
B-return_date.day_number 0.000 0.000 0.000 0.0
B-arrive_time.period_mod 0.000 0.000 0.000 0.0
I-meal_code 0.000 0.000 0.000 0.0
B-toloc.country_name 1.000 1.000 1.000 1.0
B-days_code 0.000 0.000 0.000 1.0
I-arrive_time.period_of_day 0.000 0.000 0.000 0.0
I-today_relative 0.000 0.000 0.000 0.0
B-return_time.period_of_day 0.000 0.000 0.000 0.0
B-time 0.000 0.000 0.000 0.0
I-fare_basis_code 0.000 0.000 0.000 0.0
I-arrive_time.time_relative 1.000 1.000 1.000 4.0
I-depart_time.time_relative 0.000 0.000 0.000 1.0
B-today_relative 0.000 0.000 0.000 0.0
B-state_name 0.000 0.000 0.000 9.0
B-arrive_date.today_relative 0.000 0.000 0.000 0.0
B-return_time.period_mod 0.000 0.000 0.000 0.0
B-month_name 0.000 0.000 0.000 0.0
B-day_number 0.000 0.000 0.000 0.0
I-return_date.date_relative 0.667 0.667 0.667 3.0
I-return_date.today_relative 0.000 0.000 0.000 0.0
B-stoploc.airport_name 0.000 0.000 0.000 0.0
B-time_relative 0.000 0.000 0.000 0.0
I-time 0.000 0.000 0.000 0.0
I-return_date.day_number 0.000 0.000 0.000 0.0
I-meal_description 0.000 0.000 0.000 0.0
B-return_date.today_relative 0.000 0.000 0.000 0.0
B-return_date.day_name 0.000 0.000 0.000 2.0
micro avg 0.959 0.961 0.960 3657.0
macro avg 0.683 0.655 0.654 3657.0
weighted avg 0.964 0.961 0.958 3657.0
I0716 06:00:00.593960 139795883440000 interactiveshell.py:2882] Best Slot F1: 0.961
I0716 06:00:00.595951 139795883440000 estimator_training.py:186] Not using Distribute Coordinator.
I0716 06:00:00.600968 139795883440000 training.py:612] Running training and evaluation locally (non-distributed).
I0716 06:00:00.603511 139795883440000 training.py:700] Start train and evaluate loop. The evaluate will happen after every checkpoint. Checkpoint frequency is determined based on RunConfig arguments: save_checkpoints_steps 312 or save_checkpoints_secs None.
I0716 06:00:00.673551 139795883440000 estimator.py:1145] Calling model_fn.
I0716 06:00:02.042279 139795883440000 saver.py:1499] Saver not created because there are no variables in the graph to restore
I0716 06:00:02.645030 139795883440000 <ipython-input-6-d6747b40360b>:55]
[<tf.Variable 'embedding:0' shape=(750, 300) dtype=float32_ref>,
<tf.Variable 'module/aggregation/weights:0' shape=(3,) dtype=float32>,
<tf.Variable 'module/aggregation/scaling:0' shape=() dtype=float32>,
<tf.Variable 'dense/kernel:0' shape=(1324, 300) dtype=float32_ref>,
<tf.Variable 'dense/bias:0' shape=(300,) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/fw/gru_cell/gates/kernel:0' shape=(600, 600) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/fw/gru_cell/gates/bias:0' shape=(600,) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/fw/gru_cell/candidate/kernel:0' shape=(600, 300) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/fw/gru_cell/candidate/bias:0' shape=(300,) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/bw/gru_cell/gates/kernel:0' shape=(600, 600) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/bw/gru_cell/gates/bias:0' shape=(600,) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/bw/gru_cell/candidate/kernel:0' shape=(600, 300) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/bw/gru_cell/candidate/bias:0' shape=(300,) dtype=float32_ref>,
<tf.Variable 'dense_1/kernel:0' shape=(600, 23) dtype=float32_ref>,
<tf.Variable 'dense_1/bias:0' shape=(23,) dtype=float32_ref>,
<tf.Variable 'dense_2/kernel:0' shape=(600, 122) dtype=float32_ref>,
<tf.Variable 'dense_2/bias:0' shape=(122,) dtype=float32_ref>]
I0716 06:00:03.956224 139795883440000 estimator.py:1147] Done calling model_fn.
I0716 06:00:03.964102 139795883440000 basic_session_run_hooks.py:541] Create CheckpointSaverHook.
I0716 06:00:04.286618 139795883440000 monitored_session.py:240] Graph was finalized.
I0716 06:00:04.307549 139795883440000 saver.py:1280] Restoring parameters from ../model/elmo_bigru/model.ckpt-15288
I0716 06:00:05.404417 139795883440000 session_manager.py:500] Running local_init_op.
I0716 06:00:05.497148 139795883440000 session_manager.py:502] Done running local_init_op.
I0716 06:00:08.350968 139795883440000 basic_session_run_hooks.py:606] Saving checkpoints for 15288 into ../model/elmo_bigru/model.ckpt.
Reading ../data/atis.train.w-intent.iob
I0716 06:00:13.357057 139795883440000 basic_session_run_hooks.py:262] loss = 0.0077921036, step = 15288 I0716 06:00:13.358960 139795883440000 basic_session_run_hooks.py:262] lr = 5.99212e-05 I0716 06:00:40.077207 139795883440000 basic_session_run_hooks.py:692] global_step/sec: 3.74241 I0716 06:00:40.083547 139795883440000 basic_session_run_hooks.py:260] loss = 0.013172938, step = 15388 (26.726 sec) I0716 06:00:40.087269 139795883440000 basic_session_run_hooks.py:260] lr = 5.9293186e-05 (26.728 sec) I0716 06:01:07.204914 139795883440000 basic_session_run_hooks.py:692] global_step/sec: 3.68628 I0716 06:01:07.212323 139795883440000 basic_session_run_hooks.py:260] loss = 0.010772801, step = 15488 (27.129 sec) I0716 06:01:07.215197 139795883440000 basic_session_run_hooks.py:260] lr = 5.8671747e-05 (27.128 sec) I0716 06:01:32.990794 139795883440000 basic_session_run_hooks.py:692] global_step/sec: 3.87808 I0716 06:01:32.999497 139795883440000 basic_session_run_hooks.py:260] loss = 0.0069304784, step = 15588 (25.787 sec) I0716 06:01:33.001451 139795883440000 basic_session_run_hooks.py:260] lr = 5.8056834e-05 (25.786 sec) I0716 06:01:36.349722 139795883440000 basic_session_run_hooks.py:606] Saving checkpoints for 15600 into ../model/elmo_bigru/model.ckpt. I0716 06:01:38.454736 139795883440000 estimator.py:1145] Calling model_fn. I0716 06:01:40.095187 139795883440000 saver.py:1499] Saver not created because there are no variables in the graph to restore I0716 06:01:40.678136 139795883440000 estimator.py:1147] Done calling model_fn. I0716 06:01:40.711853 139795883440000 evaluation.py:255] Starting evaluation at 2019-07-16T06:01:40Z I0716 06:01:40.939875 139795883440000 monitored_session.py:240] Graph was finalized. I0716 06:01:40.958451 139795883440000 saver.py:1280] Restoring parameters from ../model/elmo_bigru/model.ckpt-15600 I0716 06:01:41.885448 139795883440000 session_manager.py:500] Running local_init_op. I0716 06:01:41.942841 139795883440000 session_manager.py:502] Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
I0716 06:02:40.721067 139795883440000 evaluation.py:275] Finished evaluation at 2019-07-16-06:02:40 I0716 06:02:40.722697 139795883440000 estimator.py:2039] Saving dict for global step 15600: global_step = 15600, loss = 0.42624673 I0716 06:02:40.728740 139795883440000 estimator.py:2099] Saving 'checkpoint_path' summary for global step 15600: ../model/elmo_bigru/model.ckpt-15600 I0716 06:02:40.827952 139795883440000 estimator.py:368] Loss for final step: 0.009893632.
Reading ../data/atis.test.w-intent.iob
I0716 06:02:41.432628 139795883440000 estimator.py:1145] Calling model_fn. I0716 06:02:42.993689 139795883440000 saver.py:1499] Saver not created because there are no variables in the graph to restore I0716 06:02:43.514800 139795883440000 estimator.py:1147] Done calling model_fn. I0716 06:02:43.729723 139795883440000 monitored_session.py:240] Graph was finalized. I0716 06:02:43.749116 139795883440000 saver.py:1280] Restoring parameters from ../model/elmo_bigru/model.ckpt-15600 I0716 06:02:44.648586 139795883440000 session_manager.py:500] Running local_init_op. I0716 06:02:44.698976 139795883440000 session_manager.py:502] Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.
'precision', 'predicted', average, warn_for)
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1439: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples.
'recall', 'true', average, warn_for)
I0716 06:03:42.711251 139795883440000 interactiveshell.py:2882]
precision recall f1-score support
atis_flight 0.984 0.991 0.987 632
atis_airfare 0.960 1.000 0.980 48
atis_ground_service 1.000 1.000 1.000 36
atis_airline 1.000 1.000 1.000 38
atis_abbreviation 0.943 1.000 0.971 33
atis_aircraft 1.000 0.889 0.941 9
atis_flight_time 0.500 1.000 0.667 1
atis_quantity 0.429 1.000 0.600 3
atis_flight#atis_airfare 0.833 0.417 0.556 12
atis_airport 0.947 1.000 0.973 18
atis_distance 1.000 1.000 1.000 10
atis_city 0.750 0.500 0.600 6
atis_ground_fare 1.000 1.000 1.000 7
atis_capacity 0.952 0.952 0.952 21
atis_flight_no 0.889 1.000 0.941 8
atis_meal 1.000 0.833 0.909 6
atis_restriction 0.000 0.000 0.000 0
atis_airline#atis_flight_no 0.000 0.000 0.000 0
atis_ground_service#atis_ground_fare 0.000 0.000 0.000 0
atis_airfare#atis_flight_time 0.000 0.000 0.000 0
atis_cheapest 0.000 0.000 0.000 0
atis_aircraft#atis_flight#atis_flight_no 0.000 0.000 0.000 0
micro avg 0.973 0.979 0.976 888
macro avg 0.645 0.663 0.640 888
weighted avg 0.975 0.979 0.975 888
I0716 06:03:42.749567 139795883440000 interactiveshell.py:2882]
precision recall f1-score support
O 0.000 0.000 0.000 0.0
B-toloc.city_name 0.977 0.994 0.985 716.0
B-fromloc.city_name 0.990 1.000 0.995 704.0
I-toloc.city_name 0.978 0.989 0.983 265.0
B-depart_date.day_name 0.991 0.991 0.991 212.0
B-airline_name 1.000 1.000 1.000 101.0
I-fromloc.city_name 0.978 1.000 0.989 177.0
B-depart_time.period_of_day 1.000 0.923 0.960 130.0
I-airline_name 1.000 1.000 1.000 65.0
B-depart_date.day_number 0.981 0.964 0.972 55.0
B-depart_date.month_name 0.982 0.964 0.973 56.0
B-depart_time.time 0.864 1.000 0.927 57.0
B-round_trip 1.000 0.986 0.993 73.0
B-cost_relative 0.973 0.973 0.973 37.0
I-round_trip 1.000 1.000 1.000 71.0
B-flight_mod 0.828 1.000 0.906 24.0
B-depart_time.time_relative 0.970 0.985 0.977 65.0
I-depart_time.time 0.945 1.000 0.972 52.0
B-stoploc.city_name 0.952 1.000 0.976 20.0
B-city_name 0.923 0.632 0.750 57.0
B-class_type 0.960 1.000 0.980 24.0
B-arrive_time.time 0.971 0.971 0.971 34.0
B-arrive_time.time_relative 0.935 0.935 0.935 31.0
I-class_type 1.000 1.000 1.000 17.0
B-flight_stop 1.000 1.000 1.000 21.0
I-arrive_time.time 1.000 0.971 0.986 35.0
B-airline_code 0.971 1.000 0.986 34.0
I-depart_date.day_number 1.000 0.933 0.966 15.0
I-fromloc.airport_name 0.441 1.000 0.612 15.0
B-fromloc.airport_name 0.462 1.000 0.632 12.0
B-arrive_date.day_name 0.786 1.000 0.880 11.0
B-toloc.state_code 1.000 1.000 1.000 18.0
B-depart_date.today_relative 1.000 0.889 0.941 9.0
B-flight_number 0.733 1.000 0.846 11.0
B-depart_date.date_relative 0.944 1.000 0.971 17.0
B-toloc.state_name 0.839 0.929 0.881 28.0
B-fare_basis_code 0.895 1.000 0.944 17.0
B-flight_time 1.000 1.000 1.000 1.0
B-or 1.000 1.000 1.000 3.0
B-arrive_time.period_of_day 0.750 1.000 0.857 6.0
B-meal_description 1.000 0.900 0.947 10.0
I-cost_relative 1.000 0.667 0.800 3.0
I-airport_name 0.846 0.379 0.524 29.0
B-fare_amount 1.000 1.000 1.000 2.0
I-fare_amount 1.000 1.000 1.000 2.0
I-city_name 1.000 0.567 0.723 30.0
I-toloc.airport_name 1.000 1.000 1.000 3.0
B-transport_type 1.000 1.000 1.000 10.0
B-arrive_date.month_name 0.714 0.833 0.769 6.0
B-arrive_date.day_number 0.714 0.833 0.769 6.0
I-stoploc.city_name 1.000 1.000 1.000 10.0
B-meal 1.000 1.000 1.000 16.0
B-fromloc.state_code 1.000 1.000 1.000 23.0
B-depart_time.period_mod 1.000 1.000 1.000 5.0
B-connect 1.000 1.000 1.000 6.0
B-flight_days 1.000 1.000 1.000 10.0
B-toloc.airport_name 1.000 1.000 1.000 3.0
B-fromloc.state_name 0.944 1.000 0.971 17.0
B-airport_name 0.667 0.381 0.485 21.0
B-economy 1.000 1.000 1.000 6.0
I-flight_time 1.000 1.000 1.000 1.0
B-aircraft_code 1.000 0.879 0.935 33.0
B-mod 1.000 0.500 0.667 2.0
B-airport_code 1.000 0.444 0.615 9.0
B-depart_time.start_time 1.000 0.667 0.800 3.0
B-depart_time.end_time 1.000 0.333 0.500 3.0
B-depart_date.year 1.000 1.000 1.000 3.0
I-transport_type 0.000 0.000 0.000 1.0
B-restriction_code 0.800 1.000 0.889 4.0
B-arrive_time.start_time 0.889 1.000 0.941 8.0
B-toloc.airport_code 1.000 1.000 1.000 4.0
B-arrive_time.end_time 0.889 1.000 0.941 8.0
I-arrive_time.end_time 0.889 1.000 0.941 8.0
I-depart_time.end_time 1.000 0.333 0.500 3.0
I-flight_stop 0.000 0.000 0.000 0.0
B-fromloc.airport_code 0.556 1.000 0.714 5.0
I-restriction_code 1.000 1.000 1.000 3.0
I-depart_time.start_time 1.000 1.000 1.000 1.0
I-toloc.state_name 1.000 1.000 1.000 1.0
I-depart_date.today_relative 0.000 0.000 0.000 0.0
B-arrive_date.date_relative 1.000 1.000 1.000 2.0
I-flight_mod 1.000 0.167 0.286 6.0
I-economy 0.000 0.000 0.000 0.0
B-return_date.date_relative 1.000 0.667 0.800 3.0
I-fromloc.state_name 1.000 1.000 1.000 1.0
B-state_code 1.000 1.000 1.000 1.0
I-arrive_time.start_time 1.000 1.000 1.000 1.0
I-arrive_date.day_number 0.000 0.000 0.000 0.0
B-meal_code 0.000 0.000 0.000 1.0
I-depart_time.period_of_day 1.000 1.000 1.000 1.0
B-day_name 1.000 0.500 0.667 2.0
B-period_of_day 1.000 0.750 0.857 4.0
B-stoploc.state_code 0.000 0.000 0.000 0.0
B-return_date.month_name 0.000 0.000 0.000 0.0
B-return_date.day_number 0.000 0.000 0.000 0.0
B-arrive_time.period_mod 0.000 0.000 0.000 0.0
I-meal_code 0.000 0.000 0.000 0.0
B-toloc.country_name 1.000 1.000 1.000 1.0
B-days_code 1.000 1.000 1.000 1.0
I-arrive_time.period_of_day 0.000 0.000 0.000 0.0
I-today_relative 0.000 0.000 0.000 0.0
B-return_time.period_of_day 0.000 0.000 0.000 0.0
B-time 0.000 0.000 0.000 0.0
I-fare_basis_code 0.000 0.000 0.000 0.0
I-arrive_time.time_relative 1.000 0.750 0.857 4.0
I-depart_time.time_relative 0.000 0.000 0.000 1.0
B-today_relative 0.000 0.000 0.000 0.0
B-state_name 0.000 0.000 0.000 9.0
B-arrive_date.today_relative 0.000 0.000 0.000 0.0
B-return_time.period_mod 0.000 0.000 0.000 0.0
B-month_name 0.000 0.000 0.000 0.0
B-day_number 0.000 0.000 0.000 0.0
I-return_date.date_relative 1.000 0.667 0.800 3.0
I-return_date.today_relative 0.000 0.000 0.000 0.0
B-stoploc.airport_name 0.000 0.000 0.000 0.0
B-time_relative 0.000 0.000 0.000 0.0
I-time 0.000 0.000 0.000 0.0
I-return_date.day_number 0.000 0.000 0.000 0.0
I-meal_description 0.000 0.000 0.000 0.0
B-return_date.today_relative 0.000 0.000 0.000 0.0
B-return_date.day_name 0.000 0.000 0.000 2.0
micro avg 0.960 0.961 0.960 3657.0
macro avg 0.694 0.663 0.665 3657.0
weighted avg 0.964 0.961 0.958 3657.0
I0716 06:03:42.772957 139795883440000 interactiveshell.py:2882] Best Slot F1: 0.961
I0716 06:03:42.774310 139795883440000 estimator_training.py:186] Not using Distribute Coordinator.
I0716 06:03:42.776245 139795883440000 training.py:612] Running training and evaluation locally (non-distributed).
I0716 06:03:42.778315 139795883440000 training.py:700] Start train and evaluate loop. The evaluate will happen after every checkpoint. Checkpoint frequency is determined based on RunConfig arguments: save_checkpoints_steps 312 or save_checkpoints_secs None.
I0716 06:03:42.850522 139795883440000 estimator.py:1145] Calling model_fn.
I0716 06:03:44.378764 139795883440000 saver.py:1499] Saver not created because there are no variables in the graph to restore
I0716 06:03:44.961277 139795883440000 <ipython-input-6-d6747b40360b>:55]
[<tf.Variable 'embedding:0' shape=(750, 300) dtype=float32_ref>,
<tf.Variable 'module/aggregation/weights:0' shape=(3,) dtype=float32>,
<tf.Variable 'module/aggregation/scaling:0' shape=() dtype=float32>,
<tf.Variable 'dense/kernel:0' shape=(1324, 300) dtype=float32_ref>,
<tf.Variable 'dense/bias:0' shape=(300,) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/fw/gru_cell/gates/kernel:0' shape=(600, 600) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/fw/gru_cell/gates/bias:0' shape=(600,) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/fw/gru_cell/candidate/kernel:0' shape=(600, 300) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/fw/gru_cell/candidate/bias:0' shape=(300,) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/bw/gru_cell/gates/kernel:0' shape=(600, 600) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/bw/gru_cell/gates/bias:0' shape=(600,) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/bw/gru_cell/candidate/kernel:0' shape=(600, 300) dtype=float32_ref>,
<tf.Variable 'bidirectional_rnn/bw/gru_cell/candidate/bias:0' shape=(300,) dtype=float32_ref>,
<tf.Variable 'dense_1/kernel:0' shape=(600, 23) dtype=float32_ref>,
<tf.Variable 'dense_1/bias:0' shape=(23,) dtype=float32_ref>,
<tf.Variable 'dense_2/kernel:0' shape=(600, 122) dtype=float32_ref>,
<tf.Variable 'dense_2/bias:0' shape=(122,) dtype=float32_ref>]
I0716 06:03:46.248441 139795883440000 estimator.py:1147] Done calling model_fn.
I0716 06:03:46.252445 139795883440000 basic_session_run_hooks.py:541] Create CheckpointSaverHook.
I0716 06:03:46.582277 139795883440000 monitored_session.py:240] Graph was finalized.
I0716 06:03:46.600440 139795883440000 saver.py:1280] Restoring parameters from ../model/elmo_bigru/model.ckpt-15600
I0716 06:03:47.670186 139795883440000 session_manager.py:500] Running local_init_op.
I0716 06:03:47.740401 139795883440000 session_manager.py:502] Done running local_init_op.
I0716 06:03:50.567635 139795883440000 basic_session_run_hooks.py:606] Saving checkpoints for 15600 into ../model/elmo_bigru/model.ckpt.
Reading ../data/atis.train.w-intent.iob
I0716 06:03:55.228627 139795883440000 basic_session_run_hooks.py:262] loss = 0.007499742, step = 15600 I0716 06:03:55.230428 139795883440000 basic_session_run_hooks.py:262] lr = 5.798347e-05 I0716 06:04:22.553234 139795883440000 basic_session_run_hooks.py:692] global_step/sec: 3.65965 I0716 06:04:22.559311 139795883440000 basic_session_run_hooks.py:260] loss = 0.0067059495, step = 15700 (27.331 sec) I0716 06:04:22.561770 139795883440000 basic_session_run_hooks.py:260] lr = 5.737576e-05 (27.331 sec) I0716 06:04:48.927580 139795883440000 basic_session_run_hooks.py:692] global_step/sec: 3.79156 I0716 06:04:48.931572 139795883440000 basic_session_run_hooks.py:260] loss = 0.009300061, step = 15800 (26.372 sec) I0716 06:04:48.935711 139795883440000 basic_session_run_hooks.py:260] lr = 5.6774414e-05 (26.374 sec) I0716 06:05:15.645237 139795883440000 basic_session_run_hooks.py:692] global_step/sec: 3.74285 I0716 06:05:15.651256 139795883440000 basic_session_run_hooks.py:260] loss = 0.007555264, step = 15900 (26.720 sec) I0716 06:05:15.653822 139795883440000 basic_session_run_hooks.py:260] lr = 5.6179386e-05 (26.718 sec) I0716 06:05:18.507963 139795883440000 basic_session_run_hooks.py:606] Saving checkpoints for 15912 into ../model/elmo_bigru/model.ckpt. I0716 06:05:20.802643 139795883440000 estimator.py:1145] Calling model_fn. I0716 06:05:22.489187 139795883440000 saver.py:1499] Saver not created because there are no variables in the graph to restore I0716 06:05:23.066193 139795883440000 estimator.py:1147] Done calling model_fn. I0716 06:05:23.095655 139795883440000 evaluation.py:255] Starting evaluation at 2019-07-16T06:05:23Z I0716 06:05:23.319568 139795883440000 monitored_session.py:240] Graph was finalized. I0716 06:05:23.338410 139795883440000 saver.py:1280] Restoring parameters from ../model/elmo_bigru/model.ckpt-15912 I0716 06:05:24.347270 139795883440000 session_manager.py:500] Running local_init_op. I0716 06:05:24.422134 139795883440000 session_manager.py:502] Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
I0716 06:06:23.721860 139795883440000 evaluation.py:275] Finished evaluation at 2019-07-16-06:06:23 I0716 06:06:23.723529 139795883440000 estimator.py:2039] Saving dict for global step 15912: global_step = 15912, loss = 0.42764378 I0716 06:06:23.731847 139795883440000 estimator.py:2099] Saving 'checkpoint_path' summary for global step 15912: ../model/elmo_bigru/model.ckpt-15912 I0716 06:06:23.813684 139795883440000 estimator.py:368] Loss for final step: 0.0038011419.
Reading ../data/atis.test.w-intent.iob
I0716 06:06:24.433973 139795883440000 estimator.py:1145] Calling model_fn. I0716 06:06:26.021326 139795883440000 saver.py:1499] Saver not created because there are no variables in the graph to restore I0716 06:06:26.532943 139795883440000 estimator.py:1147] Done calling model_fn. I0716 06:06:26.747190 139795883440000 monitored_session.py:240] Graph was finalized. I0716 06:06:26.767136 139795883440000 saver.py:1280] Restoring parameters from ../model/elmo_bigru/model.ckpt-15912 I0716 06:06:27.689567 139795883440000 session_manager.py:500] Running local_init_op. I0716 06:06:27.740319 139795883440000 session_manager.py:502] Done running local_init_op.
Reading ../data/atis.test.w-intent.iob
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.
'precision', 'predicted', average, warn_for)
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1439: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples.
'recall', 'true', average, warn_for)
I0716 06:07:25.706608 139795883440000 interactiveshell.py:2882]
precision recall f1-score support
atis_flight 0.984 0.991 0.987 632
atis_airfare 0.941 1.000 0.970 48
atis_ground_service 1.000 1.000 1.000 36
atis_airline 0.974 1.000 0.987 38
atis_abbreviation 0.943 1.000 0.971 33
atis_aircraft 1.000 0.889 0.941 9
atis_flight_time 0.500 1.000 0.667 1
atis_quantity 0.429 1.000 0.600 3
atis_flight#atis_airfare 0.833 0.417 0.556 12
atis_airport 0.944 0.944 0.944 18
atis_distance 1.000 1.000 1.000 10
atis_city 0.750 0.500 0.600 6
atis_ground_fare 1.000 0.857 0.923 7
atis_capacity 0.952 0.952 0.952 21
atis_flight_no 0.889 1.000 0.941 8
atis_meal 1.000 0.833 0.909 6
atis_restriction 0.000 0.000 0.000 0
atis_airline#atis_flight_no 0.000 0.000 0.000 0
atis_ground_service#atis_ground_fare 0.000 0.000 0.000 0
atis_airfare#atis_flight_time 0.000 0.000 0.000 0
atis_cheapest 0.000 0.000 0.000 0
atis_aircraft#atis_flight#atis_flight_no 0.000 0.000 0.000 0
micro avg 0.971 0.976 0.974 888
macro avg 0.643 0.654 0.634 888
weighted avg 0.973 0.976 0.973 888
I0716 06:07:25.744853 139795883440000 interactiveshell.py:2882]
precision recall f1-score support
O 0.000 0.000 0.000 0.0
B-toloc.city_name 0.977 0.994 0.985 716.0
B-fromloc.city_name 0.990 0.999 0.994 704.0
I-toloc.city_name 0.978 0.989 0.983 265.0
B-depart_date.day_name 0.991 0.991 0.991 212.0
B-airline_name 1.000 1.000 1.000 101.0
I-fromloc.city_name 0.973 1.000 0.986 177.0
B-depart_time.period_of_day 0.992 0.923 0.956 130.0
I-airline_name 1.000 1.000 1.000 65.0
B-depart_date.day_number 0.981 0.964 0.972 55.0
B-depart_date.month_name 0.982 0.964 0.973 56.0
B-depart_time.time 0.864 1.000 0.927 57.0
B-round_trip 1.000 1.000 1.000 73.0
B-cost_relative 0.973 0.973 0.973 37.0
I-round_trip 1.000 1.000 1.000 71.0
B-flight_mod 0.828 1.000 0.906 24.0
B-depart_time.time_relative 0.955 0.985 0.970 65.0
I-depart_time.time 0.945 1.000 0.972 52.0
B-stoploc.city_name 0.909 1.000 0.952 20.0
B-city_name 0.923 0.632 0.750 57.0
B-class_type 0.960 1.000 0.980 24.0
B-arrive_time.time 0.971 0.971 0.971 34.0
B-arrive_time.time_relative 0.967 0.935 0.951 31.0
I-class_type 1.000 1.000 1.000 17.0
B-flight_stop 1.000 1.000 1.000 21.0
I-arrive_time.time 1.000 0.971 0.986 35.0
B-airline_code 0.971 1.000 0.986 34.0
I-depart_date.day_number 1.000 0.933 0.966 15.0
I-fromloc.airport_name 0.441 1.000 0.612 15.0
B-fromloc.airport_name 0.462 1.000 0.632 12.0
B-arrive_date.day_name 0.786 1.000 0.880 11.0
B-toloc.state_code 1.000 1.000 1.000 18.0
B-depart_date.today_relative 1.000 0.889 0.941 9.0
B-flight_number 0.733 1.000 0.846 11.0
B-depart_date.date_relative 0.944 1.000 0.971 17.0
B-toloc.state_name 0.839 0.929 0.881 28.0
B-fare_basis_code 0.850 1.000 0.919 17.0
B-flight_time 1.000 1.000 1.000 1.0
B-or 1.000 1.000 1.000 3.0
B-arrive_time.period_of_day 0.750 1.000 0.857 6.0
B-meal_description 1.000 0.900 0.947 10.0
I-cost_relative 1.000 0.667 0.800 3.0
I-airport_name 0.846 0.379 0.524 29.0
B-fare_amount 1.000 1.000 1.000 2.0
I-fare_amount 1.000 1.000 1.000 2.0
I-city_name 1.000 0.533 0.696 30.0
I-toloc.airport_name 1.000 1.000 1.000 3.0
B-transport_type 1.000 1.000 1.000 10.0
B-arrive_date.month_name 0.714 0.833 0.769 6.0
B-arrive_date.day_number 0.714 0.833 0.769 6.0
I-stoploc.city_name 1.000 1.000 1.000 10.0
B-meal 1.000 1.000 1.000 16.0
B-fromloc.state_code 1.000 1.000 1.000 23.0
B-depart_time.period_mod 1.000 1.000 1.000 5.0
B-connect 1.000 1.000 1.000 6.0
B-flight_days 1.000 1.000 1.000 10.0
B-toloc.airport_name 1.000 1.000 1.000 3.0
B-fromloc.state_name 0.944 1.000 0.971 17.0
B-airport_name 0.667 0.381 0.485 21.0
B-economy 1.000 1.000 1.000 6.0
I-flight_time 1.000 1.000 1.000 1.0
B-aircraft_code 1.000 0.879 0.935 33.0
B-mod 1.000 0.500 0.667 2.0
B-airport_code 1.000 0.444 0.615 9.0
B-depart_time.start_time 1.000 0.667 0.800 3.0
B-depart_time.end_time 1.000 0.333 0.500 3.0
B-depart_date.year 1.000 1.000 1.000 3.0
I-transport_type 0.000 0.000 0.000 1.0
B-restriction_code 0.800 1.000 0.889 4.0
B-arrive_time.start_time 0.889 1.000 0.941 8.0
B-toloc.airport_code 1.000 1.000 1.000 4.0
B-arrive_time.end_time 0.889 1.000 0.941 8.0
I-arrive_time.end_time 0.889 1.000 0.941 8.0
I-depart_time.end_time 1.000 0.333 0.500 3.0
I-flight_stop 0.000 0.000 0.000 0.0
B-fromloc.airport_code 0.556 1.000 0.714 5.0
I-restriction_code 1.000 1.000 1.000 3.0
I-depart_time.start_time 1.000 1.000 1.000 1.0
I-toloc.state_name 1.000 1.000 1.000 1.0
I-depart_date.today_relative 0.000 0.000 0.000 0.0
B-arrive_date.date_relative 1.000 1.000 1.000 2.0
I-flight_mod 1.000 0.167 0.286 6.0
I-economy 0.000 0.000 0.000 0.0
B-return_date.date_relative 1.000 0.667 0.800 3.0
I-fromloc.state_name 1.000 1.000 1.000 1.0
B-state_code 1.000 1.000 1.000 1.0
I-arrive_time.start_time 1.000 1.000 1.000 1.0
I-arrive_date.day_number 0.000 0.000 0.000 0.0
B-meal_code 0.000 0.000 0.000 1.0
I-depart_time.period_of_day 1.000 1.000 1.000 1.0
B-day_name 1.000 0.500 0.667 2.0
B-period_of_day 1.000 0.750 0.857 4.0
B-stoploc.state_code 0.000 0.000 0.000 0.0
B-return_date.month_name 0.000 0.000 0.000 0.0
B-return_date.day_