This notebook show how to run the tree cover example on AWS SageMaker.
Please read these instructions on how to setup AWS SageMaker.
import os
import datetime
from os import path as op
import itertools
from eolearn.io import *
from eolearn.core import EOTask, EOPatch, LinearWorkflow, FeatureType, SaveToDisk, OverwritePermission
from sentinelhub import BBox, CRS, BBoxSplitter, MimeType, ServiceType
from tqdm import tqdm_notebook as tqdm
import matplotlib.pyplot as plt
import numpy as np
import geopandas
import gzip
from sklearn.metrics import confusion_matrix
# https://www.sentinel-hub.com/faq/where-get-instance-id
INSTANCE_ID = os.environ.get('INSTANCE_ID')
time_interval = ('2017-01-01', '2017-12-31')
img_width = 256
img_height = 256
maxcc = 0.2
crs = CRS.UTM_31N
aoi = geopandas.read_file('eastern_france.geojson')
aoi = aoi.to_crs(crs={'init':CRS.ogc_string(crs)})
aoi_shape = aoi.geometry.values[-1]
bbox_splitter = BBoxSplitter([aoi_shape], crs, (19, 10))
raster_value = {
'0%': (0, [0, 0, 0, 0]),
'10%': (1, [163, 235, 153, 255]),
'30%': (2, [119, 195, 118, 255]),
'50%': (3, [85, 160, 89, 255]),
'70%': (4, [58, 130, 64, 255]),
'90%': (5, [36, 103, 44, 255])
}
class MedianPixel(EOTask):
"""
The task returns a pixelwise median value from a time-series and stores the results in a
timeless data array.
"""
def __init__(self, feature, feature_out):
self.feature_type, self.feature_name = next(self._parse_features(feature)())
self.feature_type_out, self.feature_name_out = next(self._parse_features(feature_out)())
def execute(self, eopatch):
eopatch.add_feature(self.feature_type_out, self.feature_name_out,
np.median(eopatch[self.feature_type][self.feature_name], axis=0))
return eopatch
# initialize tasks
# task to get S2 L2A images
input_task = SentinelHubInputTask(data_collection=DataCollection.SENTINEL2_L2A,
bands_feature=(FeatureType.DATA, 'BANDS'),
resolution=10,
maxcc=0.2,
bands=['B04', 'B03', 'B02'],
time_difference=datetime.timedelta(hours=2),
additional_data=[(FeatureType.MASK, 'dataMask', 'IS_DATA')]
)
geopedia_data = AddGeopediaFeature((FeatureType.MASK_TIMELESS, 'TREE_COVER'),
layer='ttl2275', theme='QP', raster_value=raster_value)
# task to compute median values
get_median_pixel = MedianPixel((FeatureType.DATA, 'BANDS'),
feature_out=(FeatureType.DATA_TIMELESS, 'MEDIAN_PIXEL'))
# task to save to disk
save = SaveTask(op.join('data', 'eopatch'),
overwrite_permission=OverwritePermission.OVERWRITE_PATCH,
compress_level=2)
workflow = LinearWorkflow(input_task, geopedia_data, get_median_pixel, save)
def execute_workflow(index):
bbox = bbox_splitter.bbox_list[index]
info = bbox_splitter.info_list[index]
patch_name = 'eopatch_{0}_row-{1}_col-{2}'.format(index,
info['index_x'],
info['index_y'])
results = workflow.execute({input_task:{'bbox':bbox, 'time_interval':time_interval},
save:{'eopatch_folder':patch_name}
})
return list(results.values())[-1]
del results
subset_idx = len(bbox_splitter.bbox_list)
pbar = tqdm(total=subset_idx)
for idx in range(0, subset_idx):
patch = execute_workflow(idx)
pbar.update(1)
HBox(children=(IntProgress(value=0, max=190), HTML(value='')))
import sagemaker
from sagemaker import get_execution_role
sagemaker_session = sagemaker.Session()
role = get_execution_role()
INFO:sagemaker:Created S3 bucket: sagemaker-us-east-1-552819999234
inputs = sagemaker_session.upload_data(path='data/eopatch/', key_prefix='testing/eo-learn')
inputs
INFO:sagemaker:Created S3 bucket: sagemaker-us-east-1-552819999234
's3://sagemaker-us-east-1-552819999234/testing/eo-learn'
%%file train_eo.py
import os
from os import path as op
from glob import glob
import gzip
import numpy as np
import tensorflow as tf
from tensorflow.keras.preprocessing.image import ImageDataGenerator
from tensorflow.keras import backend as K
from tensorflow.keras.models import *
from tensorflow.keras.layers import *
from tensorflow.keras.optimizers import *
from tensorflow.keras.utils import to_categorical
K.clear_session()
from tensorflow.python.estimator.export.export import build_raw_serving_input_receiver_fn
from tensorflow.python.estimator.export.export_output import PredictOutput
INPUT_TENSOR_NAME = 'input_1'
HEIGHT = 256
WIDTH = 256
DEPTH = 3
raster_value = {
'0%': (0, [0, 0, 0, 0]),
'10%': (1, [163, 235, 153, 255]),
'30%': (2, [119, 195, 118, 255]),
'50%': (3, [85, 160, 89, 255]),
'70%': (4, [58, 130, 64, 255]),
'90%': (5, [36, 103, 44, 255])
}
data_path = '/opt/ml/input/data/training'
patches = glob('%s/*' % data_path)
x_train_raw = np.empty((len(patches), HEIGHT, WIDTH, DEPTH))
y_train_raw = np.empty((len(patches), HEIGHT, WIDTH, 1))
for i, patch in enumerate(patches):
x_file = gzip.GzipFile(op.join(patch, 'data_timeless', 'MEDIAN_PIXEL.npy.gz'), 'r')
y_file = gzip.GzipFile(op.join(patch, 'mask_timeless', 'TREE_COVER.npy.gz'), 'r')
x_train_raw[i] = np.load(x_file)[20:276,0:256,:]
y_train_raw[i] = np.load(y_file)[20:276,0:256,:]
img_mean = np.mean(x_train_raw, axis=(0, 1, 2))
img_std = np.std(x_train_raw, axis=(0, 1, 2))
x_train_mean = x_train_raw - img_mean
x_train = x_train_mean - img_std
train_gen = ImageDataGenerator(
horizontal_flip=True,
vertical_flip=True,
rotation_range=180
)
y_train = to_categorical(y_train_raw, len(raster_value))
def _weighted_bce_loss(y_true, y_pred, weight):
# avoiding overflow
epsilon = 1e-7
y_pred = K.clip(y_pred, epsilon, 1. - epsilon)
logit_y_pred = K.log(y_pred / (1. - y_pred))
# https://www.tensorflow.org/api_docs/python/tf/nn/weighted_cross_entropy_with_logits
loss = (1. - y_true) * logit_y_pred + (1. + (weight - 1.) * y_true) * \
(K.log(1. + K.exp(-K.abs(logit_y_pred))) + K.maximum(-logit_y_pred, 0.))
return K.sum(loss) / K.sum(weight)
def _weighted_dice_loss(y_true, y_pred, weight):
smooth = 1.
w, m1, m2 = weight * weight, y_true, y_pred
intersection = (m1 * m2)
score = (2. * K.sum(w * intersection) + smooth) / (K.sum(w * m1) + K.sum(w * m2) + smooth)
loss = 1. - K.sum(score)
return loss
def _weighted_bce_dice_loss(y_true, y_pred):
y_true = K.cast(y_true, 'float32')
y_pred = K.cast(y_pred, 'float32')
# if we want to get same size of output, kernel size must be odd number
averaged_mask = K.pool2d(
y_true, pool_size=(11, 11), strides=(1, 1), padding='same', pool_mode='avg')
border = K.cast(K.greater(averaged_mask, 0.005), 'float32') * K.cast(K.less(averaged_mask, 0.995), 'float32')
weight = K.ones_like(averaged_mask)
w0 = K.sum(weight)
weight += border * 2
w1 = K.sum(weight)
weight *= (w0 / w1)
loss = _weighted_bce_loss(y_true, y_pred, weight) + \
_weighted_dice_loss(y_true, y_pred, weight)
return loss
def keras_model_fn(hyperparameters):
inputs = Input((HEIGHT, WIDTH, DEPTH))
conv1 = Conv2D(64, 3, activation = 'relu', padding = 'same',
kernel_initializer = 'he_normal')(inputs)
conv1 = Conv2D(64, 3, activation = 'relu', padding = 'same',
kernel_initializer = 'he_normal')(conv1)
pool1 = MaxPooling2D(pool_size=(2, 2))(conv1)
conv2 = Conv2D(128, 3, activation = 'relu', padding = 'same',
kernel_initializer = 'he_normal')(pool1)
conv2 = Conv2D(128, 3, activation = 'relu', padding = 'same',
kernel_initializer = 'he_normal')(conv2)
pool2 = MaxPooling2D(pool_size=(2, 2))(conv2)
conv3 = Conv2D(256, 3, activation = 'relu', padding = 'same',
kernel_initializer = 'he_normal')(pool2)
conv3 = Conv2D(256, 3, activation = 'relu', padding = 'same',
kernel_initializer = 'he_normal')(conv3)
pool3 = MaxPooling2D(pool_size=(2, 2))(conv3)
conv4 = Conv2D(512, 3, activation = 'relu', padding = 'same',
kernel_initializer = 'he_normal')(pool3)
conv4 = Conv2D(512, 3, activation = 'relu', padding = 'same',
kernel_initializer = 'he_normal')(conv4)
drop4 = Dropout(0.5)(conv4)
pool4 = MaxPooling2D(pool_size=(2, 2))(drop4)
conv5 = Conv2D(1024, 3, activation = 'relu', padding = 'same',
kernel_initializer = 'he_normal')(pool4)
conv5 = Conv2D(1024, 3, activation = 'relu', padding = 'same',
kernel_initializer = 'he_normal')(conv5)
drop5 = Dropout(0.5)(conv5)
up6 = Conv2D(512, 2, activation = 'relu', padding = 'same',
kernel_initializer = 'he_normal')(UpSampling2D(size = (2,2))(drop5))
merge6 = concatenate([drop4,up6])
conv6 = Conv2D(512, 3, activation = 'relu', padding = 'same',
kernel_initializer = 'he_normal')(merge6)
conv6 = Conv2D(512, 3, activation = 'relu', padding = 'same',
kernel_initializer = 'he_normal')(conv6)
up7 = Conv2D(256, 2, activation = 'relu', padding = 'same',
kernel_initializer = 'he_normal')(UpSampling2D(size = (2,2))(conv6))
merge7 = concatenate([conv3,up7])
conv7 = Conv2D(256, 3, activation = 'relu', padding = 'same',
kernel_initializer = 'he_normal')(merge7)
conv7 = Conv2D(256, 3, activation = 'relu', padding = 'same',
kernel_initializer = 'he_normal')(conv7)
up8 = Conv2D(128, 2, activation = 'relu', padding = 'same',
kernel_initializer = 'he_normal')(UpSampling2D(size = (2,2))(conv7))
merge8 = concatenate([conv2,up8])
conv8 = Conv2D(128, 3, activation = 'relu', padding = 'same',
kernel_initializer = 'he_normal')(merge8)
conv8 = Conv2D(128, 3, activation = 'relu', padding = 'same',
kernel_initializer = 'he_normal')(conv8)
up9 = Conv2D(64, 2, activation = 'relu', padding = 'same',
kernel_initializer = 'he_normal')(UpSampling2D(size = (2,2))(conv8))
merge9 = concatenate([conv1,up9])
conv9 = Conv2D(64, 3, activation = 'relu', padding = 'same',
kernel_initializer = 'he_normal')(merge9)
conv9 = Conv2D(64, 3, activation = 'relu', padding = 'same',
kernel_initializer = 'he_normal')(conv9)
conv10 = Conv2D(len(raster_value), 1, activation = 'softmax')(conv9)
model = Model(inputs = inputs, outputs = conv10)
model.compile(optimizer = Adam(lr = 1e-4),
loss = _weighted_bce_dice_loss,
metrics = ['accuracy'])
return model
def serving_input_fn(hyperparameters):
"""This function defines the placeholders that will be added to the model during serving.
The function returns a tf.estimator.export.ServingInputReceiver object, which packages the
placeholders and the resulting feature Tensors together.
For more information: https://github.com/aws/sagemaker-python-sdk/blob/master/src/sagemaker/tensorflow/README.rst#creating-a-serving_input_fn
Args:
hyperparameters: The hyperparameters passed to SageMaker TrainingJob that runs your TensorFlow
training script.
Returns: ServingInputReceiver or fn that returns a ServingInputReceiver
"""
# Notice that the input placeholder has the same input shape as the Keras model input
tensor = tf.placeholder(tf.float32, shape=[None, HEIGHT, WIDTH, DEPTH])
# The inputs key INPUT_TENSOR_NAME matches the Keras InputLayer name
inputs = {INPUT_TENSOR_NAME: tensor}
return build_raw_serving_input_receiver_fn({INPUT_TENSOR_NAME: tensor})()
return tf.estimator.export.ServingInputReceiver(inputs, inputs)
def train_input_fn(training_dir, hyperparameters):
"""Returns input function that would feed the model during training"""
return _input(tf.estimator.ModeKeys.TRAIN,
batch_size=16, data_dir=training_dir)
def eval_input_fn(training_dir, hyperparameters):
"""Returns input function that would feed the model during evaluation"""
return _input(tf.estimator.ModeKeys.EVAL,
batch_size=16, data_dir=training_dir)
def _input(mode, batch_size, data_dir):
# example has no differentiation between train and eval
gen_iter = train_gen.flow(x=x_train, y=y_train, batch_size=batch_size)
images, labels = gen_iter.next()
return {INPUT_TENSOR_NAME: images}, labels
Overwriting train_eo.py
from sagemaker.tensorflow import TensorFlow
custom_estimator = TensorFlow(entry_point='train_eo.py',
role=role,
framework_version='1.12.0',
training_steps= 1000,
evaluation_steps= 100,
hyperparameters={'learning_rate': 0.001},
train_instance_count=1,
train_instance_type='ml.p2.xlarge')
custom_estimator.fit(inputs)
INFO:sagemaker:Created S3 bucket: sagemaker-us-east-1-552819999234 INFO:sagemaker:Creating training-job with name: sagemaker-tensorflow-2019-01-31-16-05-25-509
2019-01-31 16:05:25 Starting - Starting the training job... 2019-01-31 16:05:27 Starting - Launching requested ML instances......... 2019-01-31 16:07:06 Starting - Preparing the instances for training... 2019-01-31 16:07:50 Downloading - Downloading input data...... 2019-01-31 16:08:46 Training - Downloading the training image.. 2019-01-31 16:09:02,894 INFO - root - running container entrypoint 2019-01-31 16:09:02,894 INFO - root - starting train task 2019-01-31 16:09:02,917 INFO - container_support.training - Training starting Downloading s3://sagemaker-us-east-1-552819999234/sagemaker-tensorflow-2019-01-31-16-05-25-509/source/sourcedir.tar.gz to /tmp/script.tar.gz 2019-01-31 16:09:10,496 INFO - tf_container - ----------------------TF_CONFIG-------------------------- 2019-01-31 16:09:10,496 INFO - tf_container - {"environment": "cloud", "cluster": {"master": ["algo-1:2222"]}, "task": {"index": 0, "type": "master"}} 2019-01-31 16:09:10,496 INFO - tf_container - --------------------------------------------------------- 2019-01-31 16:09:10,497 INFO - tf_container - creating RunConfig: 2019-01-31 16:09:10,497 INFO - tf_container - {'save_checkpoints_secs': 300} 2019-01-31 16:09:10,497 INFO - tensorflow - TF_CONFIG environment variable: {u'environment': u'cloud', u'cluster': {u'master': [u'algo-1:2222']}, u'task': {u'index': 0, u'type': u'master'}} 2019-01-31 16:09:10,497 INFO - tf_container - invoking the user-provided keras_model_fn 2019-01-31 16:09:11,008 INFO - tensorflow - Using the Keras model provided. 2019-01-31 16:09:02 Training - Training image download completed. Training in progress.2019-01-31 16:09:21,690 INFO - tensorflow - Using config: {'_save_checkpoints_secs': 300, '_keep_checkpoint_max': 5, '_task_type': u'master', '_cluster_spec': <tensorflow.python.training.server_lib.ClusterSpec object at 0x7efc6c392fd0>, '_keep_checkpoint_every_n_hours': 10000, '_service': None, '_num_ps_replicas': 0, '_tf_random_seed': None, '_device_fn': None, '_num_worker_replicas': 1, '_task_id': 0, '_log_step_count_steps': 100, '_evaluation_master': '', '_eval_distribute': None, '_train_distribute': None, '_session_config': device_filters: "/job:ps" device_filters: "/job:master" allow_soft_placement: true graph_options { rewrite_options { meta_optimizer_iterations: ONE } } , '_global_id_in_cluster': 0, '_is_chief': True, '_protocol': None, '_save_checkpoints_steps': None, '_experimental_distribute': None, '_save_summary_steps': 100, '_model_dir': u's3://sagemaker-us-east-1-552819999234/sagemaker-tensorflow-2019-01-31-16-05-25-509/checkpoints', '_master': ''} 2019-01-31 16:09:21,714 INFO - tensorflow - Not using Distribute Coordinator. 2019-01-31 16:09:21,715 INFO - tensorflow - Skip starting Tensorflow server as there is only one node in the cluster. 2019-01-31 16:09:22,098 INFO - tensorflow - Calling model_fn. 2019-01-31 16:09:25,115 INFO - tensorflow - Done calling model_fn. 2019-01-31 16:09:25,116 INFO - tensorflow - Warm-starting with WarmStartSettings: WarmStartSettings(ckpt_to_initialize_from=u's3://sagemaker-us-east-1-552819999234/sagemaker-tensorflow-2019-01-31-16-05-25-509/checkpoints/keras/keras_model.ckpt', vars_to_warm_start='.*', var_name_to_vocab_info={}, var_name_to_prev_var_name={}) 2019-01-31 16:09:25,116 INFO - tensorflow - Warm-starting from: (u's3://sagemaker-us-east-1-552819999234/sagemaker-tensorflow-2019-01-31-16-05-25-509/checkpoints/keras/keras_model.ckpt',) 2019-01-31 16:09:25,118 INFO - tensorflow - Warm-starting variable: training/Adam/Variable_118; prev_var_name: Unchanged 2019-01-31 16:09:25,374 INFO - tensorflow - Warm-starting variable: conv2d_14/kernel; prev_var_name: Unchanged 2019-01-31 16:09:25,550 INFO - tensorflow - Warm-starting variable: training/Adam/Variable_119; prev_var_name: Unchanged 2019-01-31 16:09:25,819 INFO - tensorflow - Warm-starting variable: training/Adam/Variable_127; prev_var_name: Unchanged 2019-01-31 16:09:26,033 INFO - tensorflow - Warm-starting variable: training/Adam/Variable; prev_var_name: Unchanged 2019-01-31 16:09:26,263 INFO - tensorflow - Warm-starting variable: training/Adam/Variable_125; prev_var_name: Unchanged 2019-01-31 16:09:26,563 INFO - tensorflow - Warm-starting variable: training/Adam/Variable_124; prev_var_name: Unchanged 2019-01-31 16:09:27,082 INFO - tensorflow - Warm-starting variable: training/Adam/Variable_123; prev_var_name: Unchanged 2019-01-31 16:09:27,512 INFO - tensorflow - Warm-starting variable: training/Adam/Variable_122; prev_var_name: Unchanged 2019-01-31 16:09:27,805 INFO - tensorflow - Warm-starting variable: training/Adam/Variable_121; prev_var_name: Unchanged 2019-01-31 16:09:27,991 INFO - tensorflow - Warm-starting variable: training/Adam/Variable_117; prev_var_name: Unchanged 2019-01-31 16:09:28,191 INFO - tensorflow - Warm-starting variable: training/Adam/Variable_129; prev_var_name: Unchanged 2019-01-31 16:09:28,385 INFO - tensorflow - Warm-starting variable: training/Adam/Variable_128; prev_var_name: Unchanged 2019-01-31 16:09:28,739 INFO - tensorflow - Warm-starting variable: conv2d_3/bias; prev_var_name: Unchanged 2019-01-31 16:09:29,075 INFO - tensorflow - Warm-starting variable: training/Adam/Variable_131; prev_var_name: Unchanged 2019-01-31 16:09:29,277 INFO - tensorflow - Warm-starting variable: conv2d_3/kernel; prev_var_name: Unchanged 2019-01-31 16:09:29,636 INFO - tensorflow - Warm-starting variable: conv2d_19/kernel; prev_var_name: Unchanged 2019-01-31 16:09:29,904 INFO - tensorflow - Warm-starting variable: conv2d_16/kernel; prev_var_name: Unchanged 2019-01-31 16:09:30,097 INFO - tensorflow - Warm-starting variable: conv2d_8/kernel; prev_var_name: Unchanged 2019-01-31 16:09:30,438 INFO - tensorflow - Warm-starting variable: conv2d_20/kernel; prev_var_name: Unchanged 2019-01-31 16:09:30,635 INFO - tensorflow - Warm-starting variable: Adam/decay; prev_var_name: Unchanged 2019-01-31 16:09:30,811 INFO - tensorflow - Warm-starting variable: conv2d_13/bias; prev_var_name: Unchanged 2019-01-31 16:09:31,411 INFO - tensorflow - Warm-starting variable: conv2d_18/bias; prev_var_name: Unchanged 2019-01-31 16:09:31,597 INFO - tensorflow - Warm-starting variable: conv2d_22/kernel; prev_var_name: Unchanged 2019-01-31 16:09:32,202 INFO - tensorflow - Warm-starting variable: training/Adam/Variable_58; prev_var_name: Unchanged 2019-01-31 16:09:32,398 INFO - tensorflow - Warm-starting variable: training/Adam/Variable_59; prev_var_name: Unchanged 2019-01-31 16:09:32,739 INFO - tensorflow - Warm-starting variable: conv2d_9/kernel; prev_var_name: Unchanged 2019-01-31 16:09:33,094 INFO - tensorflow - Warm-starting variable: training/Adam/Variable_126; prev_var_name: Unchanged 2019-01-31 16:09:33,380 INFO - tensorflow - Warm-starting variable: training/Adam/Variable_50; prev_var_name: Unchanged 2019-01-31 16:09:33,616 INFO - tensorflow - Warm-starting variable: training/Adam/Variable_51; prev_var_name: Unchanged 2019-01-31 16:09:33,839 INFO - tensorflow - Warm-starting variable: training/Adam/Variable_52; prev_var_name: Unchanged 2019-01-31 16:09:34,099 INFO - tensorflow - Warm-starting variable: training/Adam/Variable_53; prev_var_name: Unchanged 2019-01-31 16:09:34,436 INFO - tensorflow - Warm-starting variable: training/Adam/Variable_54; prev_var_name: Unchanged 2019-01-31 16:09:34,767 INFO - tensorflow - Warm-starting variable: training/Adam/Variable_55; prev_var_name: Unchanged 2019-01-31 16:09:35,059 INFO - tensorflow - Warm-starting variable: training/Adam/Variable_56; prev_var_name: Unchanged 2019-01-31 16:09:35,284 INFO - tensorflow - Warm-starting variable: training/Adam/Variable_57; prev_var_name: Unchanged 2019-01-31 16:09:35,593 INFO - tensorflow - Warm-starting variable: conv2d_17/kernel; prev_var_name: Unchanged 2019-01-31 16:09:35,818 INFO - tensorflow - Warm-starting variable: conv2d_18/kernel; prev_var_name: Unchanged 2019-01-31 16:09:36,072 INFO - tensorflow - Warm-starting variable: conv2d_1/kernel; prev_var_name: Unchanged 2019-01-31 16:09:36,286 INFO - tensorflow - Warm-starting variable: conv2d_21/kernel; prev_var_name: Unchanged 2019-01-31 16:09:36,465 INFO - tensorflow - Warm-starting variable: training/Adam/Variable_120; prev_var_name: Unchanged 2019-01-31 16:09:36,823 INFO - tensorflow - Warm-starting variable: conv2d_22/bias; prev_var_name: Unchanged 2019-01-31 16:09:36,986 INFO - tensorflow - Warm-starting variable: conv2d_2/kernel; prev_var_name: Unchanged 2019-01-31 16:09:37,236 INFO - tensorflow - Warm-starting variable: training/Adam/Variable_97; prev_var_name: Unchanged 2019-01-31 16:09:37,459 INFO - tensorflow - Warm-starting variable: training/Adam/Variable_29; prev_var_name: Unchanged 2019-01-31 16:09:37,631 INFO - tensorflow - Warm-starting variable: training/Adam/Variable_28; prev_var_name: Unchanged 2019-01-31 16:09:37,956 INFO - tensorflow - Warm-starting variable: training/Adam/Variable_25; prev_var_name: Unchanged 2019-01-31 16:09:38,390 INFO - tensorflow - Warm-starting variable: training/Adam/Variable_24; prev_var_name: Unchanged 2019-01-31 16:09:38,589 INFO - tensorflow - Warm-starting variable: training/Adam/Variable_27; prev_var_name: Unchanged 2019-01-31 16:09:38,840 INFO - tensorflow - Warm-starting variable: training/Adam/Variable_26; prev_var_name: Unchanged 2019-01-31 16:09:39,133 INFO - tensorflow - Warm-starting variable: training/Adam/Variable_21; prev_var_name: Unchanged 2019-01-31 16:09:39,346 INFO - tensorflow - Warm-starting variable: training/Adam/Variable_20; prev_var_name: Unchanged 2019-01-31 16:09:39,615 INFO - tensorflow - Warm-starting variable: training/Adam/Variable_23; prev_var_name: Unchanged 2019-01-31 16:09:39,832 INFO - tensorflow - Warm-starting variable: training/Adam/Variable_22; prev_var_name: Unchanged 2019-01-31 16:09:40,109 INFO - tensorflow - Warm-starting variable: conv2d_8/bias; prev_var_name: Unchanged 2019-01-31 16:09:40,421 INFO - tensorflow - Warm-starting variable: conv2d/bias; prev_var_name: Unchanged 2019-01-31 16:09:40,695 INFO - tensorflow - Warm-starting variable: conv2d_4/kernel; prev_var_name: Unchanged 2019-01-31 16:09:41,103 INFO - tensorflow - Warm-starting variable: training/Adam/Variable_89; prev_var_name: Unchanged 2019-01-31 16:09:41,535 INFO - tensorflow - Warm-starting variable: training/Adam/Variable_88; prev_var_name: Unchanged 2019-01-31 16:09:41,926 INFO - tensorflow - Warm-starting variable: training/Adam/Variable_83; prev_var_name: Unchanged 2019-01-31 16:09:42,120 INFO - tensorflow - Warm-starting variable: training/Adam/Variable_82; prev_var_name: Unchanged 2019-01-31 16:09:42,335 INFO - tensorflow - Warm-starting variable: training/Adam/Variable_81; prev_var_name: Unchanged 2019-01-31 16:09:42,757 INFO - tensorflow - Warm-starting variable: training/Adam/Variable_80; prev_var_name: Unchanged 2019-01-31 16:09:43,058 INFO - tensorflow - Warm-starting variable: training/Adam/Variable_87; prev_var_name: Unchanged 2019-01-31 16:09:43,334 INFO - tensorflow - Warm-starting variable: Adam/lr; prev_var_name: Unchanged 2019-01-31 16:09:43,746 INFO - tensorflow - Warm-starting variable: training/Adam/Variable_85; prev_var_name: Unchanged 2019-01-31 16:09:44,039 INFO - tensorflow - Warm-starting variable: training/Adam/Variable_84; prev_var_name: Unchanged 2019-01-31 16:09:44,300 INFO - tensorflow - Warm-starting variable: training/Adam/Variable_47; prev_var_name: Unchanged 2019-01-31 16:09:44,579 INFO - tensorflow - Warm-starting variable: conv2d_7/bias; prev_var_name: Unchanged 2019-01-31 16:09:44,871 INFO - tensorflow - Warm-starting variable: conv2d_1/bias; prev_var_name: Unchanged 2019-01-31 16:09:45,355 INFO - tensorflow - Warm-starting variable: training/Adam/Variable_38; prev_var_name: Unchanged 2019-01-31 16:09:45,564 INFO - tensorflow - Warm-starting variable: training/Adam/Variable_39; prev_var_name: Unchanged 2019-01-31 16:09:45,786 INFO - tensorflow - Warm-starting variable: training/Adam/Variable_36; prev_var_name: Unchanged 2019-01-31 16:09:46,007 INFO - tensorflow - Warm-starting variable: training/Adam/Variable_37; prev_var_name: Unchanged 2019-01-31 16:09:46,201 INFO - tensorflow - Warm-starting variable: training/Adam/Variable_34; prev_var_name: Unchanged 2019-01-31 16:09:46,472 INFO - tensorflow - Warm-starting variable: training/Adam/Variable_35; prev_var_name: Unchanged 2019-01-31 16:09:46,686 INFO - tensorflow - Warm-starting variable: training/Adam/Variable_32; prev_var_name: Unchanged 2019-01-31 16:09:46,868 INFO - tensorflow - Warm-starting variable: training/Adam/Variable_33; prev_var_name: Unchanged 2019-01-31 16:09:47,192 INFO - tensorflow - Warm-starting variable: training/Adam/Variable_30; prev_var_name: Unchanged 2019-01-31 16:09:47,913 INFO - tensorflow - Warm-starting variable: training/Adam/Variable_31; prev_var_name: Unchanged 2019-01-31 16:09:48,315 INFO - tensorflow - Warm-starting variable: training/Adam/Variable_133; prev_var_name: Unchanged 2019-01-31 16:09:48,586 INFO - tensorflow - Warm-starting variable: training/Adam/Variable_134; prev_var_name: Unchanged 2019-01-31 16:09:48,803 INFO - tensorflow - Warm-starting variable: training/Adam/Variable_135; prev_var_name: Unchanged 2019-01-31 16:09:49,164 INFO - tensorflow - Warm-starting variable: conv2d_14/bias; prev_var_name: Unchanged 2019-01-31 16:09:49,818 INFO - tensorflow - Warm-starting variable: training/Adam/Variable_136; prev_var_name: Unchanged 2019-01-31 16:09:50,031 INFO - tensorflow - Warm-starting variable: conv2d_21/bias; prev_var_name: Unchanged 2019-01-31 16:09:50,319 INFO - tensorflow - Warm-starting variable: training/Adam/Variable_137; prev_var_name: Unchanged 2019-01-31 16:09:50,619 INFO - tensorflow - Warm-starting variable: conv2d_10/kernel; prev_var_name: Unchanged 2019-01-31 16:09:50,928 INFO - tensorflow - Warm-starting variable: training/Adam/Variable_107; prev_var_name: Unchanged 2019-01-31 16:09:51,239 INFO - tensorflow - Warm-starting variable: conv2d_16/bias; prev_var_name: Unchanged 2019-01-31 16:09:51,442 INFO - tensorflow - Warm-starting variable: training/Adam/Variable_98; prev_var_name: Unchanged 2019-01-31 16:09:51,658 INFO - tensorflow - Warm-starting variable: training/Adam/Variable_99; prev_var_name: Unchanged 2019-01-31 16:09:51,949 INFO - tensorflow - Warm-starting variable: training/Adam/Variable_94; prev_var_name: Unchanged 2019-01-31 16:09:52,165 INFO - tensorflow - Warm-starting variable: training/Adam/Variable_95; prev_var_name: Unchanged 2019-01-31 16:09:52,448 INFO - tensorflow - Warm-starting variable: training/Adam/Variable_96; prev_var_name: Unchanged 2019-01-31 16:09:53,119 INFO - tensorflow - Warm-starting variable: conv2d_15/kernel; prev_var_name: Unchanged 2019-01-31 16:09:53,310 INFO - tensorflow - Warm-starting variable: training/Adam/Variable_90; prev_var_name: Unchanged 2019-01-31 16:09:53,559 INFO - tensorflow - Warm-starting variable: training/Adam/Variable_91; prev_var_name: Unchanged 2019-01-31 16:09:53,780 INFO - tensorflow - Warm-starting variable: training/Adam/Variable_92; prev_var_name: Unchanged 2019-01-31 16:09:54,706 INFO - tensorflow - Warm-starting variable: training/Adam/Variable_93; prev_var_name: Unchanged 2019-01-31 16:09:55,104 INFO - tensorflow - Warm-starting variable: conv2d_7/kernel; prev_var_name: Unchanged 2019-01-31 16:09:55,328 INFO - tensorflow - Warm-starting variable: training/Adam/Variable_86; prev_var_name: Unchanged 2019-01-31 16:09:55,592 INFO - tensorflow - Warm-starting variable: training/Adam/Variable_6; prev_var_name: Unchanged 2019-01-31 16:09:55,789 INFO - tensorflow - Warm-starting variable: training/Adam/Variable_7; prev_var_name: Unchanged 2019-01-31 16:09:56,107 INFO - tensorflow - Warm-starting variable: training/Adam/Variable_4; prev_var_name: Unchanged 2019-01-31 16:09:56,439 INFO - tensorflow - Warm-starting variable: training/Adam/Variable_5; prev_var_name: Unchanged 2019-01-31 16:09:56,696 INFO - tensorflow - Warm-starting variable: training/Adam/Variable_2; prev_var_name: Unchanged 2019-01-31 16:09:57,009 INFO - tensorflow - Warm-starting variable: training/Adam/Variable_3; prev_var_name: Unchanged 2019-01-31 16:09:57,247 INFO - tensorflow - Warm-starting variable: training/Adam/Variable_1; prev_var_name: Unchanged 2019-01-31 16:09:57,725 INFO - tensorflow - Warm-starting variable: conv2d_12/bias; prev_var_name: Unchanged 2019-01-31 16:09:57,912 INFO - tensorflow - Warm-starting variable: training/Adam/Variable_8; prev_var_name: Unchanged 2019-01-31 16:09:58,110 INFO - tensorflow - Warm-starting variable: training/Adam/Variable_9; prev_var_name: Unchanged 2019-01-31 16:09:58,363 INFO - tensorflow - Warm-starting variable: training/Adam/Variable_116; prev_var_name: Unchanged 2019-01-31 16:09:58,588 INFO - tensorflow - Warm-starting variable: conv2d/kernel; prev_var_name: Unchanged 2019-01-31 16:09:58,849 INFO - tensorflow - Warm-starting variable: conv2d_9/bias; prev_var_name: Unchanged 2019-01-31 16:09:59,075 INFO - tensorflow - Warm-starting variable: training/Adam/Variable_61; prev_var_name: Unchanged 2019-01-31 16:09:59,283 INFO - tensorflow - Warm-starting variable: training/Adam/Variable_60; prev_var_name: Unchanged 2019-01-31 16:09:59,466 INFO - tensorflow - Warm-starting variable: training/Adam/Variable_63; prev_var_name: Unchanged 2019-01-31 16:09:59,668 INFO - tensorflow - Warm-starting variable: training/Adam/Variable_62; prev_var_name: Unchanged 2019-01-31 16:09:59,970 INFO - tensorflow - Warm-starting variable: training/Adam/Variable_65; prev_var_name: Unchanged 2019-01-31 16:10:00,475 INFO - tensorflow - Warm-starting variable: training/Adam/Variable_64; prev_var_name: Unchanged 2019-01-31 16:10:02,010 INFO - tensorflow - Warm-starting variable: training/Adam/Variable_67; prev_var_name: Unchanged 2019-01-31 16:10:02,329 INFO - tensorflow - Warm-starting variable: training/Adam/Variable_66; prev_var_name: Unchanged 2019-01-31 16:10:02,746 INFO - tensorflow - Warm-starting variable: training/Adam/Variable_69; prev_var_name: Unchanged 2019-01-31 16:10:03,052 INFO - tensorflow - Warm-starting variable: training/Adam/Variable_68; prev_var_name: Unchanged 2019-01-31 16:10:03,276 INFO - tensorflow - Warm-starting variable: training/Adam/Variable_114; prev_var_name: Unchanged 2019-01-31 16:10:03,560 INFO - tensorflow - Warm-starting variable: training/Adam/Variable_115; prev_var_name: Unchanged 2019-01-31 16:10:03,748 INFO - tensorflow - Warm-starting variable: training/Adam/Variable_112; prev_var_name: Unchanged 2019-01-31 16:10:03,917 INFO - tensorflow - Warm-starting variable: training/Adam/Variable_113; prev_var_name: Unchanged 2019-01-31 16:10:04,165 INFO - tensorflow - Warm-starting variable: training/Adam/Variable_110; prev_var_name: Unchanged 2019-01-31 16:10:04,373 INFO - tensorflow - Warm-starting variable: training/Adam/Variable_111; prev_var_name: Unchanged 2019-01-31 16:10:04,572 INFO - tensorflow - Warm-starting variable: training/Adam/Variable_14; prev_var_name: Unchanged 2019-01-31 16:10:05,033 INFO - tensorflow - Warm-starting variable: training/Adam/Variable_15; prev_var_name: Unchanged 2019-01-31 16:10:05,366 INFO - tensorflow - Warm-starting variable: training/Adam/Variable_16; prev_var_name: Unchanged 2019-01-31 16:10:05,571 INFO - tensorflow - Warm-starting variable: training/Adam/Variable_17; prev_var_name: Unchanged 2019-01-31 16:10:05,820 INFO - tensorflow - Warm-starting variable: training/Adam/Variable_10; prev_var_name: Unchanged 2019-01-31 16:10:06,040 INFO - tensorflow - Warm-starting variable: training/Adam/Variable_11; prev_var_name: Unchanged 2019-01-31 16:10:06,256 INFO - tensorflow - Warm-starting variable: training/Adam/Variable_12; prev_var_name: Unchanged 2019-01-31 16:10:06,562 INFO - tensorflow - Warm-starting variable: training/Adam/Variable_13; prev_var_name: Unchanged 2019-01-31 16:10:06,831 INFO - tensorflow - Warm-starting variable: conv2d_17/bias; prev_var_name: Unchanged 2019-01-31 16:10:07,039 INFO - tensorflow - Warm-starting variable: training/Adam/Variable_18; prev_var_name: Unchanged 2019-01-31 16:10:07,243 INFO - tensorflow - Warm-starting variable: training/Adam/Variable_19; prev_var_name: Unchanged 2019-01-31 16:10:07,516 INFO - tensorflow - Warm-starting variable: Adam/beta_1; prev_var_name: Unchanged 2019-01-31 16:10:07,732 INFO - tensorflow - Warm-starting variable: Adam/beta_2; prev_var_name: Unchanged 2019-01-31 16:10:07,917 INFO - tensorflow - Warm-starting variable: training/Adam/Variable_130; prev_var_name: Unchanged 2019-01-31 16:10:08,089 INFO - tensorflow - Warm-starting variable: training/Adam/Variable_109; prev_var_name: Unchanged 2019-01-31 16:10:08,713 INFO - tensorflow - Warm-starting variable: training/Adam/Variable_108; prev_var_name: Unchanged 2019-01-31 16:10:08,946 INFO - tensorflow - Warm-starting variable: training/Adam/Variable_74; prev_var_name: Unchanged 2019-01-31 16:10:09,238 INFO - tensorflow - Warm-starting variable: conv2d_11/kernel; prev_var_name: Unchanged 2019-01-31 16:10:09,455 INFO - tensorflow - Warm-starting variable: training/Adam/Variable_72; prev_var_name: Unchanged 2019-01-31 16:10:09,712 INFO - tensorflow - Warm-starting variable: training/Adam/Variable_73; prev_var_name: Unchanged 2019-01-31 16:10:10,043 INFO - tensorflow - Warm-starting variable: training/Adam/Variable_70; prev_var_name: Unchanged 2019-01-31 16:10:10,296 INFO - tensorflow - Warm-starting variable: training/Adam/Variable_71; prev_var_name: Unchanged 2019-01-31 16:10:10,536 INFO - tensorflow - Warm-starting variable: training/Adam/Variable_76; prev_var_name: Unchanged 2019-01-31 16:10:10,812 INFO - tensorflow - Warm-starting variable: training/Adam/Variable_77; prev_var_name: Unchanged 2019-01-31 16:10:11,028 INFO - tensorflow - Warm-starting variable: conv2d_6/kernel; prev_var_name: Unchanged 2019-01-31 16:10:11,239 INFO - tensorflow - Warm-starting variable: training/Adam/Variable_75; prev_var_name: Unchanged 2019-01-31 16:10:11,409 INFO - tensorflow - Warm-starting variable: training/Adam/Variable_105; prev_var_name: Unchanged 2019-01-31 16:10:11,644 INFO - tensorflow - Warm-starting variable: training/Adam/Variable_104; prev_var_name: Unchanged 2019-01-31 16:10:12,183 INFO - tensorflow - Warm-starting variable: training/Adam/Variable_78; prev_var_name: Unchanged 2019-01-31 16:10:12,479 INFO - tensorflow - Warm-starting variable: training/Adam/Variable_79; prev_var_name: Unchanged 2019-01-31 16:10:12,677 INFO - tensorflow - Warm-starting variable: training/Adam/Variable_101; prev_var_name: Unchanged 2019-01-31 16:10:13,078 INFO - tensorflow - Warm-starting variable: training/Adam/Variable_100; prev_var_name: Unchanged 2019-01-31 16:10:13,639 INFO - tensorflow - Warm-starting variable: training/Adam/Variable_103; prev_var_name: Unchanged 2019-01-31 16:10:13,862 INFO - tensorflow - Warm-starting variable: training/Adam/Variable_102; prev_var_name: Unchanged 2019-01-31 16:10:14,169 INFO - tensorflow - Warm-starting variable: training/Adam/Variable_106; prev_var_name: Unchanged 2019-01-31 16:10:14,463 INFO - tensorflow - Warm-starting variable: conv2d_5/bias; prev_var_name: Unchanged 2019-01-31 16:10:14,658 INFO - tensorflow - Warm-starting variable: Adam/iterations; prev_var_name: Unchanged 2019-01-31 16:10:14,907 INFO - tensorflow - Warm-starting variable: training/Adam/Variable_132; prev_var_name: Unchanged 2019-01-31 16:10:15,099 INFO - tensorflow - Warm-starting variable: conv2d_12/kernel; prev_var_name: Unchanged 2019-01-31 16:10:15,409 INFO - tensorflow - Warm-starting variable: conv2d_15/bias; prev_var_name: Unchanged 2019-01-31 16:10:15,712 INFO - tensorflow - Warm-starting variable: conv2d_19/bias; prev_var_name: Unchanged 2019-01-31 16:10:15,950 INFO - tensorflow - Warm-starting variable: conv2d_4/bias; prev_var_name: Unchanged 2019-01-31 16:10:16,303 INFO - tensorflow - Warm-starting variable: conv2d_5/kernel; prev_var_name: Unchanged 2019-01-31 16:10:16,479 INFO - tensorflow - Warm-starting variable: conv2d_13/kernel; prev_var_name: Unchanged 2019-01-31 16:10:16,667 INFO - tensorflow - Warm-starting variable: conv2d_6/bias; prev_var_name: Unchanged 2019-01-31 16:10:16,836 INFO - tensorflow - Warm-starting variable: training/Adam/Variable_46; prev_var_name: Unchanged 2019-01-31 16:10:17,021 INFO - tensorflow - Warm-starting variable: training/Adam/Variable_45; prev_var_name: Unchanged 2019-01-31 16:10:17,250 INFO - tensorflow - Warm-starting variable: training/Adam/Variable_44; prev_var_name: Unchanged 2019-01-31 16:10:17,804 INFO - tensorflow - Warm-starting variable: training/Adam/Variable_43; prev_var_name: Unchanged 2019-01-31 16:10:17,999 INFO - tensorflow - Warm-starting variable: training/Adam/Variable_42; prev_var_name: Unchanged 2019-01-31 16:10:18,239 INFO - tensorflow - Warm-starting variable: training/Adam/Variable_41; prev_var_name: Unchanged 2019-01-31 16:10:18,409 INFO - tensorflow - Warm-starting variable: training/Adam/Variable_40; prev_var_name: Unchanged 2019-01-31 16:10:18,595 INFO - tensorflow - Warm-starting variable: training/Adam/Variable_49; prev_var_name: Unchanged 2019-01-31 16:10:18,778 INFO - tensorflow - Warm-starting variable: training/Adam/Variable_48; prev_var_name: Unchanged 2019-01-31 16:10:18,958 INFO - tensorflow - Warm-starting variable: conv2d_11/bias; prev_var_name: Unchanged 2019-01-31 16:10:19,199 INFO - tensorflow - Warm-starting variable: conv2d_20/bias; prev_var_name: Unchanged 2019-01-31 16:10:19,424 INFO - tensorflow - Warm-starting variable: conv2d_2/bias; prev_var_name: Unchanged 2019-01-31 16:10:19,785 INFO - tensorflow - Warm-starting variable: conv2d_10/bias; prev_var_name: Unchanged 2019-01-31 16:10:20,014 INFO - tensorflow - Create CheckpointSaverHook. 2019-01-31 16:10:22,090 INFO - tensorflow - Graph was finalized. 2019-01-31 16:10:33,425 INFO - tensorflow - Running local_init_op. 2019-01-31 16:10:33,459 INFO - tensorflow - Done running local_init_op. 2019-01-31 16:10:41,108 INFO - tensorflow - Saving checkpoints for 0 into s3://sagemaker-us-east-1-552819999234/sagemaker-tensorflow-2019-01-31-16-05-25-509/checkpoints/model.ckpt. 2019-01-31 16:11:01.145100: E tensorflow/core/common_runtime/executor.cc:623] Executor failed to create kernel. Invalid argument: Default AvgPoolingOp only supports NHWC on device type CPU #011 [[{{node loss/conv2d_22_loss/AvgPool}} = AvgPool[T=DT_FLOAT, data_format="NCHW", ksize=[1, 1, 11, 11], padding="SAME", strides=[1, 1, 1, 1], _device="/job:localhost/replica:0/task:0/device:GPU:0"](loss/conv2d_22_loss/AvgPool-0-TransposeNHWCToNCHW-LayoutOptimizer)]] 2019-01-31 16:11:01.191318: E tensorflow/core/common_runtime/executor.cc:623] Executor failed to create kernel. Invalid argument: Default AvgPoolingOp only supports NHWC on device type CPU #011 [[{{node loss/conv2d_22_loss/AvgPool}} = AvgPool[T=DT_FLOAT, data_format="NCHW", ksize=[1, 1, 11, 11], padding="SAME", strides=[1, 1, 1, 1], _device="/job:localhost/replica:0/task:0/device:GPU:0"](loss/conv2d_22_loss/AvgPool-0-TransposeNHWCToNCHW-LayoutOptimizer)]] 2019-01-31 16:11:18,847 INFO - tensorflow - loss = 1.3605741, step = 1 2019-01-31 16:11:27.911881: E tensorflow/core/common_runtime/executor.cc:623] Executor failed to create kernel. Invalid argument: Default AvgPoolingOp only supports NHWC on device type CPU #011 [[{{node loss/conv2d_22_loss/AvgPool}} = AvgPool[T=DT_FLOAT, data_format="NCHW", ksize=[1, 1, 11, 11], padding="SAME", strides=[1, 1, 1, 1], _device="/job:localhost/replica:0/task:0/device:GPU:0"](loss/conv2d_22_loss/AvgPool-0-TransposeNHWCToNCHW-LayoutOptimizer)]] 2019-01-31 16:11:27.958816: E tensorflow/core/common_runtime/executor.cc:623] Executor failed to create kernel. Invalid argument: Default AvgPoolingOp only supports NHWC on device type CPU #011 [[{{node loss/conv2d_22_loss/AvgPool}} = AvgPool[T=DT_FLOAT, data_format="NCHW", ksize=[1, 1, 11, 11], padding="SAME", strides=[1, 1, 1, 1], _device="/job:localhost/replica:0/task:0/device:GPU:0"](loss/conv2d_22_loss/AvgPool-0-TransposeNHWCToNCHW-LayoutOptimizer)]] 2019-01-31 16:15:10.062381: E tensorflow/core/common_runtime/executor.cc:623] Executor failed to create kernel. Invalid argument: Default AvgPoolingOp only supports NHWC on device type CPU #011 [[{{node loss/conv2d_22_loss/AvgPool}} = AvgPool[T=DT_FLOAT, data_format="NCHW", ksize=[1, 1, 11, 11], padding="SAME", strides=[1, 1, 1, 1], _device="/job:localhost/replica:0/task:0/device:GPU:0"](loss/conv2d_22_loss/AvgPool-0-TransposeNHWCToNCHW-LayoutOptimizer)]] 2019-01-31 16:15:10.110668: E tensorflow/core/common_runtime/executor.cc:623] Executor failed to create kernel. Invalid argument: Default AvgPoolingOp only supports NHWC on device type CPU #011 [[{{node loss/conv2d_22_loss/AvgPool}} = AvgPool[T=DT_FLOAT, data_format="NCHW", ksize=[1, 1, 11, 11], padding="SAME", strides=[1, 1, 1, 1], _device="/job:localhost/replica:0/task:0/device:GPU:0"](loss/conv2d_22_loss/AvgPool-0-TransposeNHWCToNCHW-LayoutOptimizer)]] 2019-01-31 16:15:22.346252: E tensorflow/core/common_runtime/executor.cc:623] Executor failed to create kernel. Invalid argument: Default AvgPoolingOp only supports NHWC on device type CPU #011 [[{{node loss/conv2d_22_loss/AvgPool}} = AvgPool[T=DT_FLOAT, data_format="NCHW", ksize=[1, 1, 11, 11], padding="SAME", strides=[1, 1, 1, 1], _device="/job:localhost/replica:0/task:0/device:GPU:0"](loss/conv2d_22_loss/AvgPool-0-TransposeNHWCToNCHW-LayoutOptimizer)]] 2019-01-31 16:15:22.397598: E tensorflow/core/common_runtime/executor.cc:623] Executor failed to create kernel. Invalid argument: Default AvgPoolingOp only supports NHWC on device type CPU #011 [[{{node loss/conv2d_22_loss/AvgPool}} = AvgPool[T=DT_FLOAT, data_format="NCHW", ksize=[1, 1, 11, 11], padding="SAME", strides=[1, 1, 1, 1], _device="/job:localhost/replica:0/task:0/device:GPU:0"](loss/conv2d_22_loss/AvgPool-0-TransposeNHWCToNCHW-LayoutOptimizer)]] 2019-01-31 16:15:24,607 INFO - tensorflow - global_step/sec: 0.406899 2019-01-31 16:15:24,608 INFO - tensorflow - loss = 0.7990469, step = 101 (245.761 sec) 2019-01-31 16:15:52,448 INFO - tensorflow - Saving checkpoints for 114 into s3://sagemaker-us-east-1-552819999234/sagemaker-tensorflow-2019-01-31-16-05-25-509/checkpoints/model.ckpt. 2019-01-31 16:16:05,546 INFO - tensorflow - Calling model_fn. 2019-01-31 16:16:06,378 INFO - tensorflow - Done calling model_fn. 2019-01-31 16:16:06,400 INFO - tensorflow - Starting evaluation at 2019-01-31-16:16:06 2019-01-31 16:16:06,587 INFO - tensorflow - Graph was finalized. 2019-01-31 16:16:06,711 INFO - tensorflow - Restoring parameters from s3://sagemaker-us-east-1-552819999234/sagemaker-tensorflow-2019-01-31-16-05-25-509/checkpoints/model.ckpt-114 2019-01-31 16:16:08,665 INFO - tensorflow - Running local_init_op. 2019-01-31 16:16:08,677 INFO - tensorflow - Done running local_init_op. 2019-01-31 16:16:12.804822: E tensorflow/core/common_runtime/executor.cc:623] Executor failed to create kernel. Invalid argument: Default AvgPoolingOp only supports NHWC on device type CPU #011 [[{{node loss/conv2d_22_loss/AvgPool}} = AvgPool[T=DT_FLOAT, data_format="NCHW", ksize=[1, 1, 11, 11], padding="SAME", strides=[1, 1, 1, 1], _device="/job:localhost/replica:0/task:0/device:GPU:0"](loss/conv2d_22_loss/AvgPool-0-TransposeNHWCToNCHW-LayoutOptimizer)]] 2019-01-31 16:16:12.849267: E tensorflow/core/common_runtime/executor.cc:623] Executor failed to create kernel. Invalid argument: Default AvgPoolingOp only supports NHWC on device type CPU #011 [[{{node loss/conv2d_22_loss/AvgPool}} = AvgPool[T=DT_FLOAT, data_format="NCHW", ksize=[1, 1, 11, 11], padding="SAME", strides=[1, 1, 1, 1], _device="/job:localhost/replica:0/task:0/device:GPU:0"](loss/conv2d_22_loss/AvgPool-0-TransposeNHWCToNCHW-LayoutOptimizer)]] 2019-01-31 16:16:20,041 INFO - tensorflow - Evaluation [10/100] 2019-01-31 16:16:27,204 INFO - tensorflow - Evaluation [20/100] 2019-01-31 16:16:34,407 INFO - tensorflow - Evaluation [30/100] 2019-01-31 16:16:41,625 INFO - tensorflow - Evaluation [40/100] 2019-01-31 16:16:48,867 INFO - tensorflow - Evaluation [50/100] 2019-01-31 16:16:56,119 INFO - tensorflow - Evaluation [60/100] 2019-01-31 16:17:03,383 INFO - tensorflow - Evaluation [70/100] 2019-01-31 16:17:10,652 INFO - tensorflow - Evaluation [80/100] 2019-01-31 16:17:17,917 INFO - tensorflow - Evaluation [90/100] 2019-01-31 16:17:25,184 INFO - tensorflow - Evaluation [100/100] 2019-01-31 16:17:25,204 INFO - tensorflow - Finished evaluation at 2019-01-31-16:17:25 2019-01-31 16:17:25,204 INFO - tensorflow - Saving dict for global step 114: accuracy = 0.5057605, global_step = 114, loss = 1.0482408 2019-01-31 16:17:30,059 INFO - tensorflow - Saving 'checkpoint_path' summary for global step 114: s3://sagemaker-us-east-1-552819999234/sagemaker-tensorflow-2019-01-31-16-05-25-509/checkpoints/model.ckpt-114 2019-01-31 16:17:31,075 INFO - tensorflow - Calling model_fn. 2019-01-31 16:17:31,499 INFO - tensorflow - Done calling model_fn. 2019-01-31 16:17:31,500 INFO - tensorflow - Signatures INCLUDED in export for Eval: None 2019-01-31 16:17:31,500 INFO - tensorflow - Signatures INCLUDED in export for Classify: None 2019-01-31 16:17:31,500 INFO - tensorflow - Signatures INCLUDED in export for Regress: None 2019-01-31 16:17:31,500 INFO - tensorflow - Signatures INCLUDED in export for Predict: ['serving_default'] 2019-01-31 16:17:31,500 INFO - tensorflow - Signatures INCLUDED in export for Train: None 2019-01-31 16:17:31,650 INFO - tensorflow - Restoring parameters from s3://sagemaker-us-east-1-552819999234/sagemaker-tensorflow-2019-01-31-16-05-25-509/checkpoints/model.ckpt-114 2019-01-31 16:17:34,198 WARNING - tensorflow - From /usr/local/lib/python2.7/dist-packages/tensorflow/python/estimator/estimator.py:1044: calling add_meta_graph_and_variables (from tensorflow.python.saved_model.builder_impl) with legacy_init_op is deprecated and will be removed in a future version. Instructions for updating: Pass your op to the equivalent parameter main_op instead. 2019-01-31 16:17:34,198 INFO - tensorflow - Assets added to graph. 2019-01-31 16:17:34,198 INFO - tensorflow - No assets to write. 2019-01-31 16:17:38,582 INFO - tensorflow - SavedModel written to: s3://sagemaker-us-east-1-552819999234/sagemaker-tensorflow-2019-01-31-16-05-25-509/checkpoints/export/Servo/1548951450/saved_model.pb 2019-01-31 16:20:49,331 INFO - tensorflow - global_step/sec: 0.307954 2019-01-31 16:20:49,331 INFO - tensorflow - loss = 0.5964133, step = 201 (324.724 sec) 2019-01-31 16:20:53,739 INFO - tensorflow - Saving checkpoints for 203 into s3://sagemaker-us-east-1-552819999234/sagemaker-tensorflow-2019-01-31-16-05-25-509/checkpoints/model.ckpt. 2019-01-31 16:21:08,213 INFO - tensorflow - Skip the current checkpoint eval due to throttle secs (600 secs). 2019-01-31 16:24:43,074 INFO - tensorflow - global_step/sec: 0.42782 2019-01-31 16:24:43,075 INFO - tensorflow - loss = 0.47347695, step = 301 (233.743 sec) 2019-01-31 16:25:55,494 INFO - tensorflow - Saving checkpoints for 334 into s3://sagemaker-us-east-1-552819999234/sagemaker-tensorflow-2019-01-31-16-05-25-509/checkpoints/model.ckpt. 2019-01-31 16:26:09,754 INFO - tensorflow - Calling model_fn. 2019-01-31 16:26:10,388 INFO - tensorflow - Done calling model_fn. 2019-01-31 16:26:10,411 INFO - tensorflow - Starting evaluation at 2019-01-31-16:26:10 2019-01-31 16:26:10,598 INFO - tensorflow - Graph was finalized. 2019-01-31 16:26:10,725 INFO - tensorflow - Restoring parameters from s3://sagemaker-us-east-1-552819999234/sagemaker-tensorflow-2019-01-31-16-05-25-509/checkpoints/model.ckpt-334 2019-01-31 16:26:13,009 INFO - tensorflow - Running local_init_op. 2019-01-31 16:26:13,022 INFO - tensorflow - Done running local_init_op. 2019-01-31 16:26:16.851432: E tensorflow/core/common_runtime/executor.cc:623] Executor failed to create kernel. Invalid argument: Default AvgPoolingOp only supports NHWC on device type CPU #011 [[{{node loss/conv2d_22_loss/AvgPool}} = AvgPool[T=DT_FLOAT, data_format="NCHW", ksize=[1, 1, 11, 11], padding="SAME", strides=[1, 1, 1, 1], _device="/job:localhost/replica:0/task:0/device:GPU:0"](loss/conv2d_22_loss/AvgPool-0-TransposeNHWCToNCHW-LayoutOptimizer)]] 2019-01-31 16:26:16.893278: E tensorflow/core/common_runtime/executor.cc:623] Executor failed to create kernel. Invalid argument: Default AvgPoolingOp only supports NHWC on device type CPU #011 [[{{node loss/conv2d_22_loss/AvgPool}} = AvgPool[T=DT_FLOAT, data_format="NCHW", ksize=[1, 1, 11, 11], padding="SAME", strides=[1, 1, 1, 1], _device="/job:localhost/replica:0/task:0/device:GPU:0"](loss/conv2d_22_loss/AvgPool-0-TransposeNHWCToNCHW-LayoutOptimizer)]] 2019-01-31 16:26:24,198 INFO - tensorflow - Evaluation [10/100] 2019-01-31 16:26:31,460 INFO - tensorflow - Evaluation [20/100] 2019-01-31 16:26:38,748 INFO - tensorflow - Evaluation [30/100] 2019-01-31 16:26:46,043 INFO - tensorflow - Evaluation [40/100] 2019-01-31 16:26:53,346 INFO - tensorflow - Evaluation [50/100] 2019-01-31 16:27:00,651 INFO - tensorflow - Evaluation [60/100] 2019-01-31 16:27:07,968 INFO - tensorflow - Evaluation [70/100] 2019-01-31 16:27:15,284 INFO - tensorflow - Evaluation [80/100] 2019-01-31 16:27:22,598 INFO - tensorflow - Evaluation [90/100] 2019-01-31 16:27:29,901 INFO - tensorflow - Evaluation [100/100] 2019-01-31 16:27:29,920 INFO - tensorflow - Finished evaluation at 2019-01-31-16:27:29 2019-01-31 16:27:29,920 INFO - tensorflow - Saving dict for global step 334: accuracy = 0.48024333, global_step = 334, loss = 1.325735 2019-01-31 16:27:30,483 INFO - tensorflow - Saving 'checkpoint_path' summary for global step 334: s3://sagemaker-us-east-1-552819999234/sagemaker-tensorflow-2019-01-31-16-05-25-509/checkpoints/model.ckpt-334 2019-01-31 16:27:31,094 INFO - tensorflow - Calling model_fn. 2019-01-31 16:27:31,510 INFO - tensorflow - Done calling model_fn. 2019-01-31 16:27:31,510 INFO - tensorflow - Signatures INCLUDED in export for Eval: None 2019-01-31 16:27:31,510 INFO - tensorflow - Signatures INCLUDED in export for Classify: None 2019-01-31 16:27:31,510 INFO - tensorflow - Signatures INCLUDED in export for Regress: None 2019-01-31 16:27:31,511 INFO - tensorflow - Signatures INCLUDED in export for Predict: ['serving_default'] 2019-01-31 16:27:31,511 INFO - tensorflow - Signatures INCLUDED in export for Train: None 2019-01-31 16:27:31,663 INFO - tensorflow - Restoring parameters from s3://sagemaker-us-east-1-552819999234/sagemaker-tensorflow-2019-01-31-16-05-25-509/checkpoints/model.ckpt-334 2019-01-31 16:27:33,828 INFO - tensorflow - Assets added to graph. 2019-01-31 16:27:33,828 INFO - tensorflow - No assets to write. 2019-01-31 16:27:38,308 INFO - tensorflow - SavedModel written to: s3://sagemaker-us-east-1-552819999234/sagemaker-tensorflow-2019-01-31-16-05-25-509/checkpoints/export/Servo/1548952050/saved_model.pb 2019-01-31 16:30:06,020 INFO - tensorflow - global_step/sec: 0.30965 2019-01-31 16:30:06,020 INFO - tensorflow - loss = 0.3921545, step = 401 (322.946 sec) 2019-01-31 16:30:55,779 INFO - tensorflow - Saving checkpoints for 424 into s3://sagemaker-us-east-1-552819999234/sagemaker-tensorflow-2019-01-31-16-05-25-509/checkpoints/model.ckpt. 2019-01-31 16:31:11,891 INFO - tensorflow - Skip the current checkpoint eval due to throttle secs (600 secs). 2019-01-31 16:34:04,345 INFO - tensorflow - global_step/sec: 0.419594 2019-01-31 16:34:04,346 INFO - tensorflow - loss = 0.3427617, step = 501 (238.326 sec) 2019-01-31 16:35:57,569 INFO - tensorflow - Saving checkpoints for 553 into s3://sagemaker-us-east-1-552819999234/sagemaker-tensorflow-2019-01-31-16-05-25-509/checkpoints/model.ckpt. 2019-01-31 16:36:15,930 INFO - tensorflow - Calling model_fn. 2019-01-31 16:36:16,500 INFO - tensorflow - Done calling model_fn. 2019-01-31 16:36:16,522 INFO - tensorflow - Starting evaluation at 2019-01-31-16:36:16 2019-01-31 16:36:16,708 INFO - tensorflow - Graph was finalized. 2019-01-31 16:36:17,112 INFO - tensorflow - Restoring parameters from s3://sagemaker-us-east-1-552819999234/sagemaker-tensorflow-2019-01-31-16-05-25-509/checkpoints/model.ckpt-553 2019-01-31 16:36:19,480 INFO - tensorflow - Running local_init_op. 2019-01-31 16:36:19,491 INFO - tensorflow - Done running local_init_op. 2019-01-31 16:36:23.591777: E tensorflow/core/common_runtime/executor.cc:623] Executor failed to create kernel. Invalid argument: Default AvgPoolingOp only supports NHWC on device type CPU #011 [[{{node loss/conv2d_22_loss/AvgPool}} = AvgPool[T=DT_FLOAT, data_format="NCHW", ksize=[1, 1, 11, 11], padding="SAME", strides=[1, 1, 1, 1], _device="/job:localhost/replica:0/task:0/device:GPU:0"](loss/conv2d_22_loss/AvgPool-0-TransposeNHWCToNCHW-LayoutOptimizer)]] 2019-01-31 16:36:23.635512: E tensorflow/core/common_runtime/executor.cc:623] Executor failed to create kernel. Invalid argument: Default AvgPoolingOp only supports NHWC on device type CPU #011 [[{{node loss/conv2d_22_loss/AvgPool}} = AvgPool[T=DT_FLOAT, data_format="NCHW", ksize=[1, 1, 11, 11], padding="SAME", strides=[1, 1, 1, 1], _device="/job:localhost/replica:0/task:0/device:GPU:0"](loss/conv2d_22_loss/AvgPool-0-TransposeNHWCToNCHW-LayoutOptimizer)]] 2019-01-31 16:36:30,955 INFO - tensorflow - Evaluation [10/100] 2019-01-31 16:36:38,222 INFO - tensorflow - Evaluation [20/100] 2019-01-31 16:36:45,504 INFO - tensorflow - Evaluation [30/100] 2019-01-31 16:36:52,807 INFO - tensorflow - Evaluation [40/100] 2019-01-31 16:37:00,116 INFO - tensorflow - Evaluation [50/100] 2019-01-31 16:37:07,431 INFO - tensorflow - Evaluation [60/100] 2019-01-31 16:37:14,740 INFO - tensorflow - Evaluation [70/100] 2019-01-31 16:37:22,072 INFO - tensorflow - Evaluation [80/100] 2019-01-31 16:37:29,405 INFO - tensorflow - Evaluation [90/100] 2019-01-31 16:37:36,729 INFO - tensorflow - Evaluation [100/100] 2019-01-31 16:37:36,749 INFO - tensorflow - Finished evaluation at 2019-01-31-16:37:36 2019-01-31 16:37:36,749 INFO - tensorflow - Saving dict for global step 553: accuracy = 0.44534165, global_step = 553, loss = 1.6306622 2019-01-31 16:37:37,503 INFO - tensorflow - Saving 'checkpoint_path' summary for global step 553: s3://sagemaker-us-east-1-552819999234/sagemaker-tensorflow-2019-01-31-16-05-25-509/checkpoints/model.ckpt-553 2019-01-31 16:37:38,240 INFO - tensorflow - Calling model_fn. 2019-01-31 16:37:38,659 INFO - tensorflow - Done calling model_fn. 2019-01-31 16:37:38,659 INFO - tensorflow - Signatures INCLUDED in export for Eval: None 2019-01-31 16:37:38,659 INFO - tensorflow - Signatures INCLUDED in export for Classify: None 2019-01-31 16:37:38,659 INFO - tensorflow - Signatures INCLUDED in export for Regress: None 2019-01-31 16:37:38,659 INFO - tensorflow - Signatures INCLUDED in export for Predict: ['serving_default'] 2019-01-31 16:37:38,659 INFO - tensorflow - Signatures INCLUDED in export for Train: None 2019-01-31 16:37:38,828 INFO - tensorflow - Restoring parameters from s3://sagemaker-us-east-1-552819999234/sagemaker-tensorflow-2019-01-31-16-05-25-509/checkpoints/model.ckpt-553 2019-01-31 16:37:42,131 INFO - tensorflow - Assets added to graph. 2019-01-31 16:37:42,132 INFO - tensorflow - No assets to write. 2019-01-31 16:37:46,183 INFO - tensorflow - SavedModel written to: s3://sagemaker-us-east-1-552819999234/sagemaker-tensorflow-2019-01-31-16-05-25-509/checkpoints/export/Servo/1548952658/saved_model.pb 2019-01-31 16:39:31,528 INFO - tensorflow - global_step/sec: 0.305639 2019-01-31 16:39:31,529 INFO - tensorflow - loss = 0.30985755, step = 601 (327.183 sec) 2019-01-31 16:40:59,765 INFO - tensorflow - Saving checkpoints for 641 into s3://sagemaker-us-east-1-552819999234/sagemaker-tensorflow-2019-01-31-16-05-25-509/checkpoints/model.ckpt. 2019-01-31 16:41:11,895 INFO - tensorflow - Skip the current checkpoint eval due to throttle secs (600 secs). 2019-01-31 16:43:24,674 INFO - tensorflow - global_step/sec: 0.428917 2019-01-31 16:43:24,675 INFO - tensorflow - loss = 0.27382898, step = 701 (233.146 sec) 2019-01-31 16:46:00,079 INFO - tensorflow - Saving checkpoints for 772 into s3://sagemaker-us-east-1-552819999234/sagemaker-tensorflow-2019-01-31-16-05-25-509/checkpoints/model.ckpt. 2019-01-31 16:46:14,015 INFO - tensorflow - Skip the current checkpoint eval due to throttle secs (600 secs). 2019-01-31 16:47:18,675 INFO - tensorflow - global_step/sec: 0.427349 2019-01-31 16:47:18,675 INFO - tensorflow - loss = 0.24804598, step = 801 (234.001 sec) 2019-01-31 16:51:01,830 INFO - tensorflow - Saving checkpoints for 901 into s3://sagemaker-us-east-1-552819999234/sagemaker-tensorflow-2019-01-31-16-05-25-509/checkpoints/model.ckpt. 2019-01-31 16:51:16,941 INFO - tensorflow - Calling model_fn. 2019-01-31 16:51:17,548 INFO - tensorflow - Done calling model_fn. 2019-01-31 16:51:17,570 INFO - tensorflow - Starting evaluation at 2019-01-31-16:51:17 2019-01-31 16:51:17,757 INFO - tensorflow - Graph was finalized. 2019-01-31 16:51:17,871 INFO - tensorflow - Restoring parameters from s3://sagemaker-us-east-1-552819999234/sagemaker-tensorflow-2019-01-31-16-05-25-509/checkpoints/model.ckpt-901 2019-01-31 16:51:20,039 INFO - tensorflow - Running local_init_op. 2019-01-31 16:51:20,050 INFO - tensorflow - Done running local_init_op. 2019-01-31 16:51:24.094208: E tensorflow/core/common_runtime/executor.cc:623] Executor failed to create kernel. Invalid argument: Default AvgPoolingOp only supports NHWC on device type CPU #011 [[{{node loss/conv2d_22_loss/AvgPool}} = AvgPool[T=DT_FLOAT, data_format="NCHW", ksize=[1, 1, 11, 11], padding="SAME", strides=[1, 1, 1, 1], _device="/job:localhost/replica:0/task:0/device:GPU:0"](loss/conv2d_22_loss/AvgPool-0-TransposeNHWCToNCHW-LayoutOptimizer)]] 2019-01-31 16:51:24.137650: E tensorflow/core/common_runtime/executor.cc:623] Executor failed to create kernel. Invalid argument: Default AvgPoolingOp only supports NHWC on device type CPU #011 [[{{node loss/conv2d_22_loss/AvgPool}} = AvgPool[T=DT_FLOAT, data_format="NCHW", ksize=[1, 1, 11, 11], padding="SAME", strides=[1, 1, 1, 1], _device="/job:localhost/replica:0/task:0/device:GPU:0"](loss/conv2d_22_loss/AvgPool-0-TransposeNHWCToNCHW-LayoutOptimizer)]] 2019-01-31 16:51:31,491 INFO - tensorflow - Evaluation [10/100] 2019-01-31 16:51:38,809 INFO - tensorflow - Evaluation [20/100] 2019-01-31 16:51:46,124 INFO - tensorflow - Evaluation [30/100] 2019-01-31 16:51:53,440 INFO - tensorflow - Evaluation [40/100] 2019-01-31 16:52:00,763 INFO - tensorflow - Evaluation [50/100] 2019-01-31 16:52:08,082 INFO - tensorflow - Evaluation [60/100] 2019-01-31 16:52:15,411 INFO - tensorflow - Evaluation [70/100] 2019-01-31 16:52:22,736 INFO - tensorflow - Evaluation [80/100] 2019-01-31 16:52:30,056 INFO - tensorflow - Evaluation [90/100] 2019-01-31 16:52:37,372 INFO - tensorflow - Evaluation [100/100] 2019-01-31 16:52:37,391 INFO - tensorflow - Finished evaluation at 2019-01-31-16:52:37 2019-01-31 16:52:37,392 INFO - tensorflow - Saving dict for global step 901: accuracy = 0.43437195, global_step = 901, loss = 1.9879459 2019-01-31 16:52:38,082 INFO - tensorflow - Saving 'checkpoint_path' summary for global step 901: s3://sagemaker-us-east-1-552819999234/sagemaker-tensorflow-2019-01-31-16-05-25-509/checkpoints/model.ckpt-901 2019-01-31 16:52:39,084 INFO - tensorflow - Calling model_fn. 2019-01-31 16:52:39,507 INFO - tensorflow - Done calling model_fn. 2019-01-31 16:52:39,507 INFO - tensorflow - Signatures INCLUDED in export for Eval: None 2019-01-31 16:52:39,507 INFO - tensorflow - Signatures INCLUDED in export for Classify: None 2019-01-31 16:52:39,508 INFO - tensorflow - Signatures INCLUDED in export for Regress: None 2019-01-31 16:52:39,508 INFO - tensorflow - Signatures INCLUDED in export for Predict: ['serving_default'] 2019-01-31 16:52:39,508 INFO - tensorflow - Signatures INCLUDED in export for Train: None 2019-01-31 16:52:39,747 INFO - tensorflow - Restoring parameters from s3://sagemaker-us-east-1-552819999234/sagemaker-tensorflow-2019-01-31-16-05-25-509/checkpoints/model.ckpt-901 2019-01-31 16:52:43,713 INFO - tensorflow - Assets added to graph. 2019-01-31 16:52:43,714 INFO - tensorflow - No assets to write. 2019-01-31 16:52:52,730 INFO - tensorflow - SavedModel written to: s3://sagemaker-us-east-1-552819999234/sagemaker-tensorflow-2019-01-31-16-05-25-509/checkpoints/export/Servo/1548953558/saved_model.pb 2019-01-31 16:52:52,819 INFO - tensorflow - global_step/sec: 0.299272 2019-01-31 16:52:53,663 INFO - tensorflow - loss = 0.22607544, step = 901 (334.988 sec) 2019-01-31 16:56:03,610 INFO - tensorflow - Saving checkpoints for 987 into s3://sagemaker-us-east-1-552819999234/sagemaker-tensorflow-2019-01-31-16-05-25-509/checkpoints/model.ckpt. 2019-01-31 16:56:14,839 INFO - tensorflow - Skip the current checkpoint eval due to throttle secs (600 secs). 2019-01-31 16:56:44,119 INFO - tensorflow - Saving checkpoints for 1000 into s3://sagemaker-us-east-1-552819999234/sagemaker-tensorflow-2019-01-31-16-05-25-509/checkpoints/model.ckpt. 2019-01-31 16:56:54,544 INFO - tensorflow - Skip the current checkpoint eval due to throttle secs (600 secs). 2019-01-31 16:56:54,984 INFO - tensorflow - Calling model_fn. 2019-01-31 16:56:55,592 INFO - tensorflow - Done calling model_fn. 2019-01-31 16:56:55,614 INFO - tensorflow - Starting evaluation at 2019-01-31-16:56:55 2019-01-31 16:56:55,801 INFO - tensorflow - Graph was finalized. 2019-01-31 16:56:55,875 INFO - tensorflow - Restoring parameters from s3://sagemaker-us-east-1-552819999234/sagemaker-tensorflow-2019-01-31-16-05-25-509/checkpoints/model.ckpt-1000 2019-01-31 16:56:58,024 INFO - tensorflow - Running local_init_op. 2019-01-31 16:56:58,034 INFO - tensorflow - Done running local_init_op. 2019-01-31 16:57:02.101586: E tensorflow/core/common_runtime/executor.cc:623] Executor failed to create kernel. Invalid argument: Default AvgPoolingOp only supports NHWC on device type CPU #011 [[{{node loss/conv2d_22_loss/AvgPool}} = AvgPool[T=DT_FLOAT, data_format="NCHW", ksize=[1, 1, 11, 11], padding="SAME", strides=[1, 1, 1, 1], _device="/job:localhost/replica:0/task:0/device:GPU:0"](loss/conv2d_22_loss/AvgPool-0-TransposeNHWCToNCHW-LayoutOptimizer)]] 2019-01-31 16:57:02.144682: E tensorflow/core/common_runtime/executor.cc:623] Executor failed to create kernel. Invalid argument: Default AvgPoolingOp only supports NHWC on device type CPU #011 [[{{node loss/conv2d_22_loss/AvgPool}} = AvgPool[T=DT_FLOAT, data_format="NCHW", ksize=[1, 1, 11, 11], padding="SAME", strides=[1, 1, 1, 1], _device="/job:localhost/replica:0/task:0/device:GPU:0"](loss/conv2d_22_loss/AvgPool-0-TransposeNHWCToNCHW-LayoutOptimizer)]] 2019-01-31 16:57:09,438 INFO - tensorflow - Evaluation [10/100] 2019-01-31 16:57:16,694 INFO - tensorflow - Evaluation [20/100] 2019-01-31 16:57:23,959 INFO - tensorflow - Evaluation [30/100] 2019-01-31 16:57:31,234 INFO - tensorflow - Evaluation [40/100] 2019-01-31 16:57:38,514 INFO - tensorflow - Evaluation [50/100] 2019-01-31 16:57:45,808 INFO - tensorflow - Evaluation [60/100] 2019-01-31 16:57:53,111 INFO - tensorflow - Evaluation [70/100] 2019-01-31 16:58:00,409 INFO - tensorflow - Evaluation [80/100] 2019-01-31 16:58:07,710 INFO - tensorflow - Evaluation [90/100] 2019-01-31 16:58:15,027 INFO - tensorflow - Evaluation [100/100] 2019-01-31 16:58:15,046 INFO - tensorflow - Finished evaluation at 2019-01-31-16:58:15 2019-01-31 16:58:15,046 INFO - tensorflow - Saving dict for global step 1000: accuracy = 0.53406906, global_step = 1000, loss = 1.8152299 2019-01-31 16:58:15,735 INFO - tensorflow - Saving 'checkpoint_path' summary for global step 1000: s3://sagemaker-us-east-1-552819999234/sagemaker-tensorflow-2019-01-31-16-05-25-509/checkpoints/model.ckpt-1000 2019-01-31 16:58:16,397 INFO - tensorflow - Calling model_fn. 2019-01-31 16:58:16,806 INFO - tensorflow - Done calling model_fn. 2019-01-31 16:58:16,807 INFO - tensorflow - Signatures INCLUDED in export for Eval: None 2019-01-31 16:58:16,807 INFO - tensorflow - Signatures INCLUDED in export for Classify: None 2019-01-31 16:58:16,807 INFO - tensorflow - Signatures INCLUDED in export for Regress: None 2019-01-31 16:58:16,807 INFO - tensorflow - Signatures INCLUDED in export for Predict: ['serving_default'] 2019-01-31 16:58:16,807 INFO - tensorflow - Signatures INCLUDED in export for Train: None 2019-01-31 16:58:16,968 INFO - tensorflow - Restoring parameters from s3://sagemaker-us-east-1-552819999234/sagemaker-tensorflow-2019-01-31-16-05-25-509/checkpoints/model.ckpt-1000 2019-01-31 16:58:19,161 INFO - tensorflow - Assets added to graph. 2019-01-31 16:58:19,161 INFO - tensorflow - No assets to write. 2019-01-31 16:58:29 Uploading - Uploading generated training model2019-01-31 16:58:23,562 INFO - tensorflow - SavedModel written to: s3://sagemaker-us-east-1-552819999234/sagemaker-tensorflow-2019-01-31-16-05-25-509/checkpoints/export/Servo/1548953896/saved_model.pb 2019-01-31 16:58:24,876 INFO - tensorflow - Loss for final step: 0.20883322. 2019-01-31 16:58:26,085 INFO - tf_container - Downloaded saved model at /opt/ml/model/export/Servo/1548953896 2019-01-31 16:58:50 Completed - Training job completed Billable seconds: 3061
custom_predictor = custom_estimator.deploy(initial_instance_count=1, instance_type='ml.m4.xlarge')
INFO:sagemaker:Creating model with name: sagemaker-tensorflow-2019-01-31-16-05-25-509 INFO:sagemaker:Creating endpoint with name sagemaker-tensorflow-2019-01-31-16-05-25-509
-----------------------------------------------------------------------------!
example_patch = 'eopatch_107_row-10_col-7'
example_file_x = gzip.GzipFile(op.join('data', 'eopatch', example_patch, 'data_timeless', 'MEDIAN_PIXEL.npy.gz'), 'r')
example_file_y = gzip.GzipFile(op.join('data', 'eopatch', example_patch, 'mask_timeless', 'TREE_COVER.npy.gz'), 'r')
example_file_x_array = np.load(example_file_x)[20:276,0:256,:]
example_file_y_array = np.load(example_file_y)[20:276,0:256,:]
prediction = custom_predictor.predict({ 'input_1': np.array([example_file_x_array]) })
visual_prediction = np.argmax(np.reshape(prediction['outputs']['conv2d_22']['float_val'], (256, 256, 6)), axis=2)
fig = plt.figure(figsize=(12,4))
ax1 = fig.add_subplot(1,3,1)
ax1.imshow(example_file_x_array)
ax2 = fig.add_subplot(1,3,2)
ax2.imshow(example_file_y_array[:,:,0])
ax3 = fig.add_subplot(1,3,3)
ax3.imshow(visual_prediction)
<matplotlib.image.AxesImage at 0x7f0738ee3f28>