#!/usr/bin/env python # coding: utf-8 # In[1]: get_ipython().run_line_magic('reload_ext', 'autoreload') get_ipython().run_line_magic('autoreload', '2') get_ipython().run_line_magic('matplotlib', 'inline') import os os.environ["CUDA_DEVICE_ORDER"]="PCI_BUS_ID"; os.environ["CUDA_VISIBLE_DEVICES"]="0"; import ktrain from ktrain import vision as vis from ktrain import text as txt # # Additional Features # # In this notebook, we will demonstrated some additional features of *ktrain*. To demonstrate these features, we will first load some sample data and define a simple model that trains fast. As usual, we will wrap the model and data in a *ktrain* Learner object. # In[2]: # load and prepare data as you normally would in Keras from tensorflow.keras.preprocessing import sequence from tensorflow.keras.datasets import imdb NUM_WORDS = 20000 MAXLEN = 400 def load_data(): (x_train, y_train), (x_test, y_test) = imdb.load_data(num_words=NUM_WORDS) x_train = sequence.pad_sequences(x_train, maxlen=MAXLEN) x_test = sequence.pad_sequences(x_test, maxlen=MAXLEN) return (x_train, y_train), (x_test, y_test) (x_train, y_train), (x_test, y_test) = load_data() # In[3]: # build a model as you normally would in Keras from tensorflow.keras.models import Sequential from tensorflow.keras.layers import Dense, Embedding, GlobalAveragePooling1D def get_model(): model = Sequential() model.add(Embedding(NUM_WORDS, 50, input_length=MAXLEN)) model.add(GlobalAveragePooling1D()) model.add(Dense(1, activation='sigmoid')) model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy']) return model model = get_model() # In[4]: learner = ktrain.get_learner(model, train_data=(x_train, y_train), val_data = (x_test, y_test)) # ## Debugging Keras Models # # When debugging neural networks, it is sometimes useful to [inspect the results of intermediate layers](https://stackoverflow.com/questions/42233963/debugging-keras-intermediate-layer-or-objective-variables-with-tensorflow). *ktrain* makes this easy with the ```print_layers``` and ```layer_output``` methods. The ```print_layers``` method prints the layers of your network with each layer being assigned an index. The output of ```print_layers``` also shows which layers are trainable (unfrozen) and untrainable (frozen). # # **NOTE:** This may not work with models created using the subclassing API in TensorFlow. See [this issue](https://github.com/tensorflow/tensorflow/issues/30955). # In[5]: learner.print_layers() # The ```layer_output``` method accepts a layer ID as input. By default, it will print the output of the layer associated with the layer ID using the first example in the training set. The example_id and use_val arguments can control which example is used to generate the output. In the example below, we print the output of the Dense layer of our network using the sixth example from our validation set (i.e., the example with index 5 in the x_test array). # In[6]: learner.layer_output(2, example_id=5, use_val=True) # Since the model has not yet been trained, it returns 0.5. # Here, we print the output of the GlobalAveragePooling1D layer (layer_id=1) implicitly using the first example from our training set. # In[7]: learner.layer_output(1) # ## Setting Global Weight Decay in Keras # # [Weight decay](https://machinelearningmastery.com/how-to-reduce-overfitting-in-deep-learning-with-weight-regularization/) is a form of regularization to reduce overfitting. By default, *ktrain* uses no weight decay. The `learner.set_weight_decay` method can be used to set the weight decay rate. With no arguments, this will use the default weight decay rate of `0.01`. Weight decay is implemented using the [AdamWeightDecay](https://arxiv.org/abs/1711.05101) optimizer. # # In[8]: learner.get_weight_decay() # In[9]: learner.set_weight_decay() # In[10]: learner.get_weight_decay() # In[11]: learner.set_weight_decay(wd=0.1) # using a higher wd than default # In[12]: learner.get_weight_decay() # In[13]: learner.autofit(1e-2, 1) # ## Saving the Current Model # # When issuing calls to ```autofit```, ```fit_onecycle```, and ```fit```, it is sometimes useful to periodically save the model in case additional training results in overfitting. This can be accomplished with the ```learner.save_model``` and ```learner.load_model``` methods, which are simply wrappers to ```model.save``` and ```load_model``` in Keras. Example usage is shown below: # In[9]: learner.save_model('/tmp/mymodel') # train further here # if overfitting, then do this: learner.load_model('/tmp/mymodel') # ## Built-In Callbacks # # *ktrain* enables easy access to [EarlyStopping](https://keras.io/callbacks/#earlystopping), [ModelCheckpoint](https://keras.io/callbacks/#modelcheckpoint), and [ReduceLROnPlateau](https://keras.io/callbacks/#reducelronplateau) callbacks. All of the **fit** methods of *ktrain* have both an ```early_stopping``` argument and a ```checkpoint_folder``` argument. When setting ```early_stopping=3```, for instance, the training will disconinue automatically when the validation loss fails to improve after a 3 epochs. When suppying the path to a folder using ```checkpoint_folder``` argument, files containing the weights obtained after each epoch will be saved to the folder. In the example below, we will train our model using the ```fit``` method using both arguments. # In[8]: learner.reset_weights() learner.fit(0.005, 10, cycle_len=1, cycle_mult=2, early_stopping=3, checkpoint_folder='/tmp') # The weights obtained after each of the six epochs will be available as files stored in the checkpoint_folder: # # ``` # /tmp/weights-01.hdf5 /tmp/weights-03.hdf5 /tmp/weights-05.hdf5 # /tmp/weights-02.hdf5 /tmp/weights-04.hdf5 /tmp/weights-06.hdf5 # ``` # # These weights can be easily loaded into the model as one would normally do: # ``` # learner.model.load_weights('/tmp/weights-02.hdfs') # ``` # # The ```checkpoint_folder``` argument can be used with any "fit" method (i.e., ```autofit```, ```fit_onecycle```, and ```fit```). It is particularly useful when needing to rollback to an earlier epoch in case the model overfits. # # In addition, the ```autofit``` method includes a **reduce_on_plateau** parameter, which can be used to automatically reduce the maximum (and base) learning rates in the [triangular learning rate policy](https://arxiv.org/abs/1506.01186) when the validation loss no longer improves by a factor specified by the **reduce_factor** argument. The criterion can be changed to validation accuracy using the **monitor** argument. When ```autofit``` is called without an **epochs** parameter, both the **early_stopping** and **reduce_on_plateau** are automatically enabled. # ## Custom Callbacks # The ```fit```, ```autofit```, and ```fit_onecycle``` methods of the Learner object can also accept custom callbacks, just as you can in standard calls to ```model.fit```. For instance, here we re-train our model and print the ROC-AUC score after each epoch: # In[5]: # define a custom callback for ROC-AUC from tensorflow.keras.callbacks import Callback from sklearn.metrics import roc_auc_score class RocAucEvaluation(Callback): def __init__(self, validation_data=(), interval=1): super(Callback, self).__init__() self.interval = interval self.X_val, self.y_val = validation_data def on_epoch_end(self, epoch, logs={}): if epoch % self.interval == 0: y_pred = self.model.predict(self.X_val, verbose=0) score = roc_auc_score(self.y_val, y_pred) print("\n ROC-AUC - epoch: %d - score: %.6f \n" % (epoch+1, score)) RocAuc = RocAucEvaluation(validation_data=(x_test, y_test), interval=1) # In[6]: # re-create our model from scratch and train using our custom ROC-AUC callback learner = ktrain.get_learner(get_model(), train_data=(x_train, y_train), val_data = (x_test, y_test)) learner.autofit(0.005, 2, callbacks=[RocAuc]) # ## Using TensorBoard with *ktrain* # # Since *ktrain* is a lightweight wrapper around Keras, you can use TensorBoard callbacks with all `fit` methods in *ktrain* (e.g., `autofit`, `fit_onecycyle`, `fit`) just as you would normally do with `model.fit` in Keras: # In[5]: learner.reset_weights() import tensorflow as tf # create TensorBoard calback tbCallBack = tf.keras.callbacks.TensorBoard(log_dir='/tmp/Graph', histogram_freq=0, write_graph=True, write_images=True) # supply it as custom callback to any fit method of Learner learner.autofit(0.005, 2, callbacks=[tbCallBack]) # When training is complete, TensorBoard can be started with: # # ```tensorboard --logdir /tmp/Graph``` # ## Exporting Models to TensorFlow Lite or ONNX # # `Predictor` instances allow you to export your models to ONNX and TensorFlowLite: # ```python # p = ktrain.load_predictor('/tmp/my_predictor') # p.export_model_to_tflite('/tmp/model.tflite') # p.export_model_to_onnx('/tmp/model.onnx') # ``` # See the [ktrain-ONNX-TFLite-examples.ipynb](https://github.com/amaiya/ktrain/blob/master/examples/text/ktrain-ONNX-TFLite-examples.ipynb) example notebook for more details. # # # # ## Previewing Data Augmentation # # *ktrain* allows you to preview a data augmentation prior to use in training. We will re-use the Dogs vs. Cats dataset from above. Let's set the location of the data and instantiate a data augmentation scheme using the ```ktrain.vision.get_data_aug``` function. # In[2]: DATADIR = 'data/dogscats' data_aug = vis.get_data_aug(horizontal_flip=True) # Let's examine the data augmentaiton parameters, which will be the parameters enabled by default plus horizontal_flip=True. Note that the data_aug object is simply a Keras **ImageDataGenerator** object. # In[3]: data_aug.__dict__ # Finally, let's run the data augmentation scheme for four images on a selected cat photo: # In[8]: vis.preview_data_aug(DATADIR+'/train/cats/cat.10000.jpg', data_aug, n=8, rows=2) # In[ ]: