High-level RNN CNTK Example¶

import numpy as np
import os
import sys
import cntk
from cntk.layers import Embedding, LSTM, GRU, Dense, Recurrence
from cntk import sequence
from common.params_lstm import *
from common.utils import *

def create_symbol(CUDNN=True):
    # Weight initialiser from uniform distribution
    # Activation (unless states) is None
    with cntk.layers.default_options(init = cntk.glorot_uniform()):
        x = Embedding(EMBEDSIZE)(features) # output: list of len=BATCHSIZE of arrays with shape=(MAXLEN, EMBEDSIZE)
        
        # Since we have a vanilla RNN, instead of using the more flexible Recurrence(GRU) unit, which allows for
        # example LayerNormalisation to be added to the network, we can use optimized_rnnstack which quickly
        # goes down to the CuDNN level. This is another reason not to read much into the speed comparison because
        # it becomes a measure of which framework has the fastest way to go down to CuDNN.
        if not CUDNN:
            x = Recurrence(GRU(NUMHIDDEN))(x) # output: list of len=BATCHSIZE of arrays with shape=(MAXLEN, NUMHIDDEN)
        else:
            W = cntk.parameter((cntk.InferredDimension, 4))
            x = cntk.ops.optimized_rnnstack(x, W, NUMHIDDEN, num_layers=1, bidirectional=False, recurrent_op='gru')
        
        x = sequence.last(x) #o utput: array with shape=(BATCHSIZE, NUMHIDDEN)
        x = Dense(2)(x) # output: array with shape=(BATCHSIZE, 2)
        return x

def init_model(m):
    # Loss (dense labels); check if support for sparse labels
    loss = cntk.cross_entropy_with_softmax(m, labels)  
    # ADAM, set unit_gain to False to match others
    learner = cntk.adam(m.parameters,
                        lr=cntk.learning_rate_schedule(LR, cntk.UnitType.minibatch) ,
                        momentum=cntk.momentum_schedule(BETA_1), 
                        variance_momentum=cntk.momentum_schedule(BETA_2),
                        epsilon=EPS,
                        unit_gain=False)
    trainer = cntk.Trainer(m, (loss, cntk.classification_error(m, labels)), [learner])
    return trainer