Notebook

DeepCross on Criteo Ad Dataset in TF 2.x¶

In [ ]:

!pip install tensorflow==2.5.0

In [ ]:

!pip install -q -U kaggle
!pip install --upgrade --force-reinstall --no-deps kaggle
!mkdir ~/.kaggle
!cp /content/drive/MyDrive/kaggle.json ~/.kaggle/
!chmod 600 ~/.kaggle/kaggle.json
!kaggle datasets download -d mrkmakr/criteo-dataset

Collecting kaggle
  Downloading kaggle-1.5.12.tar.gz (58 kB)
     |████████████████████████████████| 58 kB 2.6 MB/s 
Building wheels for collected packages: kaggle
  Building wheel for kaggle (setup.py) ... done
  Created wheel for kaggle: filename=kaggle-1.5.12-py3-none-any.whl size=73051 sha256=d0e4b97f111ac4c64a7c2fead5edb3baa0f1f4f21eca031e865b06b3f40f3bbb
  Stored in directory: /root/.cache/pip/wheels/62/d6/58/5853130f941e75b2177d281eb7e44b4a98ed46dd155f556dc5
Successfully built kaggle
Installing collected packages: kaggle
  Attempting uninstall: kaggle
    Found existing installation: kaggle 1.5.12
    Uninstalling kaggle-1.5.12:
      Successfully uninstalled kaggle-1.5.12
Successfully installed kaggle-1.5.12
Downloading criteo-dataset.zip to /content
100% 4.31G/4.31G [02:05<00:00, 44.7MB/s]
100% 4.31G/4.31G [02:05<00:00, 36.8MB/s]

In [ ]:

!unzip criteo-dataset.zip

Archive:  criteo-dataset.zip
  inflating: dac/readme.txt          
  inflating: dac/test.txt            
  inflating: dac/train.txt

In [ ]:

import os
import pandas as pd
import numpy as np
from sklearn.preprocessing import LabelEncoder, KBinsDiscretizer
from sklearn.model_selection import train_test_split

import tensorflow as tf
from tensorflow.keras import Model
from tensorflow.keras.layers import Layer, Input, ReLU
from tensorflow.keras.layers import Dense, Embedding, Dropout
from tensorflow.keras.regularizers import l2
from tensorflow.keras.losses import binary_crossentropy
from tensorflow.keras.callbacks import EarlyStopping
from tensorflow.keras.optimizers import Adam
from tensorflow.keras.metrics import AUC

In [ ]:

os.environ['TF_CPP_MIN_LOG_LEVEL'] = '2'
os.environ['CUDA_VISIBLE_DEVICES'] = '0'

file = 'dac/train.txt'
read_part = True
sample_num = 10000
test_size = 0.2

embed_dim = 8
dnn_dropout = 0.5
hidden_units = [256, 128, 64]

learning_rate = 0.001
batch_size = 4096
epochs = 10

In [ ]:

def sparseFeature(feat, feat_num, embed_dim=4):
    """
    create dictionary for sparse feature
    :param feat: feature name
    :param feat_num: the total number of sparse features that do not repeat
    :param embed_dim: embedding dimension
    :return:
    """
    return {'feat_name': feat, 'feat_num': feat_num, 'embed_dim': embed_dim}


def denseFeature(feat):
    """
    create dictionary for dense feature
    :param feat: dense feature name
    :return:
    """
    return {'feat_name': feat}

In [ ]:

def create_criteo_dataset(file, embed_dim=8, read_part=True, sample_num=100000, test_size=0.2):
    """
    a example about creating criteo dataset
    :param file: dataset's path
    :param embed_dim: the embedding dimension of sparse features
    :param read_part: whether to read part of it
    :param sample_num: the number of instances if read_part is True
    :param test_size: ratio of test dataset
    :return: feature columns, train, test
    """
    names = ['label', 'I1', 'I2', 'I3', 'I4', 'I5', 'I6', 'I7', 'I8', 'I9', 'I10', 'I11',
             'I12', 'I13', 'C1', 'C2', 'C3', 'C4', 'C5', 'C6', 'C7', 'C8', 'C9', 'C10', 'C11',
             'C12', 'C13', 'C14', 'C15', 'C16', 'C17', 'C18', 'C19', 'C20', 'C21', 'C22',
             'C23', 'C24', 'C25', 'C26']

    if read_part:
        data_df = pd.read_csv(file, sep='\t', iterator=True, header=None,
                          names=names)
        data_df = data_df.get_chunk(sample_num)

    else:
        data_df = pd.read_csv(file, sep='\t', header=None, names=names)

    sparse_features = ['C' + str(i) for i in range(1, 27)]
    dense_features = ['I' + str(i) for i in range(1, 14)]
    features = sparse_features + dense_features

    data_df[sparse_features] = data_df[sparse_features].fillna('-1')
    data_df[dense_features] = data_df[dense_features].fillna(0)

    # Bin continuous data into intervals.
    est = KBinsDiscretizer(n_bins=100, encode='ordinal', strategy='uniform')
    data_df[dense_features] = est.fit_transform(data_df[dense_features])

    for feat in sparse_features:
        le = LabelEncoder()
        data_df[feat] = le.fit_transform(data_df[feat])

    # ==============Feature Engineering===================

    # ====================================================
    feature_columns = [sparseFeature(feat, int(data_df[feat].max()) + 1, embed_dim=embed_dim)
                        for feat in features]
    train, test = train_test_split(data_df, test_size=test_size)

    train_X = train[features].values.astype('int32')
    train_y = train['label'].values.astype('int32')
    test_X = test[features].values.astype('int32')
    test_y = test['label'].values.astype('int32')

    return feature_columns, (train_X, train_y), (test_X, test_y)

In [ ]:

class Residual_Units(Layer):
    """
    Residual Units
    """
    def __init__(self, hidden_unit, dim_stack):
        """
        :param hidden_unit: A list. Neural network hidden units.
        :param dim_stack: A scalar. The dimension of inputs unit.
        """
        super(Residual_Units, self).__init__()
        self.layer1 = Dense(units=hidden_unit, activation='relu')
        self.layer2 = Dense(units=dim_stack, activation=None)
        self.relu = ReLU()

    def call(self, inputs, **kwargs):
        x = inputs
        x = self.layer1(x)
        x = self.layer2(x)
        outputs = self.relu(x + inputs)
        return outputs

In [ ]:

class Deep_Crossing(Model):
    def __init__(self, feature_columns, hidden_units, res_dropout=0., embed_reg=1e-6):
        """
        Deep&Crossing
        :param feature_columns: A list. sparse column feature information.
        :param hidden_units: A list. Neural network hidden units.
        :param res_dropout: A scalar. Dropout of resnet.
        :param embed_reg: A scalar. The regularizer of embedding.
        """
        super(Deep_Crossing, self).__init__()
        self.sparse_feature_columns = feature_columns
        self.embed_layers = {
            'embed_' + str(i): Embedding(input_dim=feat['feat_num'],
                                         input_length=1,
                                         output_dim=feat['embed_dim'],
                                         embeddings_initializer='random_uniform',
                                         embeddings_regularizer=l2(embed_reg))
            for i, feat in enumerate(self.sparse_feature_columns)
        }
        # the total length of embedding layers
        embed_layers_len = sum([feat['embed_dim'] for feat in self.sparse_feature_columns])
        self.res_network = [Residual_Units(unit, embed_layers_len) for unit in hidden_units]
        self.res_dropout = Dropout(res_dropout)
        self.dense = Dense(1, activation=None)

    def call(self, inputs):
        sparse_inputs = inputs
        sparse_embed = tf.concat([self.embed_layers['embed_{}'.format(i)](sparse_inputs[:, i])
                                  for i in range(sparse_inputs.shape[1])], axis=-1)
        r = sparse_embed
        for res in self.res_network:
            r = res(r)
        r = self.res_dropout(r)
        outputs = tf.nn.sigmoid(self.dense(r))
        return outputs

    def summary(self):
        sparse_inputs = Input(shape=(len(self.sparse_feature_columns),), dtype=tf.int32)
        Model(inputs=sparse_inputs, outputs=self.call(sparse_inputs)).summary()

In [ ]:

# ========================== Create dataset =======================
feature_columns, train, test = create_criteo_dataset(file=file,
                                                        embed_dim=embed_dim,
                                                        read_part=read_part,
                                                        sample_num=sample_num,
                                                        test_size=test_size)
train_X, train_y = train
test_X, test_y = test
# ============================Build Model==========================
mirrored_strategy = tf.distribute.MirroredStrategy()
with mirrored_strategy.scope():
    model = Deep_Crossing(feature_columns, hidden_units)
    model.summary()
    # =========================Compile============================
    model.compile(loss=binary_crossentropy, optimizer=Adam(learning_rate=learning_rate),
                    metrics=[AUC()])
# ============================model checkpoint======================
# check_path = 'save/deep_crossing_weights.epoch_{epoch:04d}.val_loss_{val_loss:.4f}.ckpt'
# checkpoint = tf.keras.callbacks.ModelCheckpoint(check_path, save_weights_only=True,
#                                                 verbose=1, period=5)
# ===========================Fit==============================
model.fit(
    train_X,
    train_y,
    epochs=epochs,
    callbacks=[EarlyStopping(monitor='val_loss', patience=2, restore_best_weights=True)],  # checkpoint
    batch_size=batch_size,
    validation_split=0.1
)
# ===========================Test==============================
print('test AUC: %f' % model.evaluate(test_X, test_y, batch_size=batch_size)[1])

WARNING:tensorflow:There are non-GPU devices in `tf.distribute.Strategy`, not using nccl allreduce.
WARNING:tensorflow:Collective ops is not configured at program startup. Some performance features may not be enabled.
INFO:tensorflow:Using MirroredStrategy with devices ('/job:localhost/replica:0/task:0/device:CPU:0',)
Model: "model"
__________________________________________________________________________________________________
Layer (type)                    Output Shape         Param #     Connected to                     
==================================================================================================
input_1 (InputLayer)            [(None, 39)]         0                                            
__________________________________________________________________________________________________
tf.__operators__.getitem (Slici (None,)              0           input_1[0][0]                    
__________________________________________________________________________________________________
tf.__operators__.getitem_1 (Sli (None,)              0           input_1[0][0]                    
__________________________________________________________________________________________________
tf.__operators__.getitem_2 (Sli (None,)              0           input_1[0][0]                    
__________________________________________________________________________________________________
tf.__operators__.getitem_3 (Sli (None,)              0           input_1[0][0]                    
__________________________________________________________________________________________________
tf.__operators__.getitem_4 (Sli (None,)              0           input_1[0][0]                    
__________________________________________________________________________________________________
tf.__operators__.getitem_5 (Sli (None,)              0           input_1[0][0]                    
__________________________________________________________________________________________________
tf.__operators__.getitem_6 (Sli (None,)              0           input_1[0][0]                    
__________________________________________________________________________________________________
tf.__operators__.getitem_7 (Sli (None,)              0           input_1[0][0]                    
__________________________________________________________________________________________________
tf.__operators__.getitem_8 (Sli (None,)              0           input_1[0][0]                    
__________________________________________________________________________________________________
tf.__operators__.getitem_9 (Sli (None,)              0           input_1[0][0]                    
__________________________________________________________________________________________________
tf.__operators__.getitem_10 (Sl (None,)              0           input_1[0][0]                    
__________________________________________________________________________________________________
tf.__operators__.getitem_11 (Sl (None,)              0           input_1[0][0]                    
__________________________________________________________________________________________________
tf.__operators__.getitem_12 (Sl (None,)              0           input_1[0][0]                    
__________________________________________________________________________________________________
tf.__operators__.getitem_13 (Sl (None,)              0           input_1[0][0]                    
__________________________________________________________________________________________________
tf.__operators__.getitem_14 (Sl (None,)              0           input_1[0][0]                    
__________________________________________________________________________________________________
tf.__operators__.getitem_15 (Sl (None,)              0           input_1[0][0]                    
__________________________________________________________________________________________________
tf.__operators__.getitem_16 (Sl (None,)              0           input_1[0][0]                    
__________________________________________________________________________________________________
tf.__operators__.getitem_17 (Sl (None,)              0           input_1[0][0]                    
__________________________________________________________________________________________________
tf.__operators__.getitem_18 (Sl (None,)              0           input_1[0][0]                    
__________________________________________________________________________________________________
tf.__operators__.getitem_19 (Sl (None,)              0           input_1[0][0]                    
__________________________________________________________________________________________________
tf.__operators__.getitem_20 (Sl (None,)              0           input_1[0][0]                    
__________________________________________________________________________________________________
tf.__operators__.getitem_21 (Sl (None,)              0           input_1[0][0]                    
__________________________________________________________________________________________________
tf.__operators__.getitem_22 (Sl (None,)              0           input_1[0][0]                    
__________________________________________________________________________________________________
tf.__operators__.getitem_23 (Sl (None,)              0           input_1[0][0]                    
__________________________________________________________________________________________________
tf.__operators__.getitem_24 (Sl (None,)              0           input_1[0][0]                    
__________________________________________________________________________________________________
tf.__operators__.getitem_25 (Sl (None,)              0           input_1[0][0]                    
__________________________________________________________________________________________________
tf.__operators__.getitem_26 (Sl (None,)              0           input_1[0][0]                    
__________________________________________________________________________________________________
tf.__operators__.getitem_27 (Sl (None,)              0           input_1[0][0]                    
__________________________________________________________________________________________________
tf.__operators__.getitem_28 (Sl (None,)              0           input_1[0][0]                    
__________________________________________________________________________________________________
tf.__operators__.getitem_29 (Sl (None,)              0           input_1[0][0]                    
__________________________________________________________________________________________________
tf.__operators__.getitem_30 (Sl (None,)              0           input_1[0][0]                    
__________________________________________________________________________________________________
tf.__operators__.getitem_31 (Sl (None,)              0           input_1[0][0]                    
__________________________________________________________________________________________________
tf.__operators__.getitem_32 (Sl (None,)              0           input_1[0][0]                    
__________________________________________________________________________________________________
tf.__operators__.getitem_33 (Sl (None,)              0           input_1[0][0]                    
__________________________________________________________________________________________________
tf.__operators__.getitem_34 (Sl (None,)              0           input_1[0][0]                    
__________________________________________________________________________________________________
tf.__operators__.getitem_35 (Sl (None,)              0           input_1[0][0]                    
__________________________________________________________________________________________________
tf.__operators__.getitem_36 (Sl (None,)              0           input_1[0][0]                    
__________________________________________________________________________________________________
tf.__operators__.getitem_37 (Sl (None,)              0           input_1[0][0]                    
__________________________________________________________________________________________________
tf.__operators__.getitem_38 (Sl (None,)              0           input_1[0][0]                    
__________________________________________________________________________________________________
embedding (Embedding)           (None, 8)            1400        tf.__operators__.getitem[0][0]   
__________________________________________________________________________________________________
embedding_1 (Embedding)         (None, 8)            3088        tf.__operators__.getitem_1[0][0] 
__________________________________________________________________________________________________
embedding_2 (Embedding)         (None, 8)            44168       tf.__operators__.getitem_2[0][0] 
__________________________________________________________________________________________________
embedding_3 (Embedding)         (None, 8)            32264       tf.__operators__.getitem_3[0][0] 
__________________________________________________________________________________________________
embedding_4 (Embedding)         (None, 8)            448         tf.__operators__.getitem_4[0][0] 
__________________________________________________________________________________________________
embedding_5 (Embedding)         (None, 8)            64          tf.__operators__.getitem_5[0][0] 
__________________________________________________________________________________________________
embedding_6 (Embedding)         (None, 8)            25472       tf.__operators__.getitem_6[0][0] 
__________________________________________________________________________________________________
embedding_7 (Embedding)         (None, 8)            744         tf.__operators__.getitem_7[0][0] 
__________________________________________________________________________________________________
embedding_8 (Embedding)         (None, 8)            24          tf.__operators__.getitem_8[0][0] 
__________________________________________________________________________________________________
embedding_9 (Embedding)         (None, 8)            23888       tf.__operators__.getitem_9[0][0] 
__________________________________________________________________________________________________
embedding_10 (Embedding)        (None, 8)            16672       tf.__operators__.getitem_10[0][0]
__________________________________________________________________________________________________
embedding_11 (Embedding)        (None, 8)            42272       tf.__operators__.getitem_11[0][0]
__________________________________________________________________________________________________
embedding_12 (Embedding)        (None, 8)            13800       tf.__operators__.getitem_12[0][0]
__________________________________________________________________________________________________
embedding_13 (Embedding)        (None, 8)            192         tf.__operators__.getitem_13[0][0]
__________________________________________________________________________________________________
embedding_14 (Embedding)        (None, 8)            16280       tf.__operators__.getitem_14[0][0]
__________________________________________________________________________________________________
embedding_15 (Embedding)        (None, 8)            37792       tf.__operators__.getitem_15[0][0]
__________________________________________________________________________________________________
embedding_16 (Embedding)        (None, 8)            72          tf.__operators__.getitem_16[0][0]
__________________________________________________________________________________________________
embedding_17 (Embedding)        (None, 8)            9192        tf.__operators__.getitem_17[0][0]
__________________________________________________________________________________________________
embedding_18 (Embedding)        (None, 8)            4376        tf.__operators__.getitem_18[0][0]
__________________________________________________________________________________________________
embedding_19 (Embedding)        (None, 8)            32          tf.__operators__.getitem_19[0][0]
__________________________________________________________________________________________________
embedding_20 (Embedding)        (None, 8)            40296       tf.__operators__.getitem_20[0][0]
__________________________________________________________________________________________________
embedding_21 (Embedding)        (None, 8)            64          tf.__operators__.getitem_21[0][0]
__________________________________________________________________________________________________
embedding_22 (Embedding)        (None, 8)            96          tf.__operators__.getitem_22[0][0]
__________________________________________________________________________________________________
embedding_23 (Embedding)        (None, 8)            20200       tf.__operators__.getitem_23[0][0]
__________________________________________________________________________________________________
embedding_24 (Embedding)        (None, 8)            320         tf.__operators__.getitem_24[0][0]
__________________________________________________________________________________________________
embedding_25 (Embedding)        (None, 8)            15512       tf.__operators__.getitem_25[0][0]
__________________________________________________________________________________________________
embedding_26 (Embedding)        (None, 8)            800         tf.__operators__.getitem_26[0][0]
__________________________________________________________________________________________________
embedding_27 (Embedding)        (None, 8)            800         tf.__operators__.getitem_27[0][0]
__________________________________________________________________________________________________
embedding_28 (Embedding)        (None, 8)            800         tf.__operators__.getitem_28[0][0]
__________________________________________________________________________________________________
embedding_29 (Embedding)        (None, 8)            800         tf.__operators__.getitem_29[0][0]
__________________________________________________________________________________________________
embedding_30 (Embedding)        (None, 8)            800         tf.__operators__.getitem_30[0][0]
__________________________________________________________________________________________________
embedding_31 (Embedding)        (None, 8)            800         tf.__operators__.getitem_31[0][0]
__________________________________________________________________________________________________
embedding_32 (Embedding)        (None, 8)            800         tf.__operators__.getitem_32[0][0]
__________________________________________________________________________________________________
embedding_33 (Embedding)        (None, 8)            800         tf.__operators__.getitem_33[0][0]
__________________________________________________________________________________________________
embedding_34 (Embedding)        (None, 8)            800         tf.__operators__.getitem_34[0][0]
__________________________________________________________________________________________________
embedding_35 (Embedding)        (None, 8)            800         tf.__operators__.getitem_35[0][0]
__________________________________________________________________________________________________
embedding_36 (Embedding)        (None, 8)            800         tf.__operators__.getitem_36[0][0]
__________________________________________________________________________________________________
embedding_37 (Embedding)        (None, 8)            800         tf.__operators__.getitem_37[0][0]
__________________________________________________________________________________________________
embedding_38 (Embedding)        (None, 8)            800         tf.__operators__.getitem_38[0][0]
__________________________________________________________________________________________________
tf.concat (TFOpLambda)          (None, 312)          0           embedding[0][0]                  
                                                                 embedding_1[0][0]                
                                                                 embedding_2[0][0]                
                                                                 embedding_3[0][0]                
                                                                 embedding_4[0][0]                
                                                                 embedding_5[0][0]                
                                                                 embedding_6[0][0]                
                                                                 embedding_7[0][0]                
                                                                 embedding_8[0][0]                
                                                                 embedding_9[0][0]                
                                                                 embedding_10[0][0]               
                                                                 embedding_11[0][0]               
                                                                 embedding_12[0][0]               
                                                                 embedding_13[0][0]               
                                                                 embedding_14[0][0]               
                                                                 embedding_15[0][0]               
                                                                 embedding_16[0][0]               
                                                                 embedding_17[0][0]               
                                                                 embedding_18[0][0]               
                                                                 embedding_19[0][0]               
                                                                 embedding_20[0][0]               
                                                                 embedding_21[0][0]               
                                                                 embedding_22[0][0]               
                                                                 embedding_23[0][0]               
                                                                 embedding_24[0][0]               
                                                                 embedding_25[0][0]               
                                                                 embedding_26[0][0]               
                                                                 embedding_27[0][0]               
                                                                 embedding_28[0][0]               
                                                                 embedding_29[0][0]               
                                                                 embedding_30[0][0]               
                                                                 embedding_31[0][0]               
                                                                 embedding_32[0][0]               
                                                                 embedding_33[0][0]               
                                                                 embedding_34[0][0]               
                                                                 embedding_35[0][0]               
                                                                 embedding_36[0][0]               
                                                                 embedding_37[0][0]               
                                                                 embedding_38[0][0]               
__________________________________________________________________________________________________
residual__units (Residual_Units (None, 312)          160312      tf.concat[0][0]                  
__________________________________________________________________________________________________
residual__units_1 (Residual_Uni (None, 312)          80312       residual__units[0][0]            
__________________________________________________________________________________________________
residual__units_2 (Residual_Uni (None, 312)          40312       residual__units_1[0][0]          
__________________________________________________________________________________________________
dropout (Dropout)               (None, 312)          0           residual__units_2[0][0]          
__________________________________________________________________________________________________
dense_6 (Dense)                 (None, 1)            313         dropout[0][0]                    
__________________________________________________________________________________________________
tf.math.sigmoid (TFOpLambda)    (None, 1)            0           dense_6[0][0]                    
==================================================================================================
Total params: 640,377
Trainable params: 640,377
Non-trainable params: 0
__________________________________________________________________________________________________
Epoch 1/10
2/2 [==============================] - 9s 2s/step - loss: 0.6509 - auc: 0.4894 - val_loss: 0.5833 - val_auc: 0.4670
Epoch 2/10
2/2 [==============================] - 0s 232ms/step - loss: 0.5611 - auc: 0.5267 - val_loss: 0.5340 - val_auc: 0.4906
Epoch 3/10
2/2 [==============================] - 0s 228ms/step - loss: 0.5265 - auc: 0.5544 - val_loss: 0.5617 - val_auc: 0.5473
Epoch 4/10
2/2 [==============================] - 0s 231ms/step - loss: 0.5339 - auc: 0.6971 - val_loss: 0.5361 - val_auc: 0.5915
1/1 [==============================] - 0s 113ms/step - loss: 0.5191 - auc: 0.5249
test AUC: 0.524854

In [ ]: