In [1]:

%env MKL_NUM_THREADS=12
%env OMP_NUM_THREADS=12

env: MKL_NUM_THREADS=12
env: OMP_NUM_THREADS=12

In [2]:

from collections import defaultdict

import numpy as np
import scipy as sp
import pandas as pd
from ipypb import track

from polara.evaluation import evaluation_engine as ee
from polara.evaluation.pipelines import random_grid, find_optimal_config
from polara.recommender.external.turi.turiwrapper import TuriFactorizationRecommender

from data_preprocessing import (get_amazon_data,
                                get_similarity_data,
                                prepare_data_model)
from utils import (report_results, save_results,
                   apply_config, print_data_stats,
                   save_training_time, save_cv_training_time)

%matplotlib inline

In [3]:

seed = 42

In [4]:

experiment_name = 'sgd'

Experiment setup¶

In [5]:

data_labels = ['AMZe', 'AMZvg']

In [6]:

# according to https://apple.github.io/turicreate/docs/api/generated/turicreate.recommender.ranking_factorization_recommender.RankingFactorizationRecommender.html
init_config = dict(with_data_feedback = False, # implicit case
                   ranking_optimization = True,
                   solver = 'adagrad',
                   sgd_step_size = 0, # let Turi autotune it
                   seed = seed,
                   max_iterations = 25,
                   other_tc_params = {}
                   )
mf_init_config = dict.fromkeys(data_labels, {'SGD': init_config}) # standard scenario

In [7]:

params = {
          'regularization': [1e-10, 3e-10, 1e-9, 3e-9, 1e-8, 3e-8, 1e-7, 3e-7, 1e-6, 3e-6],
          'linear_regularization': [1e-10, 3e-10, 1e-9, 3e-9, 1e-8, 3e-8, 1e-7, 3e-7, 1e-6, 3e-6],
          'rank': [40] # for initial tuning (exploration)
         }

if init_config['solver'] == 'adagrad':
    params.update({
                   'adagrad_momentum_weighting': [0.9, 0.95, 0.99]
                  })      

In [8]:

ranks_grid = [1, 5, 15, 30, 50, 75, 100, 150, 200, 250, 300, 400,
              500, 750, 1000, 1250, 1500, 1750, 2000, 2500, 3000]

In [9]:

mf_ranks = {'AMZe': ranks_grid,
            'AMZvg': ranks_grid
           }

In [10]:

topk_values = [1, 3, 10, 20, 30]

In [11]:

target_metric = 'mrr'

In [12]:

data_dict = dict.fromkeys(data_labels)
meta_dict = dict.fromkeys(data_labels)
similarities = dict.fromkeys(data_labels)
sim_indices = dict.fromkeys(data_labels)
feature_idx = dict.fromkeys(data_labels)

In [13]:

all_data = [data_dict, similarities, sim_indices, meta_dict]

Amazon Electronics¶

In [14]:

lbl = 'AMZe'

In [15]:

data_dict[lbl], meta_dict[lbl] = get_amazon_data('/data/recsys/amazon/ratings_Electronics.csv',
                                                 meta_path='/data/recsys/amazon/meta/meta_Electronics.json.gz',
                                                 implicit=True,
                                                 pcore=5,
                                                 filter_no_meta=True,
                                                 flat_categories=True) # take only bottom level categories

In [16]:

similarities[lbl], sim_indices[lbl], feature_idx[lbl] = get_similarity_data(meta_dict[lbl])

In [17]:

(meta_dict[lbl].applymap(len).sum(axis=1)==0).mean()

Out[17]:

0.0

Amazon Video Games¶

In [18]:

lbl = 'AMZvg'

In [19]:

data_dict[lbl], meta_dict[lbl] = get_amazon_data('/data/recsys/amazon/ratings_Video_Games.csv',
                                                 meta_path='/data/recsys/amazon/meta/meta_Video_Games.json.gz',
                                                 implicit=True,
                                                 pcore=5,
                                                 filter_data={'categories': ['Games']}, # filter uniformative category
                                                 filter_no_meta=True,
                                                 flat_categories=True) # take only bottom level categories

In [20]:

similarities[lbl], sim_indices[lbl], feature_idx[lbl] = get_similarity_data(meta_dict[lbl])

In [21]:

(meta_dict[lbl].applymap(len).sum(axis=1)==0).mean()

Out[21]:

0.0

Data stats¶

In [22]:

print_data_stats(data_labels, all_data)

AMZe
{'userid': 124895, 'asin': 44843}
density 0.019153791836615672
similarity matrix density 1.1054998336712965
AMZvg
{'userid': 14251, 'asin': 6858}
density 0.13281340440589384
similarity matrix density 9.081814734274188

Standard experiment¶

In [23]:

def prepare_recommender_models(data_label, data_models, config):
    data_model = data_models[data_label]
    mf = TuriFactorizationRecommender(data_model, item_side_info=None)
    mf.method = 'SGD'
    models = [mf]
    apply_config(models, config, data_label)
    return models

def fine_tune_mf(model, params, label, ntrials=60, record_time_as=None):
    param_grid, param_names = random_grid(params, n=ntrials)
    best_mf_config, mf_scores = find_optimal_config(model, param_grid, param_names,
                                                    target_metric,
                                                    return_scores=True,
                                                    force_build=True,
                                                    iterator=lambda x: track(x, label=label))
    model_config = {model.method: dict(zip(param_names, best_mf_config))}
    model_scores = {model.method: mf_scores}
    try:
        if record_time_as:
            save_training_time(f'{experiment_name}_{record_time_as}', model, mf_scores.index, label)
    finally:
        return model_config, model_scores

tuning¶

In [24]:

config = {}
scores = {}
times = {}
data_models = {}

In [25]:

mf_init_config['AMZe']['SGD']

Out[25]:

{'with_data_feedback': False,
 'ranking_optimization': True,
 'solver': 'adagrad',
 'sgd_step_size': 0,
 'seed': 42,
 'max_iterations': 25,
 'other_tc_params': {}}

In [26]:

for label in track(data_labels):
    data_models[label] = prepare_data_model(label, *all_data, seed)
    model, = prepare_recommender_models(label, data_models, mf_init_config)
    config[label], scores[label] = fine_tune_mf(model, params, label, ntrials=60, record_time_as='param')
del model

100% 2/2 [44:09<03:39, 1324.27s/it]

AMZe 100% 60/60 [40:15<00:36, 40.25s/it]

AMZvg 100% 60/60 [03:38<00:03, 3.63s/it]

In [27]:

# no meta
report_results('tuning', scores);

/opt/conda/envs/py36/lib/python3.6/site-packages/pandas/plotting/_core.py:1001: UserWarning: Attempted to set non-positive left xlim on a log-scaled axis.
Invalid limit will be ignored.
  ax.set_xlim(left, right)

In [28]:

config

Out[28]:

{'AMZe': {'SGD': {'regularization': 1e-06,
   'linear_regularization': 1e-08,
   'rank': 40,
   'adagrad_momentum_weighting': 0.9}},
 'AMZvg': {'SGD': {'regularization': 3e-06,
   'linear_regularization': 1e-07,
   'rank': 40,
   'adagrad_momentum_weighting': 0.99}}}

saving data¶

In [29]:

save_results(f'{experiment_name}_param', config=config, tuning=scores)

rank estimation¶

In [30]:

rank_config = {}
rank_scores = {}

In [31]:

for label in track(data_labels):
    model, = prepare_recommender_models(label, data_models,
                                        [mf_init_config, config]) # initiate with optimal config
    rank_config[label], rank_scores[label] = fine_tune_mf(model, {'rank': mf_ranks[label]},
                                                          label, ntrials=0, record_time_as='rank')
del model

100% 2/2 [01:30:50<07:52, 2725.15s/it]

AMZe 100% 21/21 [01:22:58<01:49, 237.07s/it]

AMZvg 100% 21/21 [07:52<00:10, 22.46s/it]

In [32]:

# no meta
report_results('rank', {lbl: v.sort_index() for lbl, scr in rank_scores.items() for k, v in scr.items()});

In [33]:

rank_config

Out[33]:

{'AMZe': {'SGD': {'rank': 1500}}, 'AMZvg': {'SGD': {'rank': 1500}}}

saving data¶

In [34]:

save_results(f'{experiment_name}_rank', config=rank_config, tuning=rank_scores)

cross-validation¶

In [35]:

result = {}
for label in track(data_labels):
    models = prepare_recommender_models(label, data_models, [mf_init_config, config, rank_config])
    result[label] = ee.run_cv_experiment(models,
                                         fold_experiment=ee.topk_test,
                                         topk_list=topk_values,
                                         ignore_feedback=True,
                                         iterator=lambda x: track(x, label=label))
    save_cv_training_time(experiment_name, models, label)

100% 2/2 [39:47<03:38, 1193.26s/it]

AMZe 100% 5/5 [36:08<07:19, 433.66s/it]

AMZvg 100% 5/5 [03:38<00:44, 43.61s/it]

In [36]:

# no meta
report_results('topn', result, target_metric);

In [37]:

pd.concat({lbl: res.mean(level='top-n').loc[10, :'ranking'] for lbl, res in result.items()}, axis=1)

Out[37]:

		AMZe	AMZvg
type	metric
relevance	hr	0.050090	0.143850
ranking	mrr	0.021451	0.062757

saving data¶

In [38]:

save_results(experiment_name, cv=result)

In [ ]: