In [1]:

%env MKL_NUM_THREADS=12
%env OMP_NUM_THREADS=12

env: MKL_NUM_THREADS=12
env: OMP_NUM_THREADS=12

In [2]:

from collections import defaultdict

import numpy as np
import scipy as sp
import pandas as pd
from ipypb import track

from polara.evaluation import evaluation_engine as ee
from polara.evaluation.pipelines import random_grid, find_optimal_config
from polara.recommender.external.turi.turiwrapper import TuriFactorizationRecommender

from data_preprocessing import (get_movielens_data,
                                get_bookcrossing_data,
                                get_similarity_data,
                                prepare_data_model)
from utils import (report_results, save_results,
                   apply_config, print_data_stats,
                   save_training_time, save_cv_training_time)

%matplotlib inline

In [3]:

seed = 42

In [4]:

experiment_name = 'sgd'

Experiment setup¶

In [5]:

data_labels = ['ML1M', 'ML10M', 'BX']

In [6]:

# according to https://apple.github.io/turicreate/docs/api/generated/turicreate.recommender.ranking_factorization_recommender.RankingFactorizationRecommender.html
init_config = dict(with_data_feedback = False, # implicit case
                   ranking_optimization = True,
                   solver = 'adagrad',
                   sgd_step_size = 0, # let Turi autotune it
                   seed = seed,
                   max_iterations = 25,
                   other_tc_params = {}
                   )
mf_init_config = dict.fromkeys(data_labels, {'SGD': init_config}) # standard scenario

In [7]:

params = {
          'regularization': [1e-10, 3e-10, 1e-9, 3e-9, 1e-8, 3e-8, 1e-7, 3e-7, 1e-6, 3e-6],
          'linear_regularization': [1e-10, 3e-10, 1e-9, 3e-9, 1e-8, 3e-8, 1e-7, 3e-7, 1e-6, 3e-6],
          'rank': [40] # for initial tuning (exploration)
         }

if init_config['solver'] == 'adagrad':
    params.update({
                   'adagrad_momentum_weighting': [0.9, 0.95, 0.99]
                  })      

In [8]:

ranks_grid = [1, 5, 10, 15, 20, 30, 50, 60, 75, 100, 125, 150, 200, 250, 300, 350, 400,
              500, 750, 1000, 1250, 1500, 1750, 2000, 2500, 3000]

In [9]:

mf_ranks = {'ML1M': [r for r in ranks_grid if r <= 1000],
             'ML10M': [r for r in ranks_grid if r <= 1000],
             'BX': [r for r in ranks_grid if r <= 2000]}

In [10]:

topk_values = [1, 3, 10, 20, 30]

In [11]:

target_metric = 'mrr'

In [12]:

data_dict = dict.fromkeys(data_labels)
meta_dict = dict.fromkeys(data_labels)
similarities = dict.fromkeys(data_labels)
sim_indices = dict.fromkeys(data_labels)

In [13]:

all_data = [data_dict, similarities, sim_indices, meta_dict]

Movielens1M¶

In [14]:

lbl = 'ML1M'

In [15]:

data_dict[lbl], meta_dict[lbl] = get_movielens_data('/data/recsys/movielens/ml-1m.zip',
                                                    meta_path='data/meta_info_ml1m.csv',
                                                    implicit=True,
                                                    filter_no_meta=True)

In [16]:

# not used actually, simply to onform with general pipeline
itemid = meta_dict[lbl].index.name
sim_indices[lbl] = {itemid: meta_dict[lbl].index}
similarities[lbl] = {itemid: sp.sparse.eye(len(meta_dict[lbl].index))}

Movielens10M¶

In [17]:

lbl = 'ML10M'

In [18]:

data_dict[lbl], meta_dict[lbl] = get_movielens_data('/data/recsys/movielens/ml-10m.zip',
                                                    meta_path='data/meta_info_ml10m.csv',
                                                    implicit=True,
                                                    filter_no_meta=True)

In [19]:

# not used actually, simply to onform with general pipeline
itemid = meta_dict[lbl].index.name
sim_indices[lbl] = {itemid: meta_dict[lbl].index}
similarities[lbl] = {itemid: sp.sparse.eye(len(meta_dict[lbl].index))}

In [20]:

(meta_dict[lbl].applymap(len).sum(axis=1)==0).mean()

Out[20]:

0.0

BookCrossing¶

In [21]:

lbl = 'BX'

In [22]:

data_dict[lbl], meta_dict[lbl] = get_bookcrossing_data('/data/recsys/bookcrossing/BX-CSV-Dump.zip',
                                                       get_books_meta=True,
                                                       implicit=True,
                                                       pcore=5,
                                                       filter_no_meta=True)

/opt/conda/envs/py36/lib/python3.6/site-packages/pandas/io/parsers.py:1995: DeprecationWarning: invalid escape sequence '\8'
  data = self._reader.read(nrows)
/opt/conda/envs/py36/lib/python3.6/site-packages/pandas/io/parsers.py:1995: DeprecationWarning: invalid escape sequence '\9'
  data = self._reader.read(nrows)

In [23]:

# not used actually, simply to onform with general pipeline
itemid = meta_dict[lbl].index.name
sim_indices[lbl] = {itemid: meta_dict[lbl].index}
similarities[lbl] = {itemid: sp.sparse.eye(len(meta_dict[lbl].index))}

In [24]:

(meta_dict[lbl].applymap(len).sum(axis=1)==0).mean()

Out[24]:

0.0

Data stats¶

In [25]:

print_data_stats(data_labels, all_data)

ML1M
{'userid': 6038, 'movieid': 3522}
density 2.699052132255699
similarity matrix density 0.028392958546280524
ML10M
{'userid': 69797, 'movieid': 10258}
density 0.6991397242349022
similarity matrix density 0.009748488984207448
BX
{'userid': 7160, 'isbn': 16273}
density 0.18925598044812894
similarity matrix density 0.0005841769822585451

Standard experiment¶

In [26]:

def prepare_recommender_models(data_label, data_models, config):
    data_model = data_models[data_label]
    mf = TuriFactorizationRecommender(data_model, item_side_info=None)
    mf.method = 'SGD'
    models = [mf]
    apply_config(models, config, data_label)
    return models

def fine_tune_mf(model, params, label, ntrials=60, record_time_as=None):
    param_grid, param_names = random_grid(params, n=ntrials)
    best_mf_config, mf_scores = find_optimal_config(model, param_grid, param_names,
                                                    target_metric,
                                                    return_scores=True,
                                                    force_build=True,
                                                    iterator=lambda x: track(x, label=label))
    model_config = {model.method: dict(zip(param_names, best_mf_config))}
    model_scores = {model.method: mf_scores}
    try:
        if record_time_as:
            save_training_time(f'{experiment_name}_{record_time_as}', model, mf_scores.index, label)
    finally:
        return model_config, model_scores

tuning¶

In [27]:

config = {}
scores = {}
times = {}
data_models = {}

In [28]:

mf_init_config['ML1M']['SGD']

Out[28]:

{'with_data_feedback': False,
 'ranking_optimization': True,
 'solver': 'adagrad',
 'sgd_step_size': 0,
 'seed': 42,
 'max_iterations': 25,
 'other_tc_params': {}}

In [29]:

for label in track(data_labels):
    data_models[label] = prepare_data_model(label, *all_data, seed)
    model, = prepare_recommender_models(label, data_models, mf_init_config)
    config[label], scores[label] = fine_tune_mf(model, params, label, ntrials=60, record_time_as='param')
del model

100% 3/3 [02:08:14<05:13, 2564.75s/it]

ML1M 100% 60/60 [12:26<00:13, 12.44s/it]

ML10M 100% 60/60 [01:50:25<01:48, 110.42s/it]

BX 100% 60/60 [05:13<00:05, 5.21s/it]

In [30]:

# no meta
report_results('tuning', scores);

/opt/conda/envs/py36/lib/python3.6/site-packages/pandas/plotting/_core.py:1001: UserWarning: Attempted to set non-positive left xlim on a log-scaled axis.
Invalid limit will be ignored.
  ax.set_xlim(left, right)

In [31]:

config

Out[31]:

{'ML1M': {'SGD': {'regularization': 3e-06,
   'linear_regularization': 1e-10,
   'rank': 40,
   'adagrad_momentum_weighting': 0.95}},
 'ML10M': {'SGD': {'regularization': 1e-06,
   'linear_regularization': 1e-07,
   'rank': 40,
   'adagrad_momentum_weighting': 0.99}},
 'BX': {'SGD': {'regularization': 3e-06,
   'linear_regularization': 1e-08,
   'rank': 40,
   'adagrad_momentum_weighting': 0.95}}}

saving data¶

In [32]:

save_results(f'{experiment_name}_param', config=config, tuning=scores)

rank estimation¶

In [33]:

rank_config = {}
rank_scores = {}

In [34]:

for label in track(data_labels):
    model, = prepare_recommender_models(label, data_models,
                                        [mf_init_config, config]) # initiate with optimal config
    rank_config[label], rank_scores[label] = fine_tune_mf(model, {'rank': mf_ranks[label]},
                                                          label, ntrials=0, record_time_as='rank')
del model

100% 3/3 [02:00:38<10:45, 2412.83s/it]

ML1M 100% 20/20 [11:07<00:10, 33.33s/it]

ML10M 100% 20/20 [01:38:47<01:33, 296.36s/it]

BX 100% 24/24 [10:45<00:19, 26.86s/it]

In [35]:

# no meta
report_results('rank', {lbl: v.sort_index() for lbl, scr in rank_scores.items() for k, v in scr.items()});

In [36]:

rank_config

Out[36]:

{'ML1M': {'SGD': {'rank': 1000}},
 'ML10M': {'SGD': {'rank': 750}},
 'BX': {'SGD': {'rank': 1500}}}

saving data¶

In [37]:

save_results(f'{experiment_name}_rank', config=rank_config, tuning=rank_scores)

cross-validation¶

In [38]:

result = {}
for label in track(data_labels):
    models = prepare_recommender_models(label, data_models, [mf_init_config, config, rank_config])
    result[label] = ee.run_cv_experiment(models,
                                         fold_experiment=ee.topk_test,
                                         topk_list=topk_values,
                                         ignore_feedback=True,
                                         iterator=lambda x: track(x, label=label))
    save_cv_training_time(experiment_name, models, label)

100% 3/3 [01:28:08<06:34, 1762.64s/it]

ML1M 100% 5/5 [10:20<02:05, 123.91s/it]

ML10M 100% 5/5 [01:11:15<13:42, 854.94s/it]

BX 100% 5/5 [06:34<01:20, 78.72s/it]

In [39]:

# no meta
report_results('topn', result, target_metric);

In [40]:

pd.concat({lbl: res.mean(level='top-n').loc[10, :'ranking'] for lbl, res in result.items()}, axis=1)

Out[40]:

		BX	ML10M	ML1M
type	metric
relevance	hr	0.050140	0.268534	0.210568
ranking	mrr	0.022984	0.116333	0.088191

saving data¶

In [41]:

save_results(experiment_name, cv=result)

In [ ]: