import functools
from functools import partial
import itertools
import logging
import math
import os
import pickle
import sys
from pathlib import Path
import matplotlib as mpl
import matplotlib.pyplot as plt
import numpy as np
import scipy as sp
import pingouin as pg
import seaborn as sns
import yaml
#import pingouin as pg
%load_ext autoreload
%autoreload 2
%matplotlib inline
sns.set_context("poster")
sns.set(rc={"figure.figsize": (16, 12.0)})
sns.set_style("whitegrid")
import numpy as np
import pandas as pd
from scipy.stats import kendalltau, rankdata, spearmanr, pearsonr, ttest_rel
import torch.nn.functional as F
pd.set_option("display.max_rows", 120)
pd.set_option("display.max_columns", 120)
logging.basicConfig(level=logging.INFO, stream=sys.stdout)
from lda4rec.datasets import Interactions, DataLoader, random_train_test_split, MOVIELENS_1M,MetaData, get_dataset, items_per_user_train_test_split
import lda4rec.evaluations as lda_eval
from lda4rec.estimators import MFEst, PopEst, LDA4RecEst, SNMFEst
from lda4rec.utils import process_ids, cmp_ranks, Config, split_along_dim_apply, plot_cat
from lda4rec import lda
import pyro
import pyro.distributions as dist
import pyro.optim as optim
import torch
from pyro.distributions import constraints
from pyro.infer import SVI, Predictive, Trace_ELBO, TraceEnum_ELBO, config_enumerate
import neptune.new as neptune
# init dummy neptune to avoid problems with logging
neptune.init(mode='offline')
WARNING:neptune.internal.hardware.gpu.gpu_monitor:Info (NVML): NVML Shared Library Not Found. GPU usage metrics may not be reported. For more information, see https://docs-legacy.neptune.ai/logging-and-managing-experiment-results/logging-experiment-data.html#hardware-consumption offline/0ad00974-db85-4182-a956-86387fcf2d43 Remember to stop your run once you’ve finished logging your metadata (https://docs.neptune.ai/api-reference/run#stop). It will be stopped automatically only when the notebook kernel/interactive console is terminated.
<neptune.new.run.Run at 0x7f92ec3370d0>
from icecream import ic, install
install()
# configure icecream
def ic_str(obj):
if hasattr(obj, "shape"):
return f"{obj} " #
else:
return str(obj)
ic.configureOutput(argToStringFunction=ic_str)
# 19 is goodbooks dim 128, 33 is movielens dim 64
cfg = Config(Path('../configs/exp_19.yaml'))
exp_cfg = cfg["experiment"]
exp_cfg
{'dataset': 'goodbooks', 'dataset_seed': 1729, 'est_params': {'batch_size': 256, 'embedding_dim': 128, 'learning_rate': 0.001, 'n_iter': 200}, 'estimator': 'MFEst', 'interaction_pivot': 0, 'max_user_interactions': 200, 'min_user_interactions': 20, 'model_seed': 1729, 'train_test_split': 'items_per_user_train_test_split'}
train, test, data_rng = lda_eval.get_train_test_data(cfg)
est = lda_eval.load_model(Path('../models'), cfg, train)
v, t, h, b = est.get_lda_params()
user_ids, log_probs = lda_eval.cohort_user_interaction_log_probs(train, v, h, rng=data_rng)
pg.ttest(log_probs[:, 1], log_probs[:, 0], paired=True, alternative='greater')
T | dof | alternative | p-val | CI95% | cohen-d | BF10 | power | |
---|---|---|---|---|---|---|---|---|
T-test | 29.975105 | 53422 | greater | 4.381105e-196 | [177798066972580.78, inf] | 0.183153 | 2.915e+191 | 1.0 |
corr = lda_eval.popularity_ranking_corr(train, b)
corr
KendalltauResult(correlation=0.22118408993158703, pvalue=8.119494492245568e-241)
emp_pops = lda_eval.get_empirical_pops(train)
corr = lda_eval.conformity_interaction_pop_ranking_corr(emp_pops, (1/t).numpy(), train)
corr
KendalltauResult(correlation=-0.03048729326166538, pvalue=4.121011716330912e-26)
corr = lda_eval.conformity_interaction_pop_ranking_corr(b, (1/t).numpy(), train)
corr
KendalltauResult(correlation=-0.05989634446473608, pvalue=8.888821688879722e-96)
user_ids, good_twins, bad_twins, rnd_twins = lda_eval.find_good_bad_rnd_twins(v, n_users=100)
good_jacs = lda_eval.get_twin_jacs(user_ids, good_twins, train)
bad_jacs = lda_eval.get_twin_jacs(user_ids, bad_twins, train)
rnd_jacs = lda_eval.get_twin_jacs(user_ids, rnd_twins, train)
pg.ttest(good_jacs, bad_jacs, paired=True, alternative='greater')
T | dof | alternative | p-val | CI95% | cohen-d | BF10 | power | |
---|---|---|---|---|---|---|---|---|
T-test | 20.157734 | 99 | greater | 4.028075e-37 | [0.13, inf] | 2.841054 | 6.249e+33 | 1.0 |
pg.ttest(good_jacs, rnd_jacs, paired=True, alternative='greater')
T | dof | alternative | p-val | CI95% | cohen-d | BF10 | power | |
---|---|---|---|---|---|---|---|---|
T-test | 14.918771 | 99 | greater | 2.258553e-27 | [0.1, inf] | 2.070825 | 1.603e+24 | 1.0 |