import numpy as np
import pandas as pd
import seaborn as sns
import matplotlib.pyplot as plt
from sklearn.linear_model import Ridge
from sklearn.svm import SVR
from sklearn.model_selection import KFold, train_test_split, cross_val_predict, GridSearchCV
from sklearn.pipeline import Pipeline
from sklearn.decomposition import PCA
from joblib import Parallel, delayed
from mlxtend.evaluate import permutation_test
sns.set(rc={"figure.figsize":(4, 2)})
sns.set_style("whitegrid")
We load functional network matrices (netmats) from the HCP1200-release, as published on connectomeDB: https://db.humanconnectome.org/ Due to licensoing issues, data is not supplied with the repository, but can be downloaded from the ConnectomeDB. See hcp_data/readme.md for more details.
# HCP data can be obtained from the connectomeDB
# data is not part of this repository
subjectIDs = pd.read_csv('hcp_data/subjectIDs.txt', header=None)
netmats_pearson = pd.read_csv('hcp_data/netmats1_correlationZ.txt',
sep=' ',
header=None)
netmats_pearson['ID'] = subjectIDs[0]
netmats_pearson.set_index('ID', drop=True, inplace=True)
netmats_parcor = pd.read_csv('hcp_data/netmats2_partial-correlation.txt',
sep=' ',
header=None)
netmats_parcor['ID'] = subjectIDs[0]
netmats_parcor.set_index('ID', drop=True, inplace=True)
behavior = pd.read_csv('hcp_data/hcp1200_behavioral_data.csv')
behavior = behavior.set_index('Subject', drop=True)
# convert age to numeric
age = []
for s in behavior['Age']:
if s == '36+':
age.append(36)
else:
split = s.split(sep='-')
age.append(np.mean((float(split[0]), float(split[1]))))
behavior['age'] = age
behavior
Release | Acquisition | Gender | Age | 3T_Full_MR_Compl | T1_Count | T2_Count | 3T_RS-fMRI_Count | 3T_RS-fMRI_PctCompl | 3T_Full_Task_fMRI | ... | Odor_Unadj | Odor_AgeAdj | PainIntens_RawScore | PainInterf_Tscore | Taste_Unadj | Taste_AgeAdj | Mars_Log_Score | Mars_Errs | Mars_Final | age | |
---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
Subject | |||||||||||||||||||||
100004 | S900 | Q06 | M | 22-25 | False | 0 | 0 | 0 | 0.0 | False | ... | 101.12 | 86.45 | 2.0 | 45.9 | 107.17 | 105.31 | 1.80 | 0.0 | 1.80 | 23.5 |
100206 | S900 | Q11 | M | 26-30 | True | 1 | 1 | 4 | 100.0 | True | ... | 108.79 | 97.19 | 1.0 | 49.7 | 72.63 | 72.03 | 1.84 | 0.0 | 1.84 | 28.0 |
100307 | Q1 | Q01 | F | 26-30 | True | 1 | 1 | 4 | 100.0 | True | ... | 101.12 | 86.45 | 0.0 | 38.6 | 71.69 | 71.76 | 1.76 | 0.0 | 1.76 | 28.0 |
100408 | Q3 | Q03 | M | 31-35 | True | 1 | 1 | 4 | 100.0 | True | ... | 108.79 | 98.04 | 2.0 | 52.6 | 114.01 | 113.59 | 1.76 | 2.0 | 1.68 | 33.0 |
100610 | S900 | Q08 | M | 26-30 | True | 2 | 1 | 4 | 100.0 | True | ... | 122.25 | 110.45 | 0.0 | 38.6 | 84.84 | 85.31 | 1.92 | 1.0 | 1.88 | 28.0 |
... | ... | ... | ... | ... | ... | ... | ... | ... | ... | ... | ... | ... | ... | ... | ... | ... | ... | ... | ... | ... | ... |
992774 | Q2 | Q02 | M | 31-35 | True | 2 | 2 | 4 | 100.0 | True | ... | 122.25 | 111.41 | 4.0 | 50.1 | 107.17 | 103.55 | 1.76 | 0.0 | 1.76 | 33.0 |
993675 | S900 | Q09 | F | 26-30 | True | 2 | 2 | 4 | 100.0 | True | ... | 122.25 | 110.45 | 0.0 | 38.6 | 84.07 | 84.25 | 1.80 | 1.0 | 1.76 | 28.0 |
994273 | S500 | Q06 | M | 26-30 | True | 1 | 1 | 4 | 100.0 | True | ... | 122.25 | 111.41 | 7.0 | 63.8 | 110.65 | 109.73 | 1.80 | 1.0 | 1.76 | 28.0 |
995174 | S1200 | Q13 | M | 22-25 | False | 1 | 1 | 2 | 0.0 | True | ... | 88.61 | 64.58 | 3.0 | 50.1 | 117.16 | 117.40 | 1.80 | 0.0 | 1.80 | 23.5 |
996782 | S900 | Q08 | F | 26-30 | True | 2 | 2 | 4 | 100.0 | True | ... | 108.79 | 97.19 | 0.0 | 38.6 | 75.43 | 73.72 | 1.84 | 0.0 | 1.84 | 28.0 |
1206 rows × 582 columns
def create_data(target='CogTotalComp_AgeAdj', feature_data=netmats_parcor):
# it's a good practice to use pandas for merging, messing up subject order can be painful
features = feature_data.columns
df = behavior
df = df.merge(feature_data, left_index=True, right_index=True, how='left')
if target:
df = df.dropna(subset = [target] + features.values.tolist())
y = df[target].values
else:
df = df.dropna(subset = features.values.tolist())
rng = np.random.default_rng(42)
y = rng.normal(0,1, len(df))
X = df[features].values
return X, y
We define a workhorse function which:
def bootstrap_workhorse(X, y, sample_size, model, random_state):
#create discovery and replication samples by random sampling from the whole dataset (without replacement)
X_discovery, X_replication, y_discovery, y_replication = train_test_split(X, y, train_size=sample_size, test_size=sample_size, shuffle=True, random_state=random_state)
# obtain cross-validated predictions in the discovery sample
predicted_discovery_cv = cross_val_predict(estimator=model, X=X_discovery, y=y_discovery, cv=10, n_jobs=1)
# correlation between the cross-validated predictions and observations in the discovery sample
# this is the correct, unbiased estimate!
r_disc_cv = np.corrcoef(predicted_discovery_cv, y_discovery)[0, 1]
# finalize model by training it on the full discovery sample (without cross-validation)
final_model = model.fit(X=X_discovery, y=y_discovery)
# obtain predictions with the final model on the discovery sample, note that this model actually overfits this sample.
# we do this only to demonstrate biased estimates
predicted_discovery_overfit = final_model.predict(X=X_discovery)
# here we obtain the biased effect size (r) estimates for demonstrational purposes
r_disc_overfit = np.corrcoef(predicted_discovery_overfit, y_discovery)[0, 1]
# We use the final model to predict the replication sample
# This is correct (no overfitting here), the final model did not see this data during training
predicted_replication = final_model.predict(X=X_replication)
# we obtain the out-of-sample prediction performance estimates
r_rep = np.corrcoef(predicted_replication, y_replication)[0, 1]
# below we calculate permutation-based p-values for all three effect size estimates (in-sample unbiased, in-sample biased, out-of-sample)
# (one sided tests, testing for positive correlation)
p_disc_cv = permutation_test(predicted_discovery_cv, y_discovery, method='approximate', num_rounds=1000, func=lambda x, y: np.corrcoef(x, y)[1][0],seed=random_state)
p_disc_overfit = permutation_test(predicted_discovery_overfit, y_discovery, method='approximate', num_rounds=1000, func=lambda x, y: np.corrcoef(x, y)[1][0],seed=random_state)
p_rep = permutation_test(predicted_replication, y_replication, method='approximate', num_rounds=1000, func=lambda x, y: np.corrcoef(x, y)[1][0],seed=random_state)
# return results
return r_disc_cv, r_disc_overfit, r_rep, p_disc_cv, p_disc_overfit, p_rep
All set, now we start the analysis.
Here we train a few different models on 100 bootstrap samples.
We aggregate the results of our workhorse function in n_bootstrap
=100 bootstrap cases (run in parallel).
The whole process is repeated for all sample sizes, fetaure_sets and target variables.
%%time
random_state = 42
n_bootstrap = 100
features = {
'netmats_parcor': netmats_parcor,
'netmats_pearson': netmats_pearson
}
models = {
'PCA_SVR': Pipeline([('pca', PCA(n_components=0.5)),
('svr', SVR())])
}
# We aggregate all results here:
df = pd.DataFrame(columns=['connectivity','model','target','n','r_discovery_cv','r_discovery_overfit','r_replication','p_discovery_cv','p_discovery_overfit','p_replication'])
for feature_set in features:
for model in models:
for target_var in ['age', 'CogTotalComp_AgeAdj', 'PMAT24_A_CR', 'Flanker_AgeAdj', 'CardSort_AgeAdj', 'PicSeq_AgeAdj']:
for sample_size in [50, 100, 200, 300, 'max']:
print('*****************************************************************')
print(feature_set, model, target_var, sample_size)
X, y = create_data(target=target_var, feature_data=features[feature_set])
if sample_size=='max':
sample_size = int(len(y)/2)
# create random seeds for each bootstrap iteration for reproducibility
rng = np.random.default_rng(random_state)
random_sates = rng.integers(np.iinfo(np.int32).max, size=n_bootstrap)
# run bootstrap iterations in parallel
r_discovery_cv, r_discovery_overfit, r_replication, p_discovery_cv, p_discovery_overfit, p_replication = zip(
*Parallel(n_jobs=-1)(
delayed(bootstrap_workhorse)(X, y, sample_size, models[model], seed) for seed in random_sates))
tmp_data_frame = pd.DataFrame({
'connectivity' : feature_set,
'model' : model,
'target' : target_var,
'n' : sample_size,
'r_discovery_cv': r_discovery_cv,
'r_discovery_overfit': r_discovery_overfit,
'r_replication': r_replication,
'p_discovery_cv': p_discovery_cv,
'p_discovery_overfit': p_discovery_overfit,
'p_replication': p_replication
})
#sns.scatterplot(x='r_replication', y='r_discovery_cv', data=tmp_data_frame)
#plt.ylabel('in-sample (r)')
#plt.xlabel('out-of-sample (r_pred)')
#plt.show()
print(tmp_data_frame.r_discovery_cv.mean(), tmp_data_frame.r_replication.mean())
for alpha in [0.05, 0.01, 0.005, 0.001]:
print('Replicability at alpha =', alpha, ':',
(tmp_data_frame.loc[tmp_data_frame['p_discovery_cv']<alpha,'p_replication']<alpha).sum() / (tmp_data_frame['p_discovery_cv']<0.05).sum() * 100, '%')
df = pd.concat((df, tmp_data_frame))
df.reset_index(drop=True, inplace=True)
df.to_csv('res_PCA_SVR.csv')
df
***************************************************************** netmats_parcor PCA_SVR age 50 0.01897346554269831 0.18901378266057708 Replicability at alpha = 0.05 : 57.14285714285714 % Replicability at alpha = 0.01 : 14.285714285714285 % Replicability at alpha = 0.005 : 0.0 % Replicability at alpha = 0.001 : 0.0 % ***************************************************************** netmats_parcor PCA_SVR age 100 0.18664851524294415 0.27347377929445293 Replicability at alpha = 0.05 : 89.55223880597015 % Replicability at alpha = 0.01 : 40.298507462686565 % Replicability at alpha = 0.005 : 31.343283582089555 % Replicability at alpha = 0.001 : 11.940298507462686 % ***************************************************************** netmats_parcor PCA_SVR age 200 0.3121257047522771 0.36175079133098437 Replicability at alpha = 0.05 : 100.0 % Replicability at alpha = 0.01 : 98.9795918367347 % Replicability at alpha = 0.005 : 95.91836734693877 % Replicability at alpha = 0.001 : 80.61224489795919 % ***************************************************************** netmats_parcor PCA_SVR age 300 0.36894676842215623 0.39438101148062005 Replicability at alpha = 0.05 : 100.0 % Replicability at alpha = 0.01 : 100.0 % Replicability at alpha = 0.005 : 100.0 % Replicability at alpha = 0.001 : 100.0 % ***************************************************************** netmats_parcor PCA_SVR age max 0.4169914586721147 0.44371738829107926 Replicability at alpha = 0.05 : 100.0 % Replicability at alpha = 0.01 : 100.0 % Replicability at alpha = 0.005 : 100.0 % Replicability at alpha = 0.001 : 100.0 % ***************************************************************** netmats_parcor PCA_SVR CogTotalComp_AgeAdj 50 -0.25822001000043204 0.14032170544200848 Replicability at alpha = 0.05 : nan % Replicability at alpha = 0.01 : nan % Replicability at alpha = 0.005 : nan % Replicability at alpha = 0.001 : nan % ***************************************************************** netmats_parcor PCA_SVR CogTotalComp_AgeAdj 100
<timed exec>:60: RuntimeWarning: invalid value encountered in long_scalars
-0.039553381304751625 0.21262859874092135 Replicability at alpha = 0.05 : 90.9090909090909 % Replicability at alpha = 0.01 : 45.45454545454545 % Replicability at alpha = 0.005 : 18.181818181818183 % Replicability at alpha = 0.001 : 9.090909090909092 % ***************************************************************** netmats_parcor PCA_SVR CogTotalComp_AgeAdj 200 0.14966202410710186 0.2713589974517626 Replicability at alpha = 0.05 : 100.0 % Replicability at alpha = 0.01 : 61.29032258064516 % Replicability at alpha = 0.005 : 53.2258064516129 % Replicability at alpha = 0.001 : 27.419354838709676 % ***************************************************************** netmats_parcor PCA_SVR CogTotalComp_AgeAdj 300 0.23474749556135074 0.3101363215395622 Replicability at alpha = 0.05 : 100.0 % Replicability at alpha = 0.01 : 94.73684210526315 % Replicability at alpha = 0.005 : 94.73684210526315 % Replicability at alpha = 0.001 : 76.84210526315789 % ***************************************************************** netmats_parcor PCA_SVR CogTotalComp_AgeAdj max 0.30682365741954354 0.34611199458096925 Replicability at alpha = 0.05 : 100.0 % Replicability at alpha = 0.01 : 100.0 % Replicability at alpha = 0.005 : 100.0 % Replicability at alpha = 0.001 : 100.0 % ***************************************************************** netmats_parcor PCA_SVR PMAT24_A_CR 50 -0.10672071098435865 0.12106221070795271 Replicability at alpha = 0.05 : 0.0 % Replicability at alpha = 0.01 : 0.0 % Replicability at alpha = 0.005 : 0.0 % Replicability at alpha = 0.001 : 0.0 % ***************************************************************** netmats_parcor PCA_SVR PMAT24_A_CR 100 0.07540309996263198 0.16660957189128314 Replicability at alpha = 0.05 : 58.06451612903226 % Replicability at alpha = 0.01 : 19.35483870967742 % Replicability at alpha = 0.005 : 3.225806451612903 % Replicability at alpha = 0.001 : 0.0 % ***************************************************************** netmats_parcor PCA_SVR PMAT24_A_CR 200 0.1818194483607081 0.24330054856845362 Replicability at alpha = 0.05 : 94.9367088607595 % Replicability at alpha = 0.01 : 62.0253164556962 % Replicability at alpha = 0.005 : 45.56962025316456 % Replicability at alpha = 0.001 : 20.253164556962027 % ***************************************************************** netmats_parcor PCA_SVR PMAT24_A_CR 300 0.23928719811339255 0.2836756329717983 Replicability at alpha = 0.05 : 100.0 % Replicability at alpha = 0.01 : 96.93877551020408 % Replicability at alpha = 0.005 : 90.81632653061224 % Replicability at alpha = 0.001 : 75.51020408163265 % ***************************************************************** netmats_parcor PCA_SVR PMAT24_A_CR max 0.30016731017540577 0.3252736273978649 Replicability at alpha = 0.05 : 100.0 % Replicability at alpha = 0.01 : 100.0 % Replicability at alpha = 0.005 : 100.0 % Replicability at alpha = 0.001 : 100.0 % ***************************************************************** netmats_parcor PCA_SVR Flanker_AgeAdj 50 -0.253682468352549 0.028663123664680764 Replicability at alpha = 0.05 : nan % Replicability at alpha = 0.01 : nan % Replicability at alpha = 0.005 : nan % Replicability at alpha = 0.001 : nan % ***************************************************************** netmats_parcor PCA_SVR Flanker_AgeAdj 100 -0.12086835280130456 0.04446606264674254 Replicability at alpha = 0.05 : 0.0 % Replicability at alpha = 0.01 : 0.0 % Replicability at alpha = 0.005 : 0.0 % Replicability at alpha = 0.001 : 0.0 % ***************************************************************** netmats_parcor PCA_SVR Flanker_AgeAdj 200 -0.04070672898990409 0.05833533752802022 Replicability at alpha = 0.05 : 33.33333333333333 % Replicability at alpha = 0.01 : 0.0 % Replicability at alpha = 0.005 : 0.0 % Replicability at alpha = 0.001 : 0.0 % ***************************************************************** netmats_parcor PCA_SVR Flanker_AgeAdj 300 0.010939993075878267 0.07671340073350896 Replicability at alpha = 0.05 : 26.666666666666668 % Replicability at alpha = 0.01 : 0.0 % Replicability at alpha = 0.005 : 0.0 % Replicability at alpha = 0.001 : 0.0 % ***************************************************************** netmats_parcor PCA_SVR Flanker_AgeAdj max 0.07368415050153652 0.12250618953881152 Replicability at alpha = 0.05 : 91.83673469387756 % Replicability at alpha = 0.01 : 26.53061224489796 % Replicability at alpha = 0.005 : 12.244897959183673 % Replicability at alpha = 0.001 : 0.0 % ***************************************************************** netmats_parcor PCA_SVR CardSort_AgeAdj 50 -0.2347808005536518 0.06074334420456127 Replicability at alpha = 0.05 : nan % Replicability at alpha = 0.01 : nan % Replicability at alpha = 0.005 : nan % Replicability at alpha = 0.001 : nan % ***************************************************************** netmats_parcor PCA_SVR CardSort_AgeAdj 100 -0.07762735321443918 0.08296991306991304 Replicability at alpha = 0.05 : 33.33333333333333 % Replicability at alpha = 0.01 : 0.0 % Replicability at alpha = 0.005 : 0.0 % Replicability at alpha = 0.001 : 0.0 % ***************************************************************** netmats_parcor PCA_SVR CardSort_AgeAdj 200 0.021921082391207124 0.10256141335871115 Replicability at alpha = 0.05 : 35.294117647058826 % Replicability at alpha = 0.01 : 5.88235294117647 % Replicability at alpha = 0.005 : 5.88235294117647 % Replicability at alpha = 0.001 : 0.0 % ***************************************************************** netmats_parcor PCA_SVR CardSort_AgeAdj 300 0.06747049534058164 0.12397123226254751 Replicability at alpha = 0.05 : 55.26315789473685 % Replicability at alpha = 0.01 : 15.789473684210526 % Replicability at alpha = 0.005 : 5.263157894736842 % Replicability at alpha = 0.001 : 5.263157894736842 % ***************************************************************** netmats_parcor PCA_SVR CardSort_AgeAdj max 0.10973316806153383 0.1495503211206215 Replicability at alpha = 0.05 : 96.1038961038961 % Replicability at alpha = 0.01 : 61.038961038961034 % Replicability at alpha = 0.005 : 37.66233766233766 % Replicability at alpha = 0.001 : 7.792207792207792 % ***************************************************************** netmats_parcor PCA_SVR PicSeq_AgeAdj 50 -0.2691222555349478 0.0751641945794097 Replicability at alpha = 0.05 : 0.0 % Replicability at alpha = 0.01 : 0.0 % Replicability at alpha = 0.005 : 0.0 % Replicability at alpha = 0.001 : 0.0 % ***************************************************************** netmats_parcor PCA_SVR PicSeq_AgeAdj 100 -0.12847580806431988 0.09875903711275212 Replicability at alpha = 0.05 : 0.0 % Replicability at alpha = 0.01 : 0.0 % Replicability at alpha = 0.005 : 0.0 % Replicability at alpha = 0.001 : 0.0 % ***************************************************************** netmats_parcor PCA_SVR PicSeq_AgeAdj 200 -0.007143177951446793 0.12405504834061054 Replicability at alpha = 0.05 : 53.333333333333336 % Replicability at alpha = 0.01 : 13.333333333333334 % Replicability at alpha = 0.005 : 6.666666666666667 % Replicability at alpha = 0.001 : 0.0 % ***************************************************************** netmats_parcor PCA_SVR PicSeq_AgeAdj 300 0.06577036071435835 0.1525445910946501 Replicability at alpha = 0.05 : 76.92307692307693 % Replicability at alpha = 0.01 : 25.64102564102564 % Replicability at alpha = 0.005 : 10.256410256410255 % Replicability at alpha = 0.001 : 0.0 % ***************************************************************** netmats_parcor PCA_SVR PicSeq_AgeAdj max 0.13437963540657485 0.1853026050732639 Replicability at alpha = 0.05 : 100.0 % Replicability at alpha = 0.01 : 80.0 % Replicability at alpha = 0.005 : 74.44444444444444 % Replicability at alpha = 0.001 : 40.0 % ***************************************************************** netmats_pearson PCA_SVR age 50 -0.12643179950265376 0.05507531631070878 Replicability at alpha = 0.05 : nan % Replicability at alpha = 0.01 : nan % Replicability at alpha = 0.005 : nan % Replicability at alpha = 0.001 : nan % ***************************************************************** netmats_pearson PCA_SVR age 100 0.009725514224550343 0.07992169563215587 Replicability at alpha = 0.05 : 10.0 % Replicability at alpha = 0.01 : 0.0 % Replicability at alpha = 0.005 : 0.0 % Replicability at alpha = 0.001 : 0.0 % ***************************************************************** netmats_pearson PCA_SVR age 200 0.0758358453025459 0.1398647058199516 Replicability at alpha = 0.05 : 64.70588235294117 % Replicability at alpha = 0.01 : 26.47058823529412 % Replicability at alpha = 0.005 : 11.76470588235294 % Replicability at alpha = 0.001 : 2.941176470588235 % ***************************************************************** netmats_pearson PCA_SVR age 300 0.13396259958675433 0.15810854528637436 Replicability at alpha = 0.05 : 89.1891891891892 % Replicability at alpha = 0.01 : 40.54054054054054 % Replicability at alpha = 0.005 : 31.08108108108108 % Replicability at alpha = 0.001 : 12.162162162162163 % ***************************************************************** netmats_pearson PCA_SVR age max 0.16461627834119258 0.19084783442850575 Replicability at alpha = 0.05 : 100.0 % Replicability at alpha = 0.01 : 85.56701030927834 % Replicability at alpha = 0.005 : 82.4742268041237 % Replicability at alpha = 0.001 : 56.70103092783505 % ***************************************************************** netmats_pearson PCA_SVR CogTotalComp_AgeAdj 50 -0.27360113131820085 0.07257458992180708 Replicability at alpha = 0.05 : nan % Replicability at alpha = 0.01 : nan % Replicability at alpha = 0.005 : nan % Replicability at alpha = 0.001 : nan % ***************************************************************** netmats_pearson PCA_SVR CogTotalComp_AgeAdj 100 -0.11427316875393931 0.06349813361022945 Replicability at alpha = 0.05 : 0.0 % Replicability at alpha = 0.01 : 0.0 % Replicability at alpha = 0.005 : 0.0 % Replicability at alpha = 0.001 : 0.0 % ***************************************************************** netmats_pearson PCA_SVR CogTotalComp_AgeAdj 200 -0.0012383369231139204 0.1061294190223969 Replicability at alpha = 0.05 : 25.0 % Replicability at alpha = 0.01 : 0.0 % Replicability at alpha = 0.005 : 0.0 % Replicability at alpha = 0.001 : 0.0 % ***************************************************************** netmats_pearson PCA_SVR CogTotalComp_AgeAdj 300 0.06987992460411578 0.13372831832326426 Replicability at alpha = 0.05 : 62.16216216216216 % Replicability at alpha = 0.01 : 18.91891891891892 % Replicability at alpha = 0.005 : 10.81081081081081 % Replicability at alpha = 0.001 : 2.7027027027027026 % ***************************************************************** netmats_pearson PCA_SVR CogTotalComp_AgeAdj max 0.1329320326293221 0.1759580210438174 Replicability at alpha = 0.05 : 100.0 % Replicability at alpha = 0.01 : 72.82608695652173 % Replicability at alpha = 0.005 : 58.69565217391305 % Replicability at alpha = 0.001 : 29.347826086956523 % ***************************************************************** netmats_pearson PCA_SVR PMAT24_A_CR 50 -0.13058241645252677 0.03937358302729429 Replicability at alpha = 0.05 : 0.0 % Replicability at alpha = 0.01 : 0.0 % Replicability at alpha = 0.005 : 0.0 % Replicability at alpha = 0.001 : 0.0 % ***************************************************************** netmats_pearson PCA_SVR PMAT24_A_CR 100 -0.018219577369671586 0.05283547742503512 Replicability at alpha = 0.05 : 18.75 % Replicability at alpha = 0.01 : 0.0 % Replicability at alpha = 0.005 : 0.0 % Replicability at alpha = 0.001 : 0.0 % ***************************************************************** netmats_pearson PCA_SVR PMAT24_A_CR 200 0.04492628597688514 0.09991315410566606 Replicability at alpha = 0.05 : 57.14285714285714 % Replicability at alpha = 0.01 : 0.0 % Replicability at alpha = 0.005 : 0.0 % Replicability at alpha = 0.001 : 0.0 % ***************************************************************** netmats_pearson PCA_SVR PMAT24_A_CR 300 0.08929516196642572 0.14153398369488263 Replicability at alpha = 0.05 : 78.72340425531915 % Replicability at alpha = 0.01 : 34.04255319148936 % Replicability at alpha = 0.005 : 27.659574468085108 % Replicability at alpha = 0.001 : 2.127659574468085 % ***************************************************************** netmats_pearson PCA_SVR PMAT24_A_CR max 0.15546061546042778 0.17931030150814908 Replicability at alpha = 0.05 : 100.0 % Replicability at alpha = 0.01 : 81.91489361702128 % Replicability at alpha = 0.005 : 76.59574468085107 % Replicability at alpha = 0.001 : 42.5531914893617 % ***************************************************************** netmats_pearson PCA_SVR Flanker_AgeAdj 50 -0.20847896699960677 0.026446681706768983 Replicability at alpha = 0.05 : 0.0 % Replicability at alpha = 0.01 : 0.0 % Replicability at alpha = 0.005 : 0.0 % Replicability at alpha = 0.001 : 0.0 % ***************************************************************** netmats_pearson PCA_SVR Flanker_AgeAdj 100 -0.10129567054697304 0.037796860369966837 Replicability at alpha = 0.05 : 0.0 % Replicability at alpha = 0.01 : 0.0 % Replicability at alpha = 0.005 : 0.0 % Replicability at alpha = 0.001 : 0.0 % ***************************************************************** netmats_pearson PCA_SVR Flanker_AgeAdj 200 -0.025107592465321495 0.04785239767856528 Replicability at alpha = 0.05 : 16.666666666666664 % Replicability at alpha = 0.01 : 0.0 % Replicability at alpha = 0.005 : 0.0 % Replicability at alpha = 0.001 : 0.0 % ***************************************************************** netmats_pearson PCA_SVR Flanker_AgeAdj 300 0.023206206455162977 0.07393753936413099 Replicability at alpha = 0.05 : 18.181818181818183 % Replicability at alpha = 0.01 : 0.0 % Replicability at alpha = 0.005 : 0.0 % Replicability at alpha = 0.001 : 0.0 % ***************************************************************** netmats_pearson PCA_SVR Flanker_AgeAdj max 0.07027346759899196 0.10203836949356383 Replicability at alpha = 0.05 : 61.224489795918366 % Replicability at alpha = 0.01 : 8.16326530612245 % Replicability at alpha = 0.005 : 4.081632653061225 % Replicability at alpha = 0.001 : 0.0 % ***************************************************************** netmats_pearson PCA_SVR CardSort_AgeAdj 50 -0.18617695057292352 0.052864559935567525 Replicability at alpha = 0.05 : nan % Replicability at alpha = 0.01 : nan % Replicability at alpha = 0.005 : nan % Replicability at alpha = 0.001 : nan % ***************************************************************** netmats_pearson PCA_SVR CardSort_AgeAdj 100 -0.05366816791298359 0.05511154998176417 Replicability at alpha = 0.05 : 60.0 % Replicability at alpha = 0.01 : 0.0 % Replicability at alpha = 0.005 : 0.0 % Replicability at alpha = 0.001 : 0.0 % ***************************************************************** netmats_pearson PCA_SVR CardSort_AgeAdj 200 0.01642082868254707 0.07325950038420452 Replicability at alpha = 0.05 : 45.0 % Replicability at alpha = 0.01 : 0.0 % Replicability at alpha = 0.005 : 0.0 % Replicability at alpha = 0.001 : 0.0 % ***************************************************************** netmats_pearson PCA_SVR CardSort_AgeAdj 300 0.042886628759160786 0.09085835147403382 Replicability at alpha = 0.05 : 38.46153846153847 % Replicability at alpha = 0.01 : 3.8461538461538463 % Replicability at alpha = 0.005 : 3.8461538461538463 % Replicability at alpha = 0.001 : 0.0 % ***************************************************************** netmats_pearson PCA_SVR CardSort_AgeAdj max 0.08332392314286416 0.10638876826754434 Replicability at alpha = 0.05 : 71.66666666666667 % Replicability at alpha = 0.01 : 10.0 % Replicability at alpha = 0.005 : 5.0 % Replicability at alpha = 0.001 : 0.0 % ***************************************************************** netmats_pearson PCA_SVR PicSeq_AgeAdj 50 -0.23345400407893277 0.03493508595660751 Replicability at alpha = 0.05 : nan % Replicability at alpha = 0.01 : nan % Replicability at alpha = 0.005 : nan % Replicability at alpha = 0.001 : nan % ***************************************************************** netmats_pearson PCA_SVR PicSeq_AgeAdj 100 -0.10664785322193493 0.06147545482107202 Replicability at alpha = 0.05 : 20.0 % Replicability at alpha = 0.01 : 0.0 % Replicability at alpha = 0.005 : 0.0 % Replicability at alpha = 0.001 : 0.0 % ***************************************************************** netmats_pearson PCA_SVR PicSeq_AgeAdj 200 -0.02801783500947512 0.06749833856170852 Replicability at alpha = 0.05 : 42.857142857142854 % Replicability at alpha = 0.01 : 14.285714285714285 % Replicability at alpha = 0.005 : 0.0 % Replicability at alpha = 0.001 : 0.0 % ***************************************************************** netmats_pearson PCA_SVR PicSeq_AgeAdj 300 0.013808903665158028 0.07458636299063129 Replicability at alpha = 0.05 : 41.66666666666667 % Replicability at alpha = 0.01 : 8.333333333333332 % Replicability at alpha = 0.005 : 8.333333333333332 % Replicability at alpha = 0.001 : 0.0 % ***************************************************************** netmats_pearson PCA_SVR PicSeq_AgeAdj max 0.05753833941897148 0.10077547292317074 Replicability at alpha = 0.05 : 56.81818181818182 % Replicability at alpha = 0.01 : 11.363636363636363 % Replicability at alpha = 0.005 : 2.272727272727273 % Replicability at alpha = 0.001 : 2.272727272727273 % CPU times: user 27.8 s, sys: 6.06 s, total: 33.9 s Wall time: 3h 7min 45s
connectivity | model | target | n | r_discovery_cv | r_discovery_overfit | r_replication | p_discovery_cv | p_discovery_overfit | p_replication | |
---|---|---|---|---|---|---|---|---|---|---|
0 | netmats_parcor | PCA_SVR | age | 50 | -0.353027 | 0.774355 | 0.204078 | 0.997003 | 0.000999 | 0.091908 |
1 | netmats_parcor | PCA_SVR | age | 50 | 0.081894 | 0.913355 | 0.360970 | 0.275724 | 0.000999 | 0.001998 |
2 | netmats_parcor | PCA_SVR | age | 50 | -0.258248 | 0.841646 | 0.265158 | 0.962038 | 0.000999 | 0.025974 |
3 | netmats_parcor | PCA_SVR | age | 50 | 0.050106 | 0.899457 | -0.206159 | 0.374625 | 0.000999 | 0.923077 |
4 | netmats_parcor | PCA_SVR | age | 50 | 0.130115 | 0.892774 | 0.282873 | 0.196803 | 0.000999 | 0.026973 |
... | ... | ... | ... | ... | ... | ... | ... | ... | ... | ... |
5995 | netmats_pearson | PCA_SVR | PicSeq_AgeAdj | 501 | 0.125791 | 0.360809 | 0.062856 | 0.004995 | 0.000999 | 0.085914 |
5996 | netmats_pearson | PCA_SVR | PicSeq_AgeAdj | 501 | 0.060623 | 0.352693 | 0.142834 | 0.085914 | 0.000999 | 0.000999 |
5997 | netmats_pearson | PCA_SVR | PicSeq_AgeAdj | 501 | 0.084630 | 0.373386 | 0.163908 | 0.028971 | 0.000999 | 0.000999 |
5998 | netmats_pearson | PCA_SVR | PicSeq_AgeAdj | 501 | 0.071084 | 0.376163 | 0.109268 | 0.065934 | 0.000999 | 0.012987 |
5999 | netmats_pearson | PCA_SVR | PicSeq_AgeAdj | 501 | 0.015929 | 0.308118 | 0.032076 | 0.347652 | 0.000999 | 0.225774 |
6000 rows × 10 columns
(no feature selection, no hyperparameter optimization) This can be expected to perform better on low samples than SVR.
%%time
random_state = 42
n_bootstrap = 100
features = {
'netmats_parcor': netmats_parcor,
'netmats_pearson': netmats_pearson
}
models = {
'ridge': Ridge()
}
# We aggregate all results here:
df = pd.DataFrame(columns=['connectivity','model','target','n','r_discovery_cv','r_discovery_overfit','r_replication','p_discovery_cv','p_discovery_overfit','p_replication'])
for feature_set in features:
for model in models:
for target_var in ['age', 'CogTotalComp_AgeAdj', 'PMAT24_A_CR', 'Flanker_AgeAdj', 'CardSort_AgeAdj', 'PicSeq_AgeAdj']:
for sample_size in [50, 100, 200, 300, 'max']:
print('*****************************************************************')
print(feature_set, model, target_var, sample_size)
X, y = create_data(target=target_var, feature_data=features[feature_set])
if sample_size=='max':
sample_size = int(len(y)/2)
# create random seeds for each bootstrap iteration for reproducibility
rng = np.random.default_rng(random_state)
random_sates = rng.integers(np.iinfo(np.int32).max, size=n_bootstrap)
# run bootstrap iterations in parallel
r_discovery_cv, r_discovery_overfit, r_replication, p_discovery_cv, p_discovery_overfit, p_replication = zip(
*Parallel(n_jobs=-1)(
delayed(bootstrap_workhorse)(X, y, sample_size, models[model], seed) for seed in random_sates))
tmp_data_frame = pd.DataFrame({
'connectivity' : feature_set,
'model' : model,
'target' : target_var,
'n' : sample_size,
'r_discovery_cv': r_discovery_cv,
'r_discovery_overfit': r_discovery_overfit,
'r_replication': r_replication,
'p_discovery_cv': p_discovery_cv,
'p_discovery_overfit': p_discovery_overfit,
'p_replication': p_replication
})
#sns.scatterplot(x='r_replication', y='r_discovery_cv', data=tmp_data_frame)
#plt.ylabel('in-sample (r)')
#plt.xlabel('out-of-sample (r_pred)')
#plt.show()
print(tmp_data_frame.r_discovery_cv.mean(), tmp_data_frame.r_replication.mean())
for alpha in [0.05, 0.01, 0.005, 0.001]:
print('Replicability at alpha =', alpha, ':',
(tmp_data_frame.loc[tmp_data_frame['p_discovery_cv']<alpha,'p_replication']<alpha).sum() / (tmp_data_frame['p_discovery_cv']<0.05).sum() * 100, '%')
df = pd.concat((df, tmp_data_frame))
df.reset_index(drop=True, inplace=True)
df.to_csv('res_ridge.csv')
df
***************************************************************** netmats_parcor ridge age 50 0.17791423402156398 0.2609198136325508 Replicability at alpha = 0.05 : 58.536585365853654 % Replicability at alpha = 0.01 : 14.634146341463413 % Replicability at alpha = 0.005 : 12.195121951219512 % Replicability at alpha = 0.001 : 7.317073170731707 % ***************************************************************** netmats_parcor ridge age 100 0.3076875117666388 0.34287580012326385 Replicability at alpha = 0.05 : 97.75280898876404 % Replicability at alpha = 0.01 : 71.91011235955057 % Replicability at alpha = 0.005 : 58.42696629213483 % Replicability at alpha = 0.001 : 38.20224719101123 % ***************************************************************** netmats_parcor ridge age 200 0.3896290546757868 0.4213171713707975 Replicability at alpha = 0.05 : 100.0 % Replicability at alpha = 0.01 : 100.0 % Replicability at alpha = 0.005 : 99.0 % Replicability at alpha = 0.001 : 97.0 % ***************************************************************** netmats_parcor ridge age 300 0.4388271357854026 0.4482536389663905 Replicability at alpha = 0.05 : 100.0 % Replicability at alpha = 0.01 : 100.0 % Replicability at alpha = 0.005 : 100.0 % Replicability at alpha = 0.001 : 100.0 % ***************************************************************** netmats_parcor ridge age max 0.47992799211519865 0.48967065593640496 Replicability at alpha = 0.05 : 100.0 % Replicability at alpha = 0.01 : 100.0 % Replicability at alpha = 0.005 : 100.0 % Replicability at alpha = 0.001 : 100.0 % ***************************************************************** netmats_parcor ridge CogTotalComp_AgeAdj 50 0.17021171106396404 0.25667317377477333 Replicability at alpha = 0.05 : 64.86486486486487 % Replicability at alpha = 0.01 : 21.62162162162162 % Replicability at alpha = 0.005 : 10.81081081081081 % Replicability at alpha = 0.001 : 0.0 % ***************************************************************** netmats_parcor ridge CogTotalComp_AgeAdj 100 0.28575343675609444 0.3244129647896814 Replicability at alpha = 0.05 : 97.5 % Replicability at alpha = 0.01 : 68.75 % Replicability at alpha = 0.005 : 63.74999999999999 % Replicability at alpha = 0.001 : 33.75 % ***************************************************************** netmats_parcor ridge CogTotalComp_AgeAdj 200 0.4017824840254701 0.4062055329971098 Replicability at alpha = 0.05 : 100.0 % Replicability at alpha = 0.01 : 99.0 % Replicability at alpha = 0.005 : 99.0 % Replicability at alpha = 0.001 : 98.0 % ***************************************************************** netmats_parcor ridge CogTotalComp_AgeAdj 300 0.4302476508313612 0.4438901556711544 Replicability at alpha = 0.05 : 100.0 % Replicability at alpha = 0.01 : 100.0 % Replicability at alpha = 0.005 : 100.0 % Replicability at alpha = 0.001 : 100.0 % ***************************************************************** netmats_parcor ridge CogTotalComp_AgeAdj max 0.4728370610628511 0.47903946590450225 Replicability at alpha = 0.05 : 100.0 % Replicability at alpha = 0.01 : 100.0 % Replicability at alpha = 0.005 : 100.0 % Replicability at alpha = 0.001 : 100.0 % ***************************************************************** netmats_parcor ridge PMAT24_A_CR 50 0.11946337463600125 0.2012726045326664 Replicability at alpha = 0.05 : 45.83333333333333 % Replicability at alpha = 0.01 : 8.333333333333332 % Replicability at alpha = 0.005 : 4.166666666666666 % Replicability at alpha = 0.001 : 0.0 % ***************************************************************** netmats_parcor ridge PMAT24_A_CR 100 0.19847768139383298 0.22748732178394315 Replicability at alpha = 0.05 : 75.40983606557377 % Replicability at alpha = 0.01 : 32.78688524590164 % Replicability at alpha = 0.005 : 22.950819672131146 % Replicability at alpha = 0.001 : 6.557377049180328 % ***************************************************************** netmats_parcor ridge PMAT24_A_CR 200 0.2650468303412575 0.27987993716710663 Replicability at alpha = 0.05 : 97.89473684210527 % Replicability at alpha = 0.01 : 87.36842105263159 % Replicability at alpha = 0.005 : 80.0 % Replicability at alpha = 0.001 : 47.368421052631575 % ***************************************************************** netmats_parcor ridge PMAT24_A_CR 300 0.2805456646124498 0.2902385092001479 Replicability at alpha = 0.05 : 100.0 % Replicability at alpha = 0.01 : 98.9795918367347 % Replicability at alpha = 0.005 : 98.9795918367347 % Replicability at alpha = 0.001 : 86.73469387755102 % ***************************************************************** netmats_parcor ridge PMAT24_A_CR max 0.2907928923672402 0.301492004778787 Replicability at alpha = 0.05 : 100.0 % Replicability at alpha = 0.01 : 100.0 % Replicability at alpha = 0.005 : 100.0 % Replicability at alpha = 0.001 : 100.0 % ***************************************************************** netmats_parcor ridge Flanker_AgeAdj 50 -0.030999346126054293 0.06932189595874623 Replicability at alpha = 0.05 : 10.0 % Replicability at alpha = 0.01 : 0.0 % Replicability at alpha = 0.005 : 0.0 % Replicability at alpha = 0.001 : 0.0 % ***************************************************************** netmats_parcor ridge Flanker_AgeAdj 100 0.042718319533254744 0.08508903100867408 Replicability at alpha = 0.05 : 21.052631578947366 % Replicability at alpha = 0.01 : 5.263157894736842 % Replicability at alpha = 0.005 : 0.0 % Replicability at alpha = 0.001 : 0.0 % ***************************************************************** netmats_parcor ridge Flanker_AgeAdj 200 0.08964481140622263 0.10467365119965867 Replicability at alpha = 0.05 : 44.44444444444444 % Replicability at alpha = 0.01 : 11.11111111111111 % Replicability at alpha = 0.005 : 2.7777777777777777 % Replicability at alpha = 0.001 : 0.0 % ***************************************************************** netmats_parcor ridge Flanker_AgeAdj 300 0.11297509234550222 0.11951226654893972 Replicability at alpha = 0.05 : 64.61538461538461 % Replicability at alpha = 0.01 : 16.923076923076923 % Replicability at alpha = 0.005 : 10.76923076923077 % Replicability at alpha = 0.001 : 0.0 % ***************************************************************** netmats_parcor ridge Flanker_AgeAdj max 0.12868470883092095 0.14022631939127772 Replicability at alpha = 0.05 : 98.87640449438202 % Replicability at alpha = 0.01 : 61.79775280898876 % Replicability at alpha = 0.005 : 42.69662921348314 % Replicability at alpha = 0.001 : 12.359550561797752 % ***************************************************************** netmats_parcor ridge CardSort_AgeAdj 50 -0.02095238133961973 0.09078700181744905 Replicability at alpha = 0.05 : 25.0 % Replicability at alpha = 0.01 : 0.0 % Replicability at alpha = 0.005 : 0.0 % Replicability at alpha = 0.001 : 0.0 % ***************************************************************** netmats_parcor ridge CardSort_AgeAdj 100 0.0433796889042241 0.10144797710118604 Replicability at alpha = 0.05 : 36.36363636363637 % Replicability at alpha = 0.01 : 0.0 % Replicability at alpha = 0.005 : 0.0 % Replicability at alpha = 0.001 : 0.0 % ***************************************************************** netmats_parcor ridge CardSort_AgeAdj 200 0.10935807078719435 0.14201690360065491 Replicability at alpha = 0.05 : 53.48837209302325 % Replicability at alpha = 0.01 : 13.953488372093023 % Replicability at alpha = 0.005 : 11.627906976744185 % Replicability at alpha = 0.001 : 2.3255813953488373 % ***************************************************************** netmats_parcor ridge CardSort_AgeAdj 300 0.1578820806839097 0.1671880335883637 Replicability at alpha = 0.05 : 89.74358974358975 % Replicability at alpha = 0.01 : 46.15384615384615 % Replicability at alpha = 0.005 : 33.33333333333333 % Replicability at alpha = 0.001 : 5.128205128205128 % ***************************************************************** netmats_parcor ridge CardSort_AgeAdj max 0.17685042894464448 0.1895769375836651 Replicability at alpha = 0.05 : 100.0 % Replicability at alpha = 0.01 : 96.90721649484536 % Replicability at alpha = 0.005 : 91.75257731958763 % Replicability at alpha = 0.001 : 65.97938144329896 % ***************************************************************** netmats_parcor ridge PicSeq_AgeAdj 50 0.03559426488441976 0.09860417658851763 Replicability at alpha = 0.05 : 10.0 % Replicability at alpha = 0.01 : 0.0 % Replicability at alpha = 0.005 : 0.0 % Replicability at alpha = 0.001 : 0.0 % ***************************************************************** netmats_parcor ridge PicSeq_AgeAdj 100 0.0709293081745233 0.1314332174937417 Replicability at alpha = 0.05 : 47.05882352941176 % Replicability at alpha = 0.01 : 0.0 % Replicability at alpha = 0.005 : 0.0 % Replicability at alpha = 0.001 : 0.0 % ***************************************************************** netmats_parcor ridge PicSeq_AgeAdj 200 0.13807364714418496 0.1703337821724498 Replicability at alpha = 0.05 : 82.25806451612904 % Replicability at alpha = 0.01 : 24.193548387096776 % Replicability at alpha = 0.005 : 22.58064516129032 % Replicability at alpha = 0.001 : 11.29032258064516 % ***************************************************************** netmats_parcor ridge PicSeq_AgeAdj 300 0.16631934131984813 0.17241277384991086 Replicability at alpha = 0.05 : 95.23809523809523 % Replicability at alpha = 0.01 : 60.71428571428571 % Replicability at alpha = 0.005 : 44.047619047619044 % Replicability at alpha = 0.001 : 16.666666666666664 % ***************************************************************** netmats_parcor ridge PicSeq_AgeAdj max 0.18026862704274887 0.185930935195939 Replicability at alpha = 0.05 : 100.0 % Replicability at alpha = 0.01 : 93.87755102040816 % Replicability at alpha = 0.005 : 90.81632653061224 % Replicability at alpha = 0.001 : 66.3265306122449 % ***************************************************************** netmats_pearson ridge age 50 0.12197882803189604 0.18481463200614492 Replicability at alpha = 0.05 : 43.333333333333336 % Replicability at alpha = 0.01 : 13.333333333333334 % Replicability at alpha = 0.005 : 3.3333333333333335 % Replicability at alpha = 0.001 : 0.0 % ***************************************************************** netmats_pearson ridge age 100 0.24179325875395313 0.26862478264613243 Replicability at alpha = 0.05 : 77.02702702702703 % Replicability at alpha = 0.01 : 37.83783783783784 % Replicability at alpha = 0.005 : 27.027027027027028 % Replicability at alpha = 0.001 : 13.513513513513514 % ***************************************************************** netmats_pearson ridge age 200 0.3263276141934756 0.36869205100237123 Replicability at alpha = 0.05 : 100.0 % Replicability at alpha = 0.01 : 96.96969696969697 % Replicability at alpha = 0.005 : 95.95959595959596 % Replicability at alpha = 0.001 : 82.82828282828282 % ***************************************************************** netmats_pearson ridge age 300 0.3966060473778811 0.4055241903617708 Replicability at alpha = 0.05 : 100.0 % Replicability at alpha = 0.01 : 100.0 % Replicability at alpha = 0.005 : 100.0 % Replicability at alpha = 0.001 : 100.0 % ***************************************************************** netmats_pearson ridge age max 0.43660520604568265 0.4411349049881781 Replicability at alpha = 0.05 : 100.0 % Replicability at alpha = 0.01 : 100.0 % Replicability at alpha = 0.005 : 100.0 % Replicability at alpha = 0.001 : 100.0 % ***************************************************************** netmats_pearson ridge CogTotalComp_AgeAdj 50 0.12743744491146608 0.21503639335855204 Replicability at alpha = 0.05 : 40.625 % Replicability at alpha = 0.01 : 9.375 % Replicability at alpha = 0.005 : 6.25 % Replicability at alpha = 0.001 : 0.0 % ***************************************************************** netmats_pearson ridge CogTotalComp_AgeAdj 100 0.23721277837894422 0.2515114436083911 Replicability at alpha = 0.05 : 78.66666666666666 % Replicability at alpha = 0.01 : 42.66666666666667 % Replicability at alpha = 0.005 : 26.666666666666668 % Replicability at alpha = 0.001 : 8.0 % ***************************************************************** netmats_pearson ridge CogTotalComp_AgeAdj 200 0.2988184784218123 0.31597434384989187 Replicability at alpha = 0.05 : 100.0 % Replicability at alpha = 0.01 : 92.85714285714286 % Replicability at alpha = 0.005 : 85.71428571428571 % Replicability at alpha = 0.001 : 67.3469387755102 % ***************************************************************** netmats_pearson ridge CogTotalComp_AgeAdj 300 0.33742587839511323 0.34932970770652483 Replicability at alpha = 0.05 : 100.0 % Replicability at alpha = 0.01 : 98.98989898989899 % Replicability at alpha = 0.005 : 98.98989898989899 % Replicability at alpha = 0.001 : 97.97979797979798 % ***************************************************************** netmats_pearson ridge CogTotalComp_AgeAdj max 0.37241510139515843 0.38191979503949747 Replicability at alpha = 0.05 : 100.0 % Replicability at alpha = 0.01 : 100.0 % Replicability at alpha = 0.005 : 100.0 % Replicability at alpha = 0.001 : 100.0 % ***************************************************************** netmats_pearson ridge PMAT24_A_CR 50 0.1183405481901954 0.1388799182188397 Replicability at alpha = 0.05 : 11.11111111111111 % Replicability at alpha = 0.01 : 0.0 % Replicability at alpha = 0.005 : 0.0 % Replicability at alpha = 0.001 : 0.0 % ***************************************************************** netmats_pearson ridge PMAT24_A_CR 100 0.17334425193992187 0.1838033314380429 Replicability at alpha = 0.05 : 70.0 % Replicability at alpha = 0.01 : 16.0 % Replicability at alpha = 0.005 : 10.0 % Replicability at alpha = 0.001 : 2.0 % ***************************************************************** netmats_pearson ridge PMAT24_A_CR 200 0.2007322580701256 0.22291863490268424 Replicability at alpha = 0.05 : 96.15384615384616 % Replicability at alpha = 0.01 : 67.94871794871796 % Replicability at alpha = 0.005 : 57.692307692307686 % Replicability at alpha = 0.001 : 23.076923076923077 % ***************************************************************** netmats_pearson ridge PMAT24_A_CR 300 0.21756192770654528 0.23218168028471226 Replicability at alpha = 0.05 : 100.0 % Replicability at alpha = 0.01 : 86.45833333333334 % Replicability at alpha = 0.005 : 76.04166666666666 % Replicability at alpha = 0.001 : 45.83333333333333 % ***************************************************************** netmats_pearson ridge PMAT24_A_CR max 0.23228024448640586 0.24325742843831086 Replicability at alpha = 0.05 : 100.0 % Replicability at alpha = 0.01 : 98.0 % Replicability at alpha = 0.005 : 98.0 % Replicability at alpha = 0.001 : 91.0 % ***************************************************************** netmats_pearson ridge Flanker_AgeAdj 50 0.000430545327785441 0.05747397295177521 Replicability at alpha = 0.05 : 6.666666666666667 % Replicability at alpha = 0.01 : 0.0 % Replicability at alpha = 0.005 : 0.0 % Replicability at alpha = 0.001 : 0.0 % ***************************************************************** netmats_pearson ridge Flanker_AgeAdj 100 0.06386901265753732 0.07234018621938115 Replicability at alpha = 0.05 : 16.0 % Replicability at alpha = 0.01 : 4.0 % Replicability at alpha = 0.005 : 4.0 % Replicability at alpha = 0.001 : 0.0 % ***************************************************************** netmats_pearson ridge Flanker_AgeAdj 200 0.07558011501857972 0.08879863978328496 Replicability at alpha = 0.05 : 25.806451612903224 % Replicability at alpha = 0.01 : 3.225806451612903 % Replicability at alpha = 0.005 : 0.0 % Replicability at alpha = 0.001 : 0.0 % ***************************************************************** netmats_pearson ridge Flanker_AgeAdj 300 0.09489079942404219 0.09971177201685358 Replicability at alpha = 0.05 : 51.92307692307693 % Replicability at alpha = 0.01 : 11.538461538461538 % Replicability at alpha = 0.005 : 9.615384615384617 % Replicability at alpha = 0.001 : 1.9230769230769231 % ***************************************************************** netmats_pearson ridge Flanker_AgeAdj max 0.12437570615761885 0.11999535001191594 Replicability at alpha = 0.05 : 87.95180722891565 % Replicability at alpha = 0.01 : 49.39759036144578 % Replicability at alpha = 0.005 : 33.734939759036145 % Replicability at alpha = 0.001 : 8.433734939759036 % ***************************************************************** netmats_pearson ridge CardSort_AgeAdj 50 0.02582911317605042 0.07921422079367145 Replicability at alpha = 0.05 : 6.666666666666667 % Replicability at alpha = 0.01 : 0.0 % Replicability at alpha = 0.005 : 0.0 % Replicability at alpha = 0.001 : 0.0 % ***************************************************************** netmats_pearson ridge CardSort_AgeAdj 100 0.06383110584635912 0.11347807860036273 Replicability at alpha = 0.05 : 37.03703703703704 % Replicability at alpha = 0.01 : 11.11111111111111 % Replicability at alpha = 0.005 : 3.7037037037037033 % Replicability at alpha = 0.001 : 0.0 % ***************************************************************** netmats_pearson ridge CardSort_AgeAdj 200 0.10851364492038919 0.12452129430914514 Replicability at alpha = 0.05 : 59.57446808510638 % Replicability at alpha = 0.01 : 19.148936170212767 % Replicability at alpha = 0.005 : 10.638297872340425 % Replicability at alpha = 0.001 : 0.0 % ***************************************************************** netmats_pearson ridge CardSort_AgeAdj 300 0.13157426569804917 0.14640649744766188 Replicability at alpha = 0.05 : 83.82352941176471 % Replicability at alpha = 0.01 : 39.705882352941174 % Replicability at alpha = 0.005 : 33.82352941176471 % Replicability at alpha = 0.001 : 7.352941176470589 % ***************************************************************** netmats_pearson ridge CardSort_AgeAdj max 0.14399064365170303 0.15294162337000017 Replicability at alpha = 0.05 : 98.91304347826086 % Replicability at alpha = 0.01 : 72.82608695652173 % Replicability at alpha = 0.005 : 64.13043478260869 % Replicability at alpha = 0.001 : 20.652173913043477 % ***************************************************************** netmats_pearson ridge PicSeq_AgeAdj 50 0.06812605964162959 0.0940846981022576 Replicability at alpha = 0.05 : 20.0 % Replicability at alpha = 0.01 : 0.0 % Replicability at alpha = 0.005 : 0.0 % Replicability at alpha = 0.001 : 0.0 % ***************************************************************** netmats_pearson ridge PicSeq_AgeAdj 100 0.08425639355450064 0.12424545727649311 Replicability at alpha = 0.05 : 25.0 % Replicability at alpha = 0.01 : 5.555555555555555 % Replicability at alpha = 0.005 : 2.7777777777777777 % Replicability at alpha = 0.001 : 0.0 % ***************************************************************** netmats_pearson ridge PicSeq_AgeAdj 200 0.13038552328924102 0.13875213635359748 Replicability at alpha = 0.05 : 69.0909090909091 % Replicability at alpha = 0.01 : 20.0 % Replicability at alpha = 0.005 : 9.090909090909092 % Replicability at alpha = 0.001 : 3.6363636363636362 % ***************************************************************** netmats_pearson ridge PicSeq_AgeAdj 300 0.1558301559850372 0.15965154764724945 Replicability at alpha = 0.05 : 84.14634146341463 % Replicability at alpha = 0.01 : 48.78048780487805 % Replicability at alpha = 0.005 : 39.02439024390244 % Replicability at alpha = 0.001 : 8.536585365853659 % ***************************************************************** netmats_pearson ridge PicSeq_AgeAdj max 0.17669686000282808 0.18968608293893335 Replicability at alpha = 0.05 : 100.0 % Replicability at alpha = 0.01 : 90.625 % Replicability at alpha = 0.005 : 86.45833333333334 % Replicability at alpha = 0.001 : 61.458333333333336 % CPU times: user 22.5 s, sys: 5.34 s, total: 27.8 s Wall time: 15min 40s
connectivity | model | target | n | r_discovery_cv | r_discovery_overfit | r_replication | p_discovery_cv | p_discovery_overfit | p_replication | |
---|---|---|---|---|---|---|---|---|---|---|
0 | netmats_parcor | ridge | age | 50 | -0.048518 | 1.0 | 0.404661 | 0.629371 | 0.000999 | 0.002997 |
1 | netmats_parcor | ridge | age | 50 | 0.111210 | 1.0 | 0.281041 | 0.210789 | 0.000999 | 0.026973 |
2 | netmats_parcor | ridge | age | 50 | 0.103342 | 1.0 | 0.107181 | 0.227772 | 0.000999 | 0.216783 |
3 | netmats_parcor | ridge | age | 50 | 0.255112 | 1.0 | -0.203981 | 0.047952 | 0.000999 | 0.909091 |
4 | netmats_parcor | ridge | age | 50 | 0.425614 | 1.0 | 0.385665 | 0.001998 | 0.000999 | 0.004995 |
... | ... | ... | ... | ... | ... | ... | ... | ... | ... | ... |
5995 | netmats_pearson | ridge | PicSeq_AgeAdj | 501 | 0.209622 | 1.0 | 0.217841 | 0.000999 | 0.000999 | 0.000999 |
5996 | netmats_pearson | ridge | PicSeq_AgeAdj | 501 | 0.237770 | 1.0 | 0.201980 | 0.000999 | 0.000999 | 0.000999 |
5997 | netmats_pearson | ridge | PicSeq_AgeAdj | 501 | 0.268639 | 1.0 | 0.203847 | 0.000999 | 0.000999 | 0.000999 |
5998 | netmats_pearson | ridge | PicSeq_AgeAdj | 501 | 0.173644 | 1.0 | 0.195714 | 0.001998 | 0.000999 | 0.000999 |
5999 | netmats_pearson | ridge | PicSeq_AgeAdj | 501 | 0.148498 | 1.0 | 0.204881 | 0.001998 | 0.000999 | 0.000999 |
6000 rows × 10 columns
To evaluate false positives with biased estimates
%%time
random_state = 42
n_bootstrap = 100
features = {
'netmats_parcor': netmats_parcor
}
models = {
'PCA_SVR': Pipeline([('pca', PCA(n_components=0.5)),
('svr', SVR())])
}
# We aggregate all results here:
df = pd.DataFrame(columns=['connectivity','model','target','n','r_discovery_cv','r_discovery_overfit','r_replication','p_discovery_cv','p_discovery_overfit','p_replication'])
for feature_set in features:
for model in models:
for target_var in [None]:
for sample_size in [50, 100, 200, 300, 'max']:
print('*****************************************************************')
print(feature_set, model, target_var, sample_size)
X, y = create_data(target=target_var, feature_data=features[feature_set]) # gives a random y when target is None
if sample_size=='max':
sample_size = int(len(y)/2)
# create random seeds for each bootstrap iteration for reproducibility
rng = np.random.default_rng(random_state)
random_sates = rng.integers(np.iinfo(np.int32).max, size=n_bootstrap)
# run bootstrap iterations in parallel
r_discovery_cv, r_discovery_overfit, r_replication, p_discovery_cv, p_discovery_overfit, p_replication = zip(
*Parallel(n_jobs=-1)(
delayed(bootstrap_workhorse)(X, y, sample_size, models[model], seed) for seed in random_sates))
tmp_data_frame = pd.DataFrame({
'connectivity' : feature_set,
'model' : model,
'target' : target_var,
'n' : sample_size,
'r_discovery_cv': r_discovery_cv,
'r_discovery_overfit': r_discovery_overfit,
'r_replication': r_replication,
'p_discovery_cv': p_discovery_cv,
'p_discovery_overfit': p_discovery_overfit,
'p_replication': p_replication
})
#sns.scatterplot(x='r_replication', y='r_discovery_cv', data=tmp_data_frame)
#plt.ylabel('in-sample (r)')
#plt.xlabel('out-of-sample (r_pred)')
#plt.show()
print(tmp_data_frame.r_discovery_cv.mean(), tmp_data_frame.r_replication.mean())
for alpha in [0.05, 0.01, 0.005, 0.001]:
print('Replicability at alpha =', alpha, ':',
(tmp_data_frame.loc[tmp_data_frame['p_discovery_cv']<alpha,'p_replication']<alpha).sum() / (tmp_data_frame['p_discovery_cv']<0.05).sum() * 100, '%')
df = pd.concat((df, tmp_data_frame))
df.reset_index(drop=True, inplace=True)
df.to_csv('res_null_PCA_SVR.csv')
df
***************************************************************** netmats_parcor PCA_SVR None 50 -0.10752243410278593 0.015330494496431592 Replicability at alpha = 0.05 : 0.0 % Replicability at alpha = 0.01 : 0.0 % Replicability at alpha = 0.005 : 0.0 % Replicability at alpha = 0.001 : 0.0 % ***************************************************************** netmats_parcor PCA_SVR None 100 -0.05119164595574469 0.018628014156943713 Replicability at alpha = 0.05 : 14.285714285714285 % Replicability at alpha = 0.01 : 0.0 % Replicability at alpha = 0.005 : 0.0 % Replicability at alpha = 0.001 : 0.0 % ***************************************************************** netmats_parcor PCA_SVR None 200 -0.026918006714795394 -0.001351453184769438 Replicability at alpha = 0.05 : 0.0 % Replicability at alpha = 0.01 : 0.0 % Replicability at alpha = 0.005 : 0.0 % Replicability at alpha = 0.001 : 0.0 % ***************************************************************** netmats_parcor PCA_SVR None 300 -0.02639998606777391 0.0025325717560960915 Replicability at alpha = 0.05 : 0.0 % Replicability at alpha = 0.01 : 0.0 % Replicability at alpha = 0.005 : 0.0 % Replicability at alpha = 0.001 : 0.0 % ***************************************************************** netmats_parcor PCA_SVR None max -0.017262467918503065 0.006362890326088215 Replicability at alpha = 0.05 : 0.0 % Replicability at alpha = 0.01 : 0.0 % Replicability at alpha = 0.005 : 0.0 % Replicability at alpha = 0.001 : 0.0 % CPU times: user 2.4 s, sys: 752 ms, total: 3.15 s Wall time: 16min 30s
connectivity | model | target | n | r_discovery_cv | r_discovery_overfit | r_replication | p_discovery_cv | p_discovery_overfit | p_replication | |
---|---|---|---|---|---|---|---|---|---|---|
0 | netmats_parcor | PCA_SVR | None | 50 | 0.019118 | 0.893737 | 0.062928 | 0.445554 | 0.000999 | 0.330669 |
1 | netmats_parcor | PCA_SVR | None | 50 | -0.319547 | 0.897756 | 0.134277 | 0.995005 | 0.000999 | 0.185814 |
2 | netmats_parcor | PCA_SVR | None | 50 | -0.120632 | 0.871983 | 0.052140 | 0.816184 | 0.000999 | 0.344655 |
3 | netmats_parcor | PCA_SVR | None | 50 | -0.224169 | 0.814265 | -0.009114 | 0.938062 | 0.000999 | 0.536464 |
4 | netmats_parcor | PCA_SVR | None | 50 | -0.175718 | 0.889370 | 0.083803 | 0.876124 | 0.000999 | 0.279720 |
... | ... | ... | ... | ... | ... | ... | ... | ... | ... | ... |
495 | netmats_parcor | PCA_SVR | None | 501 | -0.071740 | 0.883689 | 0.026235 | 0.950050 | 0.000999 | 0.282717 |
496 | netmats_parcor | PCA_SVR | None | 501 | 0.009939 | 0.877526 | 0.019956 | 0.414585 | 0.000999 | 0.338661 |
497 | netmats_parcor | PCA_SVR | None | 501 | -0.009945 | 0.875216 | 0.015069 | 0.608392 | 0.000999 | 0.349650 |
498 | netmats_parcor | PCA_SVR | None | 501 | -0.020732 | 0.876886 | -0.082211 | 0.686314 | 0.000999 | 0.969031 |
499 | netmats_parcor | PCA_SVR | None | 501 | -0.058961 | 0.881125 | 0.014305 | 0.914086 | 0.000999 | 0.374625 |
500 rows × 10 columns
Below we perform some exploratory analysis with other models, for the sake of curiosity. This doesn't need to be revied.
%%time
random_state = 42
n_bootstrap = 100
features = {
'netmats_parcor': netmats_parcor,
'netmats_pearson': netmats_pearson
}
models = {
'SVR': SVR()
}
# We aggregate all results here:
df = pd.DataFrame(columns=['connectivity','model','target','n','r_discovery_cv','r_discovery_overfit','r_replication','p_discovery_cv','p_discovery_overfit','p_replication'])
for feature_set in features:
for model in models:
for target_var in ['age', 'CogTotalComp_AgeAdj', 'PMAT24_A_CR', 'Flanker_AgeAdj', 'CardSort_AgeAdj', 'PicSeq_AgeAdj']:
for sample_size in [50, 100, 200, 300, 'max']:
print('*****************************************************************')
print(feature_set, model, target_var, sample_size)
X, y = create_data(target=target_var, feature_data=features[feature_set])
if sample_size=='max':
sample_size = int(len(y)/2)
# create random seeds for each bootstrap iteration for reproducibility
rng = np.random.default_rng(random_state)
random_sates = rng.integers(np.iinfo(np.int32).max, size=n_bootstrap)
# run bootstrap iterations in parallel
r_discovery_cv, r_discovery_overfit, r_replication, p_discovery_cv, p_discovery_overfit, p_replication = zip(
*Parallel(n_jobs=-1)(
delayed(bootstrap_workhorse)(X, y, sample_size, models[model], seed) for seed in random_sates))
tmp_data_frame = pd.DataFrame({
'connectivity' : feature_set,
'model' : model,
'target' : target_var,
'n' : sample_size,
'r_discovery_cv': r_discovery_cv,
'r_discovery_overfit': r_discovery_overfit,
'r_replication': r_replication,
'p_discovery_cv': p_discovery_cv,
'p_discovery_overfit': p_discovery_overfit,
'p_replication': p_replication
})
#sns.scatterplot(x='r_replication', y='r_discovery_cv', data=tmp_data_frame)
#plt.ylabel('in-sample (r)')
#plt.xlabel('out-of-sample (r_pred)')
#plt.show()
print(tmp_data_frame.r_discovery_cv.mean(), tmp_data_frame.r_replication.mean())
for alpha in [0.05, 0.01, 0.005, 0.001]:
print('Replicability at alpha =', alpha, ':',
(tmp_data_frame.loc[tmp_data_frame['p_discovery_cv']<alpha,'p_replication']<alpha).sum() / (tmp_data_frame['p_discovery_cv']<0.05).sum() * 100, '%')
df = pd.concat((df, tmp_data_frame))
df.reset_index(drop=True, inplace=True)
df.to_csv('res_SVR.csv')
df
***************************************************************** netmats_parcor SVR age 50 -0.01643376964637595 0.23423014405557027 Replicability at alpha = 0.05 : 45.45454545454545 % Replicability at alpha = 0.01 : 9.090909090909092 % Replicability at alpha = 0.005 : 0.0 % Replicability at alpha = 0.001 : 0.0 % ***************************************************************** netmats_parcor SVR age 100 0.2004971727400091 0.31977467957054306 Replicability at alpha = 0.05 : 95.52238805970148 % Replicability at alpha = 0.01 : 49.25373134328358 % Replicability at alpha = 0.005 : 43.28358208955223 % Replicability at alpha = 0.001 : 13.432835820895523 % ***************************************************************** netmats_parcor SVR age 200 0.34526027745129295 0.40305868339855644 Replicability at alpha = 0.05 : 100.0 % Replicability at alpha = 0.01 : 98.98989898989899 % Replicability at alpha = 0.005 : 94.94949494949495 % Replicability at alpha = 0.001 : 88.88888888888889 % ***************************************************************** netmats_parcor SVR age 300 0.40343680164347107 0.43064186589161435 Replicability at alpha = 0.05 : 100.0 % Replicability at alpha = 0.01 : 100.0 % Replicability at alpha = 0.005 : 100.0 % Replicability at alpha = 0.001 : 100.0 % ***************************************************************** netmats_parcor SVR age max 0.45195336703801325 0.478785359519327 Replicability at alpha = 0.05 : 100.0 % Replicability at alpha = 0.01 : 100.0 % Replicability at alpha = 0.005 : 100.0 % Replicability at alpha = 0.001 : 100.0 % ***************************************************************** netmats_parcor SVR CogTotalComp_AgeAdj 50 -0.32326413444463964 0.18740575993160108 Replicability at alpha = 0.05 : nan % Replicability at alpha = 0.01 : nan % Replicability at alpha = 0.005 : nan % Replicability at alpha = 0.001 : nan % ***************************************************************** netmats_parcor SVR CogTotalComp_AgeAdj 100
<timed exec>:58: RuntimeWarning: invalid value encountered in long_scalars
-0.14562995074992818 0.24088134739904876 Replicability at alpha = 0.05 : 100.0 % Replicability at alpha = 0.01 : 0.0 % Replicability at alpha = 0.005 : 0.0 % Replicability at alpha = 0.001 : 0.0 % ***************************************************************** netmats_parcor SVR CogTotalComp_AgeAdj 200 0.03603161832397271 0.2851671994147706 Replicability at alpha = 0.05 : 100.0 % Replicability at alpha = 0.01 : 43.47826086956522 % Replicability at alpha = 0.005 : 34.78260869565217 % Replicability at alpha = 0.001 : 8.695652173913043 % ***************************************************************** netmats_parcor SVR CogTotalComp_AgeAdj 300 0.1468259280416723 0.309336370533758 Replicability at alpha = 0.05 : 100.0 % Replicability at alpha = 0.01 : 71.23287671232876 % Replicability at alpha = 0.005 : 67.12328767123287 % Replicability at alpha = 0.001 : 49.31506849315068 % ***************************************************************** netmats_parcor SVR CogTotalComp_AgeAdj max 0.2549413117184034 0.33152667215407533 Replicability at alpha = 0.05 : 100.0 % Replicability at alpha = 0.01 : 100.0 % Replicability at alpha = 0.005 : 100.0 % Replicability at alpha = 0.001 : 100.0 % ***************************************************************** netmats_parcor SVR PMAT24_A_CR 50 -0.22239623119009483 0.15643323642010132 Replicability at alpha = 0.05 : 0.0 % Replicability at alpha = 0.01 : 0.0 % Replicability at alpha = 0.005 : 0.0 % Replicability at alpha = 0.001 : 0.0 % ***************************************************************** netmats_parcor SVR PMAT24_A_CR 100 -0.034263490784347574 0.1757279082959372 Replicability at alpha = 0.05 : 22.22222222222222 % Replicability at alpha = 0.01 : 0.0 % Replicability at alpha = 0.005 : 0.0 % Replicability at alpha = 0.001 : 0.0 % ***************************************************************** netmats_parcor SVR PMAT24_A_CR 200 0.1264519179930518 0.24892720515585945 Replicability at alpha = 0.05 : 92.15686274509804 % Replicability at alpha = 0.01 : 54.90196078431373 % Replicability at alpha = 0.005 : 35.294117647058826 % Replicability at alpha = 0.001 : 13.725490196078432 % ***************************************************************** netmats_parcor SVR PMAT24_A_CR 300 0.19804464630414487 0.2842987213429107 Replicability at alpha = 0.05 : 100.0 % Replicability at alpha = 0.01 : 86.81318681318682 % Replicability at alpha = 0.005 : 82.41758241758241 % Replicability at alpha = 0.001 : 57.14285714285714 % ***************************************************************** netmats_parcor SVR PMAT24_A_CR max 0.281877972297132 0.32318498565606746 Replicability at alpha = 0.05 : 100.0 % Replicability at alpha = 0.01 : 100.0 % Replicability at alpha = 0.005 : 100.0 % Replicability at alpha = 0.001 : 100.0 % ***************************************************************** netmats_parcor SVR Flanker_AgeAdj 50 -0.3204641069539859 0.030480831068828495 Replicability at alpha = 0.05 : nan % Replicability at alpha = 0.01 : nan % Replicability at alpha = 0.005 : nan % Replicability at alpha = 0.001 : nan % ***************************************************************** netmats_parcor SVR Flanker_AgeAdj 100 -0.1890903926253441 0.06595568727430982 Replicability at alpha = 0.05 : 0.0 % Replicability at alpha = 0.01 : 0.0 % Replicability at alpha = 0.005 : 0.0 % Replicability at alpha = 0.001 : 0.0 % ***************************************************************** netmats_parcor SVR Flanker_AgeAdj 200 -0.08732742503822419 0.08078012627412137 Replicability at alpha = 0.05 : 0.0 % Replicability at alpha = 0.01 : 0.0 % Replicability at alpha = 0.005 : 0.0 % Replicability at alpha = 0.001 : 0.0 % ***************************************************************** netmats_parcor SVR Flanker_AgeAdj 300 -0.02181275209991224 0.09337973998758166 Replicability at alpha = 0.05 : 66.66666666666666 % Replicability at alpha = 0.01 : 0.0 % Replicability at alpha = 0.005 : 0.0 % Replicability at alpha = 0.001 : 0.0 % ***************************************************************** netmats_parcor SVR Flanker_AgeAdj max 0.037791420905649346 0.13254327723076448 Replicability at alpha = 0.05 : 88.88888888888889 % Replicability at alpha = 0.01 : 18.51851851851852 % Replicability at alpha = 0.005 : 7.4074074074074066 % Replicability at alpha = 0.001 : 0.0 % ***************************************************************** netmats_parcor SVR CardSort_AgeAdj 50 -0.31648999016482265 0.08034666460705671 Replicability at alpha = 0.05 : nan % Replicability at alpha = 0.01 : nan % Replicability at alpha = 0.005 : nan % Replicability at alpha = 0.001 : nan % ***************************************************************** netmats_parcor SVR CardSort_AgeAdj 100 -0.1724700765060818 0.10139569898385202 Replicability at alpha = 0.05 : nan % Replicability at alpha = 0.01 : nan % Replicability at alpha = 0.005 : nan % Replicability at alpha = 0.001 : nan % ***************************************************************** netmats_parcor SVR CardSort_AgeAdj 200 -0.03266879795717834 0.12273062020169488 Replicability at alpha = 0.05 : 33.33333333333333 % Replicability at alpha = 0.01 : 0.0 % Replicability at alpha = 0.005 : 0.0 % Replicability at alpha = 0.001 : 0.0 % ***************************************************************** netmats_parcor SVR CardSort_AgeAdj 300 0.039912731726428986 0.14838044253474514 Replicability at alpha = 0.05 : 64.28571428571429 % Replicability at alpha = 0.01 : 14.285714285714285 % Replicability at alpha = 0.005 : 10.714285714285714 % Replicability at alpha = 0.001 : 3.571428571428571 % ***************************************************************** netmats_parcor SVR CardSort_AgeAdj max 0.10304816152320781 0.16557288741091614 Replicability at alpha = 0.05 : 98.59154929577466 % Replicability at alpha = 0.01 : 59.154929577464785 % Replicability at alpha = 0.005 : 40.845070422535215 % Replicability at alpha = 0.001 : 2.8169014084507045 % ***************************************************************** netmats_parcor SVR PicSeq_AgeAdj 50 -0.3208113579952884 0.08193465752885971 Replicability at alpha = 0.05 : nan % Replicability at alpha = 0.01 : nan % Replicability at alpha = 0.005 : nan % Replicability at alpha = 0.001 : nan % ***************************************************************** netmats_parcor SVR PicSeq_AgeAdj 100 -0.19464008504119243 0.11223560776725991 Replicability at alpha = 0.05 : nan % Replicability at alpha = 0.01 : nan % Replicability at alpha = 0.005 : nan % Replicability at alpha = 0.001 : nan % ***************************************************************** netmats_parcor SVR PicSeq_AgeAdj 200 -0.08624314038157287 0.142170250432876 Replicability at alpha = 0.05 : 100.0 % Replicability at alpha = 0.01 : 100.0 % Replicability at alpha = 0.005 : 50.0 % Replicability at alpha = 0.001 : 0.0 % ***************************************************************** netmats_parcor SVR PicSeq_AgeAdj 300 0.0027082380419047036 0.16087552055866783 Replicability at alpha = 0.05 : 83.33333333333334 % Replicability at alpha = 0.01 : 25.0 % Replicability at alpha = 0.005 : 8.333333333333332 % Replicability at alpha = 0.001 : 0.0 % ***************************************************************** netmats_parcor SVR PicSeq_AgeAdj max 0.0888093961587751 0.186970438550117 Replicability at alpha = 0.05 : 100.0 % Replicability at alpha = 0.01 : 61.53846153846154 % Replicability at alpha = 0.005 : 40.0 % Replicability at alpha = 0.001 : 7.6923076923076925 % ***************************************************************** netmats_pearson SVR age 50 -0.07578059047514771 0.11307723629924883 Replicability at alpha = 0.05 : 33.33333333333333 % Replicability at alpha = 0.01 : 0.0 % Replicability at alpha = 0.005 : 0.0 % Replicability at alpha = 0.001 : 0.0 % ***************************************************************** netmats_pearson SVR age 100 0.10117091818899272 0.1919678284018117 Replicability at alpha = 0.05 : 60.60606060606061 % Replicability at alpha = 0.01 : 15.151515151515152 % Replicability at alpha = 0.005 : 12.121212121212121 % Replicability at alpha = 0.001 : 0.0 % ***************************************************************** netmats_pearson SVR age 200 0.23100402523093122 0.30911095241403364 Replicability at alpha = 0.05 : 98.86363636363636 % Replicability at alpha = 0.01 : 82.95454545454545 % Replicability at alpha = 0.005 : 70.45454545454545 % Replicability at alpha = 0.001 : 50.0 % ***************************************************************** netmats_pearson SVR age 300 0.31953483250166514 0.34442934241771356 Replicability at alpha = 0.05 : 100.0 % Replicability at alpha = 0.01 : 100.0 % Replicability at alpha = 0.005 : 100.0 % Replicability at alpha = 0.001 : 97.0 % ***************************************************************** netmats_pearson SVR age max 0.4010174468125596 0.42731662316926716 Replicability at alpha = 0.05 : 100.0 % Replicability at alpha = 0.01 : 100.0 % Replicability at alpha = 0.005 : 100.0 % Replicability at alpha = 0.001 : 100.0 % ***************************************************************** netmats_pearson SVR CogTotalComp_AgeAdj 50 -0.30982472602999245 0.11287333731403326 Replicability at alpha = 0.05 : nan % Replicability at alpha = 0.01 : nan % Replicability at alpha = 0.005 : nan % Replicability at alpha = 0.001 : nan % ***************************************************************** netmats_pearson SVR CogTotalComp_AgeAdj 100 -0.13890778666026496 0.12805067469075515 Replicability at alpha = 0.05 : nan % Replicability at alpha = 0.01 : nan % Replicability at alpha = 0.005 : nan % Replicability at alpha = 0.001 : nan % ***************************************************************** netmats_pearson SVR CogTotalComp_AgeAdj 200 -7.21805212150481e-05 0.1730921499219021 Replicability at alpha = 0.05 : 61.53846153846154 % Replicability at alpha = 0.01 : 15.384615384615385 % Replicability at alpha = 0.005 : 7.6923076923076925 % Replicability at alpha = 0.001 : 0.0 % ***************************************************************** netmats_pearson SVR CogTotalComp_AgeAdj 300 0.09110523124823479 0.19749525772200202 Replicability at alpha = 0.05 : 88.23529411764706 % Replicability at alpha = 0.01 : 43.13725490196079 % Replicability at alpha = 0.005 : 25.49019607843137 % Replicability at alpha = 0.001 : 5.88235294117647 % ***************************************************************** netmats_pearson SVR CogTotalComp_AgeAdj max 0.17509369508119277 0.23880005412473712 Replicability at alpha = 0.05 : 100.0 % Replicability at alpha = 0.01 : 96.96969696969697 % Replicability at alpha = 0.005 : 91.91919191919192 % Replicability at alpha = 0.001 : 71.71717171717171 % ***************************************************************** netmats_pearson SVR PMAT24_A_CR 50 -0.18522734626282625 0.0659513189654039 Replicability at alpha = 0.05 : 0.0 % Replicability at alpha = 0.01 : 0.0 % Replicability at alpha = 0.005 : 0.0 % Replicability at alpha = 0.001 : 0.0 % ***************************************************************** netmats_pearson SVR PMAT24_A_CR 100 -0.03541889369372119 0.10663680239430814 Replicability at alpha = 0.05 : 22.22222222222222 % Replicability at alpha = 0.01 : 0.0 % Replicability at alpha = 0.005 : 0.0 % Replicability at alpha = 0.001 : 0.0 % ***************************************************************** netmats_pearson SVR PMAT24_A_CR 200 0.07480305222747267 0.16771321619124785 Replicability at alpha = 0.05 : 76.47058823529412 % Replicability at alpha = 0.01 : 23.52941176470588 % Replicability at alpha = 0.005 : 8.823529411764707 % Replicability at alpha = 0.001 : 0.0 % ***************************************************************** netmats_pearson SVR PMAT24_A_CR 300 0.14465996000988599 0.2164368604723245 Replicability at alpha = 0.05 : 94.66666666666667 % Replicability at alpha = 0.01 : 65.33333333333333 % Replicability at alpha = 0.005 : 50.66666666666667 % Replicability at alpha = 0.001 : 13.333333333333334 % ***************************************************************** netmats_pearson SVR PMAT24_A_CR max 0.23067026527717135 0.26365788015471814 Replicability at alpha = 0.05 : 100.0 % Replicability at alpha = 0.01 : 100.0 % Replicability at alpha = 0.005 : 99.0 % Replicability at alpha = 0.001 : 93.0 % ***************************************************************** netmats_pearson SVR Flanker_AgeAdj 50 -0.27036407987316635 0.03746547055481584 Replicability at alpha = 0.05 : nan % Replicability at alpha = 0.01 : nan % Replicability at alpha = 0.005 : nan % Replicability at alpha = 0.001 : nan % ***************************************************************** netmats_pearson SVR Flanker_AgeAdj 100 -0.14100630065170205 0.05657798711077884 Replicability at alpha = 0.05 : 66.66666666666666 % Replicability at alpha = 0.01 : 0.0 % Replicability at alpha = 0.005 : 0.0 % Replicability at alpha = 0.001 : 0.0 % ***************************************************************** netmats_pearson SVR Flanker_AgeAdj 200 -0.055116793515234415 0.05799142750256638 Replicability at alpha = 0.05 : 0.0 % Replicability at alpha = 0.01 : 0.0 % Replicability at alpha = 0.005 : 0.0 % Replicability at alpha = 0.001 : 0.0 % ***************************************************************** netmats_pearson SVR Flanker_AgeAdj 300 0.004240141987394162 0.08004593409063611 Replicability at alpha = 0.05 : 23.076923076923077 % Replicability at alpha = 0.01 : 0.0 % Replicability at alpha = 0.005 : 0.0 % Replicability at alpha = 0.001 : 0.0 % ***************************************************************** netmats_pearson SVR Flanker_AgeAdj max 0.05566085950364401 0.11481030338787498 Replicability at alpha = 0.05 : 75.75757575757575 % Replicability at alpha = 0.01 : 15.151515151515152 % Replicability at alpha = 0.005 : 0.0 % Replicability at alpha = 0.001 : 0.0 % ***************************************************************** netmats_pearson SVR CardSort_AgeAdj 50 -0.2643667597050225 0.07491052110663295 Replicability at alpha = 0.05 : nan % Replicability at alpha = 0.01 : nan % Replicability at alpha = 0.005 : nan % Replicability at alpha = 0.001 : nan % ***************************************************************** netmats_pearson SVR CardSort_AgeAdj 100 -0.10778687253303708 0.08392608342465463 Replicability at alpha = 0.05 : nan % Replicability at alpha = 0.01 : nan % Replicability at alpha = 0.005 : nan % Replicability at alpha = 0.001 : nan % ***************************************************************** netmats_pearson SVR CardSort_AgeAdj 200 0.00894686954658656 0.10845619502551797 Replicability at alpha = 0.05 : 35.294117647058826 % Replicability at alpha = 0.01 : 0.0 % Replicability at alpha = 0.005 : 0.0 % Replicability at alpha = 0.001 : 0.0 % ***************************************************************** netmats_pearson SVR CardSort_AgeAdj 300 0.058572453315874566 0.1317703098552681 Replicability at alpha = 0.05 : 59.375 % Replicability at alpha = 0.01 : 12.5 % Replicability at alpha = 0.005 : 6.25 % Replicability at alpha = 0.001 : 0.0 % ***************************************************************** netmats_pearson SVR CardSort_AgeAdj max 0.11015433267063868 0.1472505836113826 Replicability at alpha = 0.05 : 96.0 % Replicability at alpha = 0.01 : 52.0 % Replicability at alpha = 0.005 : 30.666666666666664 % Replicability at alpha = 0.001 : 4.0 % ***************************************************************** netmats_pearson SVR PicSeq_AgeAdj 50 -0.2938000509765391 0.061028243473976014 Replicability at alpha = 0.05 : nan % Replicability at alpha = 0.01 : nan % Replicability at alpha = 0.005 : nan % Replicability at alpha = 0.001 : nan % ***************************************************************** netmats_pearson SVR PicSeq_AgeAdj 100 -0.1642823732841255 0.08346539469408594 Replicability at alpha = 0.05 : 0.0 % Replicability at alpha = 0.01 : 0.0 % Replicability at alpha = 0.005 : 0.0 % Replicability at alpha = 0.001 : 0.0 % ***************************************************************** netmats_pearson SVR PicSeq_AgeAdj 200 -0.05458187746701462 0.10608631158346489 Replicability at alpha = 0.05 : 66.66666666666666 % Replicability at alpha = 0.01 : 0.0 % Replicability at alpha = 0.005 : 0.0 % Replicability at alpha = 0.001 : 0.0 % ***************************************************************** netmats_pearson SVR PicSeq_AgeAdj 300 0.021856737276436368 0.12677891257700286 Replicability at alpha = 0.05 : 75.0 % Replicability at alpha = 0.01 : 20.0 % Replicability at alpha = 0.005 : 10.0 % Replicability at alpha = 0.001 : 5.0 % ***************************************************************** netmats_pearson SVR PicSeq_AgeAdj max 0.09644924771734871 0.1680084950838603 Replicability at alpha = 0.05 : 95.38461538461539 % Replicability at alpha = 0.01 : 47.69230769230769 % Replicability at alpha = 0.005 : 38.46153846153847 % Replicability at alpha = 0.001 : 15.384615384615385 % CPU times: user 24.1 s, sys: 6.05 s, total: 30.2 s Wall time: 3h 47min 20s
connectivity | model | target | n | r_discovery_cv | r_discovery_overfit | r_replication | p_discovery_cv | p_discovery_overfit | p_replication | |
---|---|---|---|---|---|---|---|---|---|---|
0 | netmats_parcor | SVR | age | 50 | -0.330564 | 0.899135 | 0.288430 | 0.994006 | 0.000999 | 0.029970 |
1 | netmats_parcor | SVR | age | 50 | -0.023555 | 0.963181 | 0.291646 | 0.591409 | 0.000999 | 0.019980 |
2 | netmats_parcor | SVR | age | 50 | -0.176967 | 0.968481 | 0.092822 | 0.884116 | 0.000999 | 0.244755 |
3 | netmats_parcor | SVR | age | 50 | 0.093535 | 0.960486 | -0.161212 | 0.299700 | 0.000999 | 0.854146 |
4 | netmats_parcor | SVR | age | 50 | 0.220397 | 0.958377 | 0.310812 | 0.072927 | 0.000999 | 0.016983 |
... | ... | ... | ... | ... | ... | ... | ... | ... | ... | ... |
5995 | netmats_pearson | SVR | PicSeq_AgeAdj | 501 | 0.167868 | 0.469294 | 0.119368 | 0.000999 | 0.000999 | 0.004995 |
5996 | netmats_pearson | SVR | PicSeq_AgeAdj | 501 | 0.134985 | 0.493440 | 0.222072 | 0.001998 | 0.000999 | 0.000999 |
5997 | netmats_pearson | SVR | PicSeq_AgeAdj | 501 | 0.155819 | 0.522091 | 0.223508 | 0.000999 | 0.000999 | 0.000999 |
5998 | netmats_pearson | SVR | PicSeq_AgeAdj | 501 | 0.065164 | 0.484062 | 0.215828 | 0.081918 | 0.000999 | 0.000999 |
5999 | netmats_pearson | SVR | PicSeq_AgeAdj | 501 | 0.044413 | 0.398393 | 0.100157 | 0.139860 | 0.000999 | 0.018981 |
6000 rows × 10 columns
%%time
random_state = 42
n_bootstrap = 100
features = {
'netmats_parcor': netmats_parcor,
'netmats_pearson': netmats_pearson
}
models = {
'PCA_Ridge': Pipeline([('pca', PCA(n_components=0.5)),
('ridge', Ridge())])
}
# We aggregate all results here:
df = pd.DataFrame(columns=['connectivity','model','target','n','r_discovery_cv','r_discovery_overfit','r_replication','p_discovery_cv','p_discovery_overfit','p_replication'])
for feature_set in features:
for model in models:
for target_var in ['age', 'CogTotalComp_AgeAdj', 'PMAT24_A_CR', 'Flanker_AgeAdj', 'CardSort_AgeAdj', 'PicSeq_AgeAdj']:
for sample_size in [50, 100, 200, 300, 'max']:
print('*****************************************************************')
print(feature_set, model, target_var, sample_size)
X, y = create_data(target=target_var, feature_data=features[feature_set])
if sample_size=='max':
sample_size = int(len(y)/2)
# create random seeds for each bootstrap iteration for reproducibility
rng = np.random.default_rng(random_state)
random_sates = rng.integers(np.iinfo(np.int32).max, size=n_bootstrap)
# run bootstrap iterations in parallel
r_discovery_cv, r_discovery_overfit, r_replication, p_discovery_cv, p_discovery_overfit, p_replication = zip(
*Parallel(n_jobs=-1)(
delayed(bootstrap_workhorse)(X, y, sample_size, models[model], seed) for seed in random_sates))
tmp_data_frame = pd.DataFrame({
'connectivity' : feature_set,
'model' : model,
'target' : target_var,
'n' : sample_size,
'r_discovery_cv': r_discovery_cv,
'r_discovery_overfit': r_discovery_overfit,
'r_replication': r_replication,
'p_discovery_cv': p_discovery_cv,
'p_discovery_overfit': p_discovery_overfit,
'p_replication': p_replication
})
#sns.scatterplot(x='r_replication', y='r_discovery_cv', data=tmp_data_frame)
#plt.ylabel('in-sample (r)')
#plt.xlabel('out-of-sample (r_pred)')
#plt.show()
print(tmp_data_frame.r_discovery_cv.mean(), tmp_data_frame.r_replication.mean())
for alpha in [0.05, 0.01, 0.005, 0.001]:
print('Replicability at alpha =', alpha, ':',
(tmp_data_frame.loc[tmp_data_frame['p_discovery_cv']<alpha,'p_replication']<alpha).sum() / (tmp_data_frame['p_discovery_cv']<0.05).sum() * 100, '%')
df = pd.concat((df, tmp_data_frame))
df.reset_index(drop=True, inplace=True)
df.to_csv('res_PCA_Ridge.csv')
df
***************************************************************** netmats_parcor PCA_Ridge age 50 0.12862642959256065 0.2154842766278765 Replicability at alpha = 0.05 : 42.42424242424242 % Replicability at alpha = 0.01 : 9.090909090909092 % Replicability at alpha = 0.005 : 9.090909090909092 % Replicability at alpha = 0.001 : 3.0303030303030303 % ***************************************************************** netmats_parcor PCA_Ridge age 100 0.25477086132155136 0.31779871366485796 Replicability at alpha = 0.05 : 92.40506329113924 % Replicability at alpha = 0.01 : 55.69620253164557 % Replicability at alpha = 0.005 : 49.36708860759494 % Replicability at alpha = 0.001 : 20.253164556962027 % ***************************************************************** netmats_parcor PCA_Ridge age 200 0.35627669437007353 0.39305369552769015 Replicability at alpha = 0.05 : 100.0 % Replicability at alpha = 0.01 : 97.9381443298969 % Replicability at alpha = 0.005 : 96.90721649484536 % Replicability at alpha = 0.001 : 94.84536082474226 % ***************************************************************** netmats_parcor PCA_Ridge age 300 0.4101893461668796 0.4237162505075437 Replicability at alpha = 0.05 : 100.0 % Replicability at alpha = 0.01 : 100.0 % Replicability at alpha = 0.005 : 100.0 % Replicability at alpha = 0.001 : 100.0 % ***************************************************************** netmats_parcor PCA_Ridge age max 0.4540208319189825 0.4729022285671206 Replicability at alpha = 0.05 : 100.0 % Replicability at alpha = 0.01 : 100.0 % Replicability at alpha = 0.005 : 100.0 % Replicability at alpha = 0.001 : 100.0 % ***************************************************************** netmats_parcor PCA_Ridge CogTotalComp_AgeAdj 50 0.08038653479440205 0.2088033936345983 Replicability at alpha = 0.05 : 50.0 % Replicability at alpha = 0.01 : 8.333333333333332 % Replicability at alpha = 0.005 : 0.0 % Replicability at alpha = 0.001 : 0.0 % ***************************************************************** netmats_parcor PCA_Ridge CogTotalComp_AgeAdj 100 0.22405235212307087 0.2895046566621751 Replicability at alpha = 0.05 : 86.76470588235294 % Replicability at alpha = 0.01 : 57.35294117647059 % Replicability at alpha = 0.005 : 45.588235294117645 % Replicability at alpha = 0.001 : 20.588235294117645 % ***************************************************************** netmats_parcor PCA_Ridge CogTotalComp_AgeAdj 200 0.3469654565519294 0.37227049270379375 Replicability at alpha = 0.05 : 100.0 % Replicability at alpha = 0.01 : 98.0 % Replicability at alpha = 0.005 : 97.0 % Replicability at alpha = 0.001 : 91.0 % ***************************************************************** netmats_parcor PCA_Ridge CogTotalComp_AgeAdj 300 0.394580608114359 0.4181672274624132 Replicability at alpha = 0.05 : 100.0 % Replicability at alpha = 0.01 : 100.0 % Replicability at alpha = 0.005 : 100.0 % Replicability at alpha = 0.001 : 100.0 % ***************************************************************** netmats_parcor PCA_Ridge CogTotalComp_AgeAdj max 0.4571302109903146 0.46566830203788606 Replicability at alpha = 0.05 : 100.0 % Replicability at alpha = 0.01 : 100.0 % Replicability at alpha = 0.005 : 100.0 % Replicability at alpha = 0.001 : 100.0 % ***************************************************************** netmats_parcor PCA_Ridge PMAT24_A_CR 50 0.040451637581210696 0.1681971587692431 Replicability at alpha = 0.05 : 33.33333333333333 % Replicability at alpha = 0.01 : 11.11111111111111 % Replicability at alpha = 0.005 : 5.555555555555555 % Replicability at alpha = 0.001 : 0.0 % ***************************************************************** netmats_parcor PCA_Ridge PMAT24_A_CR 100 0.17613326967327014 0.2138185116259917 Replicability at alpha = 0.05 : 64.28571428571429 % Replicability at alpha = 0.01 : 26.785714285714285 % Replicability at alpha = 0.005 : 19.642857142857142 % Replicability at alpha = 0.001 : 7.142857142857142 % ***************************************************************** netmats_parcor PCA_Ridge PMAT24_A_CR 200 0.2568532463650073 0.2964254772498707 Replicability at alpha = 0.05 : 100.0 % Replicability at alpha = 0.01 : 89.69072164948454 % Replicability at alpha = 0.005 : 79.38144329896907 % Replicability at alpha = 0.001 : 46.391752577319586 % ***************************************************************** netmats_parcor PCA_Ridge PMAT24_A_CR 300 0.29984139811184773 0.32176088413266357 Replicability at alpha = 0.05 : 100.0 % Replicability at alpha = 0.01 : 98.98989898989899 % Replicability at alpha = 0.005 : 98.98989898989899 % Replicability at alpha = 0.001 : 95.95959595959596 % ***************************************************************** netmats_parcor PCA_Ridge PMAT24_A_CR max 0.33670350102587215 0.35937113269593696 Replicability at alpha = 0.05 : 100.0 % Replicability at alpha = 0.01 : 100.0 % Replicability at alpha = 0.005 : 100.0 % Replicability at alpha = 0.001 : 100.0 % ***************************************************************** netmats_parcor PCA_Ridge Flanker_AgeAdj 50 -0.07091853637536359 0.07720607535843556 Replicability at alpha = 0.05 : 12.5 % Replicability at alpha = 0.01 : 0.0 % Replicability at alpha = 0.005 : 0.0 % Replicability at alpha = 0.001 : 0.0 % ***************************************************************** netmats_parcor PCA_Ridge Flanker_AgeAdj 100 0.005615783005556363 0.06775518059003204 Replicability at alpha = 0.05 : 44.44444444444444 % Replicability at alpha = 0.01 : 0.0 % Replicability at alpha = 0.005 : 0.0 % Replicability at alpha = 0.001 : 0.0 % ***************************************************************** netmats_parcor PCA_Ridge Flanker_AgeAdj 200 0.06273660019572652 0.09641221172741772 Replicability at alpha = 0.05 : 32.25806451612903 % Replicability at alpha = 0.01 : 9.67741935483871 % Replicability at alpha = 0.005 : 3.225806451612903 % Replicability at alpha = 0.001 : 0.0 % ***************************************************************** netmats_parcor PCA_Ridge Flanker_AgeAdj 300 0.09087187482240065 0.11790359018573071 Replicability at alpha = 0.05 : 64.44444444444444 % Replicability at alpha = 0.01 : 13.333333333333334 % Replicability at alpha = 0.005 : 8.88888888888889 % Replicability at alpha = 0.001 : 0.0 % ***************************************************************** netmats_parcor PCA_Ridge Flanker_AgeAdj max 0.12914070542468603 0.1493514183134212 Replicability at alpha = 0.05 : 98.85057471264368 % Replicability at alpha = 0.01 : 66.66666666666666 % Replicability at alpha = 0.005 : 54.02298850574713 % Replicability at alpha = 0.001 : 22.988505747126435 % ***************************************************************** netmats_parcor PCA_Ridge CardSort_AgeAdj 50 -0.07754051034040617 0.06491679152041394 Replicability at alpha = 0.05 : 0.0 % Replicability at alpha = 0.01 : 0.0 % Replicability at alpha = 0.005 : 0.0 % Replicability at alpha = 0.001 : 0.0 % ***************************************************************** netmats_parcor PCA_Ridge CardSort_AgeAdj 100 0.018712249243648053 0.08758842616505054 Replicability at alpha = 0.05 : 29.411764705882355 % Replicability at alpha = 0.01 : 5.88235294117647 % Replicability at alpha = 0.005 : 0.0 % Replicability at alpha = 0.001 : 0.0 % ***************************************************************** netmats_parcor PCA_Ridge CardSort_AgeAdj 200 0.07035662358792007 0.11789427180625331 Replicability at alpha = 0.05 : 51.515151515151516 % Replicability at alpha = 0.01 : 12.121212121212121 % Replicability at alpha = 0.005 : 3.0303030303030303 % Replicability at alpha = 0.001 : 0.0 % ***************************************************************** netmats_parcor PCA_Ridge CardSort_AgeAdj 300 0.11580087040106785 0.13746311888636603 Replicability at alpha = 0.05 : 70.3125 % Replicability at alpha = 0.01 : 25.0 % Replicability at alpha = 0.005 : 12.5 % Replicability at alpha = 0.001 : 3.125 % ***************************************************************** netmats_parcor PCA_Ridge CardSort_AgeAdj max 0.14556125018818641 0.1631270153290279 Replicability at alpha = 0.05 : 100.0 % Replicability at alpha = 0.01 : 83.14606741573034 % Replicability at alpha = 0.005 : 75.28089887640449 % Replicability at alpha = 0.001 : 32.58426966292135 % ***************************************************************** netmats_parcor PCA_Ridge PicSeq_AgeAdj 50 -0.03465592150037136 0.09271727133157677 Replicability at alpha = 0.05 : 0.0 % Replicability at alpha = 0.01 : 0.0 % Replicability at alpha = 0.005 : 0.0 % Replicability at alpha = 0.001 : 0.0 % ***************************************************************** netmats_parcor PCA_Ridge PicSeq_AgeAdj 100 0.05238772733818926 0.13754007569229276 Replicability at alpha = 0.05 : 27.27272727272727 % Replicability at alpha = 0.01 : 0.0 % Replicability at alpha = 0.005 : 0.0 % Replicability at alpha = 0.001 : 0.0 % ***************************************************************** netmats_parcor PCA_Ridge PicSeq_AgeAdj 200 0.12670659281540955 0.16615986159805035 Replicability at alpha = 0.05 : 79.62962962962963 % Replicability at alpha = 0.01 : 33.33333333333333 % Replicability at alpha = 0.005 : 14.814814814814813 % Replicability at alpha = 0.001 : 5.555555555555555 % ***************************************************************** netmats_parcor PCA_Ridge PicSeq_AgeAdj 300 0.17949670331165068 0.19971331345691073 Replicability at alpha = 0.05 : 100.0 % Replicability at alpha = 0.01 : 73.91304347826086 % Replicability at alpha = 0.005 : 58.69565217391305 % Replicability at alpha = 0.001 : 21.73913043478261 % ***************************************************************** netmats_parcor PCA_Ridge PicSeq_AgeAdj max 0.2237731212289612 0.2395956860566576 Replicability at alpha = 0.05 : 100.0 % Replicability at alpha = 0.01 : 100.0 % Replicability at alpha = 0.005 : 98.0 % Replicability at alpha = 0.001 : 94.0 % ***************************************************************** netmats_pearson PCA_Ridge age 50 -0.07033783683513443 0.05208844407915976 Replicability at alpha = 0.05 : 25.0 % Replicability at alpha = 0.01 : 0.0 % Replicability at alpha = 0.005 : 0.0 % Replicability at alpha = 0.001 : 0.0 % ***************************************************************** netmats_pearson PCA_Ridge age 100 0.027036163090746138 0.08323450077354823 Replicability at alpha = 0.05 : 53.333333333333336 % Replicability at alpha = 0.01 : 13.333333333333334 % Replicability at alpha = 0.005 : 6.666666666666667 % Replicability at alpha = 0.001 : 0.0 % ***************************************************************** netmats_pearson PCA_Ridge age 200 0.08459880636222822 0.13029806694318494 Replicability at alpha = 0.05 : 57.49999999999999 % Replicability at alpha = 0.01 : 17.5 % Replicability at alpha = 0.005 : 15.0 % Replicability at alpha = 0.001 : 5.0 % ***************************************************************** netmats_pearson PCA_Ridge age 300 0.13871694003384683 0.15373348658846553 Replicability at alpha = 0.05 : 84.0 % Replicability at alpha = 0.01 : 40.0 % Replicability at alpha = 0.005 : 30.666666666666664 % Replicability at alpha = 0.001 : 9.333333333333334 % ***************************************************************** netmats_pearson PCA_Ridge age max 0.1716832660599727 0.19326585083737233 Replicability at alpha = 0.05 : 100.0 % Replicability at alpha = 0.01 : 86.31578947368422 % Replicability at alpha = 0.005 : 83.15789473684211 % Replicability at alpha = 0.001 : 58.94736842105262 % ***************************************************************** netmats_pearson PCA_Ridge CogTotalComp_AgeAdj 50 -0.015900042073574733 0.12537223973643288 Replicability at alpha = 0.05 : 33.33333333333333 % Replicability at alpha = 0.01 : 6.666666666666667 % Replicability at alpha = 0.005 : 0.0 % Replicability at alpha = 0.001 : 0.0 % ***************************************************************** netmats_pearson PCA_Ridge CogTotalComp_AgeAdj 100 0.07330761311582074 0.12252993727182616 Replicability at alpha = 0.05 : 38.095238095238095 % Replicability at alpha = 0.01 : 4.761904761904762 % Replicability at alpha = 0.005 : 4.761904761904762 % Replicability at alpha = 0.001 : 0.0 % ***************************************************************** netmats_pearson PCA_Ridge CogTotalComp_AgeAdj 200 0.1563379565636732 0.1867342320264146 Replicability at alpha = 0.05 : 84.375 % Replicability at alpha = 0.01 : 51.5625 % Replicability at alpha = 0.005 : 39.0625 % Replicability at alpha = 0.001 : 10.9375 % ***************************************************************** netmats_pearson PCA_Ridge CogTotalComp_AgeAdj 300 0.20081744112608085 0.23265151265990663 Replicability at alpha = 0.05 : 97.82608695652173 % Replicability at alpha = 0.01 : 86.95652173913044 % Replicability at alpha = 0.005 : 78.26086956521739 % Replicability at alpha = 0.001 : 47.82608695652174 % ***************************************************************** netmats_pearson PCA_Ridge CogTotalComp_AgeAdj max 0.25886811920206454 0.2730870648573816 Replicability at alpha = 0.05 : 100.0 % Replicability at alpha = 0.01 : 100.0 % Replicability at alpha = 0.005 : 99.0 % Replicability at alpha = 0.001 : 98.0 % ***************************************************************** netmats_pearson PCA_Ridge PMAT24_A_CR 50 -0.04298162141053135 0.07394237957184593 Replicability at alpha = 0.05 : 12.5 % Replicability at alpha = 0.01 : 0.0 % Replicability at alpha = 0.005 : 0.0 % Replicability at alpha = 0.001 : 0.0 % ***************************************************************** netmats_pearson PCA_Ridge PMAT24_A_CR 100 0.06117974861322116 0.10045635022584638 Replicability at alpha = 0.05 : 37.03703703703704 % Replicability at alpha = 0.01 : 11.11111111111111 % Replicability at alpha = 0.005 : 3.7037037037037033 % Replicability at alpha = 0.001 : 0.0 % ***************************************************************** netmats_pearson PCA_Ridge PMAT24_A_CR 200 0.1197741723595705 0.16950311446524036 Replicability at alpha = 0.05 : 88.46153846153845 % Replicability at alpha = 0.01 : 40.38461538461539 % Replicability at alpha = 0.005 : 26.923076923076923 % Replicability at alpha = 0.001 : 11.538461538461538 % ***************************************************************** netmats_pearson PCA_Ridge PMAT24_A_CR 300 0.17461673921998128 0.21369688427862735 Replicability at alpha = 0.05 : 95.06172839506173 % Replicability at alpha = 0.01 : 76.5432098765432 % Replicability at alpha = 0.005 : 70.37037037037037 % Replicability at alpha = 0.001 : 50.617283950617285 % ***************************************************************** netmats_pearson PCA_Ridge PMAT24_A_CR max 0.23378222844128282 0.2597860242096012 Replicability at alpha = 0.05 : 100.0 % Replicability at alpha = 0.01 : 100.0 % Replicability at alpha = 0.005 : 99.0 % Replicability at alpha = 0.001 : 96.0 % ***************************************************************** netmats_pearson PCA_Ridge Flanker_AgeAdj 50 -0.08966561235532806 0.0587853435603973 Replicability at alpha = 0.05 : 28.57142857142857 % Replicability at alpha = 0.01 : 0.0 % Replicability at alpha = 0.005 : 0.0 % Replicability at alpha = 0.001 : 0.0 % ***************************************************************** netmats_pearson PCA_Ridge Flanker_AgeAdj 100 -0.02831283378530495 0.06525118726066839 Replicability at alpha = 0.05 : 33.33333333333333 % Replicability at alpha = 0.01 : 8.333333333333332 % Replicability at alpha = 0.005 : 0.0 % Replicability at alpha = 0.001 : 0.0 % ***************************************************************** netmats_pearson PCA_Ridge Flanker_AgeAdj 200 0.025597944220023684 0.07428468978621386 Replicability at alpha = 0.05 : 35.294117647058826 % Replicability at alpha = 0.01 : 0.0 % Replicability at alpha = 0.005 : 0.0 % Replicability at alpha = 0.001 : 0.0 % ***************************************************************** netmats_pearson PCA_Ridge Flanker_AgeAdj 300 0.06085386596480839 0.09777610396407102 Replicability at alpha = 0.05 : 46.666666666666664 % Replicability at alpha = 0.01 : 6.666666666666667 % Replicability at alpha = 0.005 : 0.0 % Replicability at alpha = 0.001 : 0.0 % ***************************************************************** netmats_pearson PCA_Ridge Flanker_AgeAdj max 0.08870168174818309 0.12150056046458278 Replicability at alpha = 0.05 : 91.80327868852459 % Replicability at alpha = 0.01 : 31.147540983606557 % Replicability at alpha = 0.005 : 14.754098360655737 % Replicability at alpha = 0.001 : 0.0 % ***************************************************************** netmats_pearson PCA_Ridge CardSort_AgeAdj 50 -0.0987911396594271 0.06275629641084896 Replicability at alpha = 0.05 : 28.57142857142857 % Replicability at alpha = 0.01 : 0.0 % Replicability at alpha = 0.005 : 0.0 % Replicability at alpha = 0.001 : 0.0 % ***************************************************************** netmats_pearson PCA_Ridge CardSort_AgeAdj 100 -0.008839113454863066 0.08038690011764064 Replicability at alpha = 0.05 : 36.36363636363637 % Replicability at alpha = 0.01 : 0.0 % Replicability at alpha = 0.005 : 0.0 % Replicability at alpha = 0.001 : 0.0 % ***************************************************************** netmats_pearson PCA_Ridge CardSort_AgeAdj 200 0.05671788351608817 0.10368928506639426 Replicability at alpha = 0.05 : 50.0 % Replicability at alpha = 0.01 : 7.142857142857142 % Replicability at alpha = 0.005 : 3.571428571428571 % Replicability at alpha = 0.001 : 0.0 % ***************************************************************** netmats_pearson PCA_Ridge CardSort_AgeAdj 300 0.08260099397657047 0.12345495479077792 Replicability at alpha = 0.05 : 77.08333333333334 % Replicability at alpha = 0.01 : 18.75 % Replicability at alpha = 0.005 : 8.333333333333332 % Replicability at alpha = 0.001 : 0.0 % ***************************************************************** netmats_pearson PCA_Ridge CardSort_AgeAdj max 0.12050015291681769 0.13965755471539476 Replicability at alpha = 0.05 : 94.87179487179486 % Replicability at alpha = 0.01 : 53.84615384615385 % Replicability at alpha = 0.005 : 35.8974358974359 % Replicability at alpha = 0.001 : 8.974358974358974 % ***************************************************************** netmats_pearson PCA_Ridge PicSeq_AgeAdj 50 -0.06504270643700373 0.08922502836890034 Replicability at alpha = 0.05 : 14.285714285714285 % Replicability at alpha = 0.01 : 0.0 % Replicability at alpha = 0.005 : 0.0 % Replicability at alpha = 0.001 : 0.0 % ***************************************************************** netmats_pearson PCA_Ridge PicSeq_AgeAdj 100 0.02639285664309291 0.08716725122436728 Replicability at alpha = 0.05 : 7.142857142857142 % Replicability at alpha = 0.01 : 0.0 % Replicability at alpha = 0.005 : 0.0 % Replicability at alpha = 0.001 : 0.0 % ***************************************************************** netmats_pearson PCA_Ridge PicSeq_AgeAdj 200 0.05070483088487841 0.09590425206641175 Replicability at alpha = 0.05 : 54.54545454545454 % Replicability at alpha = 0.01 : 9.090909090909092 % Replicability at alpha = 0.005 : 9.090909090909092 % Replicability at alpha = 0.001 : 3.0303030303030303 % ***************************************************************** netmats_pearson PCA_Ridge PicSeq_AgeAdj 300 0.07961462886944858 0.11366934169342427 Replicability at alpha = 0.05 : 73.17073170731707 % Replicability at alpha = 0.01 : 17.073170731707318 % Replicability at alpha = 0.005 : 14.634146341463413 % Replicability at alpha = 0.001 : 2.4390243902439024 % ***************************************************************** netmats_pearson PCA_Ridge PicSeq_AgeAdj max 0.11189867260910015 0.12800719108692088 Replicability at alpha = 0.05 : 90.9090909090909 % Replicability at alpha = 0.01 : 38.961038961038966 % Replicability at alpha = 0.005 : 24.675324675324674 % Replicability at alpha = 0.001 : 7.792207792207792 % CPU times: user 27.8 s, sys: 6.18 s, total: 34 s Wall time: 3h 7min 45s
connectivity | model | target | n | r_discovery_cv | r_discovery_overfit | r_replication | p_discovery_cv | p_discovery_overfit | p_replication | |
---|---|---|---|---|---|---|---|---|---|---|
0 | netmats_parcor | PCA_Ridge | age | 50 | -0.260636 | 0.457983 | 0.225977 | 0.969031 | 0.001998 | 0.062937 |
1 | netmats_parcor | PCA_Ridge | age | 50 | 0.189752 | 0.687611 | 0.246691 | 0.098901 | 0.000999 | 0.049950 |
2 | netmats_parcor | PCA_Ridge | age | 50 | -0.014329 | 0.730443 | -0.045666 | 0.538462 | 0.000999 | 0.601399 |
3 | netmats_parcor | PCA_Ridge | age | 50 | 0.194567 | 0.710963 | -0.162378 | 0.101898 | 0.000999 | 0.861139 |
4 | netmats_parcor | PCA_Ridge | age | 50 | 0.223428 | 0.717670 | 0.304525 | 0.075924 | 0.000999 | 0.015984 |
... | ... | ... | ... | ... | ... | ... | ... | ... | ... | ... |
5995 | netmats_pearson | PCA_Ridge | PicSeq_AgeAdj | 501 | 0.174858 | 0.286206 | 0.086955 | 0.000999 | 0.000999 | 0.029970 |
5996 | netmats_pearson | PCA_Ridge | PicSeq_AgeAdj | 501 | 0.114124 | 0.261799 | 0.146330 | 0.006993 | 0.000999 | 0.000999 |
5997 | netmats_pearson | PCA_Ridge | PicSeq_AgeAdj | 501 | 0.193151 | 0.310800 | 0.164787 | 0.000999 | 0.000999 | 0.000999 |
5998 | netmats_pearson | PCA_Ridge | PicSeq_AgeAdj | 501 | 0.144112 | 0.285125 | 0.095999 | 0.002997 | 0.000999 | 0.018981 |
5999 | netmats_pearson | PCA_Ridge | PicSeq_AgeAdj | 501 | 0.028526 | 0.212140 | 0.091388 | 0.256743 | 0.000999 | 0.021978 |
6000 rows × 10 columns
(I guess for non-linear kernel ridge we really need thousands of participants)
%%time
from sklearn.kernel_ridge import KernelRidge
random_state = 42
n_bootstrap = 100
features = {
'netmats_parcor': netmats_parcor,
'netmats_pearson': netmats_pearson
}
models = {
'KernelRidge': KernelRidge()
}
# We aggregate all results here:
df = pd.DataFrame(columns=['connectivity','model','target','n','r_discovery_cv','r_discovery_overfit','r_replication','p_discovery_cv','p_discovery_overfit','p_replication'])
for feature_set in features:
for model in models:
for target_var in ['age', 'CogTotalComp_AgeAdj', 'PMAT24_A_CR', 'Flanker_AgeAdj', 'CardSort_AgeAdj', 'PicSeq_AgeAdj']:
for sample_size in [50, 100, 200, 300, 'max']:
print('*****************************************************************')
print(feature_set, model, target_var, sample_size)
X, y = create_data(target=target_var, feature_data=features[feature_set])
if sample_size=='max':
sample_size = int(len(y)/2)
# create random seeds for each bootstrap iteration for reproducibility
rng = np.random.default_rng(random_state)
random_sates = rng.integers(np.iinfo(np.int32).max, size=n_bootstrap)
# run bootstrap iterations in parallel
r_discovery_cv, r_discovery_overfit, r_replication, p_discovery_cv, p_discovery_overfit, p_replication = zip(
*Parallel(n_jobs=-1)(
delayed(bootstrap_workhorse)(X, y, sample_size, models[model], seed) for seed in random_sates))
tmp_data_frame = pd.DataFrame({
'connectivity' : feature_set,
'model' : model,
'target' : target_var,
'n' : sample_size,
'r_discovery_cv': r_discovery_cv,
'r_discovery_overfit': r_discovery_overfit,
'r_replication': r_replication,
'p_discovery_cv': p_discovery_cv,
'p_discovery_overfit': p_discovery_overfit,
'p_replication': p_replication
})
#sns.scatterplot(x='r_replication', y='r_discovery_cv', data=tmp_data_frame)
#plt.ylabel('in-sample (r)')
#plt.xlabel('out-of-sample (r_pred)')
#plt.show()
print(tmp_data_frame.r_discovery_cv.mean(), tmp_data_frame.r_replication.mean())
for alpha in [0.05, 0.01, 0.005, 0.001]:
print('Replicability at alpha =', alpha, ':',
(tmp_data_frame.loc[tmp_data_frame['p_discovery_cv']<alpha,'p_replication']<alpha).sum() / (tmp_data_frame['p_discovery_cv']<0.05).sum() * 100, '%')
df = pd.concat((df, tmp_data_frame))
df.reset_index(drop=True, inplace=True)
df.to_csv('res_KernelRidge.csv')
df
***************************************************************** netmats_parcor KernelRidge age 50 -0.057193958649027946 -0.00987408448972495 Replicability at alpha = 0.05 : 50.0 % Replicability at alpha = 0.01 : 0.0 % Replicability at alpha = 0.005 : 0.0 % Replicability at alpha = 0.001 : 0.0 % ***************************************************************** netmats_parcor KernelRidge age 100 0.09658448630512596 0.13168896630686272 Replicability at alpha = 0.05 : 42.857142857142854 % Replicability at alpha = 0.01 : 14.285714285714285 % Replicability at alpha = 0.005 : 7.142857142857142 % Replicability at alpha = 0.001 : 0.0 % ***************************************************************** netmats_parcor KernelRidge age 200 0.2614177407654067 0.28952399198813356 Replicability at alpha = 0.05 : 98.93617021276596 % Replicability at alpha = 0.01 : 91.48936170212765 % Replicability at alpha = 0.005 : 88.29787234042553 % Replicability at alpha = 0.001 : 61.702127659574465 % ***************************************************************** netmats_parcor KernelRidge age 300 0.34820647180491887 0.36392508722075023 Replicability at alpha = 0.05 : 100.0 % Replicability at alpha = 0.01 : 100.0 % Replicability at alpha = 0.005 : 100.0 % Replicability at alpha = 0.001 : 98.0 % ***************************************************************** netmats_parcor KernelRidge age max 0.4263848450153784 0.44677547947947926 Replicability at alpha = 0.05 : 100.0 % Replicability at alpha = 0.01 : 100.0 % Replicability at alpha = 0.005 : 100.0 % Replicability at alpha = 0.001 : 100.0 % ***************************************************************** netmats_parcor KernelRidge CogTotalComp_AgeAdj 50 0.16461941968123622 0.21614010860768615 Replicability at alpha = 0.05 : 51.724137931034484 % Replicability at alpha = 0.01 : 17.24137931034483 % Replicability at alpha = 0.005 : 3.4482758620689653 % Replicability at alpha = 0.001 : 0.0 % ***************************************************************** netmats_parcor KernelRidge CogTotalComp_AgeAdj 100 0.2472238434443695 0.2807486381270254 Replicability at alpha = 0.05 : 87.34177215189874 % Replicability at alpha = 0.01 : 56.9620253164557 % Replicability at alpha = 0.005 : 41.77215189873418 % Replicability at alpha = 0.001 : 7.59493670886076 % ***************************************************************** netmats_parcor KernelRidge CogTotalComp_AgeAdj 200 0.3578715556631745 0.36598311730682853 Replicability at alpha = 0.05 : 100.0 % Replicability at alpha = 0.01 : 98.0 % Replicability at alpha = 0.005 : 97.0 % Replicability at alpha = 0.001 : 92.0 % ***************************************************************** netmats_parcor KernelRidge CogTotalComp_AgeAdj 300 0.39597183183617907 0.41192906682791586 Replicability at alpha = 0.05 : 100.0 % Replicability at alpha = 0.01 : 100.0 % Replicability at alpha = 0.005 : 100.0 % Replicability at alpha = 0.001 : 100.0 % ***************************************************************** netmats_parcor KernelRidge CogTotalComp_AgeAdj max 0.44961119286733026 0.4572459608388989 Replicability at alpha = 0.05 : 100.0 % Replicability at alpha = 0.01 : 100.0 % Replicability at alpha = 0.005 : 100.0 % Replicability at alpha = 0.001 : 100.0 % ***************************************************************** netmats_parcor KernelRidge PMAT24_A_CR 50 0.14813887428146885 0.21099385633002932 Replicability at alpha = 0.05 : 43.333333333333336 % Replicability at alpha = 0.01 : 10.0 % Replicability at alpha = 0.005 : 3.3333333333333335 % Replicability at alpha = 0.001 : 0.0 % ***************************************************************** netmats_parcor KernelRidge PMAT24_A_CR 100 0.20496746063666918 0.2304758314614497 Replicability at alpha = 0.05 : 78.78787878787878 % Replicability at alpha = 0.01 : 30.303030303030305 % Replicability at alpha = 0.005 : 18.181818181818183 % Replicability at alpha = 0.001 : 6.0606060606060606 % ***************************************************************** netmats_parcor KernelRidge PMAT24_A_CR 200 0.2631501764766926 0.2775964147820703 Replicability at alpha = 0.05 : 97.87234042553192 % Replicability at alpha = 0.01 : 91.48936170212765 % Replicability at alpha = 0.005 : 81.91489361702128 % Replicability at alpha = 0.001 : 50.0 % ***************************************************************** netmats_parcor KernelRidge PMAT24_A_CR 300 0.2779663785955141 0.28969927093185616 Replicability at alpha = 0.05 : 100.0 % Replicability at alpha = 0.01 : 98.9795918367347 % Replicability at alpha = 0.005 : 97.95918367346938 % Replicability at alpha = 0.001 : 87.75510204081633 % ***************************************************************** netmats_parcor KernelRidge PMAT24_A_CR max 0.2902819304653005 0.30067634628831497 Replicability at alpha = 0.05 : 100.0 % Replicability at alpha = 0.01 : 100.0 % Replicability at alpha = 0.005 : 100.0 % Replicability at alpha = 0.001 : 100.0 % ***************************************************************** netmats_parcor KernelRidge Flanker_AgeAdj 50 0.0075846654159675165 0.04546731281631298 Replicability at alpha = 0.05 : 0.0 % Replicability at alpha = 0.01 : 0.0 % Replicability at alpha = 0.005 : 0.0 % Replicability at alpha = 0.001 : 0.0 % ***************************************************************** netmats_parcor KernelRidge Flanker_AgeAdj 100 0.028920373369964638 0.05345742707646986 Replicability at alpha = 0.05 : 10.0 % Replicability at alpha = 0.01 : 10.0 % Replicability at alpha = 0.005 : 0.0 % Replicability at alpha = 0.001 : 0.0 % ***************************************************************** netmats_parcor KernelRidge Flanker_AgeAdj 200 0.0691467268283832 0.0825846781350421 Replicability at alpha = 0.05 : 48.0 % Replicability at alpha = 0.01 : 8.0 % Replicability at alpha = 0.005 : 4.0 % Replicability at alpha = 0.001 : 0.0 % ***************************************************************** netmats_parcor KernelRidge Flanker_AgeAdj 300 0.10120923004116555 0.10112037498191373 Replicability at alpha = 0.05 : 53.2258064516129 % Replicability at alpha = 0.01 : 6.451612903225806 % Replicability at alpha = 0.005 : 3.225806451612903 % Replicability at alpha = 0.001 : 0.0 % ***************************************************************** netmats_parcor KernelRidge Flanker_AgeAdj max 0.12540117494470207 0.1348979072008615 Replicability at alpha = 0.05 : 97.72727272727273 % Replicability at alpha = 0.01 : 62.5 % Replicability at alpha = 0.005 : 39.77272727272727 % Replicability at alpha = 0.001 : 9.090909090909092 % ***************************************************************** netmats_parcor KernelRidge CardSort_AgeAdj 50 0.009317221487281207 0.023026382782829466 Replicability at alpha = 0.05 : 11.11111111111111 % Replicability at alpha = 0.01 : 0.0 % Replicability at alpha = 0.005 : 0.0 % Replicability at alpha = 0.001 : 0.0 % ***************************************************************** netmats_parcor KernelRidge CardSort_AgeAdj 100 0.014968953010153987 0.0482418884153565 Replicability at alpha = 0.05 : 10.0 % Replicability at alpha = 0.01 : 0.0 % Replicability at alpha = 0.005 : 0.0 % Replicability at alpha = 0.001 : 0.0 % ***************************************************************** netmats_parcor KernelRidge CardSort_AgeAdj 200 0.05981767317394407 0.09174048910677236 Replicability at alpha = 0.05 : 18.51851851851852 % Replicability at alpha = 0.01 : 3.7037037037037033 % Replicability at alpha = 0.005 : 0.0 % Replicability at alpha = 0.001 : 0.0 % ***************************************************************** netmats_parcor KernelRidge CardSort_AgeAdj 300 0.10093466532119588 0.11857549990968218 Replicability at alpha = 0.05 : 56.60377358490566 % Replicability at alpha = 0.01 : 11.320754716981133 % Replicability at alpha = 0.005 : 5.660377358490567 % Replicability at alpha = 0.001 : 0.0 % ***************************************************************** netmats_parcor KernelRidge CardSort_AgeAdj max 0.1293337462694885 0.14595524963198883 Replicability at alpha = 0.05 : 97.82608695652173 % Replicability at alpha = 0.01 : 60.86956521739131 % Replicability at alpha = 0.005 : 48.91304347826087 % Replicability at alpha = 0.001 : 20.652173913043477 % ***************************************************************** netmats_parcor KernelRidge PicSeq_AgeAdj 50 0.08077162930529345 0.10250886586647631 Replicability at alpha = 0.05 : 14.285714285714285 % Replicability at alpha = 0.01 : 0.0 % Replicability at alpha = 0.005 : 0.0 % Replicability at alpha = 0.001 : 0.0 % ***************************************************************** netmats_parcor KernelRidge PicSeq_AgeAdj 100 0.08682538430366779 0.1318802860588839 Replicability at alpha = 0.05 : 28.57142857142857 % Replicability at alpha = 0.01 : 4.761904761904762 % Replicability at alpha = 0.005 : 0.0 % Replicability at alpha = 0.001 : 0.0 % ***************************************************************** netmats_parcor KernelRidge PicSeq_AgeAdj 200 0.13918565120434348 0.16332217274757407 Replicability at alpha = 0.05 : 76.27118644067797 % Replicability at alpha = 0.01 : 35.59322033898305 % Replicability at alpha = 0.005 : 20.33898305084746 % Replicability at alpha = 0.001 : 8.47457627118644 % ***************************************************************** netmats_parcor KernelRidge PicSeq_AgeAdj 300 0.1616184609303758 0.1679459248689494 Replicability at alpha = 0.05 : 95.0 % Replicability at alpha = 0.01 : 62.5 % Replicability at alpha = 0.005 : 43.75 % Replicability at alpha = 0.001 : 12.5 % ***************************************************************** netmats_parcor KernelRidge PicSeq_AgeAdj max 0.17643679268263768 0.18510096875677107 Replicability at alpha = 0.05 : 100.0 % Replicability at alpha = 0.01 : 92.85714285714286 % Replicability at alpha = 0.005 : 89.79591836734694 % Replicability at alpha = 0.001 : 60.204081632653065 % ***************************************************************** netmats_pearson KernelRidge age 50 0.03807088569469099 0.051053369484442765 Replicability at alpha = 0.05 : 0.0 % Replicability at alpha = 0.01 : 0.0 % Replicability at alpha = 0.005 : 0.0 % Replicability at alpha = 0.001 : 0.0 % ***************************************************************** netmats_pearson KernelRidge age 100 0.14588536175759856 0.16265150493267025 Replicability at alpha = 0.05 : 36.95652173913043 % Replicability at alpha = 0.01 : 10.869565217391305 % Replicability at alpha = 0.005 : 4.3478260869565215 % Replicability at alpha = 0.001 : 2.1739130434782608 % ***************************************************************** netmats_pearson KernelRidge age 200 0.2512611311481705 0.29053642900482446 Replicability at alpha = 0.05 : 100.0 % Replicability at alpha = 0.01 : 87.36842105263159 % Replicability at alpha = 0.005 : 80.0 % Replicability at alpha = 0.001 : 50.526315789473685 % ***************************************************************** netmats_pearson KernelRidge age 300 0.3344382552835024 0.3422570594799024 Replicability at alpha = 0.05 : 100.0 % Replicability at alpha = 0.01 : 100.0 % Replicability at alpha = 0.005 : 99.0 % Replicability at alpha = 0.001 : 98.0 % ***************************************************************** netmats_pearson KernelRidge age max 0.38955254523281985 0.4015748165361232 Replicability at alpha = 0.05 : 100.0 % Replicability at alpha = 0.01 : 100.0 % Replicability at alpha = 0.005 : 100.0 % Replicability at alpha = 0.001 : 100.0 % ***************************************************************** netmats_pearson KernelRidge CogTotalComp_AgeAdj 50 0.10898811853225036 0.17056469744828676 Replicability at alpha = 0.05 : 25.0 % Replicability at alpha = 0.01 : 0.0 % Replicability at alpha = 0.005 : 0.0 % Replicability at alpha = 0.001 : 0.0 % ***************************************************************** netmats_pearson KernelRidge CogTotalComp_AgeAdj 100 0.20205309843639846 0.21728491008449477 Replicability at alpha = 0.05 : 65.67164179104478 % Replicability at alpha = 0.01 : 23.88059701492537 % Replicability at alpha = 0.005 : 13.432835820895523 % Replicability at alpha = 0.001 : 4.477611940298507 % ***************************************************************** netmats_pearson KernelRidge CogTotalComp_AgeAdj 200 0.27308874443055137 0.29179432013421225 Replicability at alpha = 0.05 : 98.96907216494846 % Replicability at alpha = 0.01 : 85.56701030927834 % Replicability at alpha = 0.005 : 78.35051546391753 % Replicability at alpha = 0.001 : 56.70103092783505 % ***************************************************************** netmats_pearson KernelRidge CogTotalComp_AgeAdj 300 0.31640859488930073 0.32841860688252394 Replicability at alpha = 0.05 : 100.0 % Replicability at alpha = 0.01 : 97.97979797979798 % Replicability at alpha = 0.005 : 96.96969696969697 % Replicability at alpha = 0.001 : 94.94949494949495 % ***************************************************************** netmats_pearson KernelRidge CogTotalComp_AgeAdj max 0.3575561751978506 0.3671993850928954 Replicability at alpha = 0.05 : 100.0 % Replicability at alpha = 0.01 : 100.0 % Replicability at alpha = 0.005 : 100.0 % Replicability at alpha = 0.001 : 100.0 % ***************************************************************** netmats_pearson KernelRidge PMAT24_A_CR 50 0.11997847427661017 0.1481908792506186 Replicability at alpha = 0.05 : 18.51851851851852 % Replicability at alpha = 0.01 : 3.7037037037037033 % Replicability at alpha = 0.005 : 0.0 % Replicability at alpha = 0.001 : 0.0 % ***************************************************************** netmats_pearson KernelRidge PMAT24_A_CR 100 0.1719519832482972 0.1808128528981827 Replicability at alpha = 0.05 : 65.38461538461539 % Replicability at alpha = 0.01 : 17.307692307692307 % Replicability at alpha = 0.005 : 9.615384615384617 % Replicability at alpha = 0.001 : 1.9230769230769231 % ***************************************************************** netmats_pearson KernelRidge PMAT24_A_CR 200 0.20179855570134017 0.22255128258509796 Replicability at alpha = 0.05 : 96.1038961038961 % Replicability at alpha = 0.01 : 71.42857142857143 % Replicability at alpha = 0.005 : 57.14285714285714 % Replicability at alpha = 0.001 : 25.97402597402597 % ***************************************************************** netmats_pearson KernelRidge PMAT24_A_CR 300 0.21903026447013152 0.23393566883959757 Replicability at alpha = 0.05 : 100.0 % Replicability at alpha = 0.01 : 87.5 % Replicability at alpha = 0.005 : 78.125 % Replicability at alpha = 0.001 : 51.041666666666664 % ***************************************************************** netmats_pearson KernelRidge PMAT24_A_CR max 0.2351755373833093 0.24485954528661583 Replicability at alpha = 0.05 : 100.0 % Replicability at alpha = 0.01 : 98.0 % Replicability at alpha = 0.005 : 97.0 % Replicability at alpha = 0.001 : 93.0 % ***************************************************************** netmats_pearson KernelRidge Flanker_AgeAdj 50 -0.023716694800647625 0.03436903554505712 Replicability at alpha = 0.05 : 0.0 % Replicability at alpha = 0.01 : 0.0 % Replicability at alpha = 0.005 : 0.0 % Replicability at alpha = 0.001 : 0.0 % ***************************************************************** netmats_pearson KernelRidge Flanker_AgeAdj 100 0.047738595901451866 0.041321010605706816 Replicability at alpha = 0.05 : 25.0 % Replicability at alpha = 0.01 : 6.25 % Replicability at alpha = 0.005 : 6.25 % Replicability at alpha = 0.001 : 0.0 % ***************************************************************** netmats_pearson KernelRidge Flanker_AgeAdj 200 0.06940037145726707 0.07224079084275808 Replicability at alpha = 0.05 : 22.58064516129032 % Replicability at alpha = 0.01 : 3.225806451612903 % Replicability at alpha = 0.005 : 0.0 % Replicability at alpha = 0.001 : 0.0 % ***************************************************************** netmats_pearson KernelRidge Flanker_AgeAdj 300 0.0862787584310521 0.08097658093717942 Replicability at alpha = 0.05 : 37.5 % Replicability at alpha = 0.01 : 10.0 % Replicability at alpha = 0.005 : 2.5 % Replicability at alpha = 0.001 : 0.0 % ***************************************************************** netmats_pearson KernelRidge Flanker_AgeAdj max 0.1112004770125567 0.10816305123893417 Replicability at alpha = 0.05 : 73.75 % Replicability at alpha = 0.01 : 31.25 % Replicability at alpha = 0.005 : 22.5 % Replicability at alpha = 0.001 : 5.0 % ***************************************************************** netmats_pearson KernelRidge CardSort_AgeAdj 50 0.0335581506867711 0.04544280100403737 Replicability at alpha = 0.05 : 11.11111111111111 % Replicability at alpha = 0.01 : 0.0 % Replicability at alpha = 0.005 : 0.0 % Replicability at alpha = 0.001 : 0.0 % ***************************************************************** netmats_pearson KernelRidge CardSort_AgeAdj 100 0.05281205291976231 0.08257060057799995 Replicability at alpha = 0.05 : 16.666666666666664 % Replicability at alpha = 0.01 : 0.0 % Replicability at alpha = 0.005 : 0.0 % Replicability at alpha = 0.001 : 0.0 % ***************************************************************** netmats_pearson KernelRidge CardSort_AgeAdj 200 0.09027386462883383 0.09905827054440616 Replicability at alpha = 0.05 : 37.142857142857146 % Replicability at alpha = 0.01 : 2.857142857142857 % Replicability at alpha = 0.005 : 0.0 % Replicability at alpha = 0.001 : 0.0 % ***************************************************************** netmats_pearson KernelRidge CardSort_AgeAdj 300 0.10560109360465793 0.12122763077497249 Replicability at alpha = 0.05 : 67.9245283018868 % Replicability at alpha = 0.01 : 20.754716981132077 % Replicability at alpha = 0.005 : 5.660377358490567 % Replicability at alpha = 0.001 : 0.0 % ***************************************************************** netmats_pearson KernelRidge CardSort_AgeAdj max 0.11828841565176523 0.13137077010906986 Replicability at alpha = 0.05 : 94.04761904761905 % Replicability at alpha = 0.01 : 55.952380952380956 % Replicability at alpha = 0.005 : 39.285714285714285 % Replicability at alpha = 0.001 : 5.952380952380952 % ***************************************************************** netmats_pearson KernelRidge PicSeq_AgeAdj 50 0.10550764578363739 0.11506666548398405 Replicability at alpha = 0.05 : 21.73913043478261 % Replicability at alpha = 0.01 : 0.0 % Replicability at alpha = 0.005 : 0.0 % Replicability at alpha = 0.001 : 0.0 % ***************************************************************** netmats_pearson KernelRidge PicSeq_AgeAdj 100 0.10576685757034143 0.13110872942606946 Replicability at alpha = 0.05 : 25.71428571428571 % Replicability at alpha = 0.01 : 8.571428571428571 % Replicability at alpha = 0.005 : 5.714285714285714 % Replicability at alpha = 0.001 : 2.857142857142857 % ***************************************************************** netmats_pearson KernelRidge PicSeq_AgeAdj 200 0.13549566618083017 0.14498112085703446 Replicability at alpha = 0.05 : 56.666666666666664 % Replicability at alpha = 0.01 : 23.333333333333332 % Replicability at alpha = 0.005 : 11.666666666666666 % Replicability at alpha = 0.001 : 3.3333333333333335 % ***************************************************************** netmats_pearson KernelRidge PicSeq_AgeAdj 300 0.1587030766340557 0.16313167980333013 Replicability at alpha = 0.05 : 86.25 % Replicability at alpha = 0.01 : 56.25 % Replicability at alpha = 0.005 : 43.75 % Replicability at alpha = 0.001 : 12.5 % ***************************************************************** netmats_pearson KernelRidge PicSeq_AgeAdj max 0.18131959952160132 0.19307472604560239 Replicability at alpha = 0.05 : 100.0 % Replicability at alpha = 0.01 : 91.83673469387756 % Replicability at alpha = 0.005 : 85.71428571428571 % Replicability at alpha = 0.001 : 59.183673469387756 % CPU times: user 22 s, sys: 5.66 s, total: 27.7 s Wall time: 16min 37s
connectivity | model | target | n | r_discovery_cv | r_discovery_overfit | r_replication | p_discovery_cv | p_discovery_overfit | p_replication | |
---|---|---|---|---|---|---|---|---|---|---|
0 | netmats_parcor | KernelRidge | age | 50 | -0.289530 | 1.0 | 0.097963 | 0.981019 | 0.000999 | 0.253746 |
1 | netmats_parcor | KernelRidge | age | 50 | -0.418438 | 1.0 | 0.020128 | 0.999001 | 0.000999 | 0.449550 |
2 | netmats_parcor | KernelRidge | age | 50 | -0.141016 | 1.0 | -0.140765 | 0.822178 | 0.000999 | 0.838162 |
3 | netmats_parcor | KernelRidge | age | 50 | 0.131642 | 1.0 | -0.102626 | 0.183816 | 0.000999 | 0.760240 |
4 | netmats_parcor | KernelRidge | age | 50 | 0.024683 | 1.0 | 0.203918 | 0.426573 | 0.000999 | 0.079920 |
... | ... | ... | ... | ... | ... | ... | ... | ... | ... | ... |
5995 | netmats_pearson | KernelRidge | PicSeq_AgeAdj | 501 | 0.202720 | 1.0 | 0.225979 | 0.000999 | 0.000999 | 0.000999 |
5996 | netmats_pearson | KernelRidge | PicSeq_AgeAdj | 501 | 0.223386 | 1.0 | 0.217658 | 0.000999 | 0.000999 | 0.000999 |
5997 | netmats_pearson | KernelRidge | PicSeq_AgeAdj | 501 | 0.274774 | 1.0 | 0.206499 | 0.000999 | 0.000999 | 0.000999 |
5998 | netmats_pearson | KernelRidge | PicSeq_AgeAdj | 501 | 0.166591 | 1.0 | 0.203236 | 0.000999 | 0.000999 | 0.000999 |
5999 | netmats_pearson | KernelRidge | PicSeq_AgeAdj | 501 | 0.142675 | 1.0 | 0.217554 | 0.001998 | 0.000999 | 0.000999 |
6000 rows × 10 columns
More complex models are not reviewd here, as KernelRidge doesn't seem to add a lot.
See the notebook called 'plot_results.ipynb' for more detail.