Predicting DWPC Query runtime ahead of time

In [2]:
import json

import matplotlib.pyplot
import pandas
import numpy
import seaborn
import mpld3

%matplotlib inline
In [3]:
path = '../all-features/data/metapaths.json'
with open(path) as fp:
    metapaths = json.load(fp)
In [12]:
dwpc_df = pandas.read_table('../all-features/data/dwpc.tsv.bz2')
dwpc_df.head(2)
Out[12]:
hetnet compound_id disease_id metapath PC w DWPC seconds
0 hetio-ind_perm-5 DB00014 DOID:0060073 CpDpCpD 0 0.4 0.0 1.016
1 hetio-ind DB00014 DOID:1612 CpDpCpD 0 0.4 0.0 1.067
In [14]:
# Number of queries
len(dwpc_df)
Out[14]:
22933125
In [17]:
time_df = dwpc_df.groupby('metapath').seconds.mean().reset_index()
len(time_df)
Out[17]:
1215
In [18]:
cols = ['sequential_complexity', 'optimal_join_complexity', 'midpoint_join_complexity']

rows = [[
        item['abbreviation'], 
        item['join_complexities'][item['midpoint_index']], 
        item['join_complexities'][item['optimal_join_index']],
        item['join_complexities'][-1],
        item['join_complexities'][0],
    ] for item in metapaths]
complexity_df = pandas.DataFrame(rows, columns=
    ['metapath', 'midpoint_complexity', 'optimal_complexity', 'forward_complexity', 'backward_complexity'])
complexity_df = time_df.merge(complexity_df)
complexity_df['log10_seconds_per_query'] = numpy.log10(complexity_df['seconds'])
In [19]:
complexity_df.head(2)
Out[19]:
metapath seconds midpoint_complexity optimal_complexity forward_complexity backward_complexity log10_seconds_per_query
0 CbG<rG<rGaD 0.166966 3.10150 2.859092 2.859092 3.913263 -0.777372
1 CbG<rG<rGdD 0.096123 2.90328 2.640056 2.640056 3.694227 -1.017172

sequential complexity

In [22]:
matplotlib.pyplot.figure(figsize=(10, 7))
ax = seaborn.regplot('forward_complexity', 'log10_seconds_per_query', data=complexity_df,
    lowess=True, scatter_kws={'alpha': 0.5}, line_kws={'color': 'black'}, ci=False)
points = ax.collections[0]
labels = complexity_df.metapath.tolist()
tooltip = mpld3.plugins.PointLabelTooltip(points, labels)
mpld3.plugins.connect(ax.figure, tooltip)
mpld3.display()
Out[22]:

optimal join complexity

In [24]:
matplotlib.pyplot.figure(figsize=(10, 7))
ax = seaborn.regplot('optimal_complexity', 'log10_seconds_per_query', data=complexity_df,
    lowess=True, scatter_kws={'alpha': 0.5}, line_kws={'color': 'black'}, ci=False)
points = ax.collections[0]
labels = complexity_df.metapath.tolist()
tooltip = mpld3.plugins.PointLabelTooltip(points, labels)
mpld3.plugins.connect(ax.figure, tooltip)
mpld3.display()
Out[24]:

midpoint_join_complexity

In [25]:
matplotlib.pyplot.figure(figsize=(10, 7))
ax = seaborn.regplot('midpoint_complexity', 'log10_seconds_per_query', data=complexity_df,
    lowess=True, scatter_kws={'alpha': 0.5}, line_kws={'color': 'black'}, ci=False)
points = ax.collections[0]
labels = complexity_df.metapath.tolist()
tooltip = mpld3.plugins.PointLabelTooltip(points, labels)
mpld3.plugins.connect(ax.figure, tooltip)
mpld3.display()
Out[25]: