TODO: - sampling statistics

In [1]:

%load_ext autoreload
%autoreload 2

In [2]:

from microbiome.dataset import MicrobiomeDataset
from microbiome.trajectory import MicrobiomeTrajectory
import pandas as pd

This call to matplotlib.use() has no effect because the backend has already
been chosen; matplotlib.use() must be called *before* pylab, matplotlib.pyplot,
or matplotlib.backends is imported for the first time.

The backend was *originally* set to 'module://ipykernel.pylab.backend_inline' by the following code:
  File "/home/jelena/EMAN2/envs/microbiome_toolbox/lib/python3.6/runpy.py", line 193, in _run_module_as_main
    "__main__", mod_spec)
  File "/home/jelena/EMAN2/envs/microbiome_toolbox/lib/python3.6/runpy.py", line 85, in _run_code
    exec(code, run_globals)
  File "/home/jelena/EMAN2/envs/microbiome_toolbox/lib/python3.6/site-packages/ipykernel_launcher.py", line 16, in <module>
    app.launch_new_instance()
  File "/home/jelena/EMAN2/envs/microbiome_toolbox/lib/python3.6/site-packages/traitlets/config/application.py", line 664, in launch_instance
    app.start()
  File "/home/jelena/EMAN2/envs/microbiome_toolbox/lib/python3.6/site-packages/ipykernel/kernelapp.py", line 612, in start
    self.io_loop.start()
  File "/home/jelena/EMAN2/envs/microbiome_toolbox/lib/python3.6/site-packages/tornado/platform/asyncio.py", line 199, in start
    self.asyncio_loop.run_forever()
  File "/home/jelena/EMAN2/envs/microbiome_toolbox/lib/python3.6/asyncio/base_events.py", line 427, in run_forever
    self._run_once()
  File "/home/jelena/EMAN2/envs/microbiome_toolbox/lib/python3.6/asyncio/base_events.py", line 1440, in _run_once
    handle._run()
  File "/home/jelena/EMAN2/envs/microbiome_toolbox/lib/python3.6/asyncio/events.py", line 145, in _run
    self._callback(*self._args)
  File "/home/jelena/EMAN2/envs/microbiome_toolbox/lib/python3.6/site-packages/tornado/ioloop.py", line 688, in <lambda>
    lambda f: self._run_callback(functools.partial(callback, future))
  File "/home/jelena/EMAN2/envs/microbiome_toolbox/lib/python3.6/site-packages/tornado/ioloop.py", line 741, in _run_callback
    ret = callback()
  File "/home/jelena/EMAN2/envs/microbiome_toolbox/lib/python3.6/site-packages/tornado/gen.py", line 814, in inner
    self.ctx_run(self.run)
  File "/home/jelena/EMAN2/envs/microbiome_toolbox/lib/python3.6/site-packages/tornado/gen.py", line 162, in _fake_ctx_run
    return f(*args, **kw)
  File "/home/jelena/EMAN2/envs/microbiome_toolbox/lib/python3.6/site-packages/tornado/gen.py", line 775, in run
    yielded = self.gen.send(value)
  File "/home/jelena/EMAN2/envs/microbiome_toolbox/lib/python3.6/site-packages/ipykernel/kernelbase.py", line 381, in dispatch_queue
    yield self.process_one()
  File "/home/jelena/EMAN2/envs/microbiome_toolbox/lib/python3.6/site-packages/tornado/gen.py", line 250, in wrapper
    runner = Runner(ctx_run, result, future, yielded)
  File "/home/jelena/EMAN2/envs/microbiome_toolbox/lib/python3.6/site-packages/tornado/gen.py", line 741, in __init__
    self.ctx_run(self.run)
  File "/home/jelena/EMAN2/envs/microbiome_toolbox/lib/python3.6/site-packages/tornado/gen.py", line 162, in _fake_ctx_run
    return f(*args, **kw)
  File "/home/jelena/EMAN2/envs/microbiome_toolbox/lib/python3.6/site-packages/tornado/gen.py", line 775, in run
    yielded = self.gen.send(value)
  File "/home/jelena/EMAN2/envs/microbiome_toolbox/lib/python3.6/site-packages/ipykernel/kernelbase.py", line 365, in process_one
    yield gen.maybe_future(dispatch(*args))
  File "/home/jelena/EMAN2/envs/microbiome_toolbox/lib/python3.6/site-packages/tornado/gen.py", line 234, in wrapper
    yielded = ctx_run(next, result)
  File "/home/jelena/EMAN2/envs/microbiome_toolbox/lib/python3.6/site-packages/tornado/gen.py", line 162, in _fake_ctx_run
    return f(*args, **kw)
  File "/home/jelena/EMAN2/envs/microbiome_toolbox/lib/python3.6/site-packages/ipykernel/kernelbase.py", line 268, in dispatch_shell
    yield gen.maybe_future(handler(stream, idents, msg))
  File "/home/jelena/EMAN2/envs/microbiome_toolbox/lib/python3.6/site-packages/tornado/gen.py", line 234, in wrapper
    yielded = ctx_run(next, result)
  File "/home/jelena/EMAN2/envs/microbiome_toolbox/lib/python3.6/site-packages/tornado/gen.py", line 162, in _fake_ctx_run
    return f(*args, **kw)
  File "/home/jelena/EMAN2/envs/microbiome_toolbox/lib/python3.6/site-packages/ipykernel/kernelbase.py", line 545, in execute_request
    user_expressions, allow_stdin,
  File "/home/jelena/EMAN2/envs/microbiome_toolbox/lib/python3.6/site-packages/tornado/gen.py", line 234, in wrapper
    yielded = ctx_run(next, result)
  File "/home/jelena/EMAN2/envs/microbiome_toolbox/lib/python3.6/site-packages/tornado/gen.py", line 162, in _fake_ctx_run
    return f(*args, **kw)
  File "/home/jelena/EMAN2/envs/microbiome_toolbox/lib/python3.6/site-packages/ipykernel/ipkernel.py", line 306, in do_execute
    res = shell.run_cell(code, store_history=store_history, silent=silent)
  File "/home/jelena/EMAN2/envs/microbiome_toolbox/lib/python3.6/site-packages/ipykernel/zmqshell.py", line 536, in run_cell
    return super(ZMQInteractiveShell, self).run_cell(*args, **kwargs)
  File "/home/jelena/EMAN2/envs/microbiome_toolbox/lib/python3.6/site-packages/IPython/core/interactiveshell.py", line 2867, in run_cell
    raw_cell, store_history, silent, shell_futures)
  File "/home/jelena/EMAN2/envs/microbiome_toolbox/lib/python3.6/site-packages/IPython/core/interactiveshell.py", line 2895, in _run_cell
    return runner(coro)
  File "/home/jelena/EMAN2/envs/microbiome_toolbox/lib/python3.6/site-packages/IPython/core/async_helpers.py", line 68, in _pseudo_sync_runner
    coro.send(None)
  File "/home/jelena/EMAN2/envs/microbiome_toolbox/lib/python3.6/site-packages/IPython/core/interactiveshell.py", line 3072, in run_cell_async
    interactivity=interactivity, compiler=compiler, result=result)
  File "/home/jelena/EMAN2/envs/microbiome_toolbox/lib/python3.6/site-packages/IPython/core/interactiveshell.py", line 3263, in run_ast_nodes
    if (await self.run_code(code, result,  async_=asy)):
  File "/home/jelena/EMAN2/envs/microbiome_toolbox/lib/python3.6/site-packages/IPython/core/interactiveshell.py", line 3343, in run_code
    exec(code_obj, self.user_global_ns, self.user_ns)
  File "<ipython-input-2-2430da55f922>", line 1, in <module>
    from microbiome.dataset import MicrobiomeDataset
  File "/home/jelena/Desktop/microbiome2021/ssh/microbiome-toolbox/microbiome/dataset.py", line 11, in <module>
    import matplotlib.pyplot as plt
  File "/home/jelena/EMAN2/envs/microbiome_toolbox/lib/python3.6/site-packages/matplotlib/pyplot.py", line 71, in <module>
    from matplotlib.backends import pylab_setup
  File "/home/jelena/EMAN2/envs/microbiome_toolbox/lib/python3.6/site-packages/matplotlib/backends/__init__.py", line 16, in <module>
    line for line in traceback.format_stack()

In [3]:

dataset = MicrobiomeDataset(file_name="human_data") 

In [4]:

from microbiome.enumerations import TimeUnit

In [5]:

trajectory = MicrobiomeTrajectory(dataset, dataset.feature_columns)

In [6]:

trajectory.time_unit = TimeUnit.YEAR

In [7]:

results = trajectory.plot_reference_trajectory()
results["fig"]

FigureWidget({
    'data': [{'fill': 'toself',
              'fillcolor': 'rgba(26,150,65,0.3)',
             …

In [8]:

trajectory.time_unit = TimeUnit.DAY

In [9]:

results = trajectory.plot_reference_trajectory(degree=1)
results["fig"]

FigureWidget({
    'data': [{'fill': 'toself',
              'fillcolor': 'rgba(26,150,65,0.3)',
             …

In [11]:

# human dataset doesn't have reference and non-reference
# results = trajectory.plot_reference_groups()
# results["fig"]

In [13]:

# human dataset doesn't have reference and non-reference
# results = trajectory.plot_reference_groups(degree=1)
# results["fig"]

In [14]:

trajectory.dataset.df.group.value_counts()

Out[14]:

Healthy Twins Triplets    48
Healthy Singletons        18
Name: group, dtype: Int64

In [15]:

l = trajectory.dataset.df.group.value_counts().index.values[:2]
l

Out[15]:

array(['Healthy Twins Triplets', 'Healthy Singletons'], dtype=object)

In [16]:

results = trajectory.plot_groups()
results["fig"]

FigureWidget({
    'data': [{'fill': 'toself',
              'fillcolor': 'rgba(99, 110, 250,0.3)',
          …

In [17]:

results = trajectory.plot_groups(degree=1)
results["fig"]

FigureWidget({
    'data': [{'fill': 'toself',
              'fillcolor': 'rgba(99, 110, 250,0.3)',
          …

In [18]:

results["ret_val"]

Out[18]:

'<b>Performance Information</b><br>MAE Healthy Singletons: 13.423<br>R^2 Healthy Singletons: 0.897<br>MAE Healthy Twins Triplets: 13.328<br>R^2 Healthy Twins Triplets: 0.920<br><b>Linear p-value (k, n)</b>:<br>Healthy Singletons vs. Healthy Twins Triplets: (0.831, 0.878)'

In [19]:

from microbiome.enumerations import AnomalyType

In [20]:

results = trajectory.plot_anomalies(anomaly_type=AnomalyType.PREDICTION_INTERVAL)
results['fig']

FigureWidget({
    'data': [{'fill': 'toself',
              'fillcolor': 'rgba(26,150,65,0.3)',
             …

In [21]:

results = trajectory.plot_anomalies(anomaly_type=AnomalyType.LOW_PASS_FILTER)
results['fig']

FigureWidget({
    'data': [{'fill': 'toself',
              'fillcolor': 'rgba(26,150,65,0.3)',
             …

In [22]:

results = trajectory.plot_anomalies(anomaly_type=AnomalyType.ISOLATION_FOREST)
results['fig']#.write_html("file.html")

FigureWidget({
    'data': [{'fill': 'toself',
              'fillcolor': 'rgba(26,150,65,0.3)',
             …

In [23]:

results = trajectory.plot_timeboxes(layout_settings=dict(hoverdistance=None), time_block_ranges=[10,10,10,10,10, 10, 20])
results['fig']

FigureWidget({
    'data': [{'fill': 'toself',
              'fillcolor': 'rgba(26,150,65,0.3)',
             …

In [24]:

results = trajectory.plot_animated_longitudinal_information()
results["fig"]

In [26]:

from microbiome.enumerations import FeatureExtraction

In [27]:

dataset.normalized = True
trajectory = MicrobiomeTrajectory(dataset, dataset.feature_columns, feature_extraction=FeatureExtraction.NEAR_ZERO_VARIANCE)
trajectory.feature_columns_plot

In [28]:

dataset.normalized = False
trajectory = MicrobiomeTrajectory(dataset, dataset.feature_columns, feature_extraction=FeatureExtraction.NEAR_ZERO_VARIANCE)
trajectory.feature_columns_plot

In [29]:

dataset.normalized = True
trajectory = MicrobiomeTrajectory(dataset, dataset.feature_columns, feature_extraction=FeatureExtraction.CORRELATION)
trajectory.feature_columns_plot

In [30]:

dataset.normalized = False
trajectory = MicrobiomeTrajectory(dataset, dataset.feature_columns, feature_extraction=FeatureExtraction.CORRELATION)
trajectory.feature_columns_plot

In [31]:

dataset.normalized = True
trajectory = MicrobiomeTrajectory(dataset, dataset.feature_columns, feature_extraction=FeatureExtraction.TOP_K_IMPORTANT)
trajectory.feature_columns_plot

In [32]:

dataset.normalized = False
trajectory = MicrobiomeTrajectory(dataset, dataset.feature_columns, feature_extraction=FeatureExtraction.TOP_K_IMPORTANT)
trajectory.feature_columns_plot

In [33]:

from microbiome.dataset import ReferenceGroup,FeatureColumnsType

In [34]:

dataset = MicrobiomeDataset(file_name="human_data", feature_columns=FeatureColumnsType.BACTERIA_AND_METADATA) 
trajectory = MicrobiomeTrajectory(dataset, dataset.feature_columns)

In [35]:

dataset.normalized = True
trajectory = MicrobiomeTrajectory(dataset, dataset.feature_columns, feature_extraction=FeatureExtraction.TOP_K_IMPORTANT)
trajectory.feature_columns_plot

In [ ]: