In [1]:

import sys
sys.path.append('../scripts')

In [2]:

from pathlib import Path
import pandas as pd

In [3]:

import hydra
from hydra import compose, initialize

hydra.core.global_hydra.GlobalHydra.instance().clear()
initialize(config_path=Path('..'), job_name='foo', version_base='1.1')
config = compose(config_name='experiment.yaml')

In [4]:

from dataset import load_data

base_path = Path('..')
train_df, val_df, test_df = load_data(base_path / config.data.cnf_tsv_path, base_path / config.data.controls_tsv_path)

In [5]:

all_df = pd.concat([train_df, val_df, test_df])

In [6]:

len(train_df), len(val_df), len(test_df), len(all_df)

Out[6]:

(5336, 1201, 1159, 7696)

In [7]:

# Ellipses
(~train_df.controls).sum(), (~val_df.controls).sum(), (~test_df.controls).sum(), (~all_df.controls).sum()

Out[7]:

(2715, 590, 543, 3848)

In [8]:

# Controls
(train_df.controls).sum(), (val_df.controls).sum(), (test_df.controls).sum(), (all_df.controls).sum()

Out[8]:

(2621, 611, 616, 3848)

In [ ]: