%matplotlib inline
import pandas as pd
import numpy as np

# Set some Pandas options
pd.set_option('html', False)
pd.set_option('max_columns', 30)
pd.set_option('max_rows', 20)

a_list = range(1000)
an_array = np.arange(1000)

an_array[:10]

type(an_array)

timeit [i**2 for i in a_list]

timeit an_array**2

an_array[5:10]

a_list[0] = 'a string inside a list'
a_list[:10]

an_array[0] = 'a string inside an array'

an_array.dtype

an_array[0] = 1.234
an_array[:10]

np.linspace(0, 1, num=5)

np.logspace(1, 4, num=4)


np.random.randn(5)

norm_10 = np.random.normal(loc=10, scale=3, size=10)
norm_10

mask = norm_10 > 9
mask

norm_10[mask]

norm_10[mask] = 0
print norm_10

norm_10[np.nonzero(norm_10)]

array_2d = np.array([[1, 2], [3, 4]])
array_2d.shape

array_2d[0,1]

md_array = np.arange(8).reshape(2,4)
print md_array

md_array[1, 2:4]

md_array[:, 2]

md_array[1]

md_array.dtype

md_array.shape

md_array.ndim

md_array.nbytes

md_array.min(), md_array.max()

md_array.sum(), md_array.prod()

md_array.mean(), md_array.std()

md_array.sum(axis=0)

md_array.sum(axis=1)

random_array = np.random.random((3,2,3,4))
random_array

random_array.sum(2).shape

first_array = np.random.randn(4)
second_array = np.random.randn(4)

first_array, second_array

first_array * second_array

first_array * 5

md_array

md_array * first_array

md_array * np.array([-1, 2.3])

md_array * np.array([-1, 2.3])[:, np.newaxis]

md_array.T * np.array([-1, 2.3])

md_array.dot(first_array)

np.dot(md_array, first_array)

from IPython.core.display import HTML
HTML("<iframe src=http://pandas.pydata.org width=800 height=350></iframe>")

counts = pd.Series([632, 1638, 569, 115])
counts

counts.values

counts.index

bacteria = pd.Series([632, 1638, 569, 115], 
    index=['Firmicutes', 'Proteobacteria', 'Actinobacteria', 'Bacteroidetes'])

bacteria

bacteria['Actinobacteria']

bacteria[[name.endswith('bacteria') for name in bacteria.index]]

[name.endswith('bacteria') for name in bacteria.index]

bacteria[0]

bacteria.name = 'counts'
bacteria.index.name = 'phylum'
bacteria

np.log(bacteria)

bacteria[bacteria>1000]

bacteria_dict = {'Firmicutes': 632, 'Proteobacteria': 1638, 'Actinobacteria': 569, 'Bacteroidetes': 115}
pd.Series(bacteria_dict)

bacteria2 = pd.Series(bacteria_dict, index=['Cyanobacteria','Firmicutes','Proteobacteria','Actinobacteria'])
bacteria2

bacteria2.isnull()

bacteria + bacteria2

data = pd.DataFrame({'value':[632, 1638, 569, 115, 433, 1130, 754, 555],
                     'patient':[1, 1, 1, 1, 2, 2, 2, 2],
                     'phylum':['Firmicutes', 'Proteobacteria', 'Actinobacteria', 
    'Bacteroidetes', 'Firmicutes', 'Proteobacteria', 'Actinobacteria', 'Bacteroidetes']})
data

data[['phylum','value','patient']]

data.columns

data['value']

data.value

type(data.value)

type(data[['value']])

data.ix[3]

vals = data.value
vals

vals[5] = 0
vals

data

vals = data.value.copy()
vals[5] = 1000
data

data.value[3] = 14
data

data['year'] = 2013
data

data.treatment = 1
data

data.treatment

treatment = pd.Series([0]*4 + [1]*2)
treatment

data['treatment'] = treatment
data

month = ['Jan', 'Feb', 'Mar', 'Apr']
data['month'] = month

data.values

data.index

data.index[0] = 15

bacteria2.index = bacteria.index

bacteria2

!cat data/microbiome.csv

mb = pd.read_csv("data/microbiome.csv")
mb

pd.read_csv("data/microbiome.csv", header=None).head()

mb = pd.read_table("data/microbiome.csv", sep=',')

mb = pd.read_csv("data/microbiome.csv", index_col=['Taxon','Patient'])
mb.head()

pd.read_csv("data/microbiome.csv", skiprows=[3,4,6]).head()

pd.read_csv("data/microbiome.csv", nrows=4)

data_chunks = pd.read_csv("data/microbiome.csv", chunksize=15)

mean_tissue = {chunk.Taxon[0]:chunk.Tissue.mean() for chunk in data_chunks}
    
mean_tissue

!cat data/microbiome_missing.csv

pd.read_csv("data/microbiome_missing.csv").head(20)

pd.isnull(pd.read_csv("data/microbiome_missing.csv")).head(20)

pd.read_csv("data/microbiome_missing.csv", na_values=['?', -99999]).head(20)

baseball = pd.read_csv("data/baseball.csv", index_col='id')
baseball.head()

player_id = baseball.player + baseball.year.astype(str)
baseball_newind = baseball.copy()
baseball_newind.index = player_id
baseball_newind.head()

baseball_newind.index.is_unique

baseball_newind.ix['wickmbo012007']

baseball.reindex(baseball.index[::-1]).head()

id_range = range(baseball.index.values.min(), baseball.index.values.max())
baseball.reindex(id_range).head()

baseball.reindex(id_range, method='ffill', columns=['player','year']).head()

baseball.reindex(id_range, fill_value='mr.nobody', columns=['player']).head()

baseball.shape

baseball.drop([89525, 89526])

baseball.drop(['ibb','hbp'], axis=1)

# Sample Series object
hits = baseball_newind.h
hits

# Numpy-style indexing
hits[:3]

# Indexing by label
hits[['womacto012006','schilcu012006']]

hits.ix['womacto012006':'gonzalu012006']

hits['womacto012006':'gonzalu012006'] = 5
hits

baseball_newind[['h','ab']]

baseball_newind.ix['gonzalu012006', ['h','X2b', 'X3b', 'hr']]

baseball_newind.ix[['gonzalu012006','finlest012006'], 5:8]

baseball_newind.ix[:'myersmi012006', 'hr']

hr2006 = baseball[baseball.year==2006].xs('hr', axis=1)
hr2006.index = baseball.player[baseball.year==2006]

hr2007 = baseball[baseball.year==2007].xs('hr', axis=1)
hr2007.index = baseball.player[baseball.year==2007]

hr2006 = pd.Series(baseball.hr[baseball.year==2006].values, index=baseball.player[baseball.year==2006])
hr2007 = pd.Series(baseball.hr[baseball.year==2007].values, index=baseball.player[baseball.year==2007])

hr_total = hr2006 + hr2007
hr_total

hr_total[hr_total.notnull()]

hr2007.add(hr2006, fill_value=0)

baseball.hr - baseball.hr.max()

baseball.ix[89521]["player"]

stats = baseball[['h','X2b', 'X3b', 'hr']]
diff = stats - stats.xs(89521)
diff[:10]

stats.apply(np.median)

stat_range = lambda x: x.max() - x.min()
stats.apply(stat_range)

slg = lambda x: (x['h']-x['X2b']-x['X3b']-x['hr'] + 2*x['X2b'] + 3*x['X3b'] + 4*x['hr'])/(x['ab']+1e-6)
baseball.apply(slg, axis=1).apply(lambda x: '%.3f' % x)

baseball_newind.sort_index().head()

baseball_newind.sort_index(ascending=False).head()

baseball_newind.sort_index(axis=1).head()

baseball.hr.order(ascending=False)

baseball[['player','sb','cs']].sort_index(ascending=[False,True], by=['sb', 'cs']).head(10)

baseball.hr.rank()

pd.Series([100,100]).rank()

baseball.hr.rank(method='first')

baseball.rank(ascending=False).head()

baseball[['r','h','hr']].rank(ascending=False).head()

foo = pd.Series([np.nan, -3, None, 'foobar'])
foo

foo.isnull()

bacteria2

bacteria2.dropna()

bacteria2[bacteria2.notnull()]

data

data.dropna()

data.dropna(how='all')

data.ix[7, 'year'] = float('Nan')
data

data.dropna(thresh=4)

data.dropna(axis=1)

bacteria2.fillna(0)

data.fillna({'year': 2013, 'treatment':2})

data

_ = data.year.fillna(2013, inplace=True)
data

bacteria2.fillna(method='bfill')

bacteria2.fillna(bacteria2.mean())

segments = pd.read_csv("data/AIS/transit_segments.csv")
segments.head()

vessels = pd.read_csv("data/AIS/vessel_information.csv", index_col='mmsi')
vessels.head()

vessels.type.value_counts()

df1 = pd.DataFrame(dict(id=range(4), age=np.random.randint(18, 31, size=4)))
df2 = pd.DataFrame(dict(id=range(3)+range(3), score=np.random.random(size=6)))

df1, df2

pd.merge(df1, df2)

pd.merge(df1, df2, how='outer')

segments.head(1)

vessels.head(1)

segments_merged = pd.merge(vessels, segments, left_index=True, right_on='mmsi')

segments_merged.head()

vessels.merge(segments, left_index=True, right_on='mmsi').head()

segments['type'] = 'foo'
pd.merge(vessels, segments, left_index=True, right_on='mmsi').head()

np.concatenate([np.random.random(5), np.random.random(5)])

np.r_[np.random.random(5), np.random.random(5)]

np.c_[np.random.random(5), np.random.random(5)]

mb1 = pd.read_excel('data/microbiome/MID1.xls', 'Sheet 1', index_col=0, header=None)
mb2 = pd.read_excel('data/microbiome/MID2.xls', 'Sheet 1', index_col=0, header=None)
mb1.shape, mb2.shape

mb1.head()

mb1.columns = mb2.columns = ['Count']

mb1.index.name = mb2.index.name = 'Taxon'

mb1.head()

mb1.index[:3]

mb1.index.is_unique

pd.concat([mb1, mb2], axis=0).shape

pd.concat([mb1, mb2], axis=0).index.is_unique

pd.concat([mb1, mb2], axis=1).shape

pd.concat([mb1, mb2], axis=1).head()

pd.concat([mb1, mb2], axis=1).values[:5]

pd.concat([mb1, mb2], axis=1, join='inner').head()

mb1.combine_first(mb2).head()

pd.concat(dict(patient1=mb1, patient2=mb2), axis=1).head()

cdystonia = pd.read_csv("data/cdystonia.csv", index_col=None)
cdystonia.head()

stacked = cdystonia.stack()
stacked

stacked.unstack().head()

cdystonia2 = cdystonia.set_index(['patient','obs'])
cdystonia2.head()

cdystonia2.index.is_unique

twstrs_wide = cdystonia2['twstrs'].unstack('obs')
twstrs_wide.head()

cdystonia_long = cdystonia[['patient','site','id','treat','age','sex']].drop_duplicates().merge(
                    twstrs_wide, right_index=True, left_on='patient', how='inner').head()
cdystonia_long

cdystonia.set_index(['patient','site','id','treat','age','sex','week'])['twstrs'].unstack('week').head()

pd.melt(cdystonia_long, id_vars=['patient','site','id','treat','age','sex'], 
        var_name='obs', value_name='twsters').head()

vessels.duplicated(cols='names')

vessels.drop_duplicates(['names'])

cdystonia.treat.value_counts()

treatment_map = {'Placebo': 0, '5000U': 1, '10000U': 2}

cdystonia['treatment'] = cdystonia.treat.map(treatment_map)
cdystonia.treatment

cdystonia2.treat.replace({'Placebo': 0, '5000U': 1, '10000U': 2})

cdystonia_grouped = cdystonia.groupby(cdystonia.patient)

cdystonia_grouped

for patient, group in cdystonia_grouped:
    print patient
    print group
    print

cdystonia_grouped.agg(np.mean).head()

cdystonia_grouped.mean().head()

cdystonia_grouped.mean().add_suffix('_mean').head()

# The median of the `twstrs` variable
cdystonia_grouped['twstrs'].quantile(0.5)

cdystonia.groupby(['week','site']).mean().head()

normalize = lambda x: (x - x.mean())/x.std()

cdystonia_grouped.transform(normalize).head()

cdystonia_grouped['twstrs'].mean().head()

chunks = dict(list(cdystonia_grouped))
chunks[4]

dict(list(cdystonia.groupby(cdystonia.dtypes, axis=1)))

def top(df, column, n=5):
    return df.sort_index(by=column, ascending=False)[:n]

top3segments = segments_merged.groupby('mmsi').apply(top, column='seg_length', n=3)[['names', 'seg_length']]
top3segments

top3segments.head(20)