Notebook

CHAPTER 8¶

Ploting and Visualization¶

A brief matplotlib API primer¶

In [6]:

%matplotlib inline
import matplotlib.pyplot as plt
import numpy as np

In [7]:

plt.plot(np.arange(10))

Out[7]:

[<matplotlib.lines.Line2D at 0x10f6fbc88>]

Figures and Subplots¶

In [8]:

fig = plt.figure()

<matplotlib.figure.Figure at 0x10f749f60>

In [9]:

ax1 = fig.add_subplot(2, 2, 1)

In [10]:

ax2 = fig.add_subplot(2, 2, 2)

In [11]:

ax3 = fig.add_subplot(2, 2, 3)

In [12]:

from numpy.random import randn

In [13]:

plt.plot(randn(50).cumsum(), 'k--')

Out[13]:

[<matplotlib.lines.Line2D at 0x10f9071d0>]

In [14]:

_ = ax1.hist(randn(100), bins = 20, color = 'k', alpha = 0.3)

In [15]:

ax2.scatter(np.arange(30), np.arange(30) + 3*randn(30))

Out[15]:

<matplotlib.collections.PathCollection at 0x10f907fd0>

In [16]:

fig, axes = plt.subplots(2, 3)

In [17]:

axes

Out[17]:

array([[<matplotlib.axes._subplots.AxesSubplot object at 0x10f9f1b38>,
        <matplotlib.axes._subplots.AxesSubplot object at 0x10fa3e860>,
        <matplotlib.axes._subplots.AxesSubplot object at 0x10fa7b828>],
       [<matplotlib.axes._subplots.AxesSubplot object at 0x10fac4860>,
        <matplotlib.axes._subplots.AxesSubplot object at 0x10fad4390>,
        <matplotlib.axes._subplots.AxesSubplot object at 0x10fb55160>]], dtype=object)

In [18]:

axes[0, 1]

Out[18]:

<matplotlib.axes._subplots.AxesSubplot at 0x10fa3e860>

Adjusting the spacing around subplots¶

In [19]:

fig, axes = plt.subplots(2, 2, sharex=True, sharey=True)
for i in range(2):
    for j in range(2):
        axes[i, j].hist(randn(500), bins = 50, color = 'k', alpha = 0.5)
plt.subplots_adjust(wspace = 0, hspace = 0)

Colors, Markers, and Line Styles¶

In [20]:

plt.plot(randn(30).cumsum(), 'g--')

Out[20]:

[<matplotlib.lines.Line2D at 0x1101b5518>]

In [21]:

plt.plot(randn(30).cumsum(), linestyle = '--', color = 'g')

Out[21]:

[<matplotlib.lines.Line2D at 0x1102a12e8>]

In [22]:

plt.plot(randn(30).cumsum(), 'ko--')

Out[22]:

[<matplotlib.lines.Line2D at 0x1103949b0>]

In [23]:

plt.plot(randn(30).cumsum(), 'ko--', linestyle = 'dashed', marker = 'o')

Out[23]:

[<matplotlib.lines.Line2D at 0x1104940f0>]

In [24]:

data = randn(30).cumsum()

In [25]:

plt.plot(data, 'k--', label = 'Default')

Out[25]:

[<matplotlib.lines.Line2D at 0x110586c88>]

In [26]:

plt.plot(data, 'k-', drawstyle = 'steps-post', label = 'steps-post')

Out[26]:

[<matplotlib.lines.Line2D at 0x110687940>]

In [28]:

plt.legend(loc = 'best', label = '...')

/usr/local/lib/python3.5/site-packages/matplotlib/axes/_axes.py:519: UserWarning: No labelled objects found. Use label='...' kwarg on individual plots.
  warnings.warn("No labelled objects found. "

Ticks, Labels and Legends¶

Seting the title, axis labels, ticks, and ticklabels¶

In [29]:

fig = plt.figure(); ax = fig.add_subplot(1, 1, 1)

In [30]:

ax.plot(randn(1000).cumsum())

Out[30]:

[<matplotlib.lines.Line2D at 0x1109fd1d0>]

In [31]:

ticks = ax.set_xticks([0, 250, 500, 750, 100])

In [32]:

labels = ax.set_xticklabels(['one', 'two', 'three', 'four', 'five'],
                            rotation = 30, fontsize = 'small')

In [33]:

ax.set_title('My first matplotlib plot')

Out[33]:

<matplotlib.text.Text at 0x11093c4e0>

In [34]:

ax.set_xlabel('Stages')

Out[34]:

<matplotlib.text.Text at 0x1108cfa90>

Adding legend¶

In [35]:

fig = plt.figure(); ax = fig.add_subplot(1, 1, 1)

In [37]:

ax.plot(randn(1000).cumsum(), 'k', label = 'one')

Out[37]:

[<matplotlib.lines.Line2D at 0x110abfa90>]

In [38]:

ax.plot(randn(1000).cumsum(), 'k--', label = 'two')

Out[38]:

[<matplotlib.lines.Line2D at 0x110abfac8>]

In [39]:

ax.plot(randn(1000).cumsum(), 'k.', label = 'two')

Out[39]:

[<matplotlib.lines.Line2D at 0x110827cc0>]

In [40]:

ax.legend(loc = 'best')

Out[40]:

<matplotlib.legend.Legend at 0x110ac42e8>

Anotations and Driving on a subplot¶

ax.text(x, y, 'hello world!', family = 'monospace', fontsize = 10)

In [42]:

import pandas as pd
from datetime import datetime

fig = plt.figure()
ax = fig.add_subplot(1, 1, 1)

data = pd.read_csv('ch08/spx.csv', index_col = 0, parse_dates=True)
spx = data['SPX']

spx.plot(ax = ax, style = 'k-')

crisis_data = [
               (datetime(2007, 10, 11), 'Peak of bull market'),
               (datetime(2008, 3, 12), 'Bear Stearns Fails'),
               (datetime(2008, 9, 15), 'Lehman Bankruptcy')
               ]

for date, label in crisis_data:
    ax.annotate(label, xy = (date, spx.asof(date) + 50),
                xytext = (date, spx.asof(date) + 200),
                arrowprops = dict(facecolor = 'black'),
                horizontalalignment = 'left', verticalalignment = 'top')
    
ax.set_xlim(['1/1/2007', '1/1/2011'])
ax.set_ylim([600, 1800])

ax.set_title('Important dates in 2008-2009 financial crisis')

Out[42]:

<matplotlib.text.Text at 0x111b1e8d0>

In [43]:

fig = plt.figure()
ax = fig.add_subplot(1, 1, 1)

rect = plt.Rectangle((0.2, 0.75), 0.4, 0.15, color = 'k', alpha = 0.3)
circ = plt.Circle((0.7, 0.2), 0.15, color = 'b', alpha = 0.3)
pgin = plt.Polygon([[0.15, 0.15], [0.35, 0.4], [0.2, 0.6]], color = 'g', alpha = 0.5)

ax.add_patch(rect)
ax.add_patch(circ)
ax.add_patch(pgin)

Out[43]:

<matplotlib.patches.Polygon at 0x111c606d8>

Saving plot to file¶

plt.savefig('figpath.svg')plt.savefig('figpath.png', dpi = 400, bbox_inches = 'tight')from io import StringIO buffer = StringIO() plt.savefig(buffer) plot_data = buffer.getvalue()

matplotlib configute¶

In [44]:

plt.rc('figure', figsize = (10, 10))

font_options = {'family': 'monospace', 'weight': 'bold', 'size': 'small'} plt.rc('font', **font_options)

Plotting functions in pandas¶

Line plots¶

In [45]:

from pandas import Series
from pandas import DataFrame
import pandas as pd

In [46]:

s = Series(np.random.randn(10).cumsum(), index = np.arange(0, 100, 10))

In [47]:

s.plot()

Out[47]:

<matplotlib.axes._subplots.AxesSubplot at 0x110bc7e10>

In [48]:

df = DataFrame(np.random.randn(10, 4).cumsum(0),
               columns = ['A', 'B', 'C', 'D'],
               index = np.arange(0, 100, 10))

In [49]:

df.plot()

Out[49]:

<matplotlib.axes._subplots.AxesSubplot at 0x111ca2400>

Bar Plots¶

In [50]:

fig, axes = plt.subplots(2, 1)

In [51]:

data = Series(np.random.rand(16), index = list('abcdefghijklmnop'))

In [52]:

data.plot(kind = 'bar', ax = axes[0], color = 'k', alpha = 0.7)

Out[52]:

<matplotlib.axes._subplots.AxesSubplot at 0x111cd8da0>

In [53]:

data.plot(kind = 'barh', ax = axes[0], color = 'k', alpha = 0.7)

Out[53]:

<matplotlib.axes._subplots.AxesSubplot at 0x111cd8da0>

In [54]:

df = DataFrame(np.random.rand(6, 4),
               index = ['one', 'two', 'three', 'four', 'five', 'sex'],
               columns = pd.Index(['A', 'B', 'C', 'D'], name = 'Genus'))

In [55]:

df.plot(kind = 'bar')

Out[55]:

<matplotlib.axes._subplots.AxesSubplot at 0x11001a390>

In [56]:

df.plot(kind = 'barh')

Out[56]:

<matplotlib.axes._subplots.AxesSubplot at 0x11003fdd8>

In [57]:

df.plot(kind = 'barh', stacked = True, alpha = 0.5)

Out[57]:

<matplotlib.axes._subplots.AxesSubplot at 0x111d4de48>

In [58]:

tips = pd.read_csv('ch08/tips.csv')

In [62]:

tips.head()

Out[62]:

	total_bill	tip	sex	smoker	day	time	size
0	16.99	1.01	Female	No	Sun	Dinner	2
1	10.34	1.66	Male	No	Sun	Dinner	3
2	21.01	3.50	Male	No	Sun	Dinner	3
3	23.68	3.31	Male	No	Sun	Dinner	2
4	24.59	3.61	Female	No	Sun	Dinner	4

In [63]:

party_counts = pd.crosstab(tips.day, tips.size)
party_counts

Out[63]:

col_0	1708
day
Fri	19
Sat	87
Sun	76
Thur	62

In [64]:

party_counts = party_counts.ix[:, 2:5]
party_counts

Out[64]:

col_0
day
Fri
Sat
Sun
Thur

In [24]:

party_pcts = party_counts.div(party_counts.sum(1).astype(float), axis = 0)
party_pcts

Out[24]:

size	2	3	4	5
day
Fri	0.888889	0.055556	0.055556	0.000000
Sat	0.623529	0.211765	0.152941	0.011765
Sun	0.520000	0.200000	0.240000	0.040000
Thur	0.827586	0.068966	0.086207	0.017241

4 rows × 4 columns

In [25]:

party_pcts.plot(kind = 'bar', stacked = True)

Out[25]:

<matplotlib.axes.AxesSubplot at 0x7fab064c0f50>

Histograms and Density Plots¶

In [67]:

tips.head()

Out[67]:

	total_bill	tip	sex	smoker	day	time	size
0	16.99	1.01	Female	No	Sun	Dinner	2
1	10.34	1.66	Male	No	Sun	Dinner	3
2	21.01	3.50	Male	No	Sun	Dinner	3
3	23.68	3.31	Male	No	Sun	Dinner	2
4	24.59	3.61	Female	No	Sun	Dinner	4

In [68]:

tips['tip_pct'] = tips['tip']/tips['total_bill']

In [72]:

tips['tip_pct'].hist(bins = 50)

Out[72]:

<matplotlib.axes._subplots.AxesSubplot at 0x10fda7f28>

In [73]:

import scipy
tips['tip_pct'].plot(kind = 'kde')

Out[73]:

<matplotlib.axes._subplots.AxesSubplot at 0x110026978>

In [74]:

comp1 = np.random.normal(0, 1, size = 200)
comp2 = np.random.normal(10, 2, size = 200)

In [75]:

values = Series(np.concatenate([comp1, comp2]))

In [76]:

values.hist(bins = 100, alpha = 0.3, color = 'k', normed = True)

Out[76]:

<matplotlib.axes._subplots.AxesSubplot at 0x1138dddd8>

In [77]:

values.plot(kind = 'kde', style = 'k--')

Out[77]:

<matplotlib.axes._subplots.AxesSubplot at 0x1152a36a0>

Scatter Plots¶

In [78]:

macro = pd.read_csv('ch08/macrodata.csv')
macro.head()

Out[78]:

	year	quarter	realgdp	realcons	realinv	realgovt	realdpi	cpi	m1	tbilrate	unemp	pop	infl	realint
0	1959.0	1.0	2710.349	1707.4	286.898	470.045	1886.9	28.98	139.7	2.82	5.8	177.146	0.00	0.00
1	1959.0	2.0	2778.801	1733.7	310.859	481.301	1919.7	29.15	141.7	3.08	5.1	177.830	2.34	0.74
2	1959.0	3.0	2775.488	1751.8	289.226	491.260	1916.4	29.35	140.5	3.82	5.3	178.657	2.74	1.09
3	1959.0	4.0	2785.204	1753.7	299.356	484.052	1931.3	29.37	140.0	4.33	5.6	179.386	0.27	4.06
4	1960.0	1.0	2847.699	1770.5	331.722	462.199	1955.5	29.54	139.6	3.50	5.2	180.007	2.31	1.19

In [79]:

data = macro[['cpi', 'm1', 'tbilrate', 'unemp']]

In [80]:

trans_data = np.log(data).diff().dropna()

In [81]:

trans_data[:5]

Out[81]:

	cpi	m1	tbilrate	unemp
1	0.005849	0.014215	0.088193	-0.128617
2	0.006838	-0.008505	0.215321	0.038466
3	0.000681	-0.003565	0.125317	0.055060
4	0.005772	-0.002861	-0.212805	-0.074108
5	0.000338	0.004289	-0.266946	0.000000

In [84]:

plt.scatter(trans_data['m1'], trans_data['unemp'])
plt.title('Change in log %s vs. log %s' %('m1', 'unemp'))

Out[84]:

<matplotlib.text.Text at 0x1132f7898>

In [85]:

pd.scatter_matrix(trans_data, diagonal = 'kde', color = 'k', alpha = 0.3)

Out[85]:

array([[<matplotlib.axes._subplots.AxesSubplot object at 0x1132fa630>,
        <matplotlib.axes._subplots.AxesSubplot object at 0x115c99f60>,
        <matplotlib.axes._subplots.AxesSubplot object at 0x115ce8a58>,
        <matplotlib.axes._subplots.AxesSubplot object at 0x115d24400>],
       [<matplotlib.axes._subplots.AxesSubplot object at 0x115d6d438>,
        <matplotlib.axes._subplots.AxesSubplot object at 0x115dadbe0>,
        <matplotlib.axes._subplots.AxesSubplot object at 0x115dfe320>,
        <matplotlib.axes._subplots.AxesSubplot object at 0x115e32a20>],
       [<matplotlib.axes._subplots.AxesSubplot object at 0x115e81c50>,
        <matplotlib.axes._subplots.AxesSubplot object at 0x115ecc9b0>,
        <matplotlib.axes._subplots.AxesSubplot object at 0x115f11eb8>,
        <matplotlib.axes._subplots.AxesSubplot object at 0x115f61048>],
       [<matplotlib.axes._subplots.AxesSubplot object at 0x115f9c2b0>,
        <matplotlib.axes._subplots.AxesSubplot object at 0x115fe6d68>,
        <matplotlib.axes._subplots.AxesSubplot object at 0x116022550>,
        <matplotlib.axes._subplots.AxesSubplot object at 0x11606fcc0>]], dtype=object)

Plotting Maps: Visualization Haiti Earthquake Crisis Data¶

In [86]:

data = pd.read_csv('ch08/Haiti.csv')

In [87]:

data.head()

Out[87]:

	Serial	INCIDENT TITLE	INCIDENT DATE	LOCATION	DESCRIPTION	CATEGORY	LATITUDE	LONGITUDE	APPROVED	VERIFIED
0	4052	* URGENT * Type O blood donations needed in #J...	05/07/2010 17:26	Jacmel, Haiti	Birthing Clinic in Jacmel #Haiti urgently need...	1. Urgences \| Emergency, 3. Public Health,	18.233333	-72.533333	YES	NO
1	4051	Food-Aid sent to Fondwa, Haiti	28/06/2010 23:06	fondwa	Please help food-aid.org deliver more food to ...	1. Urgences \| Emergency, 2. Urgences logistiqu...	50.226029	5.729886	NO	NO
2	4050	how haiti is right now and how it was during t...	24/06/2010 16:21	centrie	i feel so bad for you i know i am supposed to ...	2. Urgences logistiques \| Vital Lines, 8. Autr...	22.278381	114.174287	NO	NO
3	4049	Lost person	20/06/2010 21:59	Genoca	We are family members of Juan Antonio Zuniga O...	1. Urgences \| Emergency,	44.407062	8.933989	NO	NO
4	4042	Citi Soleil school	18/05/2010 16:26	Citi Soleil, Haiti	We are working with Haitian (NGO) -The Christi...	1. Urgences \| Emergency,	18.571084	-72.334671	YES	NO

In [88]:

data[['INCIDENT DATE', 'LATITUDE', 'LONGITUDE']][: 10]

Out[88]:

	INCIDENT DATE	LATITUDE	LONGITUDE
0	05/07/2010 17:26	18.233333	-72.533333
1	28/06/2010 23:06	50.226029	5.729886
2	24/06/2010 16:21	22.278381	114.174287
3	20/06/2010 21:59	44.407062	8.933989
4	18/05/2010 16:26	18.571084	-72.334671
5	26/04/2010 13:14	18.593707	-72.310079
6	26/04/2010 14:19	18.482800	-73.638800
7	26/04/2010 14:27	18.415000	-73.195000
8	15/03/2010 10:58	18.517443	-72.236841
9	15/03/2010 11:00	18.547790	-72.410010

In [89]:

data['CATEGORY'][: 6]

Out[89]:

0          1. Urgences | Emergency, 3. Public Health, 
1    1. Urgences | Emergency, 2. Urgences logistiqu...
2    2. Urgences logistiques | Vital Lines, 8. Autr...
3                            1. Urgences | Emergency, 
4                            1. Urgences | Emergency, 
5                       5e. Communication lines down, 
Name: CATEGORY, dtype: object

In [90]:

data.describe()

Out[90]:

	Serial	LATITUDE	LONGITUDE
count	3593.000000	3593.000000	3593.000000
mean	2080.277484	18.611495	-72.322680
std	1171.100360	0.738572	3.650776
min	4.000000	18.041313	-74.452757
25%	1074.000000	18.524070	-72.417500
50%	2163.000000	18.539269	-72.335000
75%	3088.000000	18.561820	-72.293570
max	4052.000000	50.226029	114.174287

In [91]:

data = data[(data.LATITUDE > 18) & (data.LONGITUDE < 20) &
           (data.LONGITUDE > -75) & (data.LONGITUDE < -70)
           & data.CATEGORY.notnull()]

In [92]:

def to_cat_list(catstr):
    stripped = (x.strip() for x in catstr.split(','))
    return [x for x in stripped if x]

In [104]:

def get_all_categories(cat_series):
    cat_sets = (set(to_cat_list(x)) for x in cat_series)
    return sorted(set.union(*cat_sets))

In [105]:

def get_english(cat):
    code, names = cat.split('.')
    if '|' in names:
        names = names.split(' | ')[1]
    return code, names.strip()

In [106]:

get_english('2. Urgences logistiques | Vital Lines')

Out[106]:

('2', 'Vital Lines')

In [107]:

all_cats = get_all_categories(data.CATEGORY)

In [108]:

english_mapping = dict(get_english(x) for x in all_cats)

In [109]:

english_mapping['2a']

Out[109]:

'Food Shortage'

In [110]:

english_mapping['6c']

Out[110]:

'Earthquake and aftershocks'

In [113]:

def get_code(seq):
    return [x.strip(',')[0] for x in seq if x]

all_codes = get_code(all_cats)

code_index = pd.Index(np.unique(all_codes))

dummpy_frame = DataFrame(np.zeros((len(data), len(code_index))),
                         index = data.index, columns = code_index)

In [116]:

dummpy_frame.ix[: 6, : 6]

Out[116]:

	1	2	3	4	5	6
0	0.0	0.0	0.0	0.0	0.0	0.0
4	0.0	0.0	0.0	0.0	0.0	0.0
5	0.0	0.0	0.0	0.0	0.0	0.0
6	0.0	0.0	0.0	0.0	0.0	0.0

In [118]:

for row, cat in zip(data.index, data.CATEGORY):
    codes = get_code(to_cat_list(cat))
    dummpy_frame.ix[row, codes] = 1

In [119]:

data = data.join(dummpy_frame.add_prefix('category_'))

In [120]:

data.ix[: 5, : 5]

Out[120]:

	Serial	INCIDENT TITLE	INCIDENT DATE	LOCATION	DESCRIPTION
0	4052	* URGENT * Type O blood donations needed in #J...	05/07/2010 17:26	Jacmel, Haiti	Birthing Clinic in Jacmel #Haiti urgently need...
4	4042	Citi Soleil school	18/05/2010 16:26	Citi Soleil, Haiti	We are working with Haitian (NGO) -The Christi...
5	4041	Radio Commerce in Sarthe	26/04/2010 13:14	Radio Commerce Shelter, Sarthe	i'm Louinel from Sarthe. I'd to know what can ...

from mpl_toolkits.basemap import Basemap import matplotlib.pyplot as plt def basic_haiti_map(ax=None, lllat=17.25, urlat=20.25, lllon=-75, urlon=-71): # create polar stereographic Basemap instance. m = Basemap(ax=ax, projection='stere', lon_0=(urlon + lllon) / 2, lat_0=(urlat + lllat) / 2, llcrnrlat=lllat, urcrnrlat=urlat, llcrnrlon=lllon, urcrnrlon=urlon, resolution='f') # draw coastlines, state and country boundaries, edge of map. m.drawcoastlines() m.drawstates() m.drawcountries() return m

	1	2	3	4	5	6
0	0.0	0.0	0.0	0.0	0.0	0.0
4	0.0	0.0	0.0	0.0	0.0	0.0
5	0.0	0.0	0.0	0.0	0.0	0.0
6	0.0	0.0	0.0	0.0	0.0	0.0

	1	2	3	4	5	6
0	0.0	0.0	0.0	0.0	0.0	0.0
4	0.0	0.0	0.0	0.0	0.0	0.0
5	0.0	0.0	0.0	0.0	0.0	0.0
6	0.0	0.0	0.0	0.0	0.0	0.0

	1	2	3	4	5	6
0	0.0	0.0	0.0	0.0	0.0	0.0
4	0.0	0.0	0.0	0.0	0.0	0.0
5	0.0	0.0	0.0	0.0	0.0	0.0
6	0.0	0.0	0.0	0.0	0.0	0.0