Exploring NSERC Awards Data¶

# For Plotting
import matplotlib.pyplot as plt
import matplotlib.ticker as mtick
!pip3 install plotly --user; #Plotly for graphing
import plotly.plotly as py
import plotly.graph_objs as go

# Note: haven't yet uploaded this to github or shared it
import numpy as np
import pandas as pd
import sys

## Bring in a selection of the NSERC awards data starting with 1995 and ending with 2016.
## Throw away as much as you can to keep the DataFrame small enough to manipulate using a laptop.

df = pd.DataFrame()

startYear = 2009
endYear   = 2017  ## The last year is not included, so if it was 2017 it means we include the 2016 collection but not 2017.

## Reads and processes the raw csv datafiles.
for year in range(startYear, endYear):
    file = 'https://s3.ca-central-1.amazonaws.com/open-data-ro/NSERC/NSERC_GRT_FYR' + str(year) + '_AWARD.csv.gz'
    df = df.append(pd.read_csv(file, compression='gzip', usecols = [9, 11, 17], encoding='latin-1'))
    
    print(year)
    
## Rename columns for better readability.
df.columns = ['FiscalYear', 'AwardAmount', 'Committee']  

# Get the data for the Mathematics and Statistics Funding
mathFundingData = df.loc[(df['Committee'] == 1508)]
mathFundingData.head()

# Make years the index so that data is easier to manipulate
# mathFundingData = mathFundingData.set_index('FiscalYear')

# Make a new column that is the sum of all Awards for each individual year
awardTotalsByYear = mathFundingData.assign(TotalAward=mathFundingData.groupby(['FiscalYear'])['AwardAmount'].transform('sum'))

# Count the number of awards given in each year and make a new column with them called 'AwardCount'
awardTotalsByYear = awardTotalsByYear.assign(AwardCount=awardTotalsByYear.groupby('FiscalYear').cumcount() + 1)
#awardTotalsByYear = 
# Make a new column that is the sum of all Awards that each individual has received 
#year2016 = year2016.assign(TotalAward=year2016.groupby(['Name'])['AwardAmount'].transform('sum'))

# Drop all but the last of duplicate years, now that we have the award amount info from them. 
# We keep the last one because that is where the final value of cumCount is.
deduplicatedData = awardTotalsByYear.drop_duplicates(subset = 'FiscalYear', keep = 'last')

deduplicatedData[['FiscalYear','TotalAward', 'AwardCount']]

# Plot Settings
data = mathFundingData

xAxis = 'FiscalYear'
yAxis = 'AwardAmount'

yAxisRange = [0,120]
yScalingFactor = 10**3
yScalingFactorString = ' In Thousands'

plotPointSizes = 3 

title = 'Graph of Awards Given by Committee 1508'

# Make the Plot
axes = plt.gca()
x = data[xAxis]
plt.xlabel(xAxis, fontsize=16)

y = data[yAxis]
plt.ylabel(yAxis+yScalingFactorString, fontsize=14)
axes.set_ylim(yAxisRange)
y = y/yScalingFactor

plt.title(title)

plot = plt.scatter(x,y,s=plotPointSizes)

plt.show()

# Plot Settings
year = 2016
data = mathFundingData.loc[(mathFundingData['FiscalYear'] == year)]

yAxis = 'FiscalYear'
xAxis = 'AwardAmount'

xScalingFactor = 10**3
xScalingFactorString = ' In Thousands'

yAxisRange = [1.99,2.1]

plotPointSizes = 3 

title = 'Graph of Awards Given by Committee 1508 in 2016'

# Make the Plot
axes = plt.gca()
x = data[xAxis]
plt.xlabel(xAxis, fontsize=16)

y = data[yAxis]
#plt.ylabel(yAxis+yScalingFactorString, fontsize=14)
axes.set_ylim(yAxisRange)
y = y/yScalingFactor
axes.set_yticklabels([]) # make it so the y axis is not numbered

plt.title(title)

plot = plt.scatter(x,y,s=plotPointSizes)

plt.show()

ax = data[[xAxis]].plot(kind='bar', title ="V comp", figsize=(15, 10), legend=True, fontsize=12)
ax.set_xlabel(yAxis, fontsize=12)
ax.set_ylabel(xAxis, fontsize=12)
plt.show()

# Plot Settings
year = 2016
data = mathFundingData.loc[(mathFundingData['FiscalYear'] == year)]

xAxis = 'FiscalYear'
#xAxis = 'AwardAmount'

xScalingFactor = 10**3
xScalingFactorString = ' In Thousands'

yAxisRange = [1.99,2.1]

plotPointSizes = 3 

title = 'Graph of Awards Given by Committee 1508 in 2016'

# Make the Plot

mathFundingData.hist(column='AwardAmount', ax=plt.gca())

# Make the Plot Interactive 
import plotly.offline as py
from plotly.offline import init_notebook_mode, iplot
import plotly.tools as tls
import matplotlib.pylab as plt
from matplotlib.backends.backend_agg import FigureCanvasAgg as FigureCanvas

fig = plt.Figure()
axes = fig.gca()

axes.plot(x,y)
canvas = FigureCanvas(fig)
plotly_fig = tls.mpl_to_plotly(fig)
py.iplot(plotly_fig)