In [1]:
import pandas as pd
import numpy as np
%matplotlib inline
import matplotlib.pyplot as plt
In [2]:
from pprint import *
In [3]:
import seaborn
seaborn.set()
In [4]:
train = pd.read_csv('train.csv')
test = pd.read_csv('test.csv')
Hypotheses¶
- Age
- Income
- Profession
- Marital Status
- Urban / Rural
- Expenses
- Education
- Total Work Experience
In [5]:
test.head()
Out[5]:
In [6]:
train.head()
Out[6]:
Missing Value Treatment¶
In [7]:
train.apply(lambda x: sum(x.isnull()))
Out[7]:
In [8]:
test.apply(lambda x: sum(x.isnull()))
Out[8]:
In [9]:
from scipy.stats import mode
Workclass¶
In [10]:
mode(train.Workclass)[0][0]
Out[10]:
In [11]:
Workclass_Null_Indices = train[train.Workclass.isnull()].index
for el in Workclass_Null_Indices: train.Workclass[el] = 'Private'
In [12]:
mode(test.Workclass)[0][0]
Out[12]:
In [13]:
Workclass_Null_Indices_test = test[test.Workclass.isnull()].index
for el in Workclass_Null_Indices_test: test.Workclass[el] = 'Private'
Occupation¶
In [14]:
mode(train.Occupation)[0][0]
Out[14]:
In [15]:
mode(test.Occupation)[0][0]
Out[15]:
In [16]:
Occupation_Null_Indices_train = train[train.Occupation.isnull()].index
for el in Occupation_Null_Indices_train: train.Occupation[el] = 'Prof-specialty'
In [17]:
Occupation_Null_Indices_test = test[test.Occupation.isnull()].index
for el in Occupation_Null_Indices_test: test.Occupation[el] = 'Prof-specialty'
Native Country¶
In [18]:
mode(train['Native.Country'])[0][0]
Out[18]:
In [19]:
mode(test['Native.Country'])[0][0]
Out[19]:
In [20]:
NativeCountry_Null_Indices_train = train[train['Native.Country'].isnull()].index
for el in NativeCountry_Null_Indices_train: train['Native.Country'][el] = 'United-States'
NativeCountry_Null_Indices_test = test[test['Native.Country'].isnull()].index
for el in NativeCountry_Null_Indices_test: test['Native.Country'][el] = 'United-States'
Univariate Analysis¶
Preprocessing Categorical Variables¶
Age
In [21]:
plt.hist(train.Age, bins = 100)
plt.show()
plt.hist(test.Age, bins = 100)
plt.show()
In [22]:
train.Age.describe()
Out[22]:
In [23]:
test.Age.describe()
Out[23]:
25% of the population is between 17 and 28
25% of the population is between 28 and 37
75% of the population is under 48
Working Class
In [24]:
train.Workclass.describe()
Out[24]:
In [25]:
test.Workclass.describe()
Out[25]:
In [26]:
unique_working_classes = list(train.Workclass.unique())
unique_working_classes
Out[26]:
In [27]:
from sklearn.preprocessing import LabelEncoder
In [28]:
number = LabelEncoder()
train.Workclass = number.fit_transform(train.Workclass.astype('str'))
test.Workclass = number.fit_transform(test.Workclass.astype('str'))
In [29]:
fig = plt.figure(figsize=(4,4), dpi=1600)
plt.hist(train.Workclass, bins = len(unique_working_classes))
plt.show()
fig = plt.figure(figsize=(4,4), dpi=1600)
plt.hist(test.Workclass, bins = len(unique_working_classes))
plt.show()
In [30]:
train.Workclass.unique()
Out[30]:
In [31]:
# Workclass Decoder
workclass_decoder = dict(zip(train.Workclass.unique(),unique_working_classes))
workclass_decoder
Out[31]:
In [32]:
# # Combine bins based on some common sense
# new_workclass_encoder = {0:0,1:0,2:1,3:2,4:2,5:2,6:0,7:1}
# new_workclass_train = [new_workclass_encoder[i] for i in train.Workclass]
# train.Workclass = new_workclass_train
# new_workclass_test = [new_workclass_encoder[i] for i in test.Workclass]
# test.Workclass = new_workclass_test
# new_workclass_decoder = {0:'Government',1:'Never Worked / Without Pay / Information NA', 2:'Private'}
In [33]:
# new_workclass_decoder
In [34]:
# fig = plt.figure(figsize=(2,5), dpi=1600)
# plt.hist(train.Workclass, bins = 3)
# fig = plt.figure(figsize=(2,5), dpi=1600)
# plt.hist(test.Workclass, bins = 3)
Education
In [35]:
education_levels_train = train.Education.unique()
print education_levels_train
In [36]:
train.Education = number.fit_transform(train.Education)
new_education_levels = train.Education.unique()
education_levels_decoder = dict(zip(education_levels_train, new_education_levels))
pprint(education_levels_decoder)
plt.hist(train.Education, bins = len(new_education_levels))
plt.show()
test.Education = number.fit_transform(test.Education)
plt.hist(test.Education, bins = len(new_education_levels))
plt.show()
...it would make sense to club bins in a more sensible way
In [37]:
pprint(education_levels_decoder)
In [38]:
# new_education_levels_encoder = {0:1,1:1,2:2,3:0,4:0,5:0,6:0, 7:5, 8:5, 9:3, 10:7, 11:2, 12:4, 13:0, 14:6, 15:3}
# new_education_levels_train = [new_education_levels_encoder[i] for i in train.Education]
# train.Education = new_education_levels_train
# new_education_levels_test = [new_education_levels_encoder[i] for i in test.Education]
# test.Education = new_education_levels_test
# new_education_levels_decoder = {0:'< 9th', 1:'High School', 2:'High School Grad', 3:'Bachelors', 4:'Masters', 5:'Assoc-acdm / Assoc-voc', 6:'Prof School', 7:'Doctorate'}
In [39]:
# plt.hist(train.Education, bins = len(new_education_levels_decoder))
# plt.show()
# plt.hist(test.Education, bins = len(new_education_levels_decoder))
# plt.show()
Income Group¶
In [40]:
# Rows for which Income is under $50k
print train[train['Income.Group'] != '<=50K'].index
In [41]:
temp = [0]*len(train)
for i in train[train['Income.Group'] != '<=50K'].index:
temp[i] = 1
train['Income.Group'] = temp
In [42]:
train.head()
Out[42]:
In [43]:
train.describe()
Out[43]:
Marital Status¶
In [44]:
train['Marital.Status'].describe()
Out[44]:
In [45]:
test['Marital.Status'].describe()
Out[45]:
In [46]:
marital_status_levels= train['Marital.Status'].unique()
# Encode marital status variable
print marital_status_levels
In [47]:
train['Marital.Status'] = number.fit_transform(train['Marital.Status'])
test['Marital.Status'] = number.fit_transform(test['Marital.Status'])
In [48]:
new_marital_status_levels = train['Marital.Status'].unique()
In [49]:
new_marital_status_levels
Out[49]:
In [50]:
fig = plt.figure(figsize=(4,4), dpi=1600)
plt.hist(train['Marital.Status'], bins = len(new_marital_status_levels))
fig = plt.figure(figsize=(4,4), dpi=1600)
plt.hist(test['Marital.Status'], bins = len(new_marital_status_levels))
Out[50]:
In [51]:
marital_level_status_decoder = dict(zip(new_marital_status_levels, marital_status_levels))
print marital_level_status_decoder
Occupation¶
In [52]:
occupation_levels = train.Occupation.unique()
print occupation_levels
In [53]:
train.Occupation = number.fit_transform(train.Occupation)
test.Occupation = number.fit_transform(test.Occupation)
new_occupation_levels = train.Occupation.unique()
new_occupation_levels
Out[53]:
In [54]:
new_occupation_levels_decoder = dict(zip(new_occupation_levels,occupation_levels))
new_occupation_levels_decoder
Out[54]:
In [55]:
plt.hist(train.Occupation, bins = len(new_occupation_levels))
plt.show()
plt.hist(test.Occupation, bins = len(new_occupation_levels))
plt.show()
Relationshop¶
In [56]:
relationship_levels = train.Relationship.unique()
print relationship_levels
In [57]:
train.Relationship = number.fit_transform(train.Relationship)
test.Relationship = number.fit_transform(test.Relationship)
In [58]:
new_relationship_levels = train.Relationship.unique()
relationship_levels_decoder = dict(zip(new_relationship_levels, relationship_levels))
relationship_levels_decoder
Out[58]:
In [59]:
plt.hist(train.Relationship, bins=len(relationship_levels))
plt.show()
plt.hist(test.Relationship, bins=len(relationship_levels))
plt.show()
Race¶
In [60]:
race_levels = train.Race.unique()
print race_levels
In [61]:
train.Race = number.fit_transform(train.Race)
test.Race = number.fit_transform(test.Race)
new_race_levels = train.Race.unique()
print new_race_levels
In [62]:
race_levels_decoder = dict(zip(new_race_levels,race_levels))
race_levels_decoder
Out[62]:
In [63]:
plt.hist(train.Race, bins = len(race_levels))
plt.show()
plt.hist(test.Race, bins = len(race_levels))
plt.show()
Sex¶
In [64]:
sex_levels = train.Sex.unique()
train.Sex = number.fit_transform(train.Sex)
test.Sex = number.fit_transform(test.Sex)
new_sex_levels = train.Sex.unique()
sex_levels_decoder = dict(zip(new_sex_levels, sex_levels))
sex_levels_decoder
Out[64]:
In [65]:
fig = plt.figure(figsize=(3,4), dpi=1600)
plt.hist(train.Sex, bins = 2)
plt.show()
fig = plt.figure(figsize=(3,4), dpi=1600)
plt.hist(test.Sex, bins = 2)
plt.show()
Native Country¶
In [66]:
native_country_levels = train['Native.Country'].unique()
train['Native.Country'] = number.fit_transform(train['Native.Country'])
test['Native.Country'] = number.fit_transform(test['Native.Country'])
new_native_country_levels = train['Native.Country'].unique()
native_country_levels_decoder = dict(zip(new_native_country_levels, native_country_levels))
In [67]:
native_country_levels_decoder
Out[67]:
In [68]:
plt.hist(train['Native.Country'], bins = 41)
plt.show()
plt.hist(test['Native.Country'], bins = 41)
plt.show()
This feature needs binning countries together. Question is, how best to do that?
In [69]:
pprint(native_country_levels_decoder)
Count the number of data points per country¶
In [70]:
country_observations_count_train = [len(train[train['Native.Country'] == i]) for i in range(41)]
country_observations_count_test = [len(test[test['Native.Country'] == i]) for i in range(41)]
print dict(zip([native_country_levels_decoder[i] for i in range(41)],country_observations_count_train))
print dict(zip([native_country_levels_decoder[i] for i in range(41)],country_observations_count_test))
let's not over_complicate. Let's club this as 2 bins. One for US and other non-US¶
In [71]:
# country_list = [0]*len(train['Native.Country'])
# US_index = list(train[train['Native.Country'] == 38].index)
# for i in US_index: country_list[i] = 1
# train['Native.Country'] = country_list
# country_list = [0]*len(test['Native.Country'])
# US_index = list(test[test['Native.Country'] == 38].index)
# for i in US_index: country_list[i] = 1
# test['Native.Country'] = country_list
In [72]:
test.head()
Out[72]:
Multivariate Analysis¶
In [73]:
sex_income_crosstab = pd.crosstab(train.Sex, train['Income.Group'], margins = True)
print sex_income_crosstab
In [74]:
sex_income_crosstab.iloc[:-1,:-1]
Out[74]:
In [75]:
fig = plt.figure(figsize=(6,6), dpi=1600)
# fig, ax = plt.subplots()
sex_income_crosstab.iloc[:-1,:-1].plot(kind = 'barh', stacked = True, color = ['red','blue'], alpha = 0.65)
Out[75]:
In [76]:
sex_income_crosstab
Out[76]:
In [77]:
def percentConvert(x):
return x / float(x[-1])
sex_income_crosstab.apply(percentConvert, axis = 0)
Out[77]:
In [78]:
sex_income_crosstab.apply(percentConvert, axis = 1)
Out[78]:
In [79]:
plt.figure(figsize=(6,10), dpi=1600)
sex_income_crosstab.apply(percentConvert, axis = 1).iloc[:-1,:-1].plot(kind='barh', stacked = True, color = ['red','blue'], alpha = 0.65)
plt.title('Income Distribution by Sex')
plt.show()
sex_income_crosstab.apply(percentConvert, axis = 0).iloc[:-1,:-1].transpose().plot(kind='barh', stacked = True, color = ['red','blue'], alpha = 0.65)
plt.title('Percentage share of Male / Female in each Income Class')
plt.show()
In [80]:
test.plot('Age', 'Hours.Per.Week', kind = 'scatter')
Out[80]:
This shows no real relationship between Age and Hours-Per-Week. Even intuitively we were not expecting any specific trend so this is good. In other cases, you might figure out interesting trends which can be exploited.
In [81]:
train[train.Age > 40]['Income.Group'].mean()
Out[81]:
Let's see how income classification changes solely based on age
In [82]:
ages = []
high_income_proportion = []
for age in range(17,91,3):
ages.append(age)
high_income_proportion.append(train[train.Age > age]['Income.Group'].mean())
In [83]:
plt.plot(ages, high_income_proportion)
plt.title('Proportion of people earning above $50K, given age')
Out[83]:
In [84]:
train.boxplot(column='Hours.Per.Week', by= 'Sex')
Out[84]:
In [85]:
test.boxplot(column='Hours.Per.Week', by= 'Sex')
Out[85]:
In [86]:
train.boxplot(column = 'Age', by = 'Income.Group')
Out[86]:
What is the percentage of males which have income <= 50K ?
In [87]:
1 - train[train.Sex == 1]['Income.Group'].mean()
Out[87]:
Predictive Modelling¶
Model I - Decision Tree Classifier¶
In [88]:
from sklearn.tree import DecisionTreeClassifier
In [89]:
# define the predictors
dependent_variable = 'Income.Group'
independent_variable = [x for x in train.columns if x not in ['ID', dependent_variable]]
print independent_variable
# Inititate the algorithm
model_01 = DecisionTreeClassifier(max_depth = 10, min_samples_leaf = 100, max_features= 'sqrt')
# Fit the algorithm
model_01.fit(train[independent_variable], train[dependent_variable])
# Model predictions
model_01_predictions = model_01.predict(test[independent_variable])
# Make submission
submission = pd.read_csv('sample_submission.csv')
submission.ID = test.ID
income_group = ['<=50K']*len(test['ID'])
for i in range(len(model_01_predictions)):
if model_01_predictions[i] == 1:
income_group[i] = '>50K'
submission['Income.Group'] = income_group
submission.to_csv('submission_01.csv', index = False)
Your score for this submission is : 0.826976229961.
Model II - Decision Tree Classifier with reduced number of features¶
In [90]:
from sklearn.tree import DecisionTreeClassifier
In [91]:
# define the predictors
dependent_variable = 'Income.Group'
independent_variable = ['Age', 'Education', 'Workclass', 'Sex', 'Hours.Per.Week']
print independent_variable
# Inititate the algorithm
model_02 = DecisionTreeClassifier(max_depth = 10, min_samples_leaf = 100, max_features= 'sqrt')
# Fit the algorithm
model_02.fit(train[independent_variable], train[dependent_variable])
# Model predictions
model_02_predictions = model_02.predict(test[independent_variable])
submission = pd.read_csv('sample_submission.csv')
submission.ID = test.ID
income_group = ['<=50K']*len(test['ID'])
for i in range(len(model_02_predictions)):
if model_02_predictions[i] == 1:
income_group[i] = '>50K'
submission['Income.Group'] = income_group
submission.to_csv('submission_02.csv', index = False)
Your score for this submission is : 0.806522940851.
Model III - Random Forest Classifier¶
In [92]:
from sklearn.ensemble import RandomForestClassifier
In [93]:
dependent_variable = 'Income.Group'
independent_variable = [x for x in train.columns if x not in ['ID', dependent_variable]]
# Inititate the algorithm
model_03 = RandomForestClassifier(max_depth = 10, min_samples_leaf = 10, max_features= 'auto')
# Fit the algorithm
model_03.fit(train[independent_variable], train[dependent_variable])
# Model predictions
model_03_predictions = model_03.predict(test[independent_variable])
# Make submission
submission = pd.read_csv('sample_submission.csv')
submission.ID = test.ID
income_group = ['<=50K']*len(test['ID'])
for i in range(len(model_03_predictions)):
if model_03_predictions[i] == 1:
income_group[i] = '>50K'
submission['Income.Group'] = income_group
submission.to_csv('submission_03.csv', index = False)
Your score for this submission is : 0.8359437381.
Model IV - K Nearest Neighbors Classifier¶
In [94]:
from sklearn.neighbors import KNeighborsClassifier
In [95]:
dependent_variable = 'Income.Group'
independent_variable = [x for x in train.columns if x not in ['ID', dependent_variable]]
# Inititate the algorithm
model_04 = KNeighborsClassifier(n_neighbors=10, leaf_size=1)
# Fit the algorithm
model_04.fit(train[independent_variable], train[dependent_variable])
# Model predictions
model_04_predictions = model_04.predict(test[independent_variable])
# Make submission
submission = pd.read_csv('sample_submission.csv')
submission.ID = test.ID
income_group = ['<=50K']*len(test['ID'])
for i in range(len(model_04_predictions)):
if model_04_predictions[i] == 1:
income_group[i] = '>50K'
submission['Income.Group'] = income_group
submission.to_csv('submission_04.csv', index = False)
Your score for this submission is : 0.811313801364.
Model V - Bagging¶
In [96]:
from sklearn.ensemble import BaggingClassifier
In [97]:
dependent_variable = 'Income.Group'
independent_variable = [x for x in train.columns if x not in ['ID', dependent_variable]]
# Inititate the algorithm
model_05 = BaggingClassifier()
# Fit the algorithm
model_05.fit(train[independent_variable], train[dependent_variable])
# Model predictions
model_05_predictions = model_05.predict(test[independent_variable])
# Make submission
submission = pd.read_csv('sample_submission.csv')
submission.ID = test.ID
income_group = ['<=50K']*len(test['ID'])
for i in range(len(model_05_predictions)):
if model_05_predictions[i] == 1:
income_group[i] = '>50K'
submission['Income.Group'] = income_group
submission.to_csv('submission_05.csv', index = False)
Model VI - Ada Boost¶
In [98]:
from sklearn.ensemble import AdaBoostClassifier
In [99]:
dependent_variable = 'Income.Group'
independent_variable = [x for x in train.columns if x not in ['ID', dependent_variable]]
# Inititate the algorithm
model_06 = AdaBoostClassifier(n_estimators=160, learning_rate= 0.5)
# Fit the algorithm
model_06.fit(train[independent_variable], train[dependent_variable])
# Model predictions
model_06_predictions = model_06.predict(test[independent_variable])
# Make submission
submission = pd.read_csv('sample_submission.csv')
submission.ID = test.ID
income_group = ['<=50K']*len(test['ID'])
for i in range(len(model_06_predictions)):
if model_06_predictions[i] == 1:
income_group[i] = '>50K'
submission['Income.Group'] = income_group
submission.to_csv('submission_06.csv', index = False)
Your score for this submission is : 0.836496529697.
Model VII - Gradient Boosting¶
In [100]:
from sklearn.ensemble import GradientBoostingClassifier
In [101]:
dependent_variable = 'Income.Group'
independent_variable = [x for x in train.columns if x not in ['ID', dependent_variable]]
# Inititate the algorithm
# model_07 = GradientBoostingClassifier(learning_rate=0.5, n_estimators=1700, max_depth=3)
model_07 = GradientBoostingClassifier(learning_rate=0.2, n_estimators=195, max_depth=3)
# Fit the algorithm
model_07.fit(train[independent_variable], train[dependent_variable])
# Model predictions
model_07_predictions = model_07.predict(test[independent_variable])
# Make submission
submission = pd.read_csv('sample_submission.csv')
submission.ID = test.ID
income_group = ['<=50K']*len(test['ID'])
for i in range(len(model_07_predictions)):
if model_07_predictions[i] == 1:
income_group[i] = '>50K'
submission['Income.Group'] = income_group
submission.to_csv('submission_07.csv', index = False)
Your score for this submission is : 0.841471654075.
Model VIII - Extra Trees Classifier¶
In [102]:
from sklearn.ensemble import ExtraTreesClassifier
In [103]:
dependent_variable = 'Income.Group'
independent_variable = [x for x in train.columns if x not in ['ID', dependent_variable]]
# Inititate the algorithm
model_08 = ExtraTreesClassifier()
# Fit the algorithm
model_08.fit(train[independent_variable], train[dependent_variable])
# Model predictions
model_08_predictions = model_08.predict(test[independent_variable])
# Make submission
submission = pd.read_csv('sample_submission.csv')
submission.ID = test.ID
income_group = ['<=50K']*len(test['ID'])
for i in range(len(model_08_predictions)):
if model_08_predictions[i] == 1:
income_group[i] = '>50K'
submission['Income.Group'] = income_group
submission.to_csv('submission_08.csv', index = False)
Your score for this submission is : 0.81057.