#!/usr/bin/env python
# coding: utf-8

# ## Data Exploration: Migration in Europe

# In[1]:


import numpy as np
import pandas as pd
from functools import reduce
import seaborn as sns
import matplotlib.pyplot as plt
get_ipython().run_line_magic('matplotlib', 'inline')
import matplotlib.patches as pat
from matplotlib import cm


# ## Immigration & emigration overview
# 
# 

# In[2]:


# TOTAL IMMIGRATION DATA
# https://ec.europa.eu/eurostat/cache/metadata/en/migr_immi_esms.htm
df_imm_total = pd.read_excel('migr_imm2ctz.xlsx', sheet_name='total')
df_imm_total


# In[3]:


df_imm_total.info()


# In[4]:


df_imm_total.keys()[0]


# In[5]:


# Set index
df_imm_total.set_index('GEO/TIME', inplace=True)

# Replace colons with missing value NaNs and set datatype
df_imm_total = df_imm_total.apply(pd.to_numeric, errors='coerce')

# Drop rows with no data
df_imm_total.dropna(how='all', inplace=True)

# Explore remaining missing values
df_imm_total[df_imm_total.isnull().sum(axis=1) > 0]


# In[6]:


# Backfill missing values
df_imm_total.bfill(axis=1, inplace=True)

# Rename Germany
df_imm_total.index = [s.replace('Germany (until 1990 former territory of the FRG)', 'Germany') for s in df_imm_total.index]

# Strip column names
df_imm_total.columns = [col.strip() for col in df_imm_total.columns]

df_imm_total.head()


# In[7]:


# Eurostat dataset cleaning function
countries = df_imm_total.index.tolist()

def clean_eurostat_excel(file_name, sheet):
    data = pd.read_excel(file_name, sheet_name = sheet, index_col=0)

    # rename Germany
    data.index = [str(s).replace('Germany (until 1990 former territory of the FRG)', 'Germany') for s in data.index]

    # replace ':' with NaNs, set datatype, and impute missing data with backfill/frontfill
    data = data.apply(pd.to_numeric, errors='coerce').bfill(axis=1).ffill(axis=1)
    
    # drop rows with no data
    data.dropna(how='all', inplace=True)

    # strip column names of extraneous spaces
    data.columns = [col.strip() for col in data.columns]
    
    # reduce rows to country list
    data = data.loc[countries]

    return data;


# In[8]:


df_imm_total = clean_eurostat_excel('migr_imm2ctz.xlsx', 'total')

df_imm_total.head()


# In[9]:


# TOTAL EMIGRATION DATA
# https://ec.europa.eu/eurostat/cache/metadata/en/migr_immi_esms.htm
df_emi_total = clean_eurostat_excel('migr_emi1ctz.xlsx', 'total')

df_emi_total.head()


# In[10]:


# Pivot transformation to long-form for visual analysis
def df_to_longform(df, data_col_name):
    
    df = pd.melt(df.reset_index(), id_vars='index')

    df.columns=('country','year', data_col_name)

    df.sort_values(by=['country','year'], inplace=True)

    df['year'] = df['year'].astype(str)

    return df;


# In[11]:


dfg_imm_total = df_to_longform(df_imm_total, 'immigrants')

dfg_imm_total.head()


# In[12]:


dfg_imm_total.info()


# In[13]:


# Dataframe to long-form for graphing
dfg_emi_total = df_to_longform(df_emi_total, 'emigrants')

dfg_emi_total.head()


# In[14]:


# Combining long-form dataframes to graph
dfg = pd.merge(dfg_imm_total, dfg_emi_total, on=['country', 'year'])

dfg.head()


# In[15]:


# Style
sns.set_style('whitegrid')

# Pairplot by country
pal = sns.cubehelix_palette(10, start=0.3, rot=-0.8)
g = sns.pairplot(dfg, vars=dfg.iloc[:,2:4],
            height=5, palette=pal, kind='reg', hue='country')

# Title
plt.subplots_adjust(top=0.85)
g.fig.suptitle('Immigration & Emigration by Country', size=16, weight='demi');


# In[16]:


# Pairplot by years
pal = sns.cubehelix_palette(10, start=0.3, rot=-0.8)
g = sns.pairplot(dfg, vars=dfg.iloc[:,2:4],
            height=5, palette=pal, kind='reg', hue='year')

# Title
plt.subplots_adjust(top=0.85)
g.fig.suptitle('Immigration & Emigration by Year', size=16, weight='demi');


# In[17]:


# Initialize FacetGrid object
g = sns.FacetGrid(dfg_imm_total, col='country', col_wrap=4, 
                  height=2,aspect=2).set(xticks=np.arange(0,10,3))

# Create plot
g.map(plt.plot, 'year', 'immigrants',  color='darkorange')
g.map(plt.fill_between, 'year', 'immigrants',  color='darkorange', alpha=0.5)
g.map(plt.axhline, y=0, lw=2, color='darkorange')

# Facet titles
for ax in g.axes:
    g.set_titles("{col_name}")

# Formatting axes
g.set(yticks=[])
g.despine(bottom=True, left=True)

# Legend
color_key = {'Immigrants': 'darkorange'}
patches =  [pat.Patch(color=v, label=k) for k,v in color_key.items()]

g.fig.legend(handles=patches, bbox_to_anchor =
             [0.49, 0.93]).get_frame().set_edgecolor('1.0')

# Title
plt.subplots_adjust(top=0.85)
g.fig.suptitle('Total Immigration', size=16, weight='demi');


# In[18]:


# FIRST GENERATION IMMIGRANTS REASONS DATA 2014
# https://ec.europa.eu/eurostat/cache/metadata/en/lfso_14_esms.htm
df_imm_fgen_bythous = clean_eurostat_excel('lfso_14b1dr.xlsx', 'Reason first gen imm 2014')

df_imm_fgen_bythous[df_imm_fgen_bythous.isnull().sum(axis=1) > 0]


# In[19]:


# Drop countries with no data
df_imm_fgen_bythous.dropna(how='all', inplace=True)

# Order dataframe by total first generation immigrants
df_imm_fgen_bythous.sort_values(by=['Total'], inplace=True, ascending=False)

fgen_order = df_imm_fgen_bythous.index

# Drop total - only graphing reasons
df_imm_fgen_bythous.drop('Total', axis=1, inplace=True)

# Dataframe to long-form for graphing
dfg_imm_fgen_bythous = df_to_longform(df_imm_fgen_bythous, 'x')

dfg_imm_fgen_bythous.columns = ['country', 'Reason', 'thousands']


# In[20]:


# Set plots and style
sns.set(rc={'figure.figsize':(15,30)}, style='whitegrid')

# Plot
ax = sns.barplot(data=dfg_imm_fgen_bythous, x='thousands', y='country', hue='Reason',
                 palette=sns.color_palette('deep', 7), order=fgen_order, 
                 hue_order=['Family reasons', 'Work, no job found before migrating', 
                      'Work, job found before migrating', 
                       'International protection or asylum', 'Education reasons', 
                       'Other', 'No response']) #

# Legend
plt.legend(loc='center right')

# Title
ax.set_title('First Generation Immigrants: Reasons for Migration (2014)', fontsize='large', fontweight='demi', y=1.02)
plt.figtext(0.5,0.893, 'Countries in descending order of total immigrants in 2014', ha="center", va="top", fontsize=12, color='grey'); 


# In[21]:


# Create boolean criteria for subplots
df_imm_fgen_bythous['Fam500k+'] = df_imm_fgen_bythous['Family reasons'] > 500
df_imm_fgen_bythous['Fam100k+'] = df_imm_fgen_bythous['Family reasons'] > 100

df_imm_fgen_bythous.head()


# In[22]:


# Creating sub-dataframes for zoomed-in subplots
df_imm_fgen_bythous_lar = df_imm_fgen_bythous.loc[df_imm_fgen_bythous['Fam500k+'] == True].drop(
    ['Fam500k+', 'Fam100k+'], axis=1)

df_imm_fgen_bythous_med = df_imm_fgen_bythous.loc[(df_imm_fgen_bythous['Fam500k+'] == False) & (
    df_imm_fgen_bythous['Fam100k+'] == True)].drop(
    ['Fam500k+', 'Fam100k+'], axis=1)

df_imm_fgen_bythous_sma = df_imm_fgen_bythous.loc[df_imm_fgen_bythous['Fam100k+'] == False].drop(
    ['Fam500k+', 'Fam100k+'], axis=1)

df_imm_fgen_bythous_lar.head()


# In[23]:


# Data to long-form for graphing
dfg_imm_fgen_bythous_lar = df_to_longform(df_imm_fgen_bythous_lar, 'x')
dfg_imm_fgen_bythous_med = df_to_longform(df_imm_fgen_bythous_med, 'x')
dfg_imm_fgen_bythous_sma = df_to_longform(df_imm_fgen_bythous_sma, 'x')

dfg_imm_fgen_bythous_lar.columns = ['country', 'Reason', 'thousands']
dfg_imm_fgen_bythous_med.columns = ['country', 'Reason', 'thousands']
dfg_imm_fgen_bythous_sma.columns = ['country', 'Reason', 'thousands']

dfg_imm_fgen_bythous_lar.head()


# In[24]:


# Sort for graphing order
dfg_imm_fgen_bythous_lar.sort_values(by=['thousands'], ascending=False, inplace=True)

dfg_imm_fgen_bythous_med.sort_values(by=['thousands'], ascending=False, inplace=True)

dfg_imm_fgen_bythous_sma.sort_values(by=['thousands'], ascending=False, inplace=True)


# In[25]:


# Set plots and style
fig, axs = plt.subplots(3, figsize=(10, 20), gridspec_kw={'height_ratios': [3, 2, 4]})

# Set style
sns.set_style('whitegrid')

# First subplot (> 500k family reasons)
p0 = sns.barplot(data=dfg_imm_fgen_bythous_lar, x='thousands', y='country', hue='Reason',
                 palette=sns.color_palette('deep', 7),
                 hue_order=['Family reasons', 'Work, no job found before migrating', 
                      'Work, job found before migrating', 
                       'International protection or asylum', 'Education reasons', 
                       'Other', 'No response'], ax=axs[0])
# Second subplot (200k > 500k family reasons)
p1 = sns.barplot(data=dfg_imm_fgen_bythous_med, x='thousands', y='country', hue='Reason',
                palette=sns.color_palette('deep', 7),
                 hue_order=['Family reasons', 'Work, no job found before migrating', 
                      'Work, job found before migrating', 
                       'International protection or asylum', 'Education reasons', 
                       'Other', 'No response'], ax=axs[1])

# Third subplot (< 200k family reasons)
p2 = sns.barplot(data=dfg_imm_fgen_bythous_sma, x='thousands', y='country', hue='Reason',
                 palette=sns.color_palette('deep', 7),
                 hue_order=['Family reasons', 'Work, no job found before migrating', 
                      'Work, job found before migrating', 
                       'International protection or asylum', 'Education reasons', 
                       'Other', 'No response'], ax=axs[2])


# Formatting
fig.tight_layout()
axs[0].set(xlabel='', ylabel='')
axs[1].set(xlabel='', ylabel='')
axs[2].set(xlabel='First generation immigrants', ylabel='')

# Legend
handles, labels = axs[2].get_legend_handles_labels()
p0.legend(loc='lower right')
p1.legend(handles[:0], labels[:0])
p2.legend(handles[:0], labels[:0])

# Title
plt.subplots_adjust(top=0.98)
fig.suptitle('First Generation Immigrants: Reasons for Migration (2014)', fontsize='large', fontweight='demi', y=1.01); 


# In[26]:


dfi_imm_fgen_bythous = df_imm_fgen_bythous.reset_index()

dfi_imm_fgen_bythous.head()


# In[27]:


# Set up PairGrid
g = sns.PairGrid(data=dfi_imm_fgen_bythous,
                x_vars=dfi_imm_fgen_bythous.columns[1:6], y_vars=['index'],
                height=10, aspect=0.3)

# Create stripplot
g.map(sns.stripplot, size=10, orient='h', palette=sns.cubehelix_palette(32, start=0.5, rot=-0.8, reverse=True),
     linewidth=1, edgecolor='w')

# Set x-axis limits on all columns
g.set(xlim=(-150, 4000), xlabel="Immigrants in thousands", ylabel="")

# Column titles
titles = ['Family', 'Education', 'Work (job prev.)', 'Work (no job prev.)', 'Asylum']

for ax, title in zip(g.axes.flat, titles):

    # Set a different title for each axes
    ax.set(title=title)

    # Make the grid horizontal instead of vertical
    ax.xaxis.grid(False)
    ax.yaxis.grid(True)

# Title
plt.subplots_adjust(top=0.85)
g.fig.suptitle('First Generation Immigrants: Reasons for Migration (2014)', size=16, weight='demi')
plt.figtext(0.5,0.95, 'Countries in descending order of total immigrants in 2014', ha="center", va="top", fontsize=14, color='grey')

sns.despine(left=True, bottom=True);


# In[28]:


# Initialize FacetGrid object
g = sns.FacetGrid(dfg, col='country', col_wrap=4, 
                  height=2,aspect=2).set(xticks=np.arange(0,10,3))


# Create immigration plot - set for immigrants
g.map(plt.plot, 'year', 'immigrants', color='darkorange')
g.map(plt.fill_between, 'year', 'immigrants',  color='darkorange', alpha=0.5)

g.map(plt.plot, 'year', 'emigrants', color='darkgrey')
g.map(plt.fill_between, 'year', 'emigrants',  color='darkgrey', alpha=0.5)

# Facet titles
for ax in g.axes:
    g.set_titles("{col_name}")
    g.set_axis_labels(y_var= '')

# Formatting axes
g.set(yticks=[])
g.despine(left=True)

# Legend
color_key = {'Immigrants': 'darkorange',
            'Emigrants' : 'darkgrey'}

patches =  [pat.Patch(color=v, label=k) for k,v in color_key.items()]

g.fig.legend(handles=patches, bbox_to_anchor =
             [0.525, 0.93], ncol=2).get_frame().set_edgecolor('1.0')

# Title
plt.subplots_adjust(top=0.85)
g.fig.suptitle('Total Immigration & Emigration', size=16, weight='demi');


# In[29]:


# REPORTING COUNTRY EMIGRATION DATA 
# https://ec.europa.eu/eurostat/cache/metadata/en/migr_immi_esms.htm
df_emi_reporting = clean_eurostat_excel('migr_emi1ctz.xlsx', 'reporting_country')

# Pivot transformation to long-form for visual analysis
dfg_emi_reporting = df_to_longform(df_emi_reporting, 'reporting_country_emi')

# Merge for graphing
dfg = pd.merge(dfg, dfg_emi_reporting, on=['country', 'year'])

dfg.isnull().sum()


# In[30]:


# Initialize FacetGrid object
g = sns.FacetGrid(dfg, col='country', col_wrap=4, 
                  height=2,aspect=2).set(xticks=np.arange(0,10,3))

# Create immigration plot - set for immigrants
g.map(plt.plot, 'year', 'emigrants', color='darkgrey')
g.map(plt.fill_between, 'year', 'emigrants',  color='darkgrey', alpha=0.5)

g.map(plt.plot, 'year', 'reporting_country_emi', color='cadetblue')
g.map(plt.fill_between, 'year', 'reporting_country_emi',  color='cadetblue', alpha=0.5)

# Facet titles
for ax in g.axes:
    g.set_titles("{col_name}")
    g.set_axis_labels(y_var= '')

# Formatting axes
g.set(yticks=[])
g.despine(left=True)

# Legend
color_key = {'Total emigrants': 'darkgrey',
            'Reporting country emigrants' : 'cadetblue'}

patches =  [pat.Patch(color=v, label=k) for k,v in color_key.items()]

g.fig.legend(handles=patches, bbox_to_anchor =
             [0.58, 0.93], ncol=2).get_frame().set_edgecolor('1.0')

# Title
plt.subplots_adjust(top=0.85)
g.fig.suptitle('Foreign vs. Reporting Country Emigration', size=16, weight='demi');


# In[31]:


# WORLD POPULATION DATA (IN THOUSANDS)
# https://population.un.org/wpp/Download/Standard/Population/
df_pop = pd.read_excel('world_population.xlsx')
df_pop


# In[32]:


# Select subset of dataframe
df_pop = df_pop.iloc[15:, 2:]
df_pop.head()


# In[33]:


# Create new column header
df_pop.columns = df_pop.iloc[0]
df_pop = df_pop[1:]

df_pop.head()


# In[34]:


# Cleaning up index
df_pop.set_index('Region, subregion, country or area *',inplace=True)
df_pop.drop(['Notes','Country code','Type','Parent code'],axis=1, inplace=True)
df_pop.head()


# In[35]:


# Get rid of 14 in corner and index name
del df_pop.columns.name
del df_pop.index.name

df_pop.head()


# In[36]:


# Reduce dataset to select countries and years
df_pop = df_pop.loc[countries].astype('int')

df_pop = df_pop.loc[:, '2008': '2017']

# Replace ':' with NaNs and set datatype
df_pop = df_pop.replace(':', np.nan).bfill(axis=1).ffill(axis=1)

df_pop.head()


# In[37]:


# Push index into column for graphing
dfi_pop = df_pop.reset_index()

# Drop columns for graphing
dfi_pop = dfi_pop.drop(['2009','2010','2012','2013', '2015', '2016'],axis=1)

dfi_pop.head()


# In[38]:


# Set up PairGrid
g = sns.PairGrid(data=dfi_pop.sort_values('2008', ascending=False),
                x_vars=dfi_pop.columns[1:], y_vars=['index'],
                height=10, aspect=0.3)

# Create stripplot
g.map(sns.stripplot, size=10, orient='h', palette=sns.cubehelix_palette(32, start=0.5, rot=-0.8, reverse=True),
     linewidth=1, edgecolor='w')

# Set x-axis limits on all columns
g.set(xlim=(-10000, 100000), xlabel="Population in thousands", ylabel="")

# Column titles
titles = ['2008', '2011', '2014', '2017']

for ax, title in zip(g.axes.flat, titles):

    # Set a different title for each axes
    ax.set(title=title)

    # Make the grid horizontal instead of vertical
    ax.xaxis.grid(False)
    ax.yaxis.grid(True)

# Title
plt.subplots_adjust(top=0.85)
g.fig.suptitle('Population by Country', size=16, weight='demi')

sns.despine(left=True, bottom=True);


# In[39]:


# Pivot data to long-form for visual analysis
dfg_pop = df_to_longform(df_pop, 'pop_in_thous')

dfg = pd.merge(dfg, dfg_pop, on=['country', 'year'])

dfg.info()


# In[40]:


# Create immigrants/emigrants per capita data
dfg['immigrants_per_capita'] = dfg['immigrants']/dfg['pop_in_thous']
dfg['emigrants_per_capita'] = dfg['emigrants']/dfg['pop_in_thous']

dfg.head()


# In[41]:


# Initialize FacetGrid object
g = sns.FacetGrid(dfg, col='country', col_wrap=4, 
                  height=2, aspect=2).set(xticks=np.arange(0,10,3))

# Create immigration plot - set for immigrants
g.map(plt.plot, 'year', 'immigrants_per_capita', color='darkorange')
g.map(plt.fill_between, 'year', 'immigrants_per_capita',  color='darkorange', alpha=0.5)

g.map(plt.plot, 'year', 'emigrants_per_capita', color='darkgrey')
g.map(plt.fill_between, 'year', 'emigrants_per_capita',  color='darkgrey', alpha=0.5)

# Facet titles
for ax in g.axes:
    g.set_titles("{col_name}")
    g.set_axis_labels(y_var= '')

# Formatting axes
g.set(yticks=[])
g.despine(left=True)

# Legend
color_key = {'Immigrants per capita': 'darkorange',
            'Emigrants per capita' : 'darkgrey'}

patches =  [pat.Patch(color=v, label=k) for k,v in color_key.items()]

g.fig.legend(handles=patches, bbox_to_anchor =
             [0.59, 0.93], ncol=2).get_frame().set_edgecolor('1.0')

# Title
plt.subplots_adjust(top=0.85)
g.fig.suptitle('Immigration & Emigration Per Capita', size=16, weight='demi');


# In[42]:


# GDP IN MILLION EUROS DATA
# https://ec.europa.eu/eurostat/cache/metadata/en/nama10_esms.htm
df_gdp = clean_eurostat_excel('nama_10_gdp.xlsx','GDP_million_euro')

df_gdp2 = df_gdp.drop(['2010', '2011', '2012', '2014', '2015', '2016', '2018'],axis=1)

df_gdp2 = df_gdp2.reset_index()

df_gdp2 = df_gdp2.rename(columns = {'index':'country'})

df_gdp2.head()


# In[43]:


# Style
sns.set(style='whitegrid')

# Set up PairGrid
g = sns.PairGrid(data=df_gdp2.sort_values('2009', ascending=False),
                x_vars=df_gdp2.columns[1:], y_vars=['country'],
                height=10, aspect=0.3)

# Create stripplot
g.map(sns.stripplot, size=10, orient='h', palette=sns.cubehelix_palette(32, start=0.5, rot=-0.8, reverse=True),
     linewidth=1, edgecolor='w')

# Set x-axis limits on all columns
g.set(xlim=(-150000, 3500000), xlabel="GDP", ylabel="")

# Column titles
titles =['2009', '2013', '2017']

for ax, title in zip(g.axes.flat, titles):

    # Set a different title for each axes
    ax.set(title=title)

    # Make the grid horizontal instead of vertical
    ax.xaxis.grid(False)
    ax.yaxis.grid(True)

sns.despine(left=True, bottom=True)

# Title
plt.subplots_adjust(top=0.85)
g.fig.suptitle('GDP', size=16, weight='demi');


# In[44]:


# Pivot data to long-form for visual analysis
dfg_gdp = df_to_longform(df_gdp, 'GDP')

# merge population and GDP data
dfg = pd.merge(dfg, dfg_gdp, on=['country', 'year'])

dfg.head()


# In[45]:


# Create GDP per capita variable
dfg['GDP_pc'] = dfg['GDP']/dfg['pop_in_thous']

dfg.info()


# In[46]:


# Pivot data into wide-form for graphing
df_gdp_pc = dfg.pivot(index='country', columns='year')['GDP_pc'].reset_index()

df_gdp_pc = df_gdp_pc.drop(['2010', '2011', '2012', '2014', '2015', '2016'], axis=1)

df_gdp_pc.head()


# In[47]:


# Style
sns.set(style='whitegrid')

# Set up PairGrid
g = sns.PairGrid(data=df_gdp_pc.sort_values('2009', ascending=False),
                x_vars=df_gdp_pc.columns[1:], y_vars=['country'],
                height=10, aspect=0.3)

# Create stripplot
g.map(sns.stripplot, size=10, orient='h', palette=sns.cubehelix_palette(32, start=0.5, rot=-0.8, reverse=True),
     linewidth=1, edgecolor='w')

# Set x-axis limits on all columns
g.set(xlim=(-10, 170), xlabel="GDP Per Capita", ylabel="")

# Column titles
titles =['2009', '2013', '2017']

for ax, title in zip(g.axes.flat, titles):

    # Set a different title for each axes
    ax.set(title=title)

    # Make the grid horizontal instead of vertical
    ax.xaxis.grid(False)
    ax.yaxis.grid(True)

sns.despine(left=True, bottom=True)

# Title
plt.subplots_adjust(top=0.85)
g.fig.suptitle('GDP Per Capita', size=16, weight='demi');


# In[48]:


# Create variables immigrants/emigrants per GDP
dfg['immigrants_per_GDP'] = dfg['immigrants']/dfg['GDP']
dfg['emigrants_per_GDP'] = dfg['emigrants']/dfg['GDP']

dfg.info()


# In[49]:


# Initialize FacetGrid object
g = sns.FacetGrid(dfg, col='country', col_wrap=4, height=1.8,
                  aspect=2).set(xticks=np.arange(0,10,3))

# Create immigration plot - set for immigrants
g.map(plt.plot, 'year', 'immigrants_per_GDP', color='darkorange')
g.map(plt.fill_between, 'year', 'immigrants_per_GDP',  color='darkorange', alpha=0.4)

g.map(plt.plot, 'year', 'emigrants_per_GDP', color='darkgrey')
g.map(plt.fill_between, 'year', 'emigrants_per_GDP',  color='darkgrey', alpha=0.6)

# Facet titles
for ax in g.axes:
    g.set_titles("{col_name}")
    g.set_axis_labels(y_var= '')

# Formatting axes
g.set(yticks=[])
g.despine(left=True)

# Legend
color_key = {'Immigrants per GDP': 'darkorange',
            'Emigrants per GDP' : 'darkgrey'}

patches =  [pat.Patch(color=v, label=k) for k,v in color_key.items()]

g.fig.legend(handles=patches, bbox_to_anchor =
             [0.61, 0.93], ncol=2).get_frame().set_edgecolor('1.0')

# Title
plt.subplots_adjust(top=0.85)
g.fig.suptitle('Immigration & Emigration per GDP', size=16, weight='demi');


# ### Maps of 2015 Immigration to the EU

# In[50]:


import geopandas

# GEOJSON POLYGON COUNTRY DATA
# https://datahub.io/core/geo-countries
world_map = geopandas.read_file('countries.geojson')

world_map.head()


# In[51]:


type(world_map)


# In[52]:


# Exploring map axes
world_map.plot(figsize=(30,30))
plt.axis([-13,40,30,75]);


# In[53]:


# Check if country lists of data and maps match
diff = list(set(countries) - set(world_map['ADMIN']))

diff


# In[54]:


# Find outlier
world_map[world_map['ADMIN'].str.startswith('Cz')]


# In[55]:


# Rename outlier to match data with map
world_map.iat[60,0] = 'Czechia'

world_map[world_map['ADMIN'].str.startswith('Cz')]


# In[56]:


diff = list(set(countries) - set(world_map['ADMIN']))

diff


# In[57]:


# Map data for total immigrants
europe_map_imm = pd.merge(world_map, df_imm_total, left_on=['ADMIN'], right_index=True)

europe_map_imm


# In[58]:


# Map data for immigrants per capita
df_imm_pc = dfg.pivot(index='country', columns='year')['immigrants_per_capita']

europe_map_imm_pc = pd.merge(world_map, df_imm_pc, left_on=['ADMIN'], right_index=True)

europe_map_imm_pc.head(1)


# In[59]:


# Map data for immigrants per GDP
df_imm_gdp = dfg.pivot(index='country', columns='year')['immigrants_per_GDP']

europe_map_imm_pc = pd.merge(world_map, df_imm_gdp, left_on=['ADMIN'], right_index=True)

europe_map_imm_pc.head(1)


# In[60]:


from matplotlib import cm
from matplotlib.colors import ListedColormap, LinearSegmentedColormap
from mpl_toolkits.axes_grid1 import make_axes_locatable

# Set figure and axis
fig, ax = plt.subplots(1, figsize=(15,15))
plt.axis([-13,40,30,75])

# Colormap formatting
oranges_middle = cm.get_cmap('Oranges', 512)
Oranges2 = ListedColormap(oranges_middle(np.linspace(0.25, 0.75, 256)))

# Aligning map layers
ax.set_aspect('equal')
ax.axis('off')

# Legend formatting
divider = make_axes_locatable(ax)
cax = divider.append_axes("right", size="5%", pad=0.1)

# Plots
world_map.plot(ax=ax, color='silver', edgecolor='white')

europe_map_imm.plot(ax=ax, column='2015', cmap=Oranges2, legend=True, cax=cax)

# Title
ax.set_title('European Immigration 2015', fontdict={'fontsize':'28', 'fontweight':'3'});


# In[61]:


# Set figure and axis
fig, ax = plt.subplots(1, figsize=(15,15))
plt.axis([-13,40,30,75])

# Colormap formatting
oranges_middle = cm.get_cmap('Oranges', 512)
Oranges2 = ListedColormap(oranges_middle(np.linspace(0.25, 0.75, 256)))

# Aligning map layers
ax.set_aspect('equal')
ax.axis('off')

# Legend formatting
divider = make_axes_locatable(ax)
cax = divider.append_axes("right", size="5%", pad=0.1)

# Plots
world_map.plot(ax=ax, color='silver', edgecolor='white')

europe_map_imm_pc.plot(ax=ax, column='2015', cmap=Oranges2, legend=True, cax=cax)

# Title
ax.set_title('Immigrants per Capita 2015', fontdict={'fontsize':'28', 'fontweight':'3'});


# In[62]:


# Set figure and axis
fig, ax = plt.subplots(1, figsize=(15,15))
plt.axis([-13,40,30,75])

# Colormap formatting
oranges_middle = cm.get_cmap('Oranges', 512)
Oranges2 = ListedColormap(oranges_middle(np.linspace(0.25, 0.75, 256)))

# Aligning map layers
ax.set_aspect('equal')
ax.axis('off')

# Legend formatting
divider = make_axes_locatable(ax)
cax = divider.append_axes("right", size="5%", pad=0.1)

# Plots
world_map.plot(ax=ax, color='silver', edgecolor='white')
europe_map_imm_pc.plot(ax=ax, column='2015', cmap=Oranges2, legend=True, cax=cax)

# Title
ax.set_title('Immigrants per GDP 2015', fontdict={'fontsize':'28', 'fontweight':'3'});


# ## Asylum seekers

# In[63]:


# ASYLUM APPLICATIONS DATA 
# https://ec.europa.eu/eurostat/cache/metadata/en/migr_asyapp_esms.htm
df_asylum = clean_eurostat_excel('migr_asylum_apps.xlsx', 'Data')

dfg_asylum = df_to_longform(df_asylum, 'asylum_apps')

# Explore remaining missing values
dfg_asylum[dfg_asylum.isnull().sum(axis=1) > 0]


# In[64]:


df_asylum.head()


# In[65]:


# ASYLUM DECISIONS, OUTGOING
# https://ec.europa.eu/eurostat/cache/metadata/en/migr_dub_esms.htm
df_asyl_accepted_outgoing = clean_eurostat_excel('migr_dubdo.xlsx', 'Accepted - outgoing')

dfg_asyl_accepted_outgoing = df_to_longform(df_asyl_accepted_outgoing, 'asylum_accepted_outgoing')

# ASYLUM DECISIONS, INCOMING
# https://ec.europa.eu/eurostat/cache/metadata/en/migr_dub_esms.htm
df_asyl_accepted_incoming = clean_eurostat_excel('migr_dubdi.xlsx', 'Accepted - incoming')

dfg_asyl_accepted_incoming = df_to_longform(df_asyl_accepted_incoming, 'asylum_accepted_incoming')

# Explore remaining missing values
print(dfg_asyl_accepted_incoming[dfg_asyl_accepted_incoming.isnull().sum(axis=1) > 0])
print(dfg_asyl_accepted_outgoing[dfg_asyl_accepted_outgoing.isnull().sum(axis=1) > 0])


# In[66]:


# Merge multiple into graphing dataframe
dfs = [dfg, dfg_asylum, dfg_asyl_accepted_outgoing, dfg_asyl_accepted_incoming]

dfg = reduce(lambda left, right: pd.merge(left, right, on=['country', 'year'],
                                        how='inner'), dfs)     

dfg.info()
dfg.head(1)


# In[111]:


# Initialize FacetGrid object
g = sns.FacetGrid(dfg, col='country', col_wrap=4, height=1.8,
                  aspect=2).set(xticks=np.arange(0,10,3))


# Create immigration plot - set for immigrants
g.map(plt.plot, 'year', 'immigrants', color='darkorange')
g.map(plt.fill_between, 'year', 'immigrants',  color='darkorange', alpha=0.4)

g.map(plt.plot, 'year', 'asylum_apps', color='deepskyblue')
g.map(plt.fill_between, 'year', 'asylum_apps',  color='deepskyblue', alpha=0.5)

# Facet titles
for ax in g.axes:
    g.set_titles("{col_name}")
    g.set_axis_labels(y_var= '')

# Formatting axes
g.set(yticks=[])
g.despine(left=True)

# Legend
color_key = {'Immigrants': 'darkorange',
            'Asylum applications' : 'deepskyblue'}

patches =  [pat.Patch(color=v, label=k) for k,v in color_key.items()]

g.fig.legend(handles=patches, bbox_to_anchor =
             [0.555, 0.93], ncol=2).get_frame().set_edgecolor('1.0')

# Title
plt.subplots_adjust(top=0.85)
g.fig.suptitle('Immigrants & Asylum Seekers', size=16, weight='demi');


# In[68]:


# Initialize FacetGrid object
g = sns.FacetGrid(dfg, col='country', col_wrap=4, height=2.2,
                  aspect=1.6).set(xticks=np.arange(0,10,3))

# Create immigration plot - set for immigrants
g.map(plt.plot, 'year', 'asylum_apps', color='deepskyblue')
g.map(plt.fill_between, 'year', 'asylum_apps',  color='deepskyblue', alpha=0.5)

g.map(plt.plot, 'year', 'asylum_accepted_incoming', color='indianred')
g.map(plt.fill_between, 'year', 'asylum_accepted_incoming',  color='indianred', alpha=0.5)


# Facet titles
for ax in g.axes:
    g.set_titles("{col_name}")
    g.set_axis_labels(y_var= '')

# Formatting axes
g.set(yticks=[])
g.despine(left=True)

# Legend
color_key = {'Asylum applications' : 'deepskyblue',
            'Accepted incoming applications': 'indianred'}

patches =  [pat.Patch(color=v, label=k) for k,v in color_key.items()]

g.fig.legend(handles=patches, bbox_to_anchor =
             [0.61, 0.93], ncol=2).get_frame().set_edgecolor('1.0')

# Title
plt.subplots_adjust(top=0.85)
g.fig.suptitle('Asylum Applications: Total vs. Accepted', size=16, weight='demi');


# In[69]:


# Group by country and year for graphing
dfg_grouped_country = dfg.groupby(['country']).sum().reset_index()

dfg_grouped_year = dfg.groupby(['year']).sum().reset_index()


# In[70]:


# Split by-country dataframe for clearer graphing
dfg_grouped_country['250k+'] =  dfg_grouped_country['asylum_apps'] > 250000
dfg_grouped_country['50k+'] =  dfg_grouped_country['asylum_apps'] > 50000


dfg_grouped_country.sort_values('asylum_apps', ascending=False, inplace=True)


# In[71]:


# Create sub-dataframes
dfg_grouped_country_lar = dfg_grouped_country.loc[dfg_grouped_country['250k+'] == True]
dfg_grouped_country_med = dfg_grouped_country.loc[(dfg_grouped_country['50k+'] == True) & 
                                                    (dfg_grouped_country['250k+'] == False)]
dfg_grouped_country_sma = dfg_grouped_country.loc[dfg_grouped_country['50k+'] == False]

dfg_grouped_country_sma.head()


# In[72]:


# Initialize figure
fig, ax = plt.subplots(figsize=(8, 13))

# Plot asylum apps
sns.barplot(data = dfg_grouped_country.sort_values('asylum_apps', ascending=False), x='asylum_apps', y='country',
            label='Asylum applications', color='deepskyblue', ci=None)

# Plot asylum apps accepted, incoming
sns.barplot(data = dfg_grouped_country.sort_values('asylum_apps', ascending=False), x='asylum_accepted_incoming', y='country',
            label='Accepted incoming applications', color="indianred", ci=None)

# Legend
ax.legend(bbox_to_anchor = [0.5, 0.95])

# Axis label
ax.set(ylabel='',
       xlabel='Incoming applications)')
sns.despine(left=True, bottom=True)

# Title
fig.subplots_adjust(top=0.93)
fig.suptitle('Asylum Applications: Total vs. Accepted Incoming (2009 - 2017)', size=14, weight='demi');


# In[73]:


# Set figure and axes
fig, axs = plt.subplots(3, figsize=(10, 15), gridspec_kw={'height_ratios': [1, 1.43, 2.14]})

# Set style
sns.set_style('whitegrid')

# First subplot (>250k asylum apps)
sns.barplot(data = dfg_grouped_country_lar, y='country', x='asylum_apps',
            label='Total asylum applications', color="deepskyblue", ci=None, ax=axs[0])

sns.barplot(data = dfg_grouped_country_lar, y='country', x='asylum_accepted_incoming',
            label='Accepted incoming applications', color="indianred", ci=None, ax=axs[0])

# Second subplot(<250k asylum apps)
sns.barplot(data = dfg_grouped_country_med, y='country', x='asylum_apps',
            label='Total asylum applications', color="deepskyblue", ci=None, ax=axs[1])

sns.barplot(data = dfg_grouped_country_med, y='country', x='asylum_accepted_incoming',
            label='Accepted incoming applications', color="indianred", ci=None, ax=axs[1])

# Third subplot(<50k asylum apps)
sns.barplot(data = dfg_grouped_country_sma, y='country', x='asylum_apps',
            label='Total asylum applications', color="deepskyblue", ci=None, ax=axs[2])

sns.barplot(data = dfg_grouped_country_sma, y='country', x='asylum_accepted_incoming',
            label='Accepted incoming applications', color="indianred", ci=None, ax=axs[2])


# Formatting
fig.tight_layout()
axs[0].set(xlabel='', ylabel='')
axs[1].set(xlabel='', ylabel='')
axs[2].set(xlabel='', ylabel='')

# Legend
axs[2].legend(bbox_to_anchor = [0.97, 0.13])

# Title
plt.subplots_adjust(top=0.95)
fig.suptitle('Asylum Applications: Total vs. Accepted Incoming (2009 - 2017)', size=16, weight='demi');


# In[74]:


# Initialize figure
fig, ax = plt.subplots(figsize=(15, 10))

# Plot asylum apps
sns.set_color_codes("pastel")
sns.barplot(data = dfg_grouped_year, x='year', y='asylum_apps',
            label="Total asylum applications", color="deepskyblue", ci=None)

# Plot asylum apps accepted, incoming
sns.set_color_codes('muted')
sns.barplot(data = dfg_grouped_year, x='year', y='asylum_accepted_incoming',
            label="Accepted incoming applications", color="indianred", ci=None)

# Legend
ax.legend(bbox_to_anchor = [0.25, 0.95]) #loc='upper left'

# Axis labels
ax.set(ylabel='',xlabel='Applications')
sns.despine(left=True, bottom=True)

# Title
plt.subplots_adjust(top=0.95)
fig.suptitle('Accepted (Incoming) & Total Asylum Applications', size=20, weight='demi');


# In[75]:


# Initialize FacetGrid object
g = sns.FacetGrid(dfg, col='country', col_wrap=4, height=2.2,
                  aspect=1.6).set(xticks=np.arange(0,10,3))

# Create immigration plot - set for immigrants
g.map(plt.plot, 'year', 'asylum_accepted_outgoing', color='mediumpurple')
g.map(plt.fill_between, 'year', 'asylum_accepted_outgoing',  color='mediumpurple', alpha=0.5)

g.map(plt.plot, 'year', 'asylum_accepted_incoming', color='cadetblue')
g.map(plt.fill_between, 'year', 'asylum_accepted_incoming',  color='cadetblue', alpha=0.5)


# Facet titles
for ax in g.axes:
    g.set_titles("{col_name}")
    g.set_axis_labels(y_var= '')

# Formatting axes
g.set(yticks=[])
g.despine(left=True)

# Legend
color_key = {'Accepted outgoing applications' : 'mediumpurple',
            'Accepted incoming applications': 'cadetblue'}

patches =  [pat.Patch(color=v, label=k) for k,v in color_key.items()]

g.fig.legend(handles=patches, bbox_to_anchor =
             [0.645, 0.93], ncol=2).get_frame().set_edgecolor('1.0')

# Title
plt.subplots_adjust(top=0.85)
g.fig.suptitle('Accepted Applications: Outgoing vs Incoming', size=16, weight='demi');


# In[76]:


# Create new variables for graphing
dfg['asylum_accepted_incoming_per_capita'] = dfg['asylum_accepted_incoming']/dfg['pop_in_thous']*1000
dfg['asylum_accepted_incoming_per_GDP'] = dfg['asylum_accepted_incoming']/dfg['GDP']*1000000

dfg.head()


# In[77]:


# Pivot data into wide-form for graphing
df_asylum_inc_pc = dfg.pivot(index='country', columns='year')['asylum_accepted_incoming_per_capita'].reset_index()
df_asylum_inc_pgdp = dfg.pivot(index='country', columns='year')['asylum_accepted_incoming_per_GDP'].reset_index()

df_asylum_inc_pc.head()


# In[78]:


df_asylum_inc_pc = df_asylum_inc_pc.drop(['2009', '2010', '2011', '2012'],axis=1)
df_asylum_inc_pgdp = df_asylum_inc_pgdp.drop(['2009', '2010', '2011', '2012'],axis=1)

df_asylum_inc_pc.max()


# In[79]:


# Style
sns.set(style='whitegrid')

# Set up PairGrid
g = sns.PairGrid(data=df_asylum_inc_pc.sort_values('2015', ascending=False),
                x_vars=df_asylum_inc_pc.columns[1:], y_vars=['country'],
                height=10, aspect=0.3)

# Create stripplot
g.map(sns.stripplot, size=10, orient='h', palette=sns.cubehelix_palette(32, start=0.5, rot=-0.8, reverse=True),
     linewidth=1, edgecolor='w')

# Set x-axis limits on all columns
g.set(xlim=(-100, 1900), xlabel="Accepted per capita", ylabel="")

# Column titles
titles =['2013', '2014', '2015', '2016', '2017']

for ax, title in zip(g.axes.flat, titles):

    # Set a different title for each axes
    ax.set(title=title)

    # Make the grid horizontal instead of vertical
    ax.xaxis.grid(False)
    ax.yaxis.grid(True)

sns.despine(left=True, bottom=True)

# Title
plt.subplots_adjust(top=0.85)
g.fig.suptitle('Accepted Incoming Asylum Applications per Capita', size=16, weight='demi')
plt.figtext(0.5,0.95, 'Countries in descending order of accepted applications per capita in 2015', ha="center", va="top", fontsize=14, color='grey');


# In[80]:


df_asylum_inc_pgdp.max()


# In[81]:


# Style
sns.set(style='whitegrid')

# Set up PairGrid
g = sns.PairGrid(data=df_asylum_inc_pgdp.sort_values('2015', ascending=False),
                x_vars=df_asylum_inc_pgdp.columns[1:], y_vars=['country'],
                height=10, aspect=0.3)

# Create stripplot
g.map(sns.stripplot, size=10, orient='h', palette=sns.cubehelix_palette(32, start=0.5, rot=-0.8, reverse=True),
     linewidth=1, edgecolor='w')

# Set x-axis limits on all columns
g.set(xlim=(-3000, 98000), xlabel="Accepted per GDP", ylabel="")

# Column titles
titles =['2013', '2014', '2015', '2016', '2017']

for ax, title in zip(g.axes.flat, titles):

    # Set a different title for each axes
    ax.set(title=title)

    # Make the grid horizontal instead of vertical
    ax.xaxis.grid(False)
    ax.yaxis.grid(True)

sns.despine(left=True, bottom=True)

# Title
plt.subplots_adjust(top=0.85)
g.fig.suptitle('Accepted Incoming Asylum Applications per GDP', size=16, weight='demi')
plt.figtext(0.5,0.95, 'Countries in descending order of accepted applications per GDP in 2015', ha="center", va="top", fontsize=14, color='grey');


# ## Immigration policy, enforcement, and effectiveness

# In[82]:


# THIRD COUNTRY NATIONALS (TCN) ILLEGALLY PRESENT
# https://ec.europa.eu/eurostat/cache/metadata/en/migr_eil_esms.htm
df_tcn_illegal_pres = clean_eurostat_excel('migr_illegally_present.xlsx', 'Data')

dfg_tcn_illegal_pres = df_to_longform(df_tcn_illegal_pres, 'illegally_pres')

dfg_tcn_illegal_pres.head(1)


# In[83]:


# TCN ILLEGALLY PRESENT GIVEN LEAVE ORDERS DATA
# https://ec.europa.eu/eurostat/cache/metadata/en/migr_eil_esms.htm
df_tcn_leave_order = clean_eurostat_excel('migr_leave_order.xlsx', 'migr_eiord')

dfg_tcn_leave_order = df_to_longform(df_tcn_leave_order, 'leave_order')

dfg_tcn_leave_order.head(1)


# In[84]:


# TCN REFUSED AT BORDER CROSSINGS DATA 
# https://ec.europa.eu/eurostat/cache/metadata/en/migr_eil_esms.htm
df_tcn_refused = clean_eurostat_excel('migr_refused_entry.xlsx', 'migr_eirfs')

dfg_tcn_refused = df_to_longform(df_tcn_refused, 'refused_border')

dfg_tcn_refused.head(1)


# In[85]:


# TCN ILLEGALLY PRESENT THAT LEFT COUNTRY AFTER LEAVE ORDER DATA
# https://ec.europa.eu/eurostat/cache/metadata/en/migr_eil_esms.htm
df_tcn_returned = clean_eurostat_excel('migr_eirtn.xlsx', 'total_returned')

dfg_tcn_returned = df_to_longform(df_tcn_returned, 'illegal_returned')

dfg_tcn_returned.head(1)


# In[86]:


# TCN ILLEGALLY PRESENT THAT LEFT COUNTRY AND EU AFTER LEAVE ORDER DATA
# https://ec.europa.eu/eurostat/cache/metadata/en/migr_eil_esms.htm
df_tcn_returned_third = clean_eurostat_excel('migr_eirtn.xlsx', 'returned_third_country')

dfg_tcn_returned_third = df_to_longform(df_tcn_returned_third, 'illegal_returned_thirdcountry')

dfg_tcn_returned_third.head(1)


# In[87]:


# Merge all into graphing dataframe
dfs = [dfg, dfg_tcn_illegal_pres, dfg_tcn_leave_order, dfg_tcn_refused,
       dfg_tcn_returned, dfg_tcn_returned_third]

dfg = reduce(lambda left, right: pd.merge(left, right, on=['country', 'year'],
                                        how='inner'), dfs)     

dfg.info()
dfg.head(1)


# In[107]:


# Initialize FacetGrid object
g = sns.FacetGrid(dfg, col='country', col_wrap=4, height=1.8,
                  aspect=2).set(xticks=np.arange(0,10,3))


# Create immigration plot - set for immigrants
g.map(plt.plot, 'year', 'immigrants', color='darkorange')
g.map(plt.fill_between, 'year', 'immigrants',  color='darkorange', alpha=0.4)

g.map(plt.plot, 'year', 'refused_border', color='olivedrab')
g.map(plt.fill_between, 'year', 'refused_border',  color='olivedrab', alpha=0.6)

# Facet titles
for ax in g.axes:
    g.set_titles("{col_name}")
    g.set_axis_labels(y_var= '')

# Formatting axes
g.set(yticks=[])
g.despine(left=True)

# Legend
color_key = {'Total immigrants': 'darkorange',
            'Third country nationals refused at border' : 'olivedrab'}

patches =  [pat.Patch(color=v, label=k) for k,v in color_key.items()]

g.fig.legend(handles=patches, bbox_to_anchor =
             [0.61, 0.93], ncol=2).get_frame().set_edgecolor('1.0')

# Title
plt.subplots_adjust(top=0.85)
g.fig.suptitle('Total Immigrants vs. Third Country Nationals Refused at Border', size=16, weight='demi');


# In[108]:


# Initialize FacetGrid object
g = sns.FacetGrid(dfg, col='country', col_wrap=4, height=1.8,
                  aspect=2).set(xticks=np.arange(0,10,3))


# Create immigration plot - set for immigrants
g.map(plt.plot, 'year', 'immigrants', color='darkorange')
g.map(plt.fill_between, 'year', 'immigrants',  color='darkorange', alpha=0.4)

g.map(plt.plot, 'year', 'illegally_pres', color='lightseagreen')
g.map(plt.fill_between, 'year', 'illegally_pres',  color='lightseagreen', alpha=0.6)

# Facet titles
for ax in g.axes:
    g.set_titles("{col_name}")
    g.set_axis_labels(y_var= '')

# Formatting axes
g.set(yticks=[])
g.despine(left=True)

# Legend
color_key = {'Total immigrants': 'darkorange',
            'Illegally present third country nationals' : 'lightseagreen'}

patches =  [pat.Patch(color=v, label=k) for k,v in color_key.items()]

g.fig.legend(handles=patches, bbox_to_anchor =
             [0.61, 0.93], ncol=2).get_frame().set_edgecolor('1.0')

# Title
plt.subplots_adjust(top=0.85)
g.fig.suptitle('Total Immigrants vs. Illegally Present Third Country Nationals', size=16, weight='demi');


# In[114]:


# Initialize FacetGrid object
g = sns.FacetGrid(dfg, col='country', col_wrap=4, height=1.8,
                  aspect=2).set(xticks=np.arange(0,10,3))


# Create immigration plot - set for immigrants
g.map(plt.plot, 'year', 'asylum_apps', color='goldenrod')
g.map(plt.fill_between, 'year', 'asylum_apps',  color='goldenrod', alpha=0.4)

g.map(plt.plot, 'year', 'illegally_pres', color='lightseagreen')
g.map(plt.fill_between, 'year', 'illegally_pres',  color='lightseagreen', alpha=0.6)

# Facet titles
for ax in g.axes:
    g.set_titles("{col_name}")
    g.set_axis_labels(y_var= '')

# Formatting axes
g.set(yticks=[])
g.despine(left=True)

# Legend
color_key = {'Asylum applications': 'goldenrod',
            'Illegally present third country nationals' : 'lightseagreen'}

patches =  [pat.Patch(color=v, label=k) for k,v in color_key.items()]

g.fig.legend(handles=patches, bbox_to_anchor =
             [0.62, 0.93], ncol=2).get_frame().set_edgecolor('1.0')

# Title
plt.subplots_adjust(top=0.85)
g.fig.suptitle('Asylum Applications vs. Illegally Present Third Country Nationals', size=16, weight='demi');


# In[116]:


# Initialize FacetGrid object
g = sns.FacetGrid(dfg, col='country', col_wrap=4, height=1.8,
                  aspect=2).set(xticks=np.arange(0,10,3))


# Create immigration plot - set for immigrants
g.map(plt.plot, 'year', 'leave_order', color='slateblue')
g.map(plt.fill_between, 'year', 'leave_order',  color='slateblue', alpha=0.5)

g.map(plt.plot, 'year', 'illegally_pres', color='lightseagreen')
g.map(plt.fill_between, 'year', 'illegally_pres',  color='lightseagreen', alpha=0.5)

# Facet titles
for ax in g.axes:
    g.set_titles("{col_name}")
    g.set_axis_labels(y_var= '')

# Formatting axes
g.set(yticks=[])
g.despine(left=True)

# Legend
color_key = {'Illegally present third country nationals (TCNs)' : 'lightseagreen',
            'Illegally present TCNs given leave order': 'slateblue'}

patches =  [pat.Patch(color=v, label=k) for k,v in color_key.items()]

g.fig.legend(handles=patches, bbox_to_anchor =
             [0.69, 0.93], ncol=2).get_frame().set_edgecolor('1.0')

# Title
plt.subplots_adjust(top=0.85)
g.fig.suptitle('Illegally Present Third Country Nationals & Those Given Leave Orders', size=16, weight='demi');


# In[92]:


# Find countries with NaNs
dfg[dfg[['illegal_returned']].isna().any(axis=1)]['country'].unique()


# In[93]:


# Create new dataframe and drop NaN rows
dfg2 = dfg[['country', 'year', 'leave_order', 'illegal_returned', 'illegal_returned_thirdcountry']]

dfg2.dropna(inplace=True)

dfg2.isnull().sum()


# In[121]:


# Initialize FacetGrid object
g = sns.FacetGrid(dfg2, col='country', col_wrap=4, height=1.8,
                  aspect=2).set(xticks=np.arange(0,10,3))


# Create immigration plot - set for immigrants
g.map(plt.plot, 'year', 'leave_order', color='slateblue')
g.map(plt.fill_between, 'year', 'leave_order',  color='slateblue', alpha=0.6)

g.map(plt.plot, 'year', 'illegal_returned', color='crimson')
g.map(plt.fill_between, 'year', 'illegal_returned',  color='crimson', alpha=0.6)

# Facet titles
for ax in g.axes:
    g.set_titles("{col_name}")
    g.set_axis_labels(y_var= '')

# Formatting axes
g.set(yticks=[])
g.despine(left=True)

# Legend
color_key = {'Illegally present third country nationals given leave order': 'slateblue',
            'Leave order recipients that left' : 'crimson'}

patches =  [pat.Patch(color=v, label=k) for k,v in color_key.items()]

g.fig.legend(handles=patches, bbox_to_anchor =
             [0.69, 0.93], ncol=2).get_frame().set_edgecolor('1.0')

# Title
plt.subplots_adjust(top=0.85)
g.fig.suptitle('Illegally Present Third Country Nationals: Leave Orders and Response', size=16, weight='demi');


# In[95]:


dfg3 = dfg2.groupby(['country']).sum().reset_index()

dfg3.info()


# In[96]:


# Relationship between leave orders and compliance?
# relationship is heteroskedastic
g = sns.jointplot(data=dfg3, x='leave_order', y='illegal_returned',
                                   kind='reg', color='crimson');


# In[97]:


# Resizing data for graph visual (no y-axis, only visual relative comparison)
dfg2['illegal_returned'] = dfg2['illegal_returned']/100
dfg2['illegal_returned_thirdcountry'] = dfg2['illegal_returned_thirdcountry']/100


# In[122]:


# Initialize FacetGrid object
g = sns.FacetGrid(dfg2, col='country', col_wrap=4, height=2.8,
                  aspect=2).set(xticks=np.arange(0,10,3))

# Create immigration plot - set for immigrants
g.map(plt.plot, 'year', 'illegal_returned', color='crimson')
g.map(plt.fill_between, 'year', 'illegal_returned',  color='crimson', alpha=0.6)

g.map(plt.plot, 'year', 'illegal_returned_thirdcountry', color='darkcyan')
g.map(plt.fill_between, 'year', 'illegal_returned_thirdcountry',  color='darkcyan', alpha=0.6)

# Facet titles
for ax in g.axes:
    g.set_titles("{col_name}", size=18)
    g.set_axis_labels(y_var= '')

# Formatting axes
g.set(yticks=[])
g.despine(left=True)

# Legend
color_key = {'Left country that gave leave order': 'crimson',
            'Left Europe altogether' : 'darkcyan'}

patches =  [pat.Patch(color=v, label=k) for k,v in color_key.items()]

g.fig.legend(handles=patches, bbox_to_anchor =
             [0.67, 0.93], ncol=2, fontsize='xx-large').get_frame().set_edgecolor('1.0')

# Title
plt.subplots_adjust(top=0.85)
g.fig.suptitle('Illegally Present Third Country Nationals That Left After Given Leave Order', size=26, weight='demi');


# In[ ]: