#!/usr/bin/env python
# coding: utf-8

# In[1]:


# Full width
from IPython.core.display import display, HTML
display(HTML("<style>.container { width:100% !important; }</style>"))


# In[2]:


get_ipython().run_line_magic('load_ext', 'autoreload')
get_ipython().run_line_magic('autoreload', '2')


# In[3]:


import math
import os
import subprocess

import matplotlib.pyplot as plt
import numpy as np
import pandas as pd
from IPython.display import display

# 
from lib_modeling import *
from lib_feature_engineering import *

# some settings for displaying Pandas results
pd.set_option('display.width', 2000)
pd.set_option('display.max_rows', 500)
pd.set_option('display.max_columns', 500)
pd.set_option('display.precision', 4)
pd.set_option('display.max_colwidth', -1)


# # Load data
# 
# - Load train + tvt = train_filtered for features evaluation
# - Load train/test for applying mean encoding

# In[4]:


# load train/test data
data_path = "home-credit-default-risk/application_train.csv"
pdf_train = pd.read_csv(data_path)

data_path = "home-credit-default-risk/application_test.csv"
pdf_test = pd.read_csv(data_path)

# filter by tvt code
pdf_tvt_extend = pd.read_pickle("pdf_tvt_extend.pkl", compression="bz2")
pdf_train_filtered = (pdf_tvt_extend.query("tvt_code == 'train'")
                      .merge(pdf_train[["SK_ID_CURR"]], on="SK_ID_CURR")
                      .drop(columns=["tvt_code"]))
pdf_train_filtered.head()


# In[5]:


# load previous application
data_path = "home-credit-default-risk/POS_CASH_balance.csv"
pdf_data = pd.read_csv(data_path)
print(pdf_data.shape)
pdf_data.head()


# In[6]:


# load meta data
meta_path = "../02_pandas/reports/report_POS_CASH_balance.csv"
pdf_meta = pd.read_csv(meta_path)


# In[7]:


# filter here for look up application with years
pdf_data["MONTHS_BALANCE"] = pdf_data["MONTHS_BALANCE"] * -1
# pdf_data = pdf_data[(pdf_data["MONTHS_BALANCE"] >= 12) & (pdf_data["MONTHS_BALANCE"] < 12 * 2)]
# pdf_data = pdf_data[(pdf_data["MONTHS_BALANCE"] >= 12 * 2) & (pdf_data["MONTHS_BALANCE"] < 12 * 3)]
# pdf_data = pdf_data[pdf_data["MONTHS_BALANCE"] >= 12 * 3]

print(pdf_data.shape)


# # DPD handling

# In[8]:


pdf_data["is_DPD"] = (pdf_data["SK_DPD"] > 0).astype(int)
pdf_data["is_DPD_DEF"] = (pdf_data["SK_DPD_DEF"] > 0).astype(int)

# drop columns
pdf_data.drop(columns=["SK_DPD", "SK_DPD_DEF"], inplace=True)


# # Categorical features

# In[9]:


# get list categorical attributes
ls_cate = pdf_meta.query("sub_type == 'object'")["name"].tolist()
ls_cate


# In[10]:


# construct categorical mapping
dict_onehot = {}
for cate in ls_cate:
    ls_val = pdf_data[cate].value_counts().index.tolist()
    dict_onehot[cate] = ls_val


# ### one hot

# In[11]:


get_ipython().run_cell_magic('time', '', 'pdf_onehot = gen_one_hot_feat(pdf_data, dict_onehot, main_key="SK_ID_CURR")\nprint(pdf_onehot.shape)\n')


# In[12]:


get_ipython().run_cell_magic('time', '', 'pdf_agg01 = agg_common_data(pdf_onehot, ["max", "sum", "mean"], main_key="SK_ID_CURR")\neval_agg01 = feature_evaluate(pdf_train_filtered, pdf_agg01)\ndisplay(eval_agg01)\n')


# In[13]:


eval_agg01.query("auc <= 0.501").shape


# In[14]:


sel_feat = eval_agg01.query("auc > 0.501")["name"].tolist()
pdf_agg01 = pdf_agg01[sel_feat]
print(pdf_agg01.shape)


# # Numerical features

# In[15]:


series_type = pdf_data.dtypes
ls_num = series_type[series_type == "int64"]
ls_num = [cname for cname in ls_num.index if cname not in ["SK_ID_PREV", "SK_ID_CURR"]]
ls_num


# In[16]:


get_ipython().run_cell_magic('time', '', 'pdf_agg02 = agg_common_data(pdf_data[["SK_ID_CURR"] + ls_num], ["max", "min", "sum", "mean", "std"], main_key="SK_ID_CURR")\neval_agg02 = feature_evaluate(pdf_train_filtered, pdf_agg02)\ndisplay(eval_agg02)\n')


# # Continuous features

# In[17]:


# get list continuous attributes
ls_con = pdf_meta.query("sub_type == 'float64'")["name"].tolist()
ls_con


# In[18]:


pdf_con = pdf_data[["SK_ID_PREV", "SK_ID_CURR"] + ls_con].copy()
pdf_con.head()


# In[19]:


get_ipython().run_cell_magic('time', '', 'pdf_agg03 = agg_common_data(pdf_con[["SK_ID_CURR"] + ls_con], ["max", "min", "sum", "mean", "std"], main_key="SK_ID_CURR")\neval_agg03 = feature_evaluate(pdf_train_filtered, pdf_agg03)\ndisplay(eval_agg03)\n')


# # Save features

# In[20]:


pdf_feat = pdf_agg01.join(pdf_agg02).join(pdf_agg03)
print(pdf_feat.shape)


# In[21]:


get_ipython().run_cell_magic('time', '', 'fname = "pos_cash"\n# fname = "pos_cash_in1year"\n# fname = "pos_cash_in2year"\n# fname = "pos_cash_gt3year"\n\nfname = os.path.join("features", "{}.pkl.bz2".format(fname))\npdf_feat.to_pickle(fname, compression="bz2")\nprint("Store features completed!")\n')