#!/usr/bin/env python
# coding: utf-8

# In[1]:


# Full width
from IPython.core.display import display, HTML
display(HTML("<style>.container { width:100% !important; }</style>"))


# In[2]:


get_ipython().run_line_magic('load_ext', 'autoreload')
get_ipython().run_line_magic('autoreload', '2')


# In[3]:


import math
import os
import subprocess

import matplotlib.pyplot as plt
import numpy as np
import pandas as pd
from IPython.display import display

# 
from lib_modeling import *
from lib_feature_engineering import *

# some settings for displaying Pandas results
pd.set_option('display.width', 2000)
pd.set_option('display.max_rows', 500)
pd.set_option('display.max_columns', 500)
pd.set_option('display.precision', 4)
pd.set_option('display.max_colwidth', -1)


# # Load data
# 
# - Load train + tvt = train_filtered for features evaluation
# - Load train/test for applying mean encoding

# In[4]:


# load train/test data
data_path = "home-credit-default-risk/application_train.csv"
pdf_train = pd.read_csv(data_path)

data_path = "home-credit-default-risk/application_test.csv"
pdf_test = pd.read_csv(data_path)

# filter by tvt code
pdf_tvt_extend = pd.read_pickle("pdf_tvt_extend.pkl", compression="bz2")
pdf_train_filtered = (pdf_tvt_extend.query("tvt_code == 'train'")
                      .merge(pdf_train[["SK_ID_CURR"]], on="SK_ID_CURR")
                      .drop(columns=["tvt_code"]))
pdf_train_filtered.head()


# In[5]:


# load previous application
data_path = "home-credit-default-risk/installments_payments.csv"
pdf_data = pd.read_csv(data_path)
print(pdf_data.shape)
pdf_data.head()


# In[6]:


# load meta data
meta_path = "../02_pandas/reports/report_installments_payments.csv"
pdf_meta = pd.read_csv(meta_path)


# # Preprocessing

# In[7]:


(pdf_data["DAYS_INSTALMENT"] / -365).hist()
plt.show()


# In[8]:


# convert to years
pdf_data["DAYS_INSTALMENT_TO_YEARS"] = pdf_data["DAYS_INSTALMENT"] / -365
pdf_data.drop(columns=["DAYS_INSTALMENT"], inplace=True)


# In[9]:


# calculate different amount
pdf_data["AMT_PAYMENT"].fillna(0, inplace=True)
pdf_data["diff_amount"] = pdf_data["AMT_PAYMENT"] - pdf_data["AMT_INSTALMENT"]
pdf_data["diff_amount"].describe()


# ## filter data here for looking up balance within years

# In[10]:


# pdf_data = pdf_data[(pdf_data["DAYS_INSTALMENT_TO_YEARS"] >= 1) & (pdf_data["DAYS_INSTALMENT_TO_YEARS"] < 2)]
# pdf_data = pdf_data[(pdf_data["DAYS_INSTALMENT_TO_YEARS"] >= 2) & (pdf_data["DAYS_INSTALMENT_TO_YEARS"] < 3)]
# pdf_data = pdf_data[pdf_data["DAYS_INSTALMENT_TO_YEARS"] >= 3]
# print(pdf_data.shape)


# # Numerical features

# In[11]:


get_ipython().run_cell_magic('time', '', 'pdf_agg02 = agg_common_data(pdf_data[["SK_ID_CURR", "NUM_INSTALMENT_NUMBER"]], ["max", "min", "sum", "mean", "std"], main_key="SK_ID_CURR")\neval_agg02 = feature_evaluate(pdf_train_filtered, pdf_agg02)\ndisplay(eval_agg02)\n')


# # Continuous features

# In[12]:


# get list continuous attributes
# ls_con = pdf_meta.query("sub_type == 'float64'")["name"].tolist()
series_type = pdf_data.dtypes
ls_con = series_type[series_type == "float64"].index.tolist()
ls_con


# In[13]:


pdf_con = pdf_data[["SK_ID_PREV", "SK_ID_CURR"] + ls_con].copy()
pdf_con.head()


# In[14]:


get_ipython().run_cell_magic('time', '', 'pdf_agg03 = agg_common_data(pdf_con[["SK_ID_CURR"] + ls_con], ["max", "min", "sum", "mean", "std"], main_key="SK_ID_CURR")\neval_agg03 = feature_evaluate(pdf_train_filtered, pdf_agg03)\ndisplay(eval_agg03)\n')


# # Save features

# In[15]:


pdf_feat = pdf_agg02.join(pdf_agg03)
print(pdf_feat.shape)


# In[16]:


get_ipython().run_cell_magic('time', '', 'fname = "installments_payments"\n# fname = "installments_payments_in1year"\n# fname = "installments_payments_in2year"\n# fname = "installments_payments_gt3year"\n\nfname = os.path.join("features", "{}.pkl.bz2".format(fname))\npdf_feat.to_pickle(fname, compression="bz2")\nprint("Store features completed!")\n')