1 금융 데이터와 전처리¶

import pandas as pd
import numpy as np
df_test = pd.DataFrame([[4,9]] * 3, columns=['a','b'])
df_test.apply(np.sqrt)

DATA_FILENAME = 'data/stock-samsung.pkl'
import pandas as pd
import FinanceDataReader as fdr
try:
    data_stock = pd.read_pickle(DATA_FILENAME)
except FileNotFoundError:
    data_stock = fdr.DataReader('005930', '1999-01-01')
    data_stock.to_pickle(DATA_FILENAME)
data_stock.head(3)

import numpy as np
df = data_stock.loc[:, ['Close']] # cf) .loc[:, 'Close'] : Series 추출
df.rename(columns={'Close':'adj_close'}, inplace=True)

# 추가 수익률 계산내용의 추가
df['simple_rtn'] = df.adj_close.pct_change()                  # 단순 수익률 계산 (자산기준 수익률 가중화 합산)
df['log_rtn'] = np.log(df.adj_close / df.adj_close.shift(1))  # Log 수익률 계산 (시간에 따른 수익률 log 합산)
df.head(3)

%matplotlib inline
%config InlineBackend.figure_format = 'retina'
import warnings
import matplotlib.pyplot as plt
from matplotlib.font_manager import fontManager as fm

plt.style.use('seaborn') # 'seaborn-colorblind'
plt.rc('axes', unicode_minus=False)
warnings.simplefilter(action='ignore', category=FutureWarning)
font_list_check  = ['D2Coding', 'NanumGothicCoding', 'NanumGothic']
for font in font_list_check:
    font_counts = [ f  for f in font_list_check 
        if font in [_.name for _ in fm.ttflist]]
    if len(font_counts) > 1: 
        print(f"found : {font}"); break
        
# 폰트 적용하기
plt.rc('font', family=font)        

plt.rcParams['figure.dpi'] = 300
plt.rcParams['figure.figsize'] = (20.0, 5.0)
df.log_rtn.plot(linestyle="--")
df.simple_rtn.plot(linestyle="--")
plt.legend(); plt.show()

# 기준 테이블을 생성한 뒤, 다른 데이터의 값을 덧붙이는 예시
df_all_dates = data_stock.loc['1999-12-31':'2010-12-31', :]
df_all_dates.join(df[['adj_close']], how='left').fillna(method="ffill").asfreq(freq='M').tail()

file_name = 'data/BOK_ECOS_statistics_codes.csv'
import re
import pandas as pd
code_table = pd.read_csv(file_name)
code_table.tail(3)

# 필요한 자료코드 추출하기
search_text = '인플레'
code_index = []
for column in ['통계명', '항목명1']:
    code_index += [
    no  for no, _ in enumerate(code_table[column].to_list())
    if _.find(search_text) != -1]
code_example = code_table.iloc[code_index, :].drop_duplicates().reset_index(drop=True)
code_example.iloc[:, :-1]

from data.finance import bok_eos_api
eos_table = bok_eos_api('040Y003', date_end='20210801', code2='FMB')  # 7년 단위로만 추출가능
eos_table['DATA_VALUE'] = list(map(lambda x : float(x),  eos_table['DATA_VALUE']))
eos_table = eos_table.rename(columns={'DATA_VALUE':'cpi', 'TIME':'Date'})

eos_table['Date'] = pd.to_datetime(eos_table['Date'], format='%Y%m')
eos_table = eos_table.set_index('Date')  # eos_table.head(3)
eos_table = eos_table[['cpi']]
eos_table.tail(3)

# ploting the DataSet
plt.rcParams['figure.dpi'] = 300
plt.rcParams['figure.figsize'] = (20.0, 3.0)
fig = plt.figure()
ax = fig.add_subplot(111, ylabel='BOK 기대인플레이션율 %')
eos_table.plot(ax=ax)
plt.show()

# CPI 정보를 데이블에 추가하고, 손실된 값들은 제거한다
data_stock.join(eos_table, 
    how='left').fillna(method='ffill').dropna().head(3)

# 분석에 필요한 날짜값을 생성하고
# 이를 기준으로 데이터 필터링
df_all_dates = pd.DataFrame(index=pd.date_range(start='2002-11-30', end='2020-12-31'))
df_monthly = df_all_dates.join(data_stock[['Close']], how='left').fillna(method='ffill').asfreq('M')
df_cpi = df_all_dates.join(eos_table[['cpi']], how='left').fillna(method='ffill').asfreq('M')
df_merge = df_monthly.join(df_cpi, how='left')

# Percent Change 내용 추가하기
df_merge['simple_rtn'] = df_merge.Close.pct_change()
df_merge['inflation_rate'] = df_merge.cpi.pct_change()

# 인플레이션으로 상승한 부분 축소한 값으로 보정
df_merge['real_rtn'] = (df_merge.simple_rtn + 1) / (df_merge.inflation_rate + 1) - 1
df_merge.head(3)

# Visualization
plt.rcParams['figure.figsize'] = (20.0, 4.0)
fig = plt.figure()
ax = fig.add_subplot(111, ylabel='BOK 기대인플레이션율 변화율')
df_merge.simple_rtn.plot(ax=ax)
df_merge.real_rtn.plot(ax=ax)
plt.legend(); plt.show()

# 월간 실현 변동률 계산하는 함수
import numpy as np
def realized_volatility(x):
    return np.sqrt(np.sum(x**2))

df_rv = df.groupby(pd.Grouper(freq='M')).apply(realized_volatility)
df_rv.rename(columns={'log_rtn': 'rv'}, inplace=True)
df_rv.head(3)

# log 수익룰 값을 환산하는 함수
df_rv.rv = df_rv.rv ** np.sqrt(12)
df_rv.head(3)

df.head(3)

fig, ax = plt.subplots(3,1, figsize=(20,9), sharex=True)
ax[0].plot(df.log_rtn)
ax[0].set(title='SAMSUNG 주가 수익률', ylabel='Log')
ax[1].plot(df_rv.rv)
ax[1].set(ylabel='Real Value (Log)')
ax[2].plot(df_rv.rv.cumsum(), label="")
ax[2].plot(df.log_rtn.cumsum())
ax[2].set(ylabel='Log(누적) / Log RV(누적)')
plt.show()

fig, ax = plt.subplots(2,1, figsize=(20, 6), sharex=True)
ax[0].plot(df.log_rtn.resample('M').mean())
ax[0].axhline(y=0, color ='red', linewidth=1.5, linestyle="--")
ax[0].set(title='SAMSUNG 주가 수익률 (월평균)', ylabel='Log')
ax[1].plot(df.log_rtn.resample('M').mean().cumsum())
ax[0].set( ylabel='Log (누적합)')
plt.show()

fig, ax = plt.subplots(3, 1, figsize=(20,12), sharex=True)
df.adj_close.plot(ax=ax[0])  # 주가 Table
ax[0].set(title='SAMSUNG time series', ylabel='Stock Price (원)')
df.simple_rtn.plot(ax=ax[1]) # 단순 수익률 Table
ax[1].set(ylabel='Simple Return (원)')
df.log_rtn.plot(ax=ax[2])    # log 수익률 Table
ax[2].set(xlabel='Date', ylabel='Log Return (%)')
plt.show()

import cufflinks as cf
from plotly.offline import iplot, init_notebook_mode

cf.set_config_file(world_readable=True, theme='pearl', offline=True) # set up settings (run it once)
init_notebook_mode(connected=True)  # initialize notebook display

# Plotly from DataFrame
df.iplot(subplots=True, shape=(3,1), shared_xaxes=True, title="SAMSUNG")

# 이동평균선의 평균과 표준편차를 계산한다
df_rolling = df[['simple_rtn']].rolling(window=21).agg(['mean', 'std'])
df_rolling.columns = df_rolling.columns.droplevel()
df_outliers = df.join(df_rolling)

# 특이값을 추출하는 함수
def indenify_outlisers(row, n_sigmas=3):
    x = row['simple_rtn']
    mu = row['mean']
    sigma = row['std']
    if (x> mu+3 * sigma) | (x < mu -3 * sigma): 
        return 1
    else: 
        return 0

# 특이값 추출함수를 테이블에 Apply 하기
df_outliers['outlier'] = df_outliers.apply(indenify_outlisers, axis=1)
df_outliers.head(3)

# 특이점 Point 테이블
outlier = df_outliers.loc[df_outliers['outlier']==1, ['simple_rtn']] # 1 표시된 값 추출하기
outlier.head(3)

# Visualization
fix, ax = plt.subplots()
ax.plot(df_outliers.index, df_outliers.simple_rtn, color='blue', label='Normal')
ax.scatter(outlier.index, outlier.simple_rtn, color='red', label='Anomaly')
ax.set_title('SAMSUNG stock returns')
ax.legend(loc='lower right')
plt.show()