import pandas as pd
import numpy as np

# Import the dataset
data = pd.read_csv('data/kc_house_data.csv')

# Convert date to date time format
data['date'] = pd.to_datetime(data['date'])

data['month'] = pd.DatetimeIndex(data['date']).month

# Remove outliers in prices
data = data[(data.price < data.price.quantile(.995)) & 
            (data.price > data.price.quantile(.005))] 

data.loc[(data.bedrooms == 33), 'bedrooms'] = 3 

# Replace missing values with no waterfront
data.loc[data['waterfront'].isnull(), 'waterfront'] = 'NO'

data['waterfront'] = data['waterfront'].apply(lambda x: 0 if x == 'NO' else 1) 

# Replace missing values with no view
data.loc[data.view.isnull(), 'view'] = 'NONE'

data['view'] = data['view'].replace({'NONE':0,
                                     'FAIR':1, 
                                     'AVERAGE':2, 
                                     'GOOD':3, 
                                     'EXCELLENT':4}) 

data['condition'] = data['condition'].replace({'Poor':0, 
                                               'Fair':1, 
                                               'Average':2, 
                                               'Good':3, 
                                               'Very Good':4})

data['grade'] = data['grade'].map(lambda x: int(x.split(' ')[0])) # only keep the number from string

# Replace missing values with 0 square footage of basement
data['sqft_basement'] = data['sqft_basement'].replace('?', '0').astype('float')

data['basement'] = data['sqft_basement'].apply(lambda x: 0 if x == 0 else 1)

# Replace missing values with not renovated
data.loc[data['yr_renovated'].isnull(), 'yr_renovated'] = 0

data['renovated'] = data['yr_renovated'].apply(lambda x: 0 if x == 0 else 1)

data['age'] = data['date'].dt.year - data['yr_built']

date_data = data[['date', 'price']]
date_data = data.set_index('date', drop=True).astype('float64')

# Resample data to count transactions per month
monthly_count = date_data.resample('MS').count()
monthly_count = monthly_count.drop(monthly_count.index[-1])

# Resample data to calculate mean price per month
monthly_avg = date_data.resample('MS').mean()
monthly_avg = monthly_avg.drop(monthly_avg.index[-1])

def check_correlation(data, threshold):
    '''Checks for the highest correlated features in a dataframe given a threshold'''
    df = data.corr().abs().stack().reset_index().sort_values(0, ascending=False)
    
    df['pairs'] = list(zip(df['level_0'], df['level_1']))
    df = df.set_index(['pairs']).drop(columns=['level_1', 'level_0'])
    
    df.columns = ['correlation']
    
    df = df.drop_duplicates()
    df = df[(df['correlation'] > threshold) & (df['correlation'] < 1)]
    
    return df

# Drop highly collinear features and useless feature like id
to_drop = ['id', 'date', 'yr_renovated', 'sqft_above', 'sqft_basement', 'yr_built', 'yr_renovated']

data = data.drop(columns=to_drop)

import geopandas as gpd
from shapely.geometry import Point, Polygon

def map_data(data_, title, name, colormap):
    '''Plots the data points using latitude and longitude on top of King County map'''
    plt.style.use('seaborn')
    
    fig, ax = plt.subplots(figsize=(8,8))
    
    king_county_fig = gpd.read_file('data/map/Incorporated_Areas_of_King_County___city_area.shp')
    
    geometry = [Point(xy) for xy in zip(data_['long'], data_['lat'])]
    geo_data = gpd.GeoDataFrame(data_, crs='EPSG:4326', geometry=geometry)
    
    king_county_fig.plot(ax=ax, alpha=0.4, color='grey')
    geo_data.plot(column='price', ax=ax, alpha=0.4, legend=True, markersize=8, cmap=plt.get_cmap(colormap), legend_kwds={'shrink': 0.7, 'format': '%.0f'})
    
    plt.xlim(data['long'].min(), data['long'].max())
    plt.ylim(data['lat'].min(), data['lat'].max())
    plt.ticklabel_format(style='plain')
    
    plt.title(title, fontsize=14)
    
    plt.savefig(f'data/images/{name}.png', dpi=200, transparent=True)
    plt.show()

# Create dataframe of houses sold under a million dollars
UnderMillion = data.loc[(data['price'] < 1000000)]

# Create dataframe of houses sold over a million dollars
OverMillion = data.loc[(data['price'] >= 1000000)]

# Import data from population census
population = pd.read_csv('data/population.csv')

population_density = population.loc[:, ['City Name', 'Total Population 2010', 'Population Density (Persons / Square Mile) 2010']]
population_density = population_density.rename(columns={'City Name':'city', 'Total Population 2010':'population', 'Population Density (Persons / Square Mile) 2010':'pop_density'})

king_county_cities = ['Algona', 'Auburn', 'Beaux Arts Village', 'Bellevue', 'Black Diamond', 
                      'Bothell', 'Burien', 'Carnation', 'Clyde Hill', 'Covington', 
                      'Des Moines', 'Duvall', 'Enumclaw', 'Federal Way', 'Hunts Point', 
                      'Issaquah', 'Kenmore', 'Kent', 'Kirkland', 'Lake Forest Park', 
                      'Maple Valley', 'Medina', 'Mercer Island', 'Milton', 'Newcastle', 
                      'Normandy Park', 'North Bend', 'Pacific', 'Redmond', 'Renton', 
                      'Sammamish', 'SeaTac', 'Seattle', 'Shoreline', 'Skykomish', 
                      'Snoqualmie', 'Tukwila', 'Woodinville', 'Yarrow Point']

city_names = pd.DataFrame(king_county_cities, columns=['city'])

import warnings
warnings.filterwarnings('ignore')

king_county_population = pd.merge(population_density, city_names, on=['city'])
king_county_population = king_county_population.append({'city':'Vashon','population':10624, 'pop_density':287.22}, ignore_index=True)

zipcodes = pd.read_csv('data/zipcodes.csv')

zipcodes['city'] = zipcodes['zipcode'].apply(lambda x: x.split('(')[1].strip('()'))
zipcodes['zipcode'] = zipcodes['zipcode'].apply(lambda x: int(x.split()[0]))

# Map population data to cities
king_county_zipcode = pd.merge(king_county_population, zipcodes, on=['city'])

engineered_data = pd.merge(data, king_county_zipcode, how='inner', on=['zipcode'])

from haversine import haversine, Unit

def find_distance(center, lat, long):
    '''Uses haversine function to find distance between a given center and a point's latitude and longitude'''
    house = (lat, long)
    distance = haversine(center, house, unit=Unit.MILES)
    return distance

def add_distance(data):
    '''Applies the find_distance function to dataframe to add columns for distances to Seattle and Redmond'''
    seattle = (47.608013, -122.335167)
    redmond = (47.673988, -122.121513)
    
    data['to_seattle'] = data.apply(lambda x: find_distance(seattle, x['lat'], x['long']), axis=1)
    data['to_redmond'] = data.apply(lambda x: find_distance(redmond, x['lat'], x['long']), axis=1)
    data['distance'] = data['to_seattle'] + data['to_redmond']

    data = data.drop(columns=['lat', 'long'])
    return data

engineered_data = add_distance(engineered_data)

from sklearn.model_selection import train_test_split

X = data.drop(columns=['price'])
y = data['price']

X_tr, X_te, y_tr, y_te = train_test_split(X, y, random_state=112221)

# Simple linear regression using one independent variable
from sklearn.linear_model import LinearRegression

lr_model = LinearRegression()
lr_model.fit(X_tr[['sqft_living']], y_tr)

baseline_pred = lr_model.predict(X_te[['sqft_living']])

# Evaluate using R squared and RMSE scores
from sklearn.model_selection import cross_val_score
from sklearn.metrics import mean_squared_error

baseline_cross_val = cross_val_score(lr_model, X_tr[['sqft_living']], y_tr, scoring='neg_root_mean_squared_error', cv=10)
baseline_cross_val_mean = -(baseline_cross_val.mean())

baseline_rmse = mean_squared_error(y_te, baseline_pred, squared=False)
baseline_score = lr_model.score(X_te[['sqft_living']], y_te)

metric_table = pd.DataFrame(columns = ['model', 'cv_rmse', 'test_rmse', 'test_score'])
metric_table = metric_table.append({'model': 'simple_lr', 
                                    'cv_rmse': f'{baseline_cross_val_mean:.2f}', 
                                    'test_rmse': f'{baseline_rmse:.2f}', 
                                    'test_score': f'{baseline_score:.2f}'}, ignore_index=True)

def run_model(regressor, name, X_train=X_tr, X_test=X_te, y_train=y_tr, y_test=y_te):
    '''Fits a regression model on train set, predicts on the test set, and returns cross validation and test scores'''
    regressor.fit(X_train, y_train)
    y_pred = regressor.predict(X_test)
    
    cross_val = cross_val_score(regressor, X_train, y_train, scoring='neg_root_mean_squared_error', cv=10)
    cross_val_mean = -(cross_val.mean())
    
    rmse = mean_squared_error(y_test, y_pred, squared=False)
    score = regressor.score(X_test, y_test)
    
    return metric_table.append({'model': name, 
                                'cv_rmse': f'{cross_val_mean:.2f}', 
                                'test_rmse': f'{rmse:.2f}', 
                                'test_score': f'{score:.2f}'}, ignore_index=True)

# Data with engineered features such as population and distance
X_ = engineered_data.drop(columns=['price'])
y_ = engineered_data['price']

X_train, X_test, y_train, y_test = train_test_split(X_, y_, random_state=112221)

from sklearn.pipeline import Pipeline
from sklearn.impute import SimpleImputer
from sklearn.preprocessing import OneHotEncoder

num_pipe = Pipeline(steps=[
    ('num_imputer', SimpleImputer(strategy='median')),
    ('scaler', StandardScaler())
])

# Perform one hot encoding on categorical features
cat_pipe = Pipeline(steps=[
    ('cat_imputer', SimpleImputer(strategy='most_frequent')),
    ('ohe', OneHotEncoder(handle_unknown='ignore'))
])

from sklearn.compose import ColumnTransformer

num_features = ['bedrooms', 'bathrooms', 'sqft_living', 'sqft_lot', 'floors', 'sqft_living15', 'sqft_lot15', 
                'age', 'population', 'pop_density', 'to_seattle', 'to_redmond', 'distance']
cat_features = ['waterfront', 'view', 'condition', 'grade', 'zipcode', 'month', 'basement', 'renovated', 'city']

col_transformer = ColumnTransformer([
    ('num_transformer', num_pipe, num_features),
    ('cat_transformer', cat_pipe, cat_features)
])

# The best model with the best estimate 
bestimate_model = tuned_rf
bestimate_model.fit(X_train, y_train)
bestimate_pred = bestimate_model.predict(X_test)

from sklearn.inspection import permutation_importance
result = permutation_importance(bestimate_model, X_test, y_test, n_repeats=10, random_state=112221, n_jobs=2)
sorted_idx = result.importances_mean.argsort()