import pandas as pd
import matplotlib.pyplot as plt
import plotly.express as px
import numpy as np
import missingno as msno
import seaborn as sns
from scipy import stats as st
from itertools import combinations
from collections import Counter
from sklearn.metrics import classification_report, confusion_matrix, accuracy_score, precision_score, recall_score
import scikitplot as skplt
from sklearn.model_selection import train_test_split
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.naive_bayes import MultinomialNB
from sklearn.pipeline import make_pipeline
import warnings
warnings.filterwarnings('ignore')
pd.set_option('max_colwidth', 400)
%matplotlib inline

def data_preprocessing(dataset_path):
    try:
        df = pd.read_csv(dataset_path, sep ="\t")
        df.columns= df.columns.str.lower()
        df.description=df.description.str.lower()
        df.drop_duplicates(inplace=True)
        df["invoicedate"]= pd.to_datetime(df["invoicedate"], format = "%m/%d/%Y %H:%M")
        df["date"] = df["invoicedate"].astype("datetime64[D]")
        df=df.dropna(subset=["description"])
        df=df.query('unitprice > 0')
        df["revenue"]= df["quantity"] * df["unitprice"]
        drop_desc =["amazon fee","postage", "manual", "samples","carriage", "cruk commission", "discount", "bank charges","dotcom postage"]
        df =df[~df["description"].isin(drop_desc)]
    except:
        print("data does not fit automation")
    return df
data = data_preprocessing("ecommerce_dataset_us.csv") 
print("Data size:", data.shape[0])
print()
print("First 5 rows of data:")
display(data.head()) 
print()
print ("Data Description")
display(data.describe(include="all").T)

def categories(classification):
    """This function assigns categories to the product descriptions"""
    description = classification["description"]
    list_of_list=[[kitchenware, home_decor, event_and_party, plant_and_accessories, bags_and_toys, others]]
    for i in list_of_list:
        if description in i[0]:
            return "kitchenware"
        if description in i[1]:
            return "home_decor"
        if description in i[2]:
            return "event_and_party"
        if description in i[3]:
            return "plant_and_accessories"
        if description in i[4]:
            return "bags_and_toys"
        if description in i[5]:
            return "others"
        
        return "undefined"
# Applying the function to the dataframe.
data["sample_categories"] = data.apply(categories, axis =1)
data.head(10)

display(data.query("sample_categories!='undefined'")["sample_categories"].value_counts().sum())
display(round(data.query("sample_categories!='undefined'")["sample_categories"].value_counts().sum()/data.shape[0],3))

data_selected= data.query("sample_categories!='undefined'")
display(data_selected.head())
data_selected.sample_categories.unique()

def training_the_model(X,y):
    X_train, X_test, y_train, y_test = train_test_split(X,y, test_size=.20, random_state=42)
    global nb_model
    nb_model = make_pipeline(TfidfVectorizer(), MultinomialNB()) 
    nb_model.fit(X_train, y_train) 

    label=nb_model.predict(X_test)

    mtrx= confusion_matrix(y_test,label)

    display(classification_report(y_test, label))
    print('\tAccuracy: {:.2f}'.format(accuracy_score(y_test,label)))
    print('\tPrecision: {:.2f}'.format(precision_score(y_test,label, average='weighted')))
    print('\tRecall: {:.2f}'.format(recall_score(y_test,label, average='weighted')))
    skplt.metrics.plot_confusion_matrix(y_test,label,cmap='tab20_r',figsize= (20,10), \
                                    title= "Confusion Matrix for Naive Base")
    return plt.show()
training_the_model(data_selected.description, data_selected.sample_categories)

data["categories"]= nb_model.predict(data["description"])
data.head(10)

cancelled_orders= data[data.invoiceno.str.contains("C", na=False)][data["quantity"]<0]
cancelled_orders = data.query("invoiceno in @cancelled_orders.invoiceno")
cancelled_orders.head()

orders_data= data.query("invoiceno not in @cancelled_orders.invoiceno")
orders_data.shape

top_invoices= orders_data.invoiceno.value_counts().to_frame()
top_invoices.rename(columns={"invoiceno":"invoice_count"}).head(10).T

frequent_categories= orders_data.groupby("categories")["invoiceno"].nunique().sort_values(ascending=True).to_frame().reset_index()\
.rename(columns={"invoiceno":"invoice_count"})

fig = px.bar(frequent_categories, y='categories', x='invoice_count',orientation = "h",title= "<b>Most Frequently Purchased Categories</b>", text='invoice_count')
fig.update_layout(
    showlegend=False,
    margin=dict(t=30,l=8,b=8,r=10))
fig.update_traces(texttemplate='%{text:.3s}', textposition='outside')
fig.update_yaxes(tickfont_family="Arial Black", title_text="<b>Categories</b>")
fig.update_xaxes(tickfont_family="Arial Black", title_text="<b>Invoice Count</b>")

fig.show()

fig = px.line(orders_data.groupby('date')['invoiceno'].nunique().reset_index(),x="date", y="invoiceno", 
              title="<b>Daily Orders</b>")

fig.update_layout(
    showlegend=True,
    margin=dict(t=30,l=8,b=8,r=10))
fig.update_xaxes(tickfont_family="Arial Black", title_text="<b>Invoice Date</b>")
fig.update_yaxes(tickfont_family="Arial Black", title_text="<b>Invoice Count</b>")

fig.show()

import plotly.express as px
fig = px.line(orders_data.groupby([orders_data['invoicedate'].astype("datetime64[M]")])['invoiceno'].nunique().reset_index(),
              x="invoicedate", y="invoiceno", title="<b>Monthly Orders</b>", line_shape= "spline", markers= True)

fig.update_layout(
    showlegend=True,
    margin=dict(t=30,l=8,b=8,r=10))
fig.update_xaxes(tickfont_family="Arial Black", title_text="<b>Invoice Date</b>")
fig.update_yaxes(tickfont_family="Arial Black", title_text="<b>Invoice Count</b>")

fig.show()

top_products= orders_data.groupby("description")["revenue"].sum().to_frame().sort_values(by ="revenue", ascending=True)\
.reset_index().head(10)

fig = px.bar(top_products, y='description', x='revenue',title= "<b>Top 10 Products by Revenue</b>",  text='revenue')
fig.update_layout(
    showlegend=False,
    margin=dict(t=30,l=8,b=8,r=10))
fig.update_traces(texttemplate='%{text:.3s}', textposition='outside')
fig.update_layout(xaxis_tickprefix = '£', xaxis_tickformat = ',.2f')
fig.update_yaxes(tickfont_family="Arial Black", title_text="<b>Products</b>")
fig.update_xaxes(tickfont_family="Arial Black", title_text="<b>Revenue (Million)</b>")

fig.show()

frequently_cancelled_products= cancelled_orders.groupby("description")["invoiceno"].nunique().sort_values(ascending=True).to_frame().reset_index()\
.rename(columns={"invoiceno":"invoice_count"}).tail(10)

fig = px.bar(frequently_cancelled_products, y='description', x='invoice_count',title= "<b>Most Frequently Cancelled Products</b>", text='invoice_count')
fig.update_layout(
    showlegend=False,
    margin=dict(t=30,l=8,b=8,r=10))
fig.update_traces(texttemplate='%{text:.2s}', textposition='outside')
fig.update_yaxes(tickfont_family="Arial Black", title_text="<b>Products</b>")
fig.update_xaxes(tickfont_family="Arial Black", title_text="<b>Invoice Count</b>")
fig.show()

fig = px.line(orders_data.groupby(orders_data['invoicedate'].astype("datetime64[D]"))["revenue"].sum()\
                                              .reset_index(),x="invoicedate", y="revenue",title= "<b>Daily Revenues</b>")
 
fig.update_layout(
    showlegend=True,
    margin=dict(t=30,l=8,b=8,r=10))
fig.update_layout(yaxis_tickprefix = '£')
fig.update_yaxes(tickfont_family="Arial Black", title_text="<b>Revenue</b>")
fig.update_xaxes(tickfont_family="Arial Black", title_text="<b>Invoice Date</b>")

fig.show()

fig = px.line(orders_data.groupby(orders_data[(orders_data["date"]>= "2018-12-01") & (orders_data["date"]<= "2019-11-30")]['invoicedate']\
                                  .astype("datetime64[M]"))["revenue"].sum().reset_index(),x="invoicedate", y="revenue",\
              title= "<b>Total Monthly Revenues</b>", line_shape= "spline", markers= True)
 
fig.update_layout(
    showlegend=True,
    margin=dict(t=30,l=8,b=8,r=10))
fig.update_layout(yaxis_tickprefix = '£')
fig.update_yaxes(tickfont_family="Arial Black", title_text="<b>Total Revenue</b>")
fig.update_xaxes(tickfont_family="Arial Black", title_text="<b>Invoice Date</b>")

fig.show()

leading_avg_revenues= orders_data.groupby("description")["revenue"].mean().sort_values(ascending=True).to_frame().reset_index()\
.head(10)

fig = px.bar(leading_avg_revenues, y='description', x='revenue',title= "<b>Top 10 Average Revenue by Products</b>", text='revenue')
fig.update_layout(
    showlegend=False,
    margin=dict(t=30,l=8,b=8,r=10))
fig.update_traces(texttemplate='%{text:.2s}', textposition='outside')
fig.update_layout(xaxis_tickprefix = '£')
fig.update_yaxes(tickfont_family="Arial Black", title_text="<b>Products</b>")
fig.update_xaxes(tickfont_family="Arial Black", title_text="<b>Revenue (Million)</b>")

fig.show()

leading_avg_revenues= orders_data.groupby("categories")["revenue"].mean().sort_values(ascending=True).to_frame().reset_index()\
.head(10)

fig = px.bar(leading_avg_revenues, y='categories', x='revenue', orientation = "h", title= "<b>Average Revenue by Category</b>", text='revenue')
fig.update_layout(
    showlegend=False,
    margin=dict(t=30,l=8,b=8,r=1))
fig.update_traces(texttemplate='%{text:.3s}', textposition='outside')
fig.update_layout(xaxis_tickprefix = '£', xaxis_tickformat = ',.2f')
fig.update_yaxes(tickfont_family="Arial Black", title_text="<b>Categories</b>")
fig.update_xaxes(tickfont_family="Arial Black", title_text="<b>Revenue</b>")

fig.show()

orders_data["product_bundle"]=orders_data.groupby('invoiceno')['description'].transform(lambda x: ';'.join(x))
product_bundle= orders_data[['invoiceno', "product_bundle"]].drop_duplicates()

count = Counter()
for row in product_bundle["product_bundle"]:
    row_list = row.split(';')
    count.update(Counter(combinations(row_list,2)))
paired=count.most_common(10)
df = pd.DataFrame(paired,columns=['combinations', "count"])
df

by_themselves = orders_data.groupby("invoiceno")["description"].nunique().reset_index()\
.rename(columns={"description": "item_count"}).query("item_count <2")
display(by_themselves.head())
display(by_themselves.shape[0])

main_products= orders_data.query("invoiceno in @by_themselves.invoiceno")
main_products= main_products["description"].value_counts().to_frame().reset_index().rename(columns={"index": "description", "description": "count"})
fig = px.bar(main_products.head(15).sort_values(by= 'count',ascending=True), y='description', x='count',orientation = "h",title= "<b>Top 15 Products Sold by Themselves</b>",  text='count')
fig.update_layout(
    showlegend=False,
    margin=dict(t=30,l=8,b=8,r=10))
fig.update_traces(texttemplate='%{text:.2s}', textposition='outside')
fig.update_yaxes(tickfont_family="Arial Black", title_text="<b>Products</b>")
fig.update_xaxes(tickfont_family="Arial Black", title_text="<b>Count</b>")

fig.show()
       

additional_products= orders_data.query("invoiceno  not in @by_themselves.invoiceno")
additional_products= additional_products["description"].value_counts().to_frame().reset_index().rename(columns={"index": "description", "description": "count"})
fig = px.bar(additional_products.head(15).sort_values(by= 'count',ascending=True), y='description', x='count',orientation = "h",title= "<b>Top 15 Additional Assortment Products</b>", text='count')
fig.update_layout(
    showlegend=False,
    margin=dict(t=30,l=8,b=8,r=10))
fig.update_traces(texttemplate='%{text:.2s}', textposition='outside')
fig.update_yaxes(tickfont_family="Arial Black", title_text="<b>Products</b>")
fig.update_xaxes(tickfont_family="Arial Black", title_text="<b>Count</b>")

fig.show()

print(str(round(((orders_data.shape[0] - by_themselves.shape[0])/orders_data.shape[0])*100,2))+"%")

additional_group_assortment = orders_data.groupby("invoiceno")["categories"].nunique().reset_index()\
.rename(columns={"categories": "item_count"}).query("item_count > 1")

additional_product_group= orders_data.query("invoiceno in @additional_group_assortment.invoiceno")
additional_product_group= additional_product_group["categories"].value_counts().to_frame().reset_index().rename(columns={"index": "categories", "categories": "count"})
fig = px.bar(additional_product_group.sort_values(by= 'count',ascending=True), y='categories', x='count', orientation ="h", title= "<b>Top Additional Assortment Categories</b>",text='count')
fig.update_layout(
    showlegend=False,
    margin=dict(t=30,l=2,b=5,r=1), )
fig.update_traces(texttemplate='%{text:.3s}', textposition='outside')
fig.update_yaxes(tickfont_family="Arial Black", title_text="<b>Categories</b>")
fig.update_xaxes(tickfont_family="Arial Black", title_text="<b>Count</b>")

fig.show()

orders_data["categories_bundle"]=orders_data.groupby('invoiceno')['categories'].transform(lambda x: ';'.join(x))
categories_bundle= orders_data[['invoiceno', "categories_bundle"]].drop_duplicates()
count = Counter()
for row in categories_bundle["categories_bundle"]:
    row_list = row.split(';')
    count.update(Counter(combinations(row_list,2)))
paired1=count.most_common(10)
df1 = pd.DataFrame(paired1,columns=['combinations', "count"])
df1

home_decor = orders_data[orders_data["categories"]=="home_decor"]
kitchenware = orders_data[orders_data["categories"]=="kitchenware"]

alpha = .05 
result = st.levene(home_decor["revenue"], kitchenware["revenue"])

print('p-value (levene test): {:.4f}'.format(result.pvalue))


if (result.pvalue < alpha):
        print("We reject the null hypothesis")
else:
        print("We can't reject the null hypothesis")

alpha = .05 
home_decor = orders_data[orders_data["categories"]=="home_decor"]
kitchenware = orders_data[orders_data["categories"]=="kitchenware"]
result = st.ttest_ind(home_decor["revenue"], kitchenware["revenue"],equal_var=False)

print('p-value: {:.4f}'.format(result.pvalue))

if (result.pvalue < alpha):
        print("We reject H0: there is statistically significant difference between average revenue from categoires home decor and kitchen ware.")
else:
        print("We can't reject H1: there is statistically no significant difference between average revenue from categoires home decor and kitchen ware.")

alpha = .05 

papar_craft = orders_data[orders_data["description"]=="paper craft , little birdie"]
non_paper_craft = orders_data[orders_data["description"]!="paper craft , little birdie"]
        
results = st.levene(papar_craft["revenue"], non_paper_craft["revenue"])

print('p-value (levene test): {:.4f}'.format(results.pvalue))


if (results.pvalue < alpha):
        print("We reject the null hypothesis")
else:
        print("We can't reject the null hypothesis")

alpha = .05 

result = st.ttest_ind(papar_craft["revenue"], non_paper_craft["revenue"],equal_var=True)

print('p-value: {:.4f}'.format(results.pvalue))

if (results.pvalue < alpha):
        print("We reject H0: there is statistically significant difference between average revenue from the product 'paper craft, little birdie' and all other products.")
else:
        print("We can't reject H0: there is statistically no significant difference between average revenue from the product 'paper craft, little birdie' and all other products.")