for dirname, _, filenames in os.walk('/kaggle/input'):
    for filename in filenames:
        print(os.path.join(dirname, filename))

style_df(customer_product.head(), 'Customer Sign-up and Cancellation Dates')

style_df(customer_info.head(), 'Customer Demographics')

style_df(customer_cases.head(), 'Call Center Activity')

style_df(product_info.head(), 'Product Information')

for df_name, df in [("product_info", product_info), 
                    ("customer_product", customer_product),
                    ("customer_info", customer_info), 
                    ("customer_cases", customer_cases)]:
    print(f"\n{df_name} missing values:")
    print(df.isnull().sum())
    
    print(f"\n{df_name} data types:")
    print(df.dtypes)

customer_product.isnull().sum()

print(churn_df.isnull().sum())

print(churn_df.isnull().sum())

churn_df.head()

target = 'will_churn_next_90d'
segments = ['age_group', 'tenure_group', 'support_level', 'product']

for segment in segments:
    churn_by_segment = modeling_df.groupby(segment, observed=True)[target].agg(['mean', 'count'])
    churn_by_segment['mean'] *= 100
    print(f"\nChurn rate by {segment}:")
    print(churn_by_segment.sort_values('mean', ascending=False))

modeling_df['tenure_weeks'] = (modeling_df['tenure_at_cutoff'] / 7).astype(int)
new_churn = modeling_df[modeling_df['tenure_at_cutoff'] < 90]
weekly_churn = new_churn.groupby('tenure_weeks')['will_churn_next_90d'].agg(['mean','count'])
weekly_churn['mean'] *= 100

print(weekly_churn)

new_product_churn = new_churn.groupby(['product', 'tenure_weeks'])['will_churn_next_90d'].mean() * 100

print(new_product_churn)

tenure_check = modeling_df[modeling_df['tenure_group'] == '0-3mo']
print(f"Count of customers in 0-3mo group: {len(tenure_check)}")

print(tenure_check['tenure_at_cutoff'].describe())

print(f"Churn rate in 0-3mo group: {tenure_check['will_churn_next_90d'].mean()*100:.2f}%")

print(tenure_check.groupby('product')['will_churn_next_90d'].mean()*100)

new_customer_df = modeling_df[modeling_df['new_customer']].copy()
established_customer_df = modeling_df[~modeling_df['new_customer']].copy()

print(f"New customers: {len(new_customer_df)} ({len(new_customer_df)/len(modeling_df):.1%})")
print(f"Established customers: {len(established_customer_df)} ({len(established_customer_df)/len(modeling_df):.1%}) ")

X_new = new_customer_df[new_features]
y_new = new_customer_df['will_churn_next_90d']

X_new = pd.get_dummies(X_new, drop_first=True)
   
X_train, X_test, y_train, y_test = train_test_split(
    X_new, y_new, test_size = 0.20, random_state = 0, stratify = y_new
)
new_churn_rate = y_new.mean()
print(f"New customer churn rate: {new_churn_rate: .4%}")
weights_new = {0:1, 1:3} # Non-churners get 3x the weight
print(f"Using class weights for new model: {weights_new}")

new_model = RandomForestClassifier(n_estimators = 100, class_weight = weights_new, random_state = 0)
new_model.fit(X_train, y_train)

X_established = established_customer_df[established_features]
y_established = established_customer_df['will_churn_next_90d']

X_established = pd.get_dummies(X_established, drop_first = True, dummy_na = False)

X_train_est, X_test_est, y_train_est, y_test_est = train_test_split(
    X_established, y_established,
    test_size = 0.20,
    random_state = 0,
    stratify = y_established
)

final_feature_names = X_established.columns.tolist() 

established_churn_rate = y_established.mean()
print(f"Established customer churn rate: {established_churn_rate: .2%}")
weight_est = {0:1,1:5}
print(f"Using class weights for established model: {weight_est}")

established_model = RandomForestClassifier(n_estimators=100,
                                          class_weight = weight_est,
                                          random_state = 0)

established_model.fit(X_train_est, y_train_est)

def evaluate_model(model, X_test, y_test, model_name = "Model"):
    y_pred = model.predict(X_test)
    y_prob = model.predict_proba(X_test)[:,1]

    print(f'Evaluation Report for {model_name}')
    print(classification_report(y_test, y_pred, target_names = ['No Churn (0)', 'Churn (1)']))

    roc_auc = roc_auc_score(y_test, y_prob)
    print(f'ROC AUC Score: {roc_auc:.4f}')

    precision, recall, _ = precision_recall_curve(y_test, y_prob)
    prc_auc = auc(recall, precision)
    print(f'Precision-Recall Curve AUC (PRC AUC): {prc_auc:.4f}')

    cm = confusion_matrix(y_test, y_pred)
    print("\n Confusion Matrix:")

    plt.figure(figsize = (6,4))
    sns.heatmap(cm, annot = True, fmt='d', cmap='Blues',
               xticklabels = ['Predicted No Churn', 'Predicted Churn'],
               yticklabels = ['Actual No Churn', 'Actual Churn'])
    plt.title(f'Confusion Matrix - {model_name}')
    plt.ylabel('Actual Label')
    plt.xlabel('Predicted Label')
    plt.show()

    print(""* (len(model_name) +24))

    return {
        'roc_auc': roc_auc,
        'prc_auc': prc_auc,
        'f1_churn': f1_score(y_test, y_pred, pos_label=1),
        'recall_churn': recall_score(y_test, y_pred, pos_label=1),
        'precision_churn': precision_score(y_test, y_pred, pos_label=1)
    }

print("Evaluating New Customer Model (Random Forest Baseline)...")
new_model_metrics = evaluate_model(new_model, X_test, y_test, 'New Customer RF')

print("\nEvaluating Established Customer Model (Random Forest Baseline) ...")
established_model_metrics = evaluate_model(established_model, X_test_est, y_test_est, "Established Customer RF")

baseline_metrics = {
    'New RF': new_model_metrics,
    'Established RF': established_model_metrics
}

# XGBoost

neg_count_est = (y_train_est == 0).sum()
pos_count_est = (y_train_est == 1).sum()
scale_pos_weight_est = neg_count_est / pos_count_est
print(f'\nCalculated scale_pos_weight for Established XGBoost: {scale_pos_weight_est:.2f}')

xgb_model_est = xgb.XGBClassifier(
    objective = 'binary:logistic',
    eval_metric = 'logloss',
    use_label_encoder = False,
    scale_pos_weight = scale_pos_weight_est,
    random_state = 0
)

print('\nTraining default XGBoost model for Established Customers...')
xgb_model_est.fit(X_train_est, y_train_est)

print('\nEvaluating Established Customer Model (XGBoost Default)...')
xgb_est_metrics = evaluate_model(xgb_model_est, X_test_est, y_test_est, 'Established Customer XGB')

xgb_base = xgb.XGBClassifier(
    objective = 'binary:logistic',
    eval_metric = 'logloss',
    use_label_encoder = False,
    random_state = 0
)

param_distributions = {
    'n_estimators': [int(x) for x in np.linspace(start = 100, stop=1000, num=10)],
    'max_depth': [3,4,5,6,7,8],
    'learning_rate': [0.01, 0.05, 0.1, 0.2],
    'scale_pos_weight': [5,10,15,20,24],
    'subsample': [0.7,0.8,0.9,1.0],
    'colsample_bytree': [0.7, 0.8, 0.9, 1.0]
}

n_iterations = 50

cv_folds = 3

scoring_metric = 'f1'

print(f"Starting Randomized Search with {n_iterations} iterations, {cv_folds}-fold CV, optimizing for '{scoring_metric}'...")

random_search = RandomizedSearchCV(
    estimator = xgb_base,
    param_distributions = param_distributions,
    n_iter = n_iterations,
    n_jobs = -1,
    cv = cv_folds,
    verbose = 1,
    random_state = 0
)

random_search.fit(X_train_est, y_train_est)

print('\nRandomized Search Complete')
print(f"Best Score ({scoring_metric}) found: {random_search.best_score_:.4f}")
print("Best Parameters found:")
print(random_search.best_params_)

best_xgb_model = random_search.best_estimator_

print("\nEvaluating the Best Tuned XGBoost Model on the Test Set...")
tuned_xgb_metrics = evaluate_model(best_xgb_model, X_test_est, y_test_est, 'Established Customer Tuned XGB')

print("\nComparison (Recall / Precision / F1 / PRC AUC / ROC AUC):")

print(f"Baseline RF:  {baseline_metrics['Established RF']['recall_churn']:.2f} / {baseline_metrics['Established RF']['precision_churn']:.2f} / {baseline_metrics['Established RF']['f1_churn']:.2f} / {baseline_metrics['Established RF']['prc_auc']:.4f} / {baseline_metrics['Established RF']['roc_auc']:.4f}")
print(f"Tuned XGB:    {tuned_xgb_metrics['recall_churn']:.2f} / {tuned_xgb_metrics['precision_churn']:.2f} / {tuned_xgb_metrics['f1_churn']:.2f} / {tuned_xgb_metrics['prc_auc']:.4f} / {tuned_xgb_metrics['roc_auc']:.4f}")

importance_scores = best_xgb_model.feature_importances_

feature_importance_df = pd.DataFrame({
    'Feature': final_feature_names,
    'Importance': importance_scores
})

feature_importance_df  = feature_importance_df.sort_values(by='Importance', ascending=False)

top_n = 20 
print(f"\nTop {top_n} Features by Importance (XGBoost Built-in):")
print(feature_importance_df.head(top_n))

plt.figure(figsize=(10, 8))
sns.barplot(x='Importance', y='Feature', data=feature_importance_df.head(top_n), palette='viridis')
plt.title(f'Top {top_n} Feature Importances - Tuned XGBoost (Established Customers)')
plt.xlabel('Importance Score')
plt.ylabel('Feature')
plt.tight_layout() 
plt.show()

shap.initjs()

print("Creating SHAP explainer...")
explainer = shap.TreeExplainer(best_xgb_model)

try:
    if not isinstance(X_test_est, pd.DataFrame):
        X_test_est_df = pd.DataFrame(X_test_est, columns=final_feature_names) 
    else:
        X_test_est_df = X_test_est

    shap_values = explainer.shap_values(X_test_est_df)
    print("SHAP values calculated.")

except Exception as e:
    print(f"Error calculating SHAP values: {e}")
    print("Ensure X_test_est is a pandas DataFrame or can be converted with final_feature_names.")
    shap_values = None 

  if shap_values is not None:
    print("\n--- SHAP Summary Plots ---")
    
    print("Generating SHAP Bar Plot (Mean Absolute SHAP Value)...")
    plt.figure() # Create a new figure context
    shap.summary_plot(shap_values, X_test_est_df, plot_type="bar", max_display=top_n, show=False)
    plt.title('SHAP Global Feature Importance (Bar Plot)')
    plt.tight_layout()
    plt.show()
   
    print("\nGenerating SHAP Beeswarm Plot...")
    plt.figure()
    shap.summary_plot(shap_values, X_test_est_df, max_display=top_n, show=False)
    plt.title('SHAP Feature Impact on Model Output (Beeswarm)') 
    plt.tight_layout()
    plt.show()

    print("\n--- SHAP Dependence Plots (Examples) ---")
   
    top_feature_1 = 'monthly_contact_rate' 
    top_feature_2 = 'billing_cycles_completed' 

    try:
        print(f"Generating SHAP Dependence Plot for '{top_feature_1}'...")
        plt.figure()
        shap.dependence_plot(top_feature_1, shap_values, X_test_est_df, interaction_index=None, show=False) 
        plt.title(f'SHAP Dependence Plot - {top_feature_1}')
        plt.tight_layout()
        plt.show()

        print(f"Generating SHAP Dependence Plot for '{top_feature_2}'...")
        plt.figure() # Create a new figure context
        shap.dependence_plot(top_feature_2, shap_values, X_test_est_df, interaction_index="auto", show=False) # 'auto' tries to find interaction
        plt.title(f'SHAP Dependence Plot - {top_feature_2} (Interaction colored)')
        plt.tight_layout()
        plt.show()

    except Exception as e:
        print(f"Could not generate one or more dependence plots: {e}")
        print("Check if feature names exist exactly in X_test_est_df.columns")

else:
    print("Skipping SHAP plots due to calculation error.")

churn_df['churned_within_90d'] = (
    (churn_df['cancel_date_time'].notna()) &
    (churn_df['tenure_days'] < 90)
).astype(int)

new_churn_actual_rate = churn_df['churned_within_90d'].mean()
print(f'Overall rate of churn within first 90 days: {new_churn_actual_rate:.2%}')
print(churn_df['churned_within_90d'].value_counts())

try: 
    idx_first_case_per_customer = customer_cases.loc[customer_cases.groupby('customer_id')['date_time'].idxmin()]
    first_case_details = idx_first_case_per_customer[['customer_id', 'reason','channel']].rename(columns={
        'reason':'first_case_reason',
        'channel':'first_case_channel'
    })

    if 'first_case_reason' in churn_df.columns:
        churn_df = churn_df.drop(columns=['first_case_reaon'])
    if 'first_case_channel' in churn_df.columns:
        churn_df = churn_df.drop(columns=['first_case_channel'])

    churn_df = pd.merge(churn_df, first_case_details, on = 'customer_id', how = 'left')
    churn_df['first_case_reason'] = churn_df['first_case_reason'].fillna('No_Contact')
    churn_df['first_case_channel'] = churn_df['first_case_channel'].fillna('No_Contact')

    print(churn_df['first_case_reason'].value_counts())
    print(churn_df['first_case_channel'].value_counts())

except KeyError as e:
    print(f"Error: A required column is missing. Make sure 'customer_cases' DataFrame is loaded correctly. Error: {e}")
except Exception as e:
    print(f"An unexpected error occurred during first case feature engineering: {e}")

new_churn_features = [
    'product',
    'price',
    'age_group',
    'gender',
    'signup_month',
    'signup_day_of_week',
    'contacted_within_7d',
    'days_to_first_contact',
    'first_case_reason',
    'first_case_channel'
]

print('\nColumns available for new churn features:', churn_df.columns)
X_new = churn_df[new_churn_features].copy()
y_new = churn_df['churned_within_90d']

print('\nFeatures selected for new Churn model (Revised):')
print(X_new.info())

X_new_encoded = pd.get_dummies(X_new, drop_first = True, dummy_na = False)

final_new_feature_names = X_new_encoded.columns.tolist()
print(f'Number of features after dummification: {len(final_new_feature_names)}')
print('First few new feature names:', final_new_feature_names[:10])

X_train_new, X_test_new, y_train_new, y_test_new, = train_test_split(
    X_new_encoded,
    y_new,
    test_size = 0.20,
    random_state = 0,
    stratify = y_new
)

print(f'Training set shape: {X_train_new.shape}')
print(f'Testing set shape: {X_test_new.shape}')
print(f'Training target distribution:\n{y_train_new.value_counts(normalize=True)}')
print(f'Testing target distribution:\n{y_test_new.value_counts(normalize=True)}')

neg_count_new = (y_train_new == 0).sum()
pos_count_new = (y_train_new == 1).sum()

if pos_count_new > 0:
    scale_pos_weight_new = neg_count_new / pos_count_new
else:
    scale_pos_weight_new = 1 

print(f'\nCalculated scale_pos_weight for new Churn XGBoost: {scale_pos_weight_new:.2f}')

xgb_model_new = xgb.XGBClassifier(
    objective = 'binary:logistic',
    eval_metric = 'logloss',
    use_label_encoder = False,
    scale_pos_weight = scale_pos_weight_new,
    random_state = 0
)

xgb_model_new.fit(X_train_new, y_train_new)

new_xgb_metrics = evaluate_model(xgb_model_new, X_test_new, y_test_new, 'new Churn XGB Baseline')

try:
    new_importance_scores = xgb_model_new.feature_importances_
    new_feature_importance_df = pd.DataFrame({
        'Feature': final_new_feature_names,
        'Importance': new_importance_scores
    })

    new_feature_importance_df.sort_values(by='Importance', ascending=False)

    top_n_new = 15

    print(f'\nTop {top_n_new} Features by Importance')
    print(new_feature_importance_df.head(top_n_new))

    plt.figure(figsize=(10,6))
    sns.barplot(x='Importance', y='Feature', data=new_feature_importance_df.head(top_n_new), palette='rocket')
    plt.title(f'Top {top_n_new} Feature Importances')
    plt.xlabel('Importance Score')
    plt.ylabel('Feature')
    plt.tight_layout()
    plt.show

except Exception as e:
    print(f'Could not generate feature importance plot {e}')

random_search_new.fit(X_train_new, y_train_new)

print(f'Best Score ({scoring_metric}) found: {random_search_new.best_score_:.4f}')
print(random_search_new.best_params_)

best_xgb_model_new = random_search_new.best_estimator_

tuned_xgb_new_metrics = evaluate_model(
    best_xgb_model_new,
    X_test_new,
    y_test_new,
    'new Churn Tuned XGB'
)

test_probabilities = best_xgb_model.predict_proba(X_test_est)[:,1]

results_df = pd.DataFrame({
    'Actual_Churn': y_test_est,
    'Predicted_Probability': test_probabilities
}, index = X_test_est.index)

results_df_sorted = results_df.sort_values(by='Predicted_Probability', ascending=False)

display(results_df_sorted.head(10))

risk_threshold = 0.5
high_risk_list = results_df_sorted[results_df_sorted['Predicted_Probability'] >= risk_threshold]
print(f"\nIdentified {len(high_risk_list)} customers above {risk_threshold:.2f} probability threshold.")

if 'explainer' in locals() and shap_values is not None:
    try:
        idx_high_risk = results_df_sorted.index[0]
        idx_low_risk = results_df_sorted.index[-1]

        print('idx_high_risk')
        display(shap.force_plot(explainer.expected_value,
                                shap_values[results_df_sorted.index.get_loc(idx_high_risk),:],
                                X_test_est_df.loc[idx_high_risk,:]))
        print('idx_low_risk')
        display(shap.force_plot(explainer.expected_value,
                               shap_values[results_df_sorted.index.get_loc(idx_low_risk),:],
                               X_test_est_df.loc[idx_low_risk,:]))
    except IndexError:
        print("Could not retrieve example indices. Ensure results_df_sorted is populated.")
    except Exception as e:
        print(f"Could not generate force plots: {e}")

else:
    print("SHAP explainer or values not available for force plots.")