The notebook

In [2]

# Data manipulation and analysis
import pandas as pd
import time
import numpy as np
pd.set_option('display.max_columns', None)  # Show all columns
# Suppress warnings
import warnings
warnings.filterwarnings('ignore')

from scipy import stats

# Visualization
import seaborn as sns
import matplotlib.pyplot as plt

# Machine Learning Libraries
import xgboost as xgb
# from tensorflow import keras  # Neural Network

# Sample more heavily from the edge fraud cases
# from sklearn.utils import resample

# from sklearn.preprocessing import OneHotEncoder

# from sklearn.ensemble import GradientBoostingClassifier

from sklearn.linear_model import LogisticRegression
import lightgbm as lgb
from sklearn.tree import DecisionTreeClassifier
from imblearn.under_sampling import RandomUnderSampler
# Metrics for model evaluation
from sklearn.metrics import classification_report, confusion_matrix, roc_auc_score, precision_recall_curve, accuracy_score, f1_score, auc, roc_curve, precision_score, recall_score, average_precision_score
# Data preprocessing
from sklearn.preprocessing import StandardScaler, RobustScaler  # Feature scaling
from sklearn.preprocessing import LabelEncoder    # Categorical encoding
from sklearn.model_selection import train_test_split
from sklearn.ensemble import RandomForestClassifier, VotingClassifier
from catboost import CatBoostClassifier
from sklearn.neighbors import KNeighborsClassifier
# Load the dataset
# Assuming your dataset is in CSV format and in the same directory as the notebook
df = pd.read_csv('PS_20174392719_1491204439457_log.csv')

In [10]

# Display basic information about the dataset
print("Dataset Info:")
print("-" * 50)
df.info()

In [11]

print("\nFirst few rows of the dataset:")
print("-" * 50)
display(df.head())

In [12]

print("\nBasic statistics of numerical columns:")
print("-" * 50)
display(df.describe())

In [13]

#EDA
# Check class imbalance for fraud detection
df['isFraud'].value_counts()

In [14]

# Get the value counts
fraud_counts = df['isFraud'].value_counts()

# Define labels and colors
labels = ['Non-Fraud (0)', 'Fraud (1)']
colors = ['skyblue', 'salmon']

# Create the pie chart
plt.figure(figsize=(8, 8))
plt.pie(fraud_counts.values, labels=labels, colors=colors, 
        autopct=lambda p: f'{p:.2f}%\n({int(p * sum(fraud_counts.values) / 100):,})',
        startangle=90, textprops={'fontsize': 12, 'weight': 'bold'})

# Add title
plt.title('Class Distribution: Fraud vs. Non-Fraud', fontsize=16)

plt.tight_layout()
plt.show()

In [15]

print("\nFraud Distribution:")
print(df['isFraud'].value_counts(normalize=True) * 100)
print("\nFlagged Fraud Distribution:")
print(df['isFlaggedFraud'].value_counts(normalize=True) * 100)

In [16]

# TRANSACTION TYPE ANALYSIS
print("\nTransaction Types:")
print(df['type'].value_counts())

In [17]

# 1. Transaction Type and Fraud
plt.figure(figsize=(12, 5))
sns.countplot(x='type', hue='isFraud', data=df)
plt.title('Transaction Types and Fraud')
plt.yscale('log')  # Log scale because of class imbalance
plt.legend(['Legitimate', 'Fraud'])
plt.tight_layout()
plt.savefig('transaction_type_fraud.png')
plt.show()
# 1. Check fraud distribution by transaction type
fraud_by_type = df.groupby('type')['isFraud'].mean() * 100
print("\nFraud Percentage by Transaction Type:")
print(fraud_by_type)

In [18]

# analysing fraud through transaction type
fraud_df=df[df['isFraud']==1].groupby('type')['isFraud'].sum().reset_index()
fraud_df

In [19]

plt.pie(fraud_df['isFraud'],labels=fraud_df['type'], autopct='%1.1f%%', startangle=90)
plt.title('Distribution of Fraud in Different transaction type')
plt.show()

In [20]

num_cols=['amount','oldbalanceOrg','newbalanceOrig','newbalanceDest','oldbalanceDest','isFlaggedFraud','isFraud']
corr_matrix=df[num_cols].corr()
plt.figure(figsize=(12, 8))
sns.heatmap(corr_matrix, annot=True, cmap='coolwarm', vmin=-1, vmax=1)
plt.title('Correlation Heatmap')
plt.show()

In [21]

# List of selected features to analyze
selected_features = ['amount', 'oldbalanceOrg', 'newbalanceOrig', 'oldbalanceDest', 'newbalanceDest']

# Create subplots (number of subplots is equal to the number of selected features)
fig, axes = plt.subplots(1, len(selected_features), figsize=(20, 5))

# Loop through each selected feature to create individual bar plots
for i, feature in enumerate(selected_features):
    sns.barplot(x=df['isFraud'], y=df[feature], hue=df['isFraud'], ax=axes[i], palette="Set2")
    axes[i].set_title(f'{feature} vs isFraud')  # Title for each subplot
    axes[i].set_xlabel('isFraud') 
    axes[i].set_ylabel(feature)  

# Add a title for the overall figure and adjust layout for better spacing
plt.suptitle("Comparison of Selected Features with Target (Bar Plot)", fontsize=16)
plt.tight_layout()  
plt.show()  

In [22]

plt.figure(figsize=(12, 8))
# Add a small value to handle zeros
non_zero_amounts = df['amount'][df['amount'] > 0]
plt.hist(np.log10(non_zero_amounts + 1), bins=50, color='green', edgecolor='black')
plt.title('Log-Scaled Distribution of Transaction Amounts')
plt.xlabel('Log10(Amount + 1)')
plt.ylabel('Frequency')
plt.grid(True, alpha=0.3)

In [23]

# Create log-scaled histogram for oldbalanceOrg
plt.figure(figsize=(12, 8))
# Add a small value to handle zeros
non_zero_oldbalance = df['oldbalanceOrg'][df['oldbalanceOrg'] > 0]
plt.hist(np.log10(non_zero_oldbalance + 1), bins=50, color='lightgreen', edgecolor='black')
plt.title('Log-Scaled Distribution of Original Old Balance')
plt.xlabel('Log10(oldbalanceOrg + 1)')
plt.ylabel('Frequency')
plt.grid(True, alpha=0.3)

In [24]

# Create log-scaled histogram for newbalanceOrig
plt.figure(figsize=(12, 8))
non_zero_newbalanceOrig = df['newbalanceOrig'][df['newbalanceOrig'] > 0]
plt.hist(np.log10(non_zero_newbalanceOrig + 1), bins=50, color='lightblue', edgecolor='black')
plt.title('Log-Scaled Distribution of Original New Balance')
plt.xlabel('Log10(newbalanceOrig + 1)')
plt.ylabel('Frequency')
plt.grid(True, alpha=0.3)

In [25]

# Create log-scaled histogram for oldbalanceDest
plt.figure(figsize=(12, 8))
non_zero_oldbalanceDest = df['oldbalanceDest'][df['oldbalanceDest'] > 0]
plt.hist(np.log10(non_zero_oldbalanceDest + 1), bins=50, color='salmon', edgecolor='black')
plt.title('Log-Scaled Distribution of Destination Old Balance')
plt.xlabel('Log10(oldbalanceDest + 1)')
plt.ylabel('Frequency')
plt.grid(True, alpha=0.3)

In [26]

# Create log-scaled histogram for newbalanceDest
plt.figure(figsize=(12, 8))
non_zero_newbalanceDest = df['newbalanceDest'][df['newbalanceDest'] > 0]
plt.hist(np.log10(non_zero_newbalanceDest + 1), bins=50, color='gold', edgecolor='black')
plt.title('Log-Scaled Distribution of Destination New Balance')
plt.xlabel('Log10(newbalanceDest + 1)')
plt.ylabel('Frequency')
plt.grid(True, alpha=0.3)

In [27]

# Create comparison histogram for oldbalanceOrg vs newbalanceOrig
plt.figure(figsize=(14, 8))

# Filter out zero values
non_zero_df = df[(df['oldbalanceOrg'] > 0) | (df['newbalanceOrig'] > 0)]

# Create log transformed values for better visualization
log_old = np.log10(non_zero_df['oldbalanceOrg'] + 1)
log_new = np.log10(non_zero_df['newbalanceOrig'] + 1)

# Calculate bin edges for both histograms to share
min_val = min(log_old.min(), log_new.min())
max_val = max(log_old.max(), log_new.max())
bins = np.linspace(min_val, max_val, 50)

# Plot both histograms on same axes
plt.hist(log_old, bins=bins, alpha=0.6, label='oldbalanceOrg', color='darkgreen', edgecolor='black')
plt.hist(log_new, bins=bins, alpha=0.6, label='newbalanceOrig', color='skyblue', edgecolor='black')

plt.title('Comparison of Original Old Balance vs New Balance (Log-Scaled)')
plt.xlabel('Log10(Balance + 1)')
plt.ylabel('Frequency')
plt.legend()
plt.grid(True, alpha=0.3)
plt.tight_layout() 

In [28]

# Create comparison histogram for oldbalanceDest vs newbalanceDest
plt.figure(figsize=(14, 8))

# Filter out zero values
non_zero_dest_df = df[(df['oldbalanceDest'] > 0) | (df['newbalanceDest'] > 0)]

# Create log transformed values for better visualization
log_old_dest = np.log10(non_zero_dest_df['oldbalanceDest'] + 1)
log_new_dest = np.log10(non_zero_dest_df['newbalanceDest'] + 1)

# Calculate bin edges for both histograms to share
min_val_dest = min(log_old_dest.min(), log_new_dest.min())
max_val_dest = max(log_old_dest.max(), log_new_dest.max())
bins_dest = np.linspace(min_val_dest, max_val_dest, 50)

# Plot both histograms on same axes
plt.hist(log_old_dest, bins=bins_dest, alpha=0.6, label='oldbalanceDest', color='coral', edgecolor='black')
plt.hist(log_new_dest, bins=bins_dest, alpha=0.6, label='newbalanceDest', color='purple', edgecolor='black')

plt.title('Comparison of Destination Old Balance vs New Balance (Log-Scaled)')
plt.xlabel('Log10(Balance + 1)')
plt.ylabel('Frequency')
plt.legend()
plt.grid(True, alpha=0.3)
plt.tight_layout()

In [4]

df['balance_diff'] = df['newbalanceDest'] - df['oldbalanceOrg']
sns.histplot(df['balance_diff'], bins=100, kde=True)
plt.title('Distribution of Balance Difference (newbalanceDest - oldbalanceOrg)')
plt.xlabel('Balance Difference')
plt.ylabel('Frequency')
plt.show()

In [5]

df['log_old_org'] = np.log10(df['oldbalanceOrg'] + 1)
df['log_new_dest'] = np.log10(df['newbalanceDest'] + 1)

plt.figure(figsize=(12, 6))
sns.histplot(data=df[df['isFraud'] == 0], x='log_old_org', color='green', label='oldbalanceOrg - Non-Fraud', alpha=0.4)
sns.histplot(data=df[df['isFraud'] == 1], x='log_old_org', color='red', label='oldbalanceOrg - Fraud', alpha=0.6)
plt.legend()
plt.title('Old Balance Org (Log10) by Fraud Status')
plt.xlabel('Log10(oldbalanceOrg + 1)')
plt.ylabel('Frequency')
plt.show()

In [29]

# Handle missing values
print("Missing values before handling:")
print(df.isnull().sum())

In [30]

# Check for zero values in important numeric columns
numeric_cols = ['amount', 'oldbalanceOrg', 'newbalanceOrig', 'oldbalanceDest', 'newbalanceDest']
print('\nZero values in numeric columns:')
zero_counts = {col: (df[col] == 0).sum() for col in numeric_cols}
zero_percentages = {col: (count/len(df)*100).round(2) for col, count in zero_counts.items()}
zero_df = pd.DataFrame({
    'Zero Count': zero_counts,
    'Percentage': zero_percentages
})
print(zero_df)

In [3]

# Create new features
# Flag transactions where all balance is transferred out
df['emptied_account'] = ((df['oldbalanceOrg'] > 0) & (df['newbalanceOrig'] == 0)).astype(int)
# Flag suspicious patterns (like identical amounts)
df['amount_equals_oldbalanceOrg'] = (df['amount'] == df['oldbalanceOrg']).astype(int)

In [4]

df['hour'] = df['step'] % 24  # Extract hour from step (since 1 step = 1 hour)
df['is_night'] = df['hour'].apply(lambda x: 1 if x < 6 else 0)  # Flag night transactions
df['amount_ratio'] = df['amount'] / (df['oldbalanceOrg'] + 1)

In [5]

# Create new binary columns for CASH_OUT and TRANSFER transactions
df['is_CASH_OUT'] = (df['type'] == 'CASH_OUT').astype(int)
df['is_TRANSFER'] = (df['type'] == 'TRANSFER').astype(int)
df['is_DEBIT'] = (df['type'] == 'DEBIT').astype(int)
df['is_PAYMENT'] = (df['type'] == 'PAYMENT').astype(int)
# Display the first few rows to verify the new columns
df.head()

In [34]

# Get only numeric columns - this automatically includes your type_encoded column
numeric_cols = df.select_dtypes(include=['number']).columns
corr = df[numeric_cols].corr()

# Visualize the correlation matrix
plt.figure(figsize=(14, 12))
sns.heatmap(corr, annot=True, cmap='coolwarm', linewidths=0.5, annot_kws={"size": 8})
plt.title('Correlation Matrix of Numeric Features')
plt.tight_layout()
plt.show()

In [6]

# Drop unnecessary columns
columns_to_drop = [
    'isFlaggedFraud',
    'nameOrig',    
    'nameDest',
    'type'
]

df = df.drop(columns=columns_to_drop)

In [36]

# Create a figure
plt.figure(figsize=(12, 8))

# Count fraud cases by hour
fraud_by_hour = df.groupby(['hour', 'isFraud']).size().unstack(fill_value=0)

if 1 in fraud_by_hour.columns:
    # Plot percentage of fraud transactions by hour
    fraud_percentage = fraud_by_hour[1] / (fraud_by_hour[0] + fraud_by_hour[1]) * 100
    
    plt.subplot(2, 1, 1)
    plt.bar(fraud_percentage.index, fraud_percentage.values, color='orangered')
    plt.title('Percentage of Fraudulent Transactions by Hour', fontsize=14)
    plt.xlabel('Hour of Day')
    plt.ylabel('Fraud Percentage (%)')
    plt.xticks(range(0, 24))
    plt.grid(axis='y', linestyle='--', alpha=0.7)
    
    # Annotate percentages
    for i, v in enumerate(fraud_percentage):
        plt.text(i, v + 0.1, f'{v:.2f}%', ha='center')
    
    # Plot count of fraud cases by hour
    plt.subplot(2, 1, 2)
    fraud_by_hour[1].plot(kind='bar', color='firebrick')
    plt.title('Count of Fraudulent Transactions by Hour', fontsize=14)
    plt.xlabel('Hour of Day')
    plt.ylabel('Number of Fraudulent Transactions')
    plt.grid(axis='y', linestyle='--', alpha=0.7)
    
    # Annotate counts
    for i, v in enumerate(fraud_by_hour[1]):
        plt.text(i, v + 0.5, str(v), ha='center')
else:
    print("No fraud cases found in the dataset")

plt.tight_layout()
plt.show()

In [37]

# Create a figure with stacked layout (one graph per figure for maximum size)
# First graph - Non-fraudulent transactions count
plt.figure(1)
fraud_by_hour = df.groupby(['hour', 'isFraud']).size().unstack(fill_value=0)

fraud_by_hour[0].plot(kind='bar', color='forestgreen', figsize=(20, 10))
plt.title('Count of Non-Fraudulent Transactions by Hour', fontsize=20)
plt.xlabel('Hour of Day', fontsize=16)
plt.ylabel('Number of Non-Fraudulent Transactions', fontsize=16)
plt.grid(axis='y', linestyle='--', alpha=0.7)
plt.ticklabel_format(axis='y', style='scientific', scilimits=(0,0))
plt.xticks(fontsize=14)
plt.yticks(fontsize=14)

# Annotate counts (scientific notation for large numbers)
for i, v in enumerate(fraud_by_hour[0]):
    plt.text(i, v + v*0.01, f'{v:.1e}', ha='center', fontsize=12)

plt.tight_layout(pad=1.1)

# Second graph - Combined log scale view on a separate figure
plt.figure(2, figsize=(20, 10))
fraud_by_hour.plot(kind='bar', log=True, figsize=(20, 10))
plt.title('Fraudulent vs Non-Fraudulent Transactions by Hour (Log Scale)', fontsize=20)
plt.xlabel('Hour of Day', fontsize=16)
plt.ylabel('Number of Transactions (Log Scale)', fontsize=16)
plt.grid(axis='y', linestyle='--', alpha=0.7)
plt.legend(['Non-Fraud', 'Fraud'], fontsize=16)
plt.xticks(fontsize=14)
plt.yticks(fontsize=14)

plt.tight_layout(pad=1.1)

# Display the figures in fullscreen mode
plt.show()

In [7]

# Create hour risk categories based on fraud percentages
def assign_hour_risk(hour):
    if hour in [4, 5]:  # >20% fraud
        return 3  # Very high risk
    elif hour in [3, 6]:  # 10-20% fraud
        return 2  # High risk
    elif hour in [2, 7]:  # 3-10% fraud
        return 1  # Medium risk
    else:  # <1% fraud
        return 0  # Low risk
        
df['hour_risk_level'] = df['hour'].apply(assign_hour_risk)
# Drop unnecessary columns
columns_to_drop1 = [
    "is_night",
    "hour",
    "step",
    "amount_ratio",
    "is_DEBIT",
    "oldbalanceDest",
    "newbalanceOrig",
    "is_PAYMENT",
    "amount"
]

df = df.drop(columns=columns_to_drop1)

In [39]

# df.to_excel('transformed_fraud_dataset.xlsx', index=False)
df.to_csv('transformed_fraud_dataset.csv', index=False)

In [40]

# Get only numeric columns - this automatically includes your type_encoded column
numeric_cols = df.select_dtypes(include=['number']).columns
corr = df[numeric_cols].corr()

# Visualize the correlation matrix
plt.figure(figsize=(14, 12))
sns.heatmap(corr, annot=True, cmap='coolwarm', linewidths=0.5, annot_kws={"size": 8})
plt.title('')
plt.tight_layout()
plt.show()

In [41]

# Set the style for nicer plots
sns.set_theme(style="whitegrid")
plt.rcParams.update({'font.size': 12})


# Create a figure with 1 row and 2 columns
fig, axes = plt.subplots(1, 2, figsize=(16, 8))

# 1. oldbalanceOrg boxplot
sns.boxplot(x='isFraud', y='oldbalanceOrg', data=df, ax=axes[0], 
            palette=['skyblue', 'coral'])
axes[0].set_title('oldbalanceOrg by Fraud Status', fontsize=16)
axes[0].set_xlabel('Fraud (0=No, 1=Yes)', fontsize=14)
axes[0].set_ylabel('oldbalanceOrg', fontsize=14)

# 2. newbalanceDest boxplot
sns.boxplot(x='isFraud', y='newbalanceDest', data=df, ax=axes[1], 
            palette=['skyblue', 'coral'])
axes[1].set_title('newbalanceDest by Fraud Status', fontsize=16)
axes[1].set_xlabel('Fraud (0=No, 1=Yes)', fontsize=14)
axes[1].set_ylabel('newbalanceDest', fontsize=14)

# Adjust layout and save
plt.tight_layout()
plt.show()

In [42]

df

In [43]

# Show correlations with fraud
fraud_correlations = df.corr()['isFraud'].sort_values(ascending=False)
print("Correlation of features with fraud:")
print(fraud_correlations)

In [8]

# X Data
X = df.drop(['isFraud'], axis=1)
print('X shape is : ' , X.shape)
print()

# y Data
y = df['isFraud']
print('y shape is : ' , y.shape)

In [9]

X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.3, random_state=42, shuffle=True)

# Splitted Data
print('X_train shape is ' , X_train.shape)
print('X_test shape is ' , X_test.shape)
print('y_train shape is ' , y_train.shape)
print('y_test shape is ' , y_test.shape)

In [43]

# XGBoost Model
# Scale the features
scaler = StandardScaler()
X_train_scaled = scaler.fit_transform(X_train)
X_test_scaled = scaler.transform(X_test)

# Calculate class weights since the dataset is imbalanced
fraud_ratio = len(y_train[y_train==1]) / len(y_train[y_train==0])
scale_pos_weight = 1 / fraud_ratio

# Initialize XGBoost model with basic parameters
xgb_model = xgb.XGBClassifier(
    scale_pos_weight=scale_pos_weight, # Helps address class imbalance
    learning_rate=0.01, # Learning rate for the model
    n_estimators=200, # Number of trees in the model
    max_depth=5, # Maximum depth of the trees
    min_child_weight=1, # Minimum sum of instance weights (hessians) needed in a child
    gamma=0, # Minimum loss reduction required to make a split
    subsample=0.8, # Subsample ratio of the training instance  Uses 80% of data and features for each tree
    colsample_bytree=0.8, # Subsample ratio of columns when constructing each tree
    random_state=42
)

# Simple fit without additional parameters
xgb_model.fit(X_train_scaled, y_train)

# Evaluate on training data
train_preds = xgb_model.predict(X_train_scaled)
train_probs = xgb_model.predict_proba(X_train_scaled)[:, 1]

# Evaluate on test data
test_preds = xgb_model.predict(X_test_scaled)
test_probs = xgb_model.predict_proba(X_test_scaled)[:, 1]

# Calculate metrics for both sets (correctly multiply by 100)
train_metrics = {
    'Accuracy': accuracy_score(y_train, train_preds) * 100,
    'Precision': precision_score(y_train, train_preds) * 100,
    'Recall': recall_score(y_train, train_preds) * 100,
    'F1 Score': f1_score(y_train, train_preds) * 100,
    'ROC AUC': roc_auc_score(y_train, train_probs) * 100,
    'PR AUC': average_precision_score(y_train, train_probs) * 100
}

test_metrics = {
    'Accuracy': accuracy_score(y_test, test_preds) * 100,
    'Precision': precision_score(y_test, test_preds) * 100,
    'Recall': recall_score(y_test, test_preds) * 100,
    'F1 Score': f1_score(y_test, test_preds) * 100,
    'ROC AUC': roc_auc_score(y_test, test_probs) * 100,
    'PR AUC': average_precision_score(y_test, test_probs) * 100
}

# Format metrics with 8 decimal places
print("\n---------- XGBoost Model Performance ----------")
print("\n---------- Training vs Test Performance (Overfitting Analysis) ----------")
print(f"{'Metric':<12} {'Training':>20} {'Test':>20} {'Difference':>20}")
print("-" * 74)
for metric in train_metrics.keys():
    diff = train_metrics[metric] - test_metrics[metric]
    print(f"{metric:<12} {train_metrics[metric]:>19.8f}% {test_metrics[metric]:>19.8f}% {diff:>19.8f}%")

# Calculate overall overfitting score (average difference across metrics)
overfitting_score = np.mean([train_metrics[m] - test_metrics[m] for m in train_metrics.keys()])
print(f"\nOverfitting Score: {overfitting_score:.8f}%")
if overfitting_score > 5:
    print("WARNING: Model shows signs of overfitting (score > 5%)")
elif overfitting_score < 2:
    print("Model shows good generalization (score < 2%)")
else:
    print("Model shows moderate overfitting")

# Display individual metrics with arrows for visual emphasis and 8 decimal places
print("\n---------- Detailed Test Metrics ----------")
for metric, value in test_metrics.items():
    print(f'➤➤➤ {metric}: {value:.8f} %')

# Print classification report
print("\nClassification Report:")
print(classification_report(y_test, test_preds))

# Create confusion matrix
cm = confusion_matrix(y_test, test_preds)
plt.figure(figsize=(8, 6))
sns.heatmap(cm, annot=True, fmt='d', cmap='Blues')
plt.title('Confusion Matrix - XGBoost')
plt.ylabel('True Label')
plt.xlabel('Predicted Label')
plt.show()

In [11]

# LightGBM Model

# 1. Feature Scaling
scaler = StandardScaler()
X_train_scaled = scaler.fit_transform(X_train)
X_test_scaled = scaler.transform(X_test)

# 2. Calculate class weights
fraud_samples = sum(y_train == 1)
non_fraud_samples = sum(y_train == 0)
class_weight = {0: 1.0, 1: non_fraud_samples / fraud_samples}
print(f"Class weights: {class_weight}")

In [12]

# 3. Model Training
lgb_model = lgb.LGBMClassifier(
    objective='binary',
    metric='auc',
    boosting_type='gbdt',
    learning_rate=0.01,
    num_leaves=31,
    max_depth=7,
    feature_fraction=0.8,
    bagging_fraction=0.8,
    bagging_freq=5,
    verbose=-1,
    is_unbalance=True,
    class_weight='balanced',
    random_state=42,
    n_estimators=200
)

lgb_model.fit(X_train_scaled, y_train)

In [13]

# 4. Model Evaluation

# Evaluate on training data
y_train_pred = lgb_model.predict(X_train_scaled)
y_train_pred_proba = lgb_model.predict_proba(X_train_scaled)[:, 1]

# Evaluate on test data
y_pred_lgb = lgb_model.predict(X_test_scaled)
y_pred_proba_lgb = lgb_model.predict_proba(X_test_scaled)[:, 1]

print("\n---------- LightGBM Model Evaluation ----------")
print("\nClassification Report (Test Set):")
print(classification_report(y_test, y_pred_lgb))

# Calculate metrics for both training and test sets
train_metrics = {
    'Accuracy': accuracy_score(y_train, y_train_pred) * 100,
    'Precision': precision_score(y_train, y_train_pred) * 100,
    'Recall': recall_score(y_train, y_train_pred) * 100,
    'F1 Score': f1_score(y_train, y_train_pred) * 100,
    'ROC AUC': roc_auc_score(y_train, y_train_pred_proba) * 100,
    'PR AUC': average_precision_score(y_train, y_train_pred_proba) * 100
}

test_metrics = {
    'Accuracy': accuracy_score(y_test, y_pred_lgb) * 100,
    'Precision': precision_score(y_test, y_pred_lgb) * 100,
    'Recall': recall_score(y_test, y_pred_lgb) * 100,
    'F1 Score': f1_score(y_test, y_pred_lgb) * 100,
    'ROC AUC': roc_auc_score(y_test, y_pred_proba_lgb) * 100,
    'PR AUC': average_precision_score(y_test, y_pred_proba_lgb) * 100
}

print("\n---------- Training vs Test Performance (Overfitting Analysis) ----------")
print(f"{'Metric':<12} {'Training':>12} {'Test':>12} {'Difference':>12}")
print("-" * 50)
for metric in train_metrics.keys():
    diff = train_metrics[metric] - test_metrics[metric]
    print(f"{metric:<12} {train_metrics[metric]:>11.8f}% {test_metrics[metric]:>11.8f}% {diff:>11.8f}%")

# Calculate overall overfitting score (average difference across metrics)
overfitting_score = np.mean([train_metrics[m] - test_metrics[m] for m in train_metrics.keys()])
print(f"\nOverfitting Score: {overfitting_score:.8f}%")
if overfitting_score > 5:
    print("WARNING: Model shows signs of overfitting (score > 5%)")
elif overfitting_score < 2:
    print("Model shows good generalization (score < 2%)")
else:
    print("Model shows moderate overfitting")

# Display individual metrics with arrows for visual emphasis
print("\n---------- Detailed Test Metrics ----------")
for metric, value in test_metrics.items():
    print(f'➤➤➤ {metric}: {value:.8f} %')

In [24]

# 5. Visualization
plt.figure(figsize=(8, 6))
cm = confusion_matrix(y_test, y_pred_lgb)
sns.heatmap(cm, annot=True, fmt='d', cmap='Blues')
plt.title('Confusion Matrix - LightGBM')
plt.ylabel('True Label')
plt.xlabel('Predicted Label')
plt.show()

In [14]

# Random Forest Model

# 1. Performance Tracking
start_time = time.time()

# 2. Feature Scaling
scaler = StandardScaler()
X_train_scaled = scaler.fit_transform(X_train)
X_test_scaled = scaler.transform(X_test)

# 3. Calculate class weights
fraud_samples = sum(y_train == 1)
non_fraud_samples = sum(y_train == 0)
class_weight_ratio = non_fraud_samples / fraud_samples
print(f"Class weight ratio (non-fraud:fraud): {class_weight_ratio:.2f}:1")

In [15]

# 4. Model Training
print("Training Random Forest model...")
rf_model = RandomForestClassifier(
    n_estimators=100,
    max_depth=10,
    min_samples_split=10,
    min_samples_leaf=4,
    max_features='sqrt',
    bootstrap=True,
    class_weight='balanced',
    n_jobs=-1,
    random_state=42
)

rf_model.fit(X_train_scaled, y_train)
training_time = time.time() - start_time
print(f"Training completed in {training_time:.2f} seconds")

In [16]

# 5. Model Evaluation
# Evaluate on training data
y_train_pred = rf_model.predict(X_train_scaled)
y_train_pred_proba = rf_model.predict_proba(X_train_scaled)[:, 1]

# Evaluate on test data
y_pred_rdf = rf_model.predict(X_test_scaled)
y_pred_proba_rdf = rf_model.predict_proba(X_test_scaled)[:, 1]

prediction_time = time.time() - start_time - training_time
print(f"Predictions completed in {prediction_time:.2f} seconds")

print("\n---------- Random Forest Model Evaluation ----------")
print("\nClassification Report (Test Set):")
print(classification_report(y_test, y_pred_rdf))

# Calculate metrics for both training and test sets
train_metrics = {
    'Accuracy': accuracy_score(y_train, y_train_pred) * 100,
    'Precision': precision_score(y_train, y_train_pred) * 100,
    'Recall': recall_score(y_train, y_train_pred) * 100,
    'F1 Score': f1_score(y_train, y_train_pred) * 100,
    'ROC AUC': roc_auc_score(y_train, y_train_pred_proba) * 100,
    'PR AUC': average_precision_score(y_train, y_train_pred_proba) * 100
}

test_metrics = {
    'Accuracy': accuracy_score(y_test, y_pred_rdf) * 100,
    'Precision': precision_score(y_test, y_pred_rdf) * 100,
    'Recall': recall_score(y_test, y_pred_rdf) * 100,
    'F1 Score': f1_score(y_test, y_pred_rdf) * 100,
    'ROC AUC': roc_auc_score(y_test, y_pred_proba_rdf) * 100,
    'PR AUC': average_precision_score(y_test, y_pred_proba_rdf) * 100
}

print("\n---------- Training vs Test Performance (Overfitting Analysis) ----------")
print(f"{'Metric':<12} {'Training':>12} {'Test':>12} {'Difference':>12}")
print("-" * 50)
for metric in train_metrics.keys():
    diff = train_metrics[metric] - test_metrics[metric]
    print(f"{metric:<12} {train_metrics[metric]:>11.8f}% {test_metrics[metric]:>11.8f}% {diff:>11.8f}%")

# Calculate overall overfitting score (average difference across metrics)
overfitting_score = np.mean([train_metrics[m] - test_metrics[m] for m in train_metrics.keys()])
print(f"\nOverfitting Score: {overfitting_score:.8f}%")
if overfitting_score > 5:
    print("WARNING: Model shows signs of overfitting (score > 5%)")
elif overfitting_score < 2:
    print("Model shows good generalization (score < 2%)")
else:
    print("Model shows moderate overfitting")

# Display individual metrics with arrows for visual emphasis
print("\n---------- Detailed Test Metrics ----------")
for metric, value in test_metrics.items():
    print(f'➤➤➤ {metric}: {value:.8f} %')

In [25]

# 6. Visualization
plt.figure(figsize=(8, 6))
cm = confusion_matrix(y_test, y_pred_rdf)
sns.heatmap(cm, annot=True, fmt='d', cmap='Blues')
plt.title('Confusion Matrix - Random Forest')
plt.ylabel('True Label')
plt.xlabel('Predicted Label')
plt.show()

In [17]

# 1. Performance Tracking
start_time = time.time()

# 2. Feature Scaling
scaler = StandardScaler()
X_train_scaled = scaler.fit_transform(X_train)
X_test_scaled = scaler.transform(X_test)

# 3. Class Weight Calculation
fraud_samples = sum(y_train == 1)
non_fraud_samples = sum(y_train == 0)
print(f"Training data: {non_fraud_samples} non-fraud, {fraud_samples} fraud")
print(f"Class distribution ratio (non-fraud:fraud): {non_fraud_samples / fraud_samples:.2f}:1")

# Define different class weight options
weight_options = {
    'balanced': 'balanced',
    'inverse': {
        0: 1.0,
        1: non_fraud_samples / fraud_samples
    },
    'sqrt': {
        0: 1.0,
        1: np.sqrt(non_fraud_samples / fraud_samples)
    },
    'custom': {
        0: 1.0,
        1: 2 * (non_fraud_samples / fraud_samples)
    }
}

# Select weight approach
selected_weight = weight_options['inverse']  # Choose from: 'balanced', 'inverse', 'sqrt', 'custom'
print(f"\nSelected weight approach: {selected_weight}")

In [18]

# 4. Model Training
print("\nTraining Logistic Regression model...")
lr_model = LogisticRegression(
    penalty='l2',
    C=1.0,
    class_weight=selected_weight,
    solver='liblinear',
    max_iter=1000,
    verbose=1,
    n_jobs=-1,
    random_state=42
)

lr_model.fit(X_train_scaled, y_train)
training_time = time.time() - start_time
print(f"Training completed in {training_time:.2f} seconds")

In [19]

# 5. Model Evaluation
# Evaluate on training data
y_train_pred = lr_model.predict(X_train_scaled)
y_train_pred_proba = lr_model.predict_proba(X_train_scaled)[:, 1]

# Evaluate on test data
y_pred_lr = lr_model.predict(X_test_scaled)
y_pred_proba_lr = lr_model.predict_proba(X_test_scaled)[:, 1]

prediction_time = time.time() - start_time - training_time
print(f"Predictions completed in {prediction_time:.2f} seconds")

print("\n---------- Logistic Regression Model Evaluation ----------")
print("\nClassification Report (Test Set):")
print(classification_report(y_test, y_pred_lr))

# Calculate metrics for both training and test sets
train_metrics = {
    'Accuracy': accuracy_score(y_train, y_train_pred) * 100,
    'Precision': precision_score(y_train, y_train_pred) * 100,
    'Recall': recall_score(y_train, y_train_pred) * 100,
    'F1 Score': f1_score(y_train, y_train_pred) * 100,
    'ROC AUC': roc_auc_score(y_train, y_train_pred_proba) * 100,
    'PR AUC': average_precision_score(y_train, y_train_pred_proba) * 100
}

test_metrics = {
    'Accuracy': accuracy_score(y_test, y_pred_lr) * 100,
    'Precision': precision_score(y_test, y_pred_lr) * 100,
    'Recall': recall_score(y_test, y_pred_lr) * 100,
    'F1 Score': f1_score(y_test, y_pred_lr) * 100,
    'ROC AUC': roc_auc_score(y_test, y_pred_proba_lr) * 100,
    'PR AUC': average_precision_score(y_test, y_pred_proba_lr) * 100
}

print("\n---------- Training vs Test Performance (Overfitting Analysis) ----------")
print(f"{'Metric':<12} {'Training':>12} {'Test':>12} {'Difference':>12}")
print("-" * 50)
for metric in train_metrics.keys():
    diff = train_metrics[metric] - test_metrics[metric]
    print(f"{metric:<12} {train_metrics[metric]:>11.8f}% {test_metrics[metric]:>11.8f}% {diff:>11.8f}%")

# Calculate overall overfitting score (average difference across metrics)
overfitting_score = np.mean([train_metrics[m] - test_metrics[m] for m in train_metrics.keys()])
print(f"\nOverfitting Score: {overfitting_score:.8f}%")
if overfitting_score > 5:
    print("WARNING: Model shows signs of overfitting (score > 5%)")
elif overfitting_score < 2:
    print("Model shows good generalization (score < 2%)")
else:
    print("Model shows moderate overfitting")

# Display individual metrics with arrows for visual emphasis
print("\n---------- Detailed Test Metrics ----------")
for metric, value in test_metrics.items():
    print(f'➤➤➤ {metric}: {value:.8f} %')

In [26]

# 6. Visualization
plt.figure(figsize=(8, 6))
cm = confusion_matrix(y_test, y_pred_lr)
sns.heatmap(cm, annot=True, fmt='d', cmap='Blues')
plt.title('Confusion Matrix - Logistic Regression')
plt.ylabel('True Label')
plt.xlabel('Predicted Label')
plt.show()

In [20]

# 1. Performance Tracking
start_time = time.time()

# 2. Feature Scaling
scaler = StandardScaler()
X_train_scaled = scaler.fit_transform(X_train)
X_test_scaled = scaler.transform(X_test)

# 3. Class Distribution Analysis
fraud_samples = sum(y_train == 1)
non_fraud_samples = sum(y_train == 0)
print(f"Training data: {non_fraud_samples} non-fraud, {fraud_samples} fraud")
print(f"Class distribution ratio (non-fraud:fraud): {non_fraud_samples / fraud_samples:.2f}:1")

In [21]

# 4. Model Training
print("\nTraining k-Nearest Neighbors model...")
knn_model = KNeighborsClassifier(
    n_neighbors=5,          # Number of neighbors to consider
    weights='distance',     # Weight points by inverse of their distance
    algorithm='auto',       # Automatically choose the best algorithm
    leaf_size=30,          # Leaf size passed to BallTree or KDTree
    p=2,                   # Power parameter for the Minkowski metric (2 = Euclidean distance)
    metric='minkowski',    # Distance metric to use
    n_jobs=-1             # Use all available cores
)

knn_model.fit(X_train_scaled, y_train)
training_time = time.time() - start_time
print(f"Training completed in {training_time:.2f} seconds")

In [61]

# 5. Model Evaluation
# Evaluate on training data
y_train_pred = knn_model.predict(X_train_scaled)
y_train_pred_proba = knn_model.predict_proba(X_train_scaled)[:, 1]

# Evaluate on test data
y_pred_knn = knn_model.predict(X_test_scaled)
y_pred_proba_knn = knn_model.predict_proba(X_test_scaled)[:, 1]

prediction_time = time.time() - start_time - training_time
print(f"Predictions completed in {prediction_time:.2f} seconds")

print("\n---------- k-Nearest Neighbors Model Evaluation ----------")
print("\nClassification Report (Test Set):")
print(classification_report(y_test, y_pred_knn))

# Calculate metrics for both training and test sets
train_metrics = {
    'Accuracy': accuracy_score(y_train, y_train_pred) * 100,
    'Precision': precision_score(y_train, y_train_pred) * 100,
    'Recall': recall_score(y_train, y_train_pred) * 100,
    'F1 Score': f1_score(y_train, y_train_pred) * 100,
    'ROC AUC': roc_auc_score(y_train, y_train_pred_proba) * 100
}

test_metrics = {
    'Accuracy': accuracy_score(y_test, y_pred_knn) * 100,
    'Precision': precision_score(y_test, y_pred_knn) * 100,
    'Recall': recall_score(y_test, y_pred_knn) * 100,
    'F1 Score': f1_score(y_test, y_pred_knn) * 100,
    'ROC AUC': roc_auc_score(y_test, y_pred_proba_knn) * 100
}

print("\n---------- Training vs Test Performance (Overfitting Analysis) ----------")
print(f"{'Metric':<12} {'Training':>12} {'Test':>12} {'Difference':>12}")
print("-" * 50)
for metric in train_metrics.keys():
    diff = train_metrics[metric] - test_metrics[metric]
    print(f"{metric:<12} {train_metrics[metric]:>11.8f}% {test_metrics[metric]:>11.8f}% {diff:>11.8f}%")

# Calculate overall overfitting score (average difference across metrics)
overfitting_score = np.mean([train_metrics[m] - test_metrics[m] for m in train_metrics.keys()])
print(f"\nOverfitting Score: {overfitting_score:.8f}%")
if overfitting_score > 5:
    print("WARNING: Model shows signs of overfitting (score > 5%)")
elif overfitting_score < 2:
    print("Model shows good generalization (score < 2%)")
else:
    print("Model shows moderate overfitting")

# Display individual metrics with arrows for visual emphasis
print("\n---------- Detailed Test Metrics ----------")
for metric, value in test_metrics.items():
    print(f'➤➤➤ {metric}: {value:.8f} %')

In [62]

# 6. Visualization
plt.figure(figsize=(8, 6))
cm = confusion_matrix(y_test, y_pred_knn)
sns.heatmap(cm, annot=True, fmt='d', cmap='Blues')
plt.title('Confusion Matrix - k-Nearest Neighbors')
plt.ylabel('True Label')
plt.xlabel('Predicted Label')
plt.show()

In [23]

# Scale the features (optional for Decision Trees but keeping for consistency)
scaler = StandardScaler()
X_train_scaled = scaler.fit_transform(X_train)
X_test_scaled = scaler.transform(X_test)

# Calculate class weights for imbalanced data
fraud_ratio = len(y_train[y_train==1]) / len(y_train[y_train==0])
class_weights = {0: 1, 1: 1/fraud_ratio}

# Initialize Decision Tree model with parameters
dt_model = DecisionTreeClassifier(
    class_weight=class_weights,  # Handle class imbalance
    max_depth=10,                # Control tree depth to prevent overfitting
    min_samples_split=50,        # Minimum samples required to split a node
    min_samples_leaf=20,         # Minimum samples required at a leaf node
    criterion='gini',            # Use gini impurity
    random_state=42              # For reproducibility
)

# Train the model
dt_model.fit(X_train_scaled, y_train)

# Evaluate on training data
train_preds = dt_model.predict(X_train_scaled)
train_probs = dt_model.predict_proba(X_train_scaled)[:, 1]

# Evaluate on test data
test_preds = dt_model.predict(X_test_scaled)
test_probs = dt_model.predict_proba(X_test_scaled)[:, 1]

# Calculate metrics for both sets (as percentages)
train_metrics = {
    'Accuracy': accuracy_score(y_train, train_preds) * 100,
    'Precision': precision_score(y_train, train_preds) * 100,
    'Recall': recall_score(y_train, train_preds) * 100,
    'F1 Score': f1_score(y_train, train_preds) * 100,
    'ROC AUC': roc_auc_score(y_train, train_probs) * 100,
    'PR AUC': average_precision_score(y_train, train_probs) * 100
}

test_metrics = {
    'Accuracy': accuracy_score(y_test, test_preds) * 100,
    'Precision': precision_score(y_test, test_preds) * 100,
    'Recall': recall_score(y_test, test_preds) * 100,
    'F1 Score': f1_score(y_test, test_preds) * 100,
    'ROC AUC': roc_auc_score(y_test, test_probs) * 100,
    'PR AUC': average_precision_score(y_test, test_probs) * 100
}

# Format metrics with 8 decimal places
print("\n---------- Decision Tree Model Performance ----------")
print("\n---------- Training vs Test Performance (Overfitting Analysis) ----------")
print(f"{'Metric':<12} {'Training':>20} {'Test':>20} {'Difference':>20}")
print("-" * 74)
for metric in train_metrics.keys():
    diff = train_metrics[metric] - test_metrics[metric]
    print(f"{metric:<12} {train_metrics[metric]:>19.8f}% {test_metrics[metric]:>19.8f}% {diff:>19.8f}%")

# Calculate overall overfitting score (average difference across metrics)
overfitting_score = np.mean([train_metrics[m] - test_metrics[m] for m in train_metrics.keys()])
print(f"\nOverfitting Score: {overfitting_score:.8f}%")
if overfitting_score > 5:
    print("WARNING: Model shows signs of overfitting (score > 5%)")
elif overfitting_score < 2:
    print("Model shows good generalization (score < 2%)")
else:
    print("Model shows moderate overfitting")

# Display individual metrics with arrows for visual emphasis and 8 decimal places
print("\n---------- Detailed Test Metrics ----------")
for metric, value in test_metrics.items():
    print(f'➤➤➤ {metric}: {value:.8f} %')

# Print classification report
print("\nClassification Report:")
print(classification_report(y_test, test_preds))

In [27]

# Create confusion matrix
cm = confusion_matrix(y_test, test_preds)
plt.figure(figsize=(8, 6))
sns.heatmap(cm, annot=True, fmt='d', cmap='Blues')
plt.title('Confusion Matrix - Decision Tree')
plt.ylabel('True Label')
plt.xlabel('Predicted Label')
plt.show()

In [28]

# Improved Outlier Handling
def handle_outliers(df, method='winsorize', z_threshold=4, columns_to_handle=None):
    """
    Enhanced outlier handling with column-specific treatment
    
    Parameters:
    -----------
    df : DataFrame
        The input dataframe
    method : str
        The method to handle outliers ('remove', 'winsorize', 'log', or 'robust_scale')
    z_threshold : float
        Threshold for Z-score outlier detection
    columns_to_handle : list
        Specific columns to apply outlier handling to (if None, applies to all numeric columns)
    """
    df_result = df.copy()
    
    # Select columns to process
    cols_to_process = columns_to_handle if columns_to_handle else df.select_dtypes(include=np.number).columns
    
    if method == 'remove':
        # Only calculate Z-scores for specified columns
        z_scores = np.abs(stats.zscore(df[cols_to_process]))
        outliers_z = (z_scores > z_threshold).any(axis=1)
        df_result = df[~outliers_z]
        print(f"\nRemoved {len(df) - len(df_result)} rows using Z-score method")
    
    elif method == 'winsorize':
        # Winsorization per column (capping extreme values)
        for col in cols_to_process:
            lower = df[col].quantile(0.01)  # Less aggressive lower bound
            upper = df[col].quantile(0.99)  # Less aggressive upper bound
            df_result[col] = np.clip(df[col], lower, upper)
        print(f"\nWinsorized {len(cols_to_process)} columns at 1% and 99% percentiles")
    
    elif method == 'log':
        # Log transformation for positive-valued columns
        for col in cols_to_process:
            if df[col].min() >= 0:  # Only apply to non-negative columns
                df_result[col] = np.log1p(df[col])
        print(f"\nApplied log transformation to applicable columns")
    
    elif method == 'robust_scale':
        # Robust scaling (less sensitive to outliers)
        from sklearn.preprocessing import RobustScaler
        scaler = RobustScaler()
        df_result[cols_to_process] = scaler.fit_transform(df[cols_to_process])
        print(f"\nApplied robust scaling to {len(cols_to_process)} columns")
    
    else:
        raise ValueError("Method must be 'remove', 'winsorize', 'log', or 'robust_scale'")
        
    return df_result

# Feature-specific outlier handling
def handle_outliers_by_feature(df):
    """
    Apply different treatment to different columns based on their characteristics
    """
    df_result = df.copy()
    
    # Amount-related columns (apply winsorization)
    amount_cols = ['oldbalanceOrg', 'newbalanceDest']
    for col in amount_cols:
        lower = df[col].quantile(0.005)
        upper = df[col].quantile(0.995)
        df_result[col] = np.clip(df[col], lower, upper)
    
    # Add outlier flags as features (instead of removing them)
    for col in amount_cols:
        z_scores = np.abs(stats.zscore(df[col]))
        df_result[f'{col}_is_outlier'] = (z_scores > 3).astype(int)
    
    return df_result

In [29]

# X Data
X = df.drop(['isFraud'], axis=1)
print('X shape is : ', X.shape)
print()
y = df['isFraud']
print('y shape is : ', y.shape)

# Implement improved preprocessing pipeline
def preprocess_data(X, y):
    """
    Improved preprocessing pipeline for fraud detection that preserves
    outlier information while limiting extreme values
    """
    # Create a copy to avoid modifying originals
    X_processed = X.copy()
    
    # Add outlier indicator features for important columns
    numeric_cols = X.select_dtypes(include=np.number).columns
    
    # Add outlier indicators for financial columns
    financial_cols = ['oldbalanceOrg', 'newbalanceDest']
    for col in financial_cols:
        if col in X.columns:
            z_scores = np.abs(stats.zscore(X[col]))
            X_processed[f'{col}_outlier'] = (z_scores > 3).astype(int)
    
    # Winsorize only the extreme outliers in amount columns (99.5% percentile)
    for col in financial_cols:
        if col in X.columns:
            lower = X[col].quantile(0.005)
            upper = X[col].quantile(0.995)
            X_processed[col] = np.clip(X[col], lower, upper)
    
    print(f"Preprocessing complete: Added {len(financial_cols)} outlier indicator features")
    print(f"Original shape: {X.shape}, New shape: {X_processed.shape}")
    
    return X_processed, y

# Apply the improved preprocessing
X_processed, y_processed = preprocess_data(X, y)

# Split the processed data
X_train, X_test, y_train, y_test = train_test_split(
    X_processed, y_processed, test_size=0.3, random_state=42, shuffle=True
)

# Splitted Data
print('X_train shape is ', X_train.shape)
print('X_test shape is ', X_test.shape)
print('y_train shape is ', y_train.shape)
print('y_test shape is ', y_test.shape)
# print(len(y_train[y_train==0]) / len(y_train[y_train==1]))

# class_weights = {0: 1, 1: 50}
class_weights = {0: 1, 1: len(y_train[y_train==0]) / len(y_train[y_train==1])}

In [30]

# 4. Model Training
print("\nTraining CatBoost model...")
start_time = time.time()  # Start timer (add this line if you haven't defined start_time)

catboost_model = CatBoostClassifier(
    iterations=1000,           # Number of trees
    learning_rate=0.01,       # Learning rate
    depth=6,                  # Depth of trees
    l2_leaf_reg=3,           # L2 regularization coefficient
    loss_function='Logloss',  # Loss function
    eval_metric='AUC',        # Evaluation metric
    random_seed=42,           # Random seed for reproducibility
    class_weights=class_weights,  # Class weights for imbalanced data
    verbose=100,              # Print progress every 100 iterations
    early_stopping_rounds=50, # Early stopping rounds
    use_best_model=True,     # Use best model found during training
    task_type='CPU'          # Can change to 'GPU' if available
)

# Create eval set for early stopping
eval_set = [(X_test, y_test)]

# Train the model
catboost_model.fit(
    X_train, 
    y_train,
    eval_set=eval_set,
    plot=True  # Plot training progress
)

training_time = time.time() - start_time
print(f"Training completed in {training_time:.2f} seconds")

In [31]

# 5. Model Evaluation
# Evaluate on training data
y_train_pred = catboost_model.predict(X_train)
y_train_pred_proba = catboost_model.predict_proba(X_train)[:, 1]

# Evaluate on test data
y_pred_cat = catboost_model.predict(X_test)
y_pred_proba_cat = catboost_model.predict_proba(X_test)[:, 1]

prediction_time = time.time() - start_time - training_time
print(f"Predictions completed in {prediction_time:.2f} seconds")

# Calculate metrics for both training and test sets
train_metrics = {
    'Accuracy': accuracy_score(y_train, y_train_pred) * 100,
    'Precision': precision_score(y_train, y_train_pred) * 100,
    'Recall': recall_score(y_train, y_train_pred) * 100,
    'F1 Score': f1_score(y_train, y_train_pred) * 100,
    'ROC AUC': roc_auc_score(y_train, y_train_pred_proba) * 100,
    'PR AUC': average_precision_score(y_train, y_train_pred_proba) * 100
}

test_metrics = {
    'Accuracy': accuracy_score(y_test, y_pred_cat) * 100,
    'Precision': precision_score(y_test, y_pred_cat) * 100,
    'Recall': recall_score(y_test, y_pred_cat) * 100,
    'F1 Score': f1_score(y_test, y_pred_cat) * 100,
    'ROC AUC': roc_auc_score(y_test, y_pred_proba_cat) * 100,
    'PR AUC': average_precision_score(y_test, y_pred_proba_cat) * 100
}

In [32]

print("\n---------- CatBoost Model Evaluation ----------")
print("\nClassification Report (Test Set):")
print(classification_report(y_test, y_pred_cat))

print("\n---------- Training vs Test Performance (Overfitting Analysis) ----------")
print(f"{'Metric':<12} {'Training':>12} {'Test':>12} {'Difference':>12}")
print("-" * 50)
for metric in train_metrics.keys():
    diff = train_metrics[metric] - test_metrics[metric]
    print(f"{metric:<12} {train_metrics[metric]:>11.8f}% {test_metrics[metric]:>11.8f}% {diff:>11.8f}%")

# Calculate overall overfitting score (average difference across metrics)
overfitting_score = np.mean([train_metrics[m] - test_metrics[m] for m in train_metrics.keys()])
print(f"\nOverfitting Score: {overfitting_score:.8f}%")
if overfitting_score > 5:
    print("WARNING: Model shows signs of overfitting (score > 5%)")
elif overfitting_score < 2:
    print("Model shows good generalization (score < 2%)")
else:
    print("Model shows moderate overfitting")

In [33]

# 6. Visualization

# Confusion Matrix
plt.figure(figsize=(8, 6))
cm = confusion_matrix(y_test, y_pred_cat)
sns.heatmap(cm, annot=True, fmt='d', cmap='Blues')
plt.title('Confusion Matrix - CatBoost')
plt.ylabel('True Label')
plt.xlabel('Predicted Label')
plt.show()

In [34]

# 4. Performance Tracking
start_time = time.time()

# 5. Feature Scaling - KNN is distance-based so scaling is essential
scaler = StandardScaler()
X_train_scaled = scaler.fit_transform(X_train)
X_test_scaled = scaler.transform(X_test)

# 6. Class Distribution Analysis
fraud_samples = sum(y_train == 1)
non_fraud_samples = sum(y_train == 0)
print(f"Training data: {non_fraud_samples} non-fraud, {fraud_samples} fraud")
print(f"Class distribution ratio (non-fraud:fraud): {non_fraud_samples / fraud_samples:.2f}:1")

# 7. KNN Model Training with Optimizations for Imbalanced Data
print("\nTraining k-Nearest Neighbors model...")
knn_outlier_model = KNeighborsClassifier(
    n_neighbors=5,          # Start with a small number of neighbors as fraud cases are rare
    weights='distance',     # Weight points by inverse of their distance - better for imbalanced data
    algorithm='auto',       # Automatically choose the best algorithm
    leaf_size=30,           # Leaf size passed to BallTree or KDTree
    p=2,                    # Power parameter for the Minkowski metric (2 = Euclidean distance)
    metric='minkowski',     # Distance metric to use
    n_jobs=-1               # Use all available cores
)

knn_outlier_model.fit(X_train_scaled, y_train)
training_time = time.time() - start_time
print(f"Training completed in {training_time:.2f} seconds")

In [72]

# 8. Model Evaluation
# Evaluate on training data
y_train_pred = knn_outlier_model.predict(X_train_scaled)
y_train_pred_proba = knn_outlier_model.predict_proba(X_train_scaled)[:, 1]

# Evaluate on test data
y_pred_knn = knn_outlier_model.predict(X_test_scaled)
y_pred_proba_knn = knn_outlier_model.predict_proba(X_test_scaled)[:, 1]

prediction_time = time.time() - start_time - training_time
print(f"Predictions completed in {prediction_time:.2f} seconds")

print("\n---------- k-Nearest Neighbors (with Outlier Detection) Model Evaluation ----------")
print("\nClassification Report (Test Set):")
print(classification_report(y_test, y_pred_knn))

# Calculate metrics for both training and test sets
train_metrics = {
    'Accuracy': accuracy_score(y_train, y_train_pred) * 100,
    'Precision': precision_score(y_train, y_train_pred) * 100,
    'Recall': recall_score(y_train, y_train_pred) * 100,
    'F1 Score': f1_score(y_train, y_train_pred) * 100,
    'ROC AUC': roc_auc_score(y_train, y_train_pred_proba) * 100
}

test_metrics = {
    'Accuracy': accuracy_score(y_test, y_pred_knn) * 100,
    'Precision': precision_score(y_test, y_pred_knn) * 100,
    'Recall': recall_score(y_test, y_pred_knn) * 100,
    'F1 Score': f1_score(y_test, y_pred_knn) * 100,
    'ROC AUC': roc_auc_score(y_test, y_pred_proba_knn) * 100
}

In [73]

print("\n---------- Training vs Test Performance (Overfitting Analysis) ----------")
print(f"{'Metric':<12} {'Training':>12} {'Test':>12} {'Difference':>12}")
print("-" * 50)
for metric in train_metrics.keys():
    diff = train_metrics[metric] - test_metrics[metric]
    print(f"{metric:<12} {train_metrics[metric]:>11.8f}% {test_metrics[metric]:>11.8f}% {diff:>11.8f}%")

# Calculate overall overfitting score (average difference across metrics)
overfitting_score = np.mean([train_metrics[m] - test_metrics[m] for m in train_metrics.keys()])
print(f"\nOverfitting Score: {overfitting_score:.8f}%")
if overfitting_score > 5:
    print("WARNING: Model shows signs of overfitting (score > 5%)")
elif overfitting_score < 2:
    print("Model shows good generalization (score < 2%)")
else:
    print("Model shows moderate overfitting")

# Display individual metrics with arrows for visual emphasis
print("\n---------- Detailed Test Metrics ----------")
for metric, value in test_metrics.items():
    print(f'➤➤➤ {metric}: {value:.8f} %')

In [74]

# 6. Visualization
# Confusion Matrix
plt.figure(figsize=(8, 6))
cm = confusion_matrix(y_test, y_pred_knn)
sns.heatmap(cm, annot=True, fmt='d', cmap='Blues')
plt.title('Confusion Matrix - KNN (Do outlier handling)')
plt.ylabel('True Label')
plt.xlabel('Predicted Label')
plt.show()

In [35]

# 1. Performance Tracking
start_time = time.time()

# 2. Feature Scaling - very important for Logistic Regression
scaler = StandardScaler()
X_train_scaled = scaler.fit_transform(X_train)
X_test_scaled = scaler.transform(X_test)

# 3. Class Weight Calculation
fraud_samples = sum(y_train == 1)
non_fraud_samples = sum(y_train == 0)
print(f"Training data: {non_fraud_samples} non-fraud, {fraud_samples} fraud")
print(f"Class distribution ratio (non-fraud:fraud): {non_fraud_samples / fraud_samples:.2f}:1")

# Define different class weight options
weight_options = {
    'balanced': 'balanced',
    'inverse': {
        0: 1.0,
        1: non_fraud_samples / fraud_samples
    },
    'sqrt': {
        0: 1.0,
        1: np.sqrt(non_fraud_samples / fraud_samples)
    },
    'custom': {
        0: 1.0,
        1: 2 * (non_fraud_samples / fraud_samples)
    }
}

# Select weight approach
selected_weight = weight_options['inverse']  # Choose from: 'balanced', 'inverse', 'sqrt', 'custom'
print(f"\nSelected weight approach: {selected_weight}")

# 4. Model Training
print("\nTraining Logistic Regression model...")
lr_outlier_model = LogisticRegression(
    penalty='l2',          # L2 regularization to prevent overfitting
    C=1.0,                 # Inverse of regularization strength
    class_weight=selected_weight,  # Handle class imbalance
    solver='liblinear',    # Efficient solver for small datasets
    max_iter=1000,         # Maximum iterations for convergence
    verbose=1,             # Output training progress
    n_jobs=-1,             # Use all available cores
    random_state=42        # For reproducibility
)

lr_outlier_model.fit(X_train_scaled, y_train)
training_time = time.time() - start_time
print(f"Training completed in {training_time:.2f} seconds")

In [36]

# 5. Model Evaluation
# Evaluate on training data
y_train_pred = lr_outlier_model.predict(X_train_scaled)
y_train_pred_proba = lr_outlier_model.predict_proba(X_train_scaled)[:, 1]

# Evaluate on test data
y_pred_lr = lr_outlier_model.predict(X_test_scaled)
y_pred_proba_lr = lr_outlier_model.predict_proba(X_test_scaled)[:, 1]

prediction_time = time.time() - start_time - training_time
print(f"Predictions completed in {prediction_time:.2f} seconds")

print("\n---------- Logistic Regression (with Outlier Handling) Model Evaluation ----------")
print("\nClassification Report (Test Set):")
print(classification_report(y_test, y_pred_lr))

# Calculate metrics for both training and test sets
train_metrics = {
    'Accuracy': accuracy_score(y_train, y_train_pred) * 100,
    'Precision': precision_score(y_train, y_train_pred) * 100,
    'Recall': recall_score(y_train, y_train_pred) * 100,
    'F1 Score': f1_score(y_train, y_train_pred) * 100,
    'ROC AUC': roc_auc_score(y_train, y_train_pred_proba) * 100,
    'PR AUC': average_precision_score(y_train, y_train_pred_proba) * 100
}

test_metrics = {
    'Accuracy': accuracy_score(y_test, y_pred_lr) * 100,
    'Precision': precision_score(y_test, y_pred_lr) * 100,
    'Recall': recall_score(y_test, y_pred_lr) * 100,
    'F1 Score': f1_score(y_test, y_pred_lr) * 100,
    'ROC AUC': roc_auc_score(y_test, y_pred_proba_lr) * 100,
    'PR AUC': average_precision_score(y_test, y_pred_proba_lr) * 100
}

print("\n---------- Training vs Test Performance (Overfitting Analysis) ----------")
print(f"{'Metric':<12} {'Training':>12} {'Test':>12} {'Difference':>12}")
print("-" * 50)
for metric in train_metrics.keys():
    diff = train_metrics[metric] - test_metrics[metric]
    print(f"{metric:<12} {train_metrics[metric]:>11.8f}% {test_metrics[metric]:>11.8f}% {diff:>11.8f}%")

# Calculate overall overfitting score (average difference across metrics)
overfitting_score = np.mean([train_metrics[m] - test_metrics[m] for m in train_metrics.keys()])
print(f"\nOverfitting Score: {overfitting_score:.8f}%")
if overfitting_score > 5:
    print("WARNING: Model shows signs of overfitting (score > 5%)")
elif overfitting_score < 2:
    print("Model shows good generalization (score < 2%)")
else:
    print("Model shows moderate overfitting")

# Display individual metrics with arrows for visual emphasis
print("\n---------- Detailed Test Metrics ----------")
for metric, value in test_metrics.items():
    print(f'➤➤➤ {metric}: {value:.8f} %')

In [37]

# 6. Visualization
# Confusion Matrix
plt.figure(figsize=(8, 6))
cm = confusion_matrix(y_test, y_pred_lr)
sns.heatmap(cm, annot=True, fmt='d', cmap='Blues')
plt.title('Confusion Matrix - Logistic Regression (Do outlier handling)')
plt.ylabel('True Label')
plt.xlabel('Predicted Label')
plt.show()

In [38]

# X Data
X = df.drop(['isFraud'], axis=1)
print('X shape is : ', X.shape)
print()
y = df['isFraud']
print('y shape is : ', y.shape)

# Implement improved preprocessing pipeline
def preprocess_data(X, y):
    """
    Improved preprocessing pipeline for fraud detection that preserves
    outlier information while limiting extreme values
    """
    # Create a copy to avoid modifying originals
    X_processed = X.copy()
    
    # Add outlier indicator features for important columns
    numeric_cols = X.select_dtypes(include=np.number).columns
    
    # Add outlier indicators for financial columns
    financial_cols = ['oldbalanceOrg', 'newbalanceDest']
    for col in financial_cols:
        if col in X.columns:
            z_scores = np.abs(stats.zscore(X[col]))
            X_processed[f'{col}_outlier'] = (z_scores > 3).astype(int)
    
    # Winsorize only the extreme outliers in amount columns (99.5% percentile)
    for col in financial_cols:
        if col in X.columns:
            lower = X[col].quantile(0.005)
            upper = X[col].quantile(0.995)
            X_processed[col] = np.clip(X[col], lower, upper)
    
    print(f"Preprocessing complete: Added {len(financial_cols)} outlier indicator features")
    print(f"Original shape: {X.shape}, New shape: {X_processed.shape}")
    
    return X_processed, y

# Apply the improved preprocessing
X_processed, y_processed = preprocess_data(X, y)

# Split the processed data
X_train, X_test, y_train, y_test = train_test_split(
    X_processed, y_processed, test_size=0.3, random_state=42, shuffle=True, stratify=y_processed
)

# Print initial class distribution
print("\nOriginal class distribution in training set:")
print(pd.Series(y_train).value_counts(normalize=True) * 100)

# Apply Random Under-Sampling to balance classes
# You can adjust the sampling_strategy for different levels of undersampling
rus = RandomUnderSampler(sampling_strategy=0.5, random_state=42)  # 1:2 ratio of minority:majority
X_train_rus, y_train_rus = rus.fit_resample(X_train, y_train)

# Print balanced class distribution
print("\nBalanced class distribution after RUS:")
print(pd.Series(y_train_rus).value_counts(normalize=True) * 100)
print(f"Original training samples: {X_train.shape[0]}")
print(f"After RUS: {X_train_rus.shape[0]}")

In [39]

# 4. Model Training with XGBoost
print("\nTraining XGBoost model on RUS-balanced data...")
start_time = time.time()  # Start timer

xgb_model = xgb.XGBClassifier(
    learning_rate=0.01,         # Learning rate
    n_estimators=200,           # Number of trees
    max_depth=6,                # Maximum tree depth
    min_child_weight=1,         # Minimum child weight
    gamma=0,                    # Minimum loss reduction for split
    subsample=0.8,              # Subsample ratio of training instances
    colsample_bytree=0.8,       # Subsample ratio of columns
    objective='binary:logistic', # Objective function
    scale_pos_weight=1,         # No need for additional weighting since data is balanced
    random_state=42             # Random seed
)

# Train the model on the balanced dataset
xgb_model.fit(
    X_train_rus, 
    y_train_rus,
    eval_set=[(X_test, y_test)],
    verbose=100
)

training_time = time.time() - start_time
print(f"Training completed in {training_time:.2f} seconds")

In [40]

# 5. Model Evaluation
# Evaluate on training data (original unbalanced set)
y_train_pred = xgb_model.predict(X_train)
y_train_pred_proba = xgb_model.predict_proba(X_train)[:, 1]

# Evaluate on test data
y_pred_xgb = xgb_model.predict(X_test)
y_pred_proba_xgb = xgb_model.predict_proba(X_test)[:, 1]

prediction_time = time.time() - start_time - training_time
print(f"Predictions completed in {prediction_time:.2f} seconds")

# Calculate metrics for both training and test sets
train_metrics = {
    'Accuracy': accuracy_score(y_train, y_train_pred) * 100,
    'Precision': precision_score(y_train, y_train_pred) * 100,
    'Recall': recall_score(y_train, y_train_pred) * 100,
    'F1 Score': f1_score(y_train, y_train_pred) * 100,
    'ROC AUC': roc_auc_score(y_train, y_train_pred_proba) * 100,
    'PR AUC': average_precision_score(y_train, y_train_pred_proba) * 100
}

test_metrics = {
    'Accuracy': accuracy_score(y_test, y_pred_xgb) * 100,
    'Precision': precision_score(y_test, y_pred_xgb) * 100,
    'Recall': recall_score(y_test, y_pred_xgb) * 100,
    'F1 Score': f1_score(y_test, y_pred_xgb) * 100,
    'ROC AUC': roc_auc_score(y_test, y_pred_proba_xgb) * 100,
    'PR AUC': average_precision_score(y_test, y_pred_proba_xgb) * 100
}

print("\n---------- RUS+XGBoost Model Evaluation ----------")
print("\nClassification Report (Test Set):")
print(classification_report(y_test, y_pred_xgb))

print("\n---------- Training vs Test Performance (Overfitting Analysis) ----------")
print(f"{'Metric':<12} {'Training':>12} {'Test':>12} {'Difference':>12}")
print("-" * 50)
for metric in train_metrics.keys():
    diff = train_metrics[metric] - test_metrics[metric]
    print(f"{metric:<12} {train_metrics[metric]:>11.8f}% {test_metrics[metric]:>11.8f}% {diff:>11.8f}%")

# Calculate overall overfitting score (average difference across metrics)
overfitting_score = np.mean([train_metrics[m] - test_metrics[m] for m in train_metrics.keys()])
print(f"\nOverfitting Score: {overfitting_score:.8f}%")
if overfitting_score > 5:
    print("WARNING: Model shows signs of overfitting (score > 5%)")
elif overfitting_score < 2:
    print("Model shows good generalization (score < 2%)")
else:
    print("Model shows moderate overfitting")

In [41]

# 6. Visualization
# Confusion Matrix
plt.figure(figsize=(8, 6))
cm = confusion_matrix(y_test, y_pred_xgb)
sns.heatmap(cm, annot=True, fmt='d', cmap='Blues')
plt.title('Confusion Matrix - RUS+XGBoost')
plt.ylabel('True Label')
plt.xlabel('Predicted Label')
plt.show()

In [82]

# warnings.filterwarnings('ignore')

# def evaluate_model(model, X_test_scaled, y_test, model_name, threshold=0.5):
#     """Evaluate a model using various metrics"""
#     start_time = time.time()
    
#     # Get predictions (handle different model types)
#     try:
#         y_pred_proba = model.predict_proba(X_test_scaled)[:, 1]
#     except:
#         # For models that might not have predict_proba
#         y_pred_proba = model.predict(X_test_scaled)
    
#     y_pred = (y_pred_proba >= threshold).astype(int)
    
#     # Calculate metrics
#     accuracy = accuracy_score(y_test, y_pred)
#     precision = precision_score(y_test, y_pred)
#     recall = recall_score(y_test, y_pred)
#     f1 = f1_score(y_test, y_pred)
#     roc_auc = roc_auc_score(y_test, y_pred_proba)
    
#     # Calculate precision-recall AUC
#     precision_curve, recall_curve, _ = precision_recall_curve(y_test, y_pred_proba)
#     pr_auc = auc(recall_curve, precision_curve)
    
#     # Confusion matrix
#     cm = confusion_matrix(y_test, y_pred)
    
#     # Speed test
#     inference_time = time.time() - start_time
    
#     # Return all metrics in a dictionary
#     return {
#         'model_name': model_name,
#         'accuracy': accuracy,
#         'precision': precision,
#         'recall': recall,
#         'f1_score': f1,
#         'roc_auc': roc_auc,
#         'pr_auc': pr_auc,
#         'confusion_matrix': cm,
#         'inference_time': inference_time,
#         'y_pred': y_pred,
#         'y_pred_proba': y_pred_proba
#     }

# # Use the models you've already trained (these variables should be defined in your environment)
# # If any of these variables aren't defined, remove them from the dictionary
# models_in_memory = {
#     'XGBoost': {'model': xgb_model, 'threshold': 0.5},
#     'LightGBM': {'model': lgb_model, 'threshold': 0.5},
#     'Random Forest': {'model': rf_model, 'threshold': 0.5},
#     'Logistic Regression (weighted)': {'model': lr_model, 'threshold': 0.5},
#     'K-Nearest Neighbors': {'model': knn_model, 'threshold': 0.5},
# }

# # Remove any models that aren't actually defined in your environment
# models = {}
# for name, model_info in models_in_memory.items():
#     if 'model' in model_info and model_info['model'] is not None:
#         models[name] = model_info
#         print(f"Using {name} model from memory")

# # Evaluate all models
# results = []
# for name, model_info in models.items():
#     model = model_info['model']
#     threshold = model_info['threshold']
#     print(f"Evaluating {name} model...")
    
#     eval_results = evaluate_model(model, X_test_scaled, y_test, name, threshold)
#     results.append(eval_results)
    
#     print(f"  - Accuracy: {eval_results['accuracy']:.8f}")
#     print(f"  - Precision: {eval_results['precision']:.8f}")
#     print(f"  - Recall: {eval_results['recall']:.8f}")
#     print(f"  - F1 Score: {eval_results['f1_score']:.8f}")
#     print(f"  - ROC AUC: {eval_results['roc_auc']:.8f}")
#     print(f"  - PR AUC: {eval_results['pr_auc']:.8f}")
#     print(f"  - Inference time: {eval_results['inference_time']:.8f} seconds")
#     print()

# # Create DataFrame with all results
# results_df = pd.DataFrame([{
#     'Model': r['model_name'],
#     'Accuracy (%)': r['accuracy'] * 100,
#     'Precision (%)': r['precision'] * 100,
#     'Recall (%)': r['recall'] * 100,
#     'F1 Score (%)': r['f1_score'] * 100,
#     'ROC AUC (%)': r['roc_auc'] * 100,
#     'PR AUC (%)': r['pr_auc'] * 100,
#     'Inference Time (s)': r['inference_time']
# } for r in results])

# # Display table of all metrics
# print("\n==== Model Comparison ====")
# print(results_df.to_string(index=False, float_format=lambda x: f"{x:.8f}"))

In [83]

# # Plot metrics comparison
# metrics_to_plot = ['Accuracy (%)', 'Precision (%)', 'Recall (%)', 
#                     'F1 Score (%)', 'ROC AUC (%)', 'PR AUC (%)']

# plt.figure(figsize=(14, 10))
# for i, metric in enumerate(metrics_to_plot):
#     plt.subplot(2, 3, i+1)
#     sns.barplot(x='Model', y=metric, data=results_df)
#     plt.title(metric)
#     plt.xticks(rotation=45)
#     plt.ylim(0, 100)
# plt.tight_layout()
# plt.suptitle('Model Performance Comparison', fontsize=16)
# plt.subplots_adjust(top=0.9)
# plt.show()

In [84]

# # Plot ROC curves for all models
# plt.figure(figsize=(10, 8))
# for r in results:
#     # Calculate ROC curve
#     fpr, tpr, _ = roc_curve(y_test, r['y_pred_proba'])
#     label = f"{r['model_name']} (AUC = {r['roc_auc']:.3f})"
#     plt.plot(fpr, tpr, label=label)

# plt.plot([0, 1], [0, 1], 'k--', label='Random')
# plt.xlabel('False Positive Rate')
# plt.ylabel('True Positive Rate')
# plt.title('ROC Curves for All Models')
# plt.legend()
# plt.grid(True, alpha=0.3)
# plt.show()

In [85]

# # Plot Precision-Recall curves for all models
# plt.figure(figsize=(10, 8))
# for r in results:
#     # Calculate Precision-Recall curve
#     precision, recall, _ = precision_recall_curve(y_test, r['y_pred_proba'])
#     label = f"{r['model_name']} (AUC = {r['pr_auc']:.3f})"
#     plt.plot(recall, precision, label=label)

# # Add the random baseline
# no_skill = sum(y_test == 1) / len(y_test)
# plt.plot([0, 1], [no_skill, no_skill], 'k--', label='Random')
# plt.xlabel('Recall')
# plt.ylabel('Precision')
# plt.title('Precision-Recall Curves for All Models')
# plt.legend()
# plt.grid(True, alpha=0.3)
# plt.show()

In [86]

# # Plot confusion matrices for each model
# num_models = len(results)
# fig, axes = plt.subplots(nrows=(num_models+1)//2, ncols=2, figsize=(15, 4*((num_models+1)//2)))
# axes = axes.flatten() if num_models > 1 else [axes]

# for i, r in enumerate(results):
#     if i < len(axes):
#         cm = r['confusion_matrix']
#         sns.heatmap(cm, annot=True, fmt='d', cmap='Blues', ax=axes[i])
#         axes[i].set_title(f'Confusion Matrix - {r["model_name"]}')
#         axes[i].set_ylabel('True Label')
#         axes[i].set_xlabel('Predicted Label')

# # Hide any unused subplots
# for j in range(i+1, len(axes)):
#     fig.delaxes(axes[j])

# plt.tight_layout()
# plt.show()

In [87]

# # Calculate inference time comparison
# plt.figure(figsize=(10, 6))
# sns.barplot(x='Model', y='Inference Time (s)', data=results_df)
# plt.title('Model Inference Time Comparison')
# plt.xticks(rotation=45)
# plt.ylabel('Time (seconds)')
# plt.grid(True, alpha=0.3)
# plt.show()

In [88]

# # Find the best model for each metric
# best_models = {}
# for metric in metrics_to_plot:
#     if 'Time' not in metric:
#         best_idx = results_df[metric].idxmax()
#         best_models[metric] = results_df.loc[best_idx, 'Model']
#     else:
#         best_idx = results_df[metric].idxmin()
#         best_models[metric] = results_df.loc[best_idx, 'Model']

# print("\n==== Best Models ====")
# for metric, model in best_models.items():
#     print(f"Best for {metric}: {model}")

# # Print the overall recommendation
# print("\n==== Recommendation ====")
# # Count how many times each model is the best
# model_counts = {}
# for model in best_models.values():
#     if model in model_counts:
#         model_counts[model] += 1
#     else:
#         model_counts[model] = 1

# # Find the model with the highest count
# best_overall = max(model_counts.items(), key=lambda x: x[1])[0]

# print(f"Overall recommended model: {best_overall}")
# print("This model achieves the best balance of performance metrics.")

# # If fraud detection is the priority, emphasize recall and F1
# print("\nFor fraud detection specifically:")
# print(f"Best model for recall (detecting frauds): {best_models['Recall (%)']}")
# print(f"Best model for F1 score (balance): {best_models['F1 Score (%)']}")
# print(f"Best model for PR AUC (overall ranking): {best_models['PR AUC (%)']}")

# # If interpretability is needed
# interpretable_models = ['Logistic Regression', 'Logistic Regression (weighted)']
# interpretable_present = [model for model in interpretable_models if model in models]
# if interpretable_present:
#     print("\nIf interpretability is required, consider:")
#     for model in interpretable_present:
#         print(f"- {model}")