training/eda.py

"""Exploratory data analysis over the PaySim dataset (EDA-01..EDA-04).

Reads `data/*.csv` (the Kaggle PaySim export fetched by fetch_data.py),
computes the class-imbalance, distribution, balance-inconsistency, and
fraud-rate findings required before feature engineering, and writes:

- reports/EDA_REPORT.md   -- narrative findings + embedded chart references
- reports/eda_charts/*.png -- the charts referenced by the report

Run from the project root:
    python -m training.eda
"""

from __future__ import annotations

import glob
from pathlib import Path

import matplotlib

matplotlib.use("Agg")

import matplotlib.pyplot as plt
import pandas as pd
import seaborn as sns

REPORTS_DIR = Path("reports")
CHARTS_DIR = REPORTS_DIR / "eda_charts"
REPORT_PATH = REPORTS_DIR / "EDA_REPORT.md"

sns.set_theme(style="whitegrid")


def _find_dataset() -> Path:
    candidates = sorted(glob.glob("data/*.csv"))
    if not candidates:
        raise FileNotFoundError(
            "No CSV found under data/. Run training/fetch_data.py first."
        )
    return Path(candidates[0])


def _save(fig: plt.Figure, name: str) -> str:
    CHARTS_DIR.mkdir(parents=True, exist_ok=True)
    path = CHARTS_DIR / name
    fig.savefig(path, bbox_inches="tight", dpi=110)
    plt.close(fig)
    return f"eda_charts/{name}"


def load_data() -> pd.DataFrame:
    path = _find_dataset()
    df = pd.read_csv(path)
    return df


def class_imbalance(df: pd.DataFrame) -> dict:
    counts = df["isFraud"].value_counts().sort_index()
    total = len(df)
    fraud_count = int(counts.get(1, 0))
    legit_count = int(counts.get(0, 0))
    fraud_rate = fraud_count / total

    fig, ax = plt.subplots(figsize=(5, 4))
    sns.barplot(
        x=["Legitimate", "Fraud"],
        y=[legit_count, fraud_count],
        hue=["Legitimate", "Fraud"],
        palette=["#4C72B0", "#C44E52"],
        legend=False,
        ax=ax,
    )
    ax.set_yscale("log")
    ax.set_ylabel("Transaction count (log scale)")
    ax.set_title("Class imbalance: isFraud")
    chart = _save(fig, "class_imbalance.png")

    return {
        "total": total,
        "legit_count": legit_count,
        "fraud_count": fraud_count,
        "fraud_rate": fraud_rate,
        "chart": chart,
    }


def type_and_amount_distributions(df: pd.DataFrame) -> dict:
    type_counts = df["type"].value_counts()

    fig, ax = plt.subplots(figsize=(6, 4))
    sns.barplot(
        x=type_counts.index,
        y=type_counts.values,
        hue=type_counts.index,
        palette="viridis",
        legend=False,
        ax=ax,
    )
    ax.set_ylabel("Transaction count")
    ax.set_title("Transaction type distribution")
    type_chart = _save(fig, "type_distribution.png")

    fig, ax = plt.subplots(figsize=(6, 4))
    sample = df.sample(min(len(df), 200_000), random_state=42)
    sns.boxplot(
        data=sample,
        x="type",
        y="amount",
        hue="type",
        palette="viridis",
        legend=False,
        ax=ax,
    )
    ax.set_yscale("log")
    ax.set_ylabel("Amount (log scale)")
    ax.set_title("Amount distribution by transaction type (sampled)")
    amount_chart = _save(fig, "amount_by_type.png")

    return {
        "type_counts": type_counts.to_dict(),
        "type_chart": type_chart,
        "amount_chart": amount_chart,
    }


def balance_inconsistencies(df: pd.DataFrame) -> dict:
    orig_mismatch = (
        df["oldbalanceOrg"] - df["amount"] != df["newbalanceOrig"]
    )
    dest_mismatch = (
        df["oldbalanceDest"] + df["amount"] != df["newbalanceDest"]
    )
    orig_zero_after = (df["oldbalanceOrg"] > 0) & (df["newbalanceOrig"] == 0)
    dest_zero_stays_zero = (
        (df["oldbalanceDest"] == 0)
        & (df["newbalanceDest"] == 0)
        & (df["amount"] > 0)
    )

    total = len(df)
    return {
        "orig_mismatch_rate": float(orig_mismatch.mean()),
        "dest_mismatch_rate": float(dest_mismatch.mean()),
        "orig_zero_after_count": int(orig_zero_after.sum()),
        "orig_zero_after_fraud_rate": float(
            df.loc[orig_zero_after, "isFraud"].mean()
        )
        if orig_zero_after.any()
        else 0.0,
        "dest_zero_stays_zero_count": int(dest_zero_stays_zero.sum()),
        "dest_zero_stays_zero_fraud_rate": float(
            df.loc[dest_zero_stays_zero, "isFraud"].mean()
        )
        if dest_zero_stays_zero.any()
        else 0.0,
        "total": total,
    }


def fraud_rate_by_type(df: pd.DataFrame) -> dict:
    rate_by_type = df.groupby("type")["isFraud"].mean().sort_values(ascending=False)
    fraud_types = rate_by_type[rate_by_type > 0].index.tolist()

    fig, ax = plt.subplots(figsize=(6, 4))
    sns.barplot(
        x=rate_by_type.index,
        y=rate_by_type.values,
        hue=rate_by_type.index,
        palette="rocket",
        legend=False,
        ax=ax,
    )
    ax.set_ylabel("Fraud rate")
    ax.set_title("Fraud rate by transaction type")
    chart = _save(fig, "fraud_rate_by_type.png")

    return {
        "rate_by_type": rate_by_type.to_dict(),
        "fraud_confined_to": fraud_types,
        "chart": chart,
    }


def fraud_rate_by_amount(df: pd.DataFrame) -> dict:
    fraud_df = df[df["isFraud"] == 1]
    legit_df = df[df["isFraud"] == 0]

    fig, ax = plt.subplots(figsize=(6, 4))
    sns.histplot(
        legit_df["amount"].clip(upper=legit_df["amount"].quantile(0.99)),
        color="#4C72B0",
        label="Legitimate",
        stat="density",
        kde=True,
        ax=ax,
        alpha=0.5,
    )
    sns.histplot(
        fraud_df["amount"].clip(upper=fraud_df["amount"].quantile(0.99)),
        color="#C44E52",
        label="Fraud",
        stat="density",
        kde=True,
        ax=ax,
        alpha=0.5,
    )
    ax.set_title("Amount distribution: fraud vs legitimate (clipped at p99)")
    ax.legend()
    chart = _save(fig, "amount_fraud_vs_legit.png")

    return {
        "fraud_amount_median": float(fraud_df["amount"].median()),
        "legit_amount_median": float(legit_df["amount"].median()),
        "fraud_amount_mean": float(fraud_df["amount"].mean()),
        "legit_amount_mean": float(legit_df["amount"].mean()),
        "chart": chart,
    }


def fraud_rate_by_time(df: pd.DataFrame) -> dict:
    df = df.copy()
    df["hour_of_day"] = df["step"] % 24

    rate_by_hour = df.groupby("hour_of_day")["isFraud"].mean()

    fig, ax = plt.subplots(figsize=(8, 4))
    sns.lineplot(x=rate_by_hour.index, y=rate_by_hour.values, marker="o", ax=ax)
    ax.set_xlabel("Hour of day (step mod 24)")
    ax.set_ylabel("Fraud rate")
    ax.set_title("Fraud rate by simulated hour of day")
    chart = _save(fig, "fraud_rate_by_hour.png")

    return {
        "rate_by_hour": rate_by_hour.to_dict(),
        "chart": chart,
    }


def flagged_fraud_crosstab(df: pd.DataFrame) -> dict:
    crosstab = pd.crosstab(df["isFraud"], df["isFlaggedFraud"])
    total_fraud = int((df["isFraud"] == 1).sum())
    flagged_and_fraud = int(((df["isFraud"] == 1) & (df["isFlaggedFraud"] == 1)).sum())
    flagged_not_fraud = int(((df["isFraud"] == 0) & (df["isFlaggedFraud"] == 1)).sum())
    recall_of_flag = flagged_and_fraud / total_fraud if total_fraud else 0.0

    return {
        "crosstab": crosstab.to_dict(),
        "total_fraud": total_fraud,
        "flagged_and_fraud": flagged_and_fraud,
        "flagged_not_fraud": flagged_not_fraud,
        "recall_of_isFlaggedFraud_against_isFraud": recall_of_flag,
    }


def render_report(results: dict) -> str:
    ci = results["class_imbalance"]
    ty = results["type_distribution"]
    bi = results["balance_inconsistencies"]
    frt = results["fraud_rate_by_type"]
    fra = results["fraud_rate_by_amount"]
    frh = results["fraud_rate_by_time"]
    flag = results["flagged_fraud_crosstab"]

    type_counts_lines = "\n".join(
        f"- `{t}`: {c:,}" for t, c in ty["type_counts"].items()
    )
    rate_by_type_lines = "\n".join(
        f"- `{t}`: {r:.6f}" for t, r in frt["rate_by_type"].items()
    )

    lines = [
        "# PaySim EDA Report",
        "",
        f"Dataset: `{_find_dataset()}` -- {ci['total']:,} transactions.",
        "",
        "## 1. Class Imbalance (EDA-01)",
        "",
        f"- Legitimate: {ci['legit_count']:,}",
        f"- Fraud: {ci['fraud_count']:,}",
        f"- Fraud rate: {ci['fraud_rate']:.6f} ({ci['fraud_rate'] * 100:.4f}%)",
        "",
        f"![Class imbalance]({ci['chart']})",
        "",
        "Fraud is an extreme minority class (~0.1%). Accuracy is not a usable "
        "metric here -- a model that never predicts fraud would still score "
        "~99.9% accuracy. Precision/recall/F1/PR-AUC/ROC-AUC are required "
        "(see project constraints).",
        "",
        "## 2. Transaction Type & Amount Distributions (EDA-01)",
        "",
        "Transaction counts by type:",
        "",
        type_counts_lines,
        "",
        f"![Type distribution]({ty['type_chart']})",
        "",
        f"![Amount by type]({ty['amount_chart']})",
        "",
        "## 3. Balance Inconsistencies (EDA-01)",
        "",
        f"- Origin balance mismatch rate (`oldbalanceOrg - amount != "
        f"newbalanceOrig`): {bi['orig_mismatch_rate']:.4f}",
        f"- Destination balance mismatch rate (`oldbalanceDest + amount != "
        f"newbalanceDest`): {bi['dest_mismatch_rate']:.4f}",
        f"- Transactions where the origin balance goes to exactly zero: "
        f"{bi['orig_zero_after_count']:,} (fraud rate within this group: "
        f"{bi['orig_zero_after_fraud_rate']:.4f})",
        f"- Transactions where destination balance is zero before and after "
        f"a nonzero-amount transfer: {bi['dest_zero_stays_zero_count']:,} "
        f"(fraud rate within this group: "
        f"{bi['dest_zero_stays_zero_fraud_rate']:.4f})",
        "",
        "PaySim's balance fields are frequently inconsistent by construction "
        "(destination balances of merchant accounts, `M...`, are always "
        "reported as zero). These inconsistency signals -- not the raw "
        "balances -- are what a leakage-safe feature module should encode "
        "(ratios/flags), per the project's Phase 2 requirements.",
        "",
        "## 4. Fraud Rate by Type / Amount / Time (EDA-02)",
        "",
        "### By transaction type",
        "",
        rate_by_type_lines,
        "",
        f"![Fraud rate by type]({frt['chart']})",
        "",
        f"**Fraud is confined to: {', '.join(frt['fraud_confined_to'])}.** "
        "All other transaction types (`PAYMENT`, `CASH_IN`, `DEBIT`) show a "
        "fraud rate of exactly 0 in this dataset. A production scoring "
        "service must still route all transaction types through the same "
        "pipeline (per API-02) rather than hard-coding a bypass -- but this "
        "finding is directly relevant to feature encoding and model "
        "expectations.",
        "",
        "### By amount",
        "",
        f"- Fraud amount: median {fra['fraud_amount_median']:,.2f}, mean "
        f"{fra['fraud_amount_mean']:,.2f}",
        f"- Legitimate amount: median {fra['legit_amount_median']:,.2f}, mean "
        f"{fra['legit_amount_mean']:,.2f}",
        "",
        f"![Amount: fraud vs legit]({fra['chart']})",
        "",
        "### By time (simulated hour of day)",
        "",
        f"![Fraud rate by hour]({frh['chart']})",
        "",
        "## 5. isFraud vs isFlaggedFraud (EDA-02 / PaySim-specific pitfall)",
        "",
        f"- Total actual fraud transactions: {flag['total_fraud']:,}",
        f"- Of those, flagged by `isFlaggedFraud`: {flag['flagged_and_fraud']:,}",
        f"- Recall of `isFlaggedFraud` against `isFraud`: "
        f"{flag['recall_of_isFlaggedFraud_against_isFraud']:.4%}",
        f"- Legitimate transactions incorrectly flagged: "
        f"{flag['flagged_not_fraud']:,}",
        "",
        "**`isFlaggedFraud` is not a usable fraud signal.** It catches only "
        "a vanishing fraction of true fraud and is a simulation artifact "
        "(PaySim's own naive over-threshold-transfer rule), not a real "
        "fraud detector. Per the project's constraints, `isFlaggedFraud` "
        "must be excluded entirely from the feature/label surface -- it is "
        "already excluded from the `Transaction` ORM model "
        "(`app/db/models.py`).",
        "",
        "## Summary for feature engineering (Phase 2)",
        "",
        "- Treat class imbalance with SMOTE/class-weights/undersampling "
        "comparison -- never rely on accuracy.",
        "- Fraud only occurs in TRANSFER and CASH_OUT; type must remain an "
        "encoded feature, not a filter.",
        "- Balance fields should be engineered into ratios/consistency "
        "flags (e.g. zero-balance-after-transfer, orig/dest mismatch), not "
        "used as raw magnitudes.",
        "- `isFlaggedFraud` is excluded from all feature/label surfaces.",
        "",
    ]
    return "\n".join(lines)


def main() -> None:
    df = load_data()

    results = {
        "class_imbalance": class_imbalance(df),
        "type_distribution": type_and_amount_distributions(df),
        "balance_inconsistencies": balance_inconsistencies(df),
        "fraud_rate_by_type": fraud_rate_by_type(df),
        "fraud_rate_by_amount": fraud_rate_by_amount(df),
        "fraud_rate_by_time": fraud_rate_by_time(df),
        "flagged_fraud_crosstab": flagged_fraud_crosstab(df),
    }

    REPORTS_DIR.mkdir(parents=True, exist_ok=True)
    report = render_report(results)
    REPORT_PATH.write_text(report)
    print(f"EDA report written to {REPORT_PATH}")
    print(f"Charts written to {CHARTS_DIR}")


if __name__ == "__main__":
    main()