backend/tests/eval_harness.py

"""Evaluation harness: run prediction engine against ground truth and produce reports."""
from __future__ import annotations

import json
import sys
import time
from collections import Counter
from pathlib import Path

import numpy as np
import pandas as pd

sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
from app.services.data_service import data_service


FIELDS = [
    ("greige_epi", "Greige EPI"),
    ("greige_ppi", "Greige PPI"),
    ("finish_epi", "FINISH EPI"),
    ("finish_ppi", "FINISH PPI"),
    ("reed_count", "Reed Count"),
    ("ends_per_dent", "Ends per dent"),
    ("reed_space", "Reed space"),
    ("finish_width", "FINISH WIDTH"),
    ("target_gsm", "FINISH GSM"),
]


def percent_error(pred_value: float | None, actual_value: float | None) -> float | None:
    if pred_value is None or actual_value is None or actual_value == 0:
        return None
    return abs(pred_value - actual_value) / abs(actual_value) * 100


def run_eval(sample: pd.DataFrame, label: str) -> dict:
    """Run predictions on a sample and return metrics."""
    results = []
    errors = {fld: [] for fld, _ in FIELDS}
    conf_dist = Counter()
    case_dist = Counter()
    fields_ok = {fld: Counter() for fld, _ in FIELDS if fld != "target_gsm"}
    fields_ok["target_gsm"] = Counter()

    t0 = time.time()
    total = len(sample)

    for i, (_, row) in enumerate(sample.iterrows()):
        try:
            pred = data_service.predict_construction({
                "weave": str(row["weave"]),
                "blend": str(row["blend"]),
                "warp_count": float(row["warp_count"]),
                "weft_count": float(row["weft_count"]),
                "finish_epi": float(row["FINISH EPI"]),
                "finish_ppi": float(row["FINISH PPI"]),
                "target_gsm": float(row["FINISH GSM"]) if pd.notna(row.get("FINISH GSM")) else None,
            })
        except Exception as e:
            continue

        rec = pred.get("recommendation", {})
        sp = pred.get("search_path", {})
        dq = pred.get("data_quality", {})

        conf_dist[dq.get("confidence", "very_low")] += 1
        case_dist[sp.get("case_number", "?")] += 1

        row_res = {"article": str(row.get("master_article", "")), "case": sp.get("case_number", "?"), "range": sp.get("range", "?"), "matches": sp.get("matches_found", 0)}

        for fld, col in FIELDS:
            p = rec.get(fld)
            a = None
            if col in row.index:
                a = float(row[col]) if pd.notna(row.get(col)) else None

            err = percent_error(p, a)
            row_res[fld + "_pred"] = p
            row_res[fld + "_actual"] = a
            row_res[fld + "_err"] = err
            if err is not None:
                errors[fld].append(err)

            # Count within-threshold
            if err is not None:
                if err <= 5:
                    fields_ok[fld]["within5"] += 1
                elif err <= 10:
                    fields_ok[fld]["within10"] += 1
                elif err <= 20:
                    fields_ok[fld]["within20"] += 1
                else:
                    fields_ok[fld]["beyond20"] += 1
            else:
                fields_ok[fld]["no_actual"] += 1

        results.append(row_res)

    elapsed = time.time() - t0

    report = {
        "label": label,
        "total_articles": total,
        "predicted_articles": len(results),
        "elapsed_seconds": round(elapsed, 1),
        "avg_ms_per_article": round(elapsed / max(len(results), 1) * 1000, 1),
    }

    field_report = {}
    for fld, _ in FIELDS:
        arr = errors[fld]
        if arr:
            a = np.array(arr)
            field_report[fld] = {
                "MAE_pct": round(float(a.mean()), 3),
                "Median_pct": round(float(np.median(a)), 3),
                "P90_pct": round(float(np.percentile(a, 90)), 3),
                "P10_pct": round(float(np.percentile(a, 10)), 3),
                "Std_pct": round(float(a.std()), 3),
                "Max_pct": round(float(a.max()), 3),
                "count": int(len(a)),
                "within_5pct": fields_ok[fld].get("within5", 0),
                "within_10pct": fields_ok[fld].get("within10", 0),
                "within_20pct": fields_ok[fld].get("within20", 0),
                "beyond_20pct": fields_ok[fld].get("beyond20", 0),
            }
        else:
            field_report[fld] = {"MAE_pct": None, "count": 0}

    report["fields"] = field_report
    report["confidence_dist"] = dict(conf_dist)
    report["case_dist"] = {f"Case_{k}": v for k, v in case_dist.items()}

    # Summary score (exclude finish_epi/finish_ppi: exact-by-design per case study)
    scores = []
    for fld in ["greige_epi", "greige_ppi", "target_gsm", "reed_count", "reed_space", "finish_width"]:
        e = field_report.get(fld, {}).get("MAE_pct")
        if e is not None:
            scores.append(e)
    report["overall_MAE_pct"] = round(float(np.mean(scores)), 3) if scores else None

    return report


def get_sample_A() -> pd.DataFrame:
    """Original 25 diverse articles (fixed sample)."""
    data_service.load_data()
    df = data_service.df
    test_candidates = df.dropna(subset=[
        "weave", "blend", "warp_count", "weft_count",
        "Greige EPI", "Greige PPI", "FINISH EPI", "FINISH PPI",
        "Reed Count", "Ends per dent", "Reed space", "FINISH GSM", "FINISH WIDTH",
    ])
    test_set = test_candidates[
        test_candidates["weave"].isin([
            "PLAIN", "2/1 S TWILL", "3/1 S TWILL", "4/1 S SATIN", "OXFORD", "DOBBY",
        ])
    ]
    test_set = test_set[test_set["blend"].isin(["100%CO", "65%PES 35%CO", "97%CO 3%EA", "60%CO 40%PES"])]
    test_set = test_set.sort_values("FINISH EPI").reset_index(drop=True)
    idx = [int(i * len(test_set) / 26) for i in range(1, 26)]
    return test_set.iloc[idx].copy()


def get_sample_B(seed: int) -> pd.DataFrame:
    """Random 25 articles (different per seed)."""
    data_service.load_data()
    df = data_service.df
    test_candidates = df.dropna(subset=[
        "weave", "blend", "warp_count", "weft_count",
        "Greige EPI", "Greige PPI", "FINISH EPI", "FINISH PPI",
        "Reed Count", "Ends per dent", "Reed space", "FINISH GSM", "FINISH WIDTH",
    ])
    test_set = test_candidates[
        test_candidates["weave"].isin([
            "PLAIN", "2/1 S TWILL", "3/1 S TWILL", "4/1 S SATIN", "OXFORD", "DOBBY",
        ])
    ]
    test_set = test_set[test_set["blend"].isin(["100%CO", "65%PES 35%CO", "97%CO 3%EA", "60%CO 40%PES"])]
    sample = test_set.sample(n=25, random_state=seed)
    return sample.copy()


def get_sample_C() -> dict:
    """Full validation report (200 articles)."""
    return data_service.get_validation_report(sample_size=200, seed=42)


def compare_reports(baseline: dict, current: dict, label: str) -> str:
    """Generate comparison table between two reports."""
    lines = [f"\n{'='*100}", f"  COMPARISON: {label}", f"{'='*100}\n"]
    lines.append(f"{'Field':18s} {'Baseline MAE%':>15s} {'Current MAE%':>15s} {'Delta':>12s} {'P90 Base':>10s} {'P90 Curr':>10s} {'Δ P90':>10s}")
    lines.append("-" * 100)

    for fld, _ in FIELDS:
        b = baseline.get("fields", {}).get(fld, {})
        c = current.get("fields", {}).get(fld, {})
        b_mae = b.get("MAE_pct")
        c_mae = c.get("MAE_pct")
        b_p90 = b.get("P90_pct")
        c_p90 = c.get("P90_pct")

        if b_mae is not None and c_mae is not None:
            delta = c_mae - b_mae
            sign = "+" if delta > 0 else ""
            color = "IMPROVED" if delta < 0 else ("REGRESSED" if delta > 0 else "SAME")
            lines.append(f"{fld:18s} {b_mae:>14.3f}% {c_mae:>14.3f}% {sign}{delta:>10.3f}% {b_p90:>9.3f}% {c_p90:>9.3f}% {c_p90-b_p90 if b_p90 and c_p90 else 0:>+9.3f}%")
        else:
            lines.append(f"{fld:18s} {'N/A':>15s} {'N/A':>15s}")

    # Confidence distribution
    b_conf = baseline.get("confidence_dist", {})
    c_conf = current.get("confidence_dist", {})
    lines.append(f"\n{'Confidence':18s} {'Baseline':>15s} {'Current':>15s} {'Delta':>12s}")
    lines.append("-" * 60)
    for level in ["high", "medium", "low", "very_low"]:
        bv = b_conf.get(level, 0)
        cv = c_conf.get(level, 0)
        d = cv - bv
        lines.append(f"{level:18s} {bv:>15d} {cv:>15d} {d:+12d}")

    # Overall
    b_ov = baseline.get('overall_MAE_pct','-')
    c_ov = current.get('overall_MAE_pct','-')
    lines.append(f"\n{'Overall MAE%':18s} {str(b_ov):>15s} {str(c_ov):>15s}")

    return "\n".join(lines)


def print_report(report: dict, title: str = "REPORT") -> None:
    """Print a readable report."""
    print(f"\n{'='*100}")
    print(f"  {title}: {report['label']}")
    print(f"{'='*100}")
    print(f"  Articles: {report['total_articles']} | Predicted: {report['predicted_articles']} | Time: {report['elapsed_seconds']}s ({report['avg_ms_per_article']}ms/art)")
    print(f"  Overall MAE: {report.get('overall_MAE_pct', 'N/A')}%")
    print()
    print(f"  {'Field':18s} {'MAE%':>8s} {'Median%':>8s} {'P90%':>8s} {'P10%':>8s} {'≤5%':>6s} {'≤10%':>6s} {'≤20%':>6s} {'>20%':>6s}")
    print(f"  {'-'*80}")
    for fld, _ in FIELDS:
        f = report["fields"].get(fld, {})
        if f.get("MAE_pct") is not None:
            print(f"  {fld:18s} {f['MAE_pct']:>7.2f}% {f['Median_pct']:>7.2f}% {f['P90_pct']:>7.2f}% {f['P10_pct']:>7.2f}% {f['within_5pct']:>5d} {f['within_10pct']:>5d} {f['within_20pct']:>5d} {f['beyond_20pct']:>5d}")
        else:
            print(f"  {fld:18s} {'N/A':>8s}")
    print()
    print(f"  Confidence: {report.get('confidence_dist', {})}")
    print(f"  Cases: {report.get('case_dist', {})}")


if __name__ == "__main__":
    import argparse
    ap = argparse.ArgumentParser()
    ap.add_argument("--mode", choices=["baseline", "compare"], required=True)
    ap.add_argument("--seed_b", type=int, default=42)
    ap.add_argument("--baseline_file", type=str, default=None)
    ap.add_argument("--report_file", type=str, default=None)
    args = ap.parse_args()

    data_service.load_data()

    if args.mode == "baseline":
        # Test Set A: original 25 fixed articles
        sample_a = get_sample_A()
        report_a = run_eval(sample_a, "Test Set A — 25 fixed diverse articles")
        print_report(report_a, "BASELINE")

        # Test Set B: random 25 new articles
        sample_b = get_sample_B(args.seed_b)
        report_b = run_eval(sample_b, f"Test Set B — 25 random articles (seed={args.seed_b})")
        print_report(report_b, "BASELINE")

        # Test Set C: full 200 validation
        print(f"\n{'='*100}")
        print(f"  Test Set C — 200-sample validation report")
        print(f"{'='*100}")
        report_c = get_sample_C()
        print(json.dumps(report_c, indent=2))

        # Combined
        combined = {
            "set_a": report_a,
            "set_b": report_b,
            "set_c": report_c,
        }

    elif args.mode == "compare":
        with open(args.baseline_file) as f:
            baseline = json.load(f)

        # Re-evaluate with current code
        sample_a = get_sample_A()
        report_a = run_eval(sample_a, "Test Set A — 25 fixed diverse articles")
        print_report(report_a, "CURRENT")

        sample_b = get_sample_B(args.seed_b)
        report_b = run_eval(sample_b, f"Test Set B — 25 random articles (seed={args.seed_b})")
        print_report(report_b, "CURRENT")

        report_c = get_sample_C()

        # Comparison
        print(compare_reports(baseline["set_a"], report_a, "TEST SET A (Fixed 25)"))
        print(compare_reports(baseline["set_b"], report_b, "TEST SET B (Random 25)"))

        print(f"\n{'='*100}")
        print(f"  Test Set C — 200-sample validation report")
        print(f"{'='*100}")
        print(json.dumps(report_c, indent=2))

        combined = {
            "set_a": report_a,
            "set_b": report_b,
            "set_c": report_c,
        }

    if args.report_file:
        with open(args.report_file, "w") as f:
            json.dump(combined, f, indent=2)
        print(f"\nReport saved to {args.report_file}")