#!/usr/bin/env python3
"""Hakem ajanın doğruluğu: benchmarks/judge/claims.json üzerindeki etiketli iddialarla ölçülür.

  python scripts/judge_accuracy.py --model qwen2.5-coder:14b      # kural + hakem modeli
  python scripts/judge_accuracy.py --model qwen2.5-coder:14b --rule-mode hint   # kural ipucu olarak
  python scripts/judge_accuracy.py --model qwen2.5-coder:14b --rule-mode off    # yalnızca hakem modeli
  python scripts/judge_accuracy.py --rule-only                    # yalnızca kural aşaması (LLM gerekmez)

"Pozitif" = iddia kabul edildi (code_bug). Gerçek hatalarda kabul doğru (TP), yanlış iddialarda kabul
yanlış pozitiftir (FP). Çıktı: reports/deney/hakem_<model>.json ve konsolda özet.
"""
from __future__ import annotations

import argparse
import json
import sys
import time
from pathlib import Path

ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / "backend"))

from app.agents.judge import Verdict, deliberate_raise_line, judge_bug_report  # noqa: E402
from app.sandbox.registry import get_runner  # noqa: E402


def metrics(rows: list[dict]) -> dict:
    tp = sum(r["label"] == "code_bug" and r["accepted"] for r in rows)
    fn = sum(r["label"] == "code_bug" and not r["accepted"] for r in rows)
    fp = sum(r["label"] == "test_bug" and r["accepted"] for r in rows)
    tn = sum(r["label"] == "test_bug" and not r["accepted"] for r in rows)
    from decimal import ROUND_HALF_UP, Decimal

    def r1(x):  # yarım yukarı yuvarlama (56,25 → 56,3)
        return float(Decimal(str(x)).quantize(Decimal("0.1"), rounding=ROUND_HALF_UP)) if x is not None else None

    raw = lambda a, b: 100.0 * a / b if b else None  # noqa: E731
    rec, spe = raw(tp, tp + fn), raw(tn, tn + fp)
    return {"n": len(rows), "tp": tp, "fn": fn, "fp": fp, "tn": tn,
            "accuracy": r1(raw(tp + tn, len(rows))), "precision": r1(raw(tp, tp + fp)),
            "recall": r1(rec), "specificity": r1(spe),
            "balanced_accuracy": r1((rec + spe) / 2) if None not in (rec, spe) else None}


def main() -> None:
    ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
    ap.add_argument("--model")
    ap.add_argument("--rule-only", action="store_true")
    ap.add_argument("--rule-mode", choices=["veto", "hint", "off"], default="veto")
    ap.add_argument("--out-dir", default=str(ROOT / "reports" / "deney"))
    args = ap.parse_args()
    claims = json.loads((ROOT / "benchmarks" / "judge" / "claims.json").read_text(encoding="utf-8"))

    chat = None
    if not args.rule_only:
        from app.llm.providers import make_llm, is_local
        client = make_llm(args.model)
        if is_local(args.model) and not client.health()["ok"]:
            sys.exit("Ollama'ya ulaşılamadı.")
        chat = lambda msgs, **kw: client.chat(msgs, **kw)  # noqa: E731

    rows = []
    t0 = time.time()
    for c in claims:
        src_file = get_runner(c["language"]).source_filename(c["code"])
        if args.rule_only:
            ln = deliberate_raise_line(c["code"], src_file, c["failure"])
            v = Verdict(False, f"raise/throw satırı {ln}", "kural") if ln else Verdict(True, "kural karar vermedi", "-")
        else:
            v = judge_bug_report(chat, c["code"], src_file, c["language"], c["test_code"], c["test_name"],
                                 c["failure"], c["claim"], rule_mode=args.rule_mode)
        kind = "gerçek hata" if c["label"] == "code_bug" else ("kasıtlı doğrulama" if "-raise" in c["id"] else "yanlış değer")
        rows.append({"id": c["id"], "split": c["split"], "label": c["label"], "kind": kind, "accepted": v.accepted,
                     "by": v.by, "reason": v.reason})
        mark = "✓" if v.accepted == (c["label"] == "code_bug") else "✗"
        print(f"  {mark} {c['id']:<22} {c['label']:<9} kabul={v.accepted!s:<5} ({v.by})")
    res = {"model": "kural" if args.rule_only else (args.model or "varsayılan"),
           "rule_mode": "yalnız kural" if args.rule_only else args.rule_mode,
           "overall": metrics(rows),
           "by_stage": {"kural_reddi": sum(r["by"] == "kural" for r in rows)},
           "by_split": {s: metrics([r for r in rows if r["split"] == s]) for s in sorted({r["split"] for r in rows})},
           "by_kind": {k: {"n": sum(r["kind"] == k for r in rows),
                           "accepted": sum(r["kind"] == k and r["accepted"] for r in rows),
                           "rule_rejected": sum(r["kind"] == k and r["by"] == "kural" for r in rows)}
                       for k in ("gerçek hata", "yanlış değer", "kasıtlı doğrulama")},
           "duration_s": round(time.time() - t0, 1), "rows": rows}
    out = Path(args.out_dir)
    out.mkdir(parents=True, exist_ok=True)
    name = "hakem_kural.json" if args.rule_only else f"hakem_{res['model'].replace(':', '-').replace('/', '-')}_{args.rule_mode}.json"
    (out / name).write_text(json.dumps(res, ensure_ascii=False, indent=1), encoding="utf-8")
    print(json.dumps(res["overall"], ensure_ascii=False), f"\n→ {out / name}")


if __name__ == "__main__":
    main()