"""
Runs every post in eval/posts.json through TELL's text tools and scores it.

    python scripts/run_eval.py              # language + number tricks only (free)
    python scripts/run_eval.py --trace      # also source tracing (costs money, see estimate)
    python scripts/run_eval.py --only pratt-zero-votes

Writes eval/results.csv and prints the numbers that go in TRAINING.md.
Scoring uses expected_tricks, which you fill in by hand after reviewing suggested_tricks.
"""

import argparse
import csv
import json
import sys
from collections import defaultdict
from pathlib import Path

ROOT = Path(__file__).resolve().parent.parent
sys.path.insert(0, str(ROOT))

try:
    from dotenv import load_dotenv
    load_dotenv(ROOT / ".env")
except ImportError:
    pass

import features  # noqa: F401,E402
from core import pipeline  # noqa: E402
from core.post import Post  # noqa: E402

COST_PER_POST_WITH_TRACE = 0.12  # worst case: 3 claims x 4 searches x $0.01


def score(found: set, expected: set):
    return len(found & expected), len(expected - found), len(found - expected)


def main():
    ap = argparse.ArgumentParser()
    ap.add_argument("--trace", action="store_true", help="also trace claims to sources")
    ap.add_argument("--only", help="run one post by id")
    args = ap.parse_args()

    posts = json.loads((ROOT / "eval" / "posts.json").read_text())["posts"]
    if args.only:
        posts = [p for p in posts if p["id"] == args.only]
    if args.trace:
        print(f"Source tracing on: up to ${COST_PER_POST_WITH_TRACE * len(posts):.2f} for {len(posts)} posts.")
        if input("Go? [y/N] ").strip().lower() != "y":
            return

    unreviewed = [p["id"] for p in posts if not p.get("expected_tricks") and p.get("suggested_tricks")]
    if unreviewed:
        print(f"Note: {len(unreviewed)} posts have no expected_tricks yet, so they score as all false alarms.\n")

    rows = []
    totals = defaultdict(lambda: [0, 0, 0, 0])  # group -> hits, misses, false alarms, posts
    for p in posts:
        report = pipeline.run(Post(text=p["text"]), {"trace": args.trace})
        found = {f["trick"] for f in report.findings}
        expected = set(p.get("expected_tricks", []))
        hit, miss, fa = score(found, expected)

        group = "control" if p.get("control") else p["favors"].split(" ")[0]
        t = totals[group]
        t[0] += hit; t[1] += miss; t[2] += fa; t[3] += 1

        verdicts = [c["verdict"] for s in report.sections for c in s.claim_checks]
        rows.append({
            "id": p["id"], "group": group,
            "found": ";".join(sorted(found)), "expected": ";".join(sorted(expected)),
            "hits": hit, "misses": miss, "false_alarms": fa,
            "trace_verdicts": ";".join(verdicts), "expected_trace": p.get("expected_trace", ""),
        })
        print(f"{p['id']:30s} found={sorted(found)}  hits={hit} miss={miss} fa={fa}  {verdicts or ''}")

    out = ROOT / "eval" / "results.csv"
    with out.open("w", newline="") as f:
        w = csv.DictWriter(f, fieldnames=rows[0].keys())
        w.writeheader()
        w.writerows(rows)

    print("\ngroup        posts  hits  misses  false_alarms  flags/post")
    for g, (h, m, fa, n) in sorted(totals.items()):
        print(f"{g:12s} {n:5d} {h:5d} {m:7d} {fa:13d} {(h + fa) / n:11.2f}")
    print("\nIf flags/post is a lot higher for one side than the other on similar posts, look at why.")
    print(f"Saved {out.relative_to(ROOT)}")


if __name__ == "__main__":
    main()