| """ |
| Runs every post in eval/posts.json through TELL's text tools and scores it. |
| |
| python scripts/run_eval.py # language + number tricks only (free) |
| python scripts/run_eval.py --trace # also source tracing (costs money, see estimate) |
| python scripts/run_eval.py --only pratt-zero-votes |
| |
| Writes eval/results.csv and prints the numbers that go in TRAINING.md. |
| Scoring uses expected_tricks, which you fill in by hand after reviewing suggested_tricks. |
| """ |
|
|
| import argparse |
| import csv |
| import json |
| import sys |
| from collections import defaultdict |
| from pathlib import Path |
|
|
| ROOT = Path(__file__).resolve().parent.parent |
| sys.path.insert(0, str(ROOT)) |
|
|
| try: |
| from dotenv import load_dotenv |
| load_dotenv(ROOT / ".env") |
| except ImportError: |
| pass |
|
|
| import features |
| from core import pipeline |
| from core.post import Post |
|
|
| COST_PER_POST_WITH_TRACE = 0.12 |
|
|
|
|
| def score(found: set, expected: set): |
| return len(found & expected), len(expected - found), len(found - expected) |
|
|
|
|
| def main(): |
| ap = argparse.ArgumentParser() |
| ap.add_argument("--trace", action="store_true", help="also trace claims to sources") |
| ap.add_argument("--only", help="run one post by id") |
| args = ap.parse_args() |
|
|
| posts = json.loads((ROOT / "eval" / "posts.json").read_text())["posts"] |
| if args.only: |
| posts = [p for p in posts if p["id"] == args.only] |
| if args.trace: |
| print(f"Source tracing on: up to ${COST_PER_POST_WITH_TRACE * len(posts):.2f} for {len(posts)} posts.") |
| if input("Go? [y/N] ").strip().lower() != "y": |
| return |
|
|
| unreviewed = [p["id"] for p in posts if not p.get("expected_tricks") and p.get("suggested_tricks")] |
| if unreviewed: |
| print(f"Note: {len(unreviewed)} posts have no expected_tricks yet, so they score as all false alarms.\n") |
|
|
| rows = [] |
| totals = defaultdict(lambda: [0, 0, 0, 0]) |
| for p in posts: |
| report = pipeline.run(Post(text=p["text"]), {"trace": args.trace}) |
| found = {f["trick"] for f in report.findings} |
| expected = set(p.get("expected_tricks", [])) |
| hit, miss, fa = score(found, expected) |
|
|
| group = "control" if p.get("control") else p["favors"].split(" ")[0] |
| t = totals[group] |
| t[0] += hit; t[1] += miss; t[2] += fa; t[3] += 1 |
|
|
| verdicts = [c["verdict"] for s in report.sections for c in s.claim_checks] |
| rows.append({ |
| "id": p["id"], "group": group, |
| "found": ";".join(sorted(found)), "expected": ";".join(sorted(expected)), |
| "hits": hit, "misses": miss, "false_alarms": fa, |
| "trace_verdicts": ";".join(verdicts), "expected_trace": p.get("expected_trace", ""), |
| }) |
| print(f"{p['id']:30s} found={sorted(found)} hits={hit} miss={miss} fa={fa} {verdicts or ''}") |
|
|
| out = ROOT / "eval" / "results.csv" |
| with out.open("w", newline="") as f: |
| w = csv.DictWriter(f, fieldnames=rows[0].keys()) |
| w.writeheader() |
| w.writerows(rows) |
|
|
| print("\ngroup posts hits misses false_alarms flags/post") |
| for g, (h, m, fa, n) in sorted(totals.items()): |
| print(f"{g:12s} {n:5d} {h:5d} {m:7d} {fa:13d} {(h + fa) / n:11.2f}") |
| print("\nIf flags/post is a lot higher for one side than the other on similar posts, look at why.") |
| print(f"Saved {out.relative_to(ROOT)}") |
|
|
|
|
| if __name__ == "__main__": |
| main() |
|
|
