File size: 3,509 Bytes
fb4fcd0 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 | """
Runs every post in eval/posts.json through TELL's text tools and scores it.
python scripts/run_eval.py # language + number tricks only (free)
python scripts/run_eval.py --trace # also source tracing (costs money, see estimate)
python scripts/run_eval.py --only pratt-zero-votes
Writes eval/results.csv and prints the numbers that go in TRAINING.md.
Scoring uses expected_tricks, which you fill in by hand after reviewing suggested_tricks.
"""
import argparse
import csv
import json
import sys
from collections import defaultdict
from pathlib import Path
ROOT = Path(__file__).resolve().parent.parent
sys.path.insert(0, str(ROOT))
try:
from dotenv import load_dotenv
load_dotenv(ROOT / ".env")
except ImportError:
pass
import features # noqa: F401,E402
from core import pipeline # noqa: E402
from core.post import Post # noqa: E402
COST_PER_POST_WITH_TRACE = 0.12 # worst case: 3 claims x 4 searches x $0.01
def score(found: set, expected: set):
return len(found & expected), len(expected - found), len(found - expected)
def main():
ap = argparse.ArgumentParser()
ap.add_argument("--trace", action="store_true", help="also trace claims to sources")
ap.add_argument("--only", help="run one post by id")
args = ap.parse_args()
posts = json.loads((ROOT / "eval" / "posts.json").read_text())["posts"]
if args.only:
posts = [p for p in posts if p["id"] == args.only]
if args.trace:
print(f"Source tracing on: up to ${COST_PER_POST_WITH_TRACE * len(posts):.2f} for {len(posts)} posts.")
if input("Go? [y/N] ").strip().lower() != "y":
return
unreviewed = [p["id"] for p in posts if not p.get("expected_tricks") and p.get("suggested_tricks")]
if unreviewed:
print(f"Note: {len(unreviewed)} posts have no expected_tricks yet, so they score as all false alarms.\n")
rows = []
totals = defaultdict(lambda: [0, 0, 0, 0]) # group -> hits, misses, false alarms, posts
for p in posts:
report = pipeline.run(Post(text=p["text"]), {"trace": args.trace})
found = {f["trick"] for f in report.findings}
expected = set(p.get("expected_tricks", []))
hit, miss, fa = score(found, expected)
group = "control" if p.get("control") else p["favors"].split(" ")[0]
t = totals[group]
t[0] += hit; t[1] += miss; t[2] += fa; t[3] += 1
verdicts = [c["verdict"] for s in report.sections for c in s.claim_checks]
rows.append({
"id": p["id"], "group": group,
"found": ";".join(sorted(found)), "expected": ";".join(sorted(expected)),
"hits": hit, "misses": miss, "false_alarms": fa,
"trace_verdicts": ";".join(verdicts), "expected_trace": p.get("expected_trace", ""),
})
print(f"{p['id']:30s} found={sorted(found)} hits={hit} miss={miss} fa={fa} {verdicts or ''}")
out = ROOT / "eval" / "results.csv"
with out.open("w", newline="") as f:
w = csv.DictWriter(f, fieldnames=rows[0].keys())
w.writeheader()
w.writerows(rows)
print("\ngroup posts hits misses false_alarms flags/post")
for g, (h, m, fa, n) in sorted(totals.items()):
print(f"{g:12s} {n:5d} {h:5d} {m:7d} {fa:13d} {(h + fa) / n:11.2f}")
print("\nIf flags/post is a lot higher for one side than the other on similar posts, look at why.")
print(f"Saved {out.relative_to(ROOT)}")
if __name__ == "__main__":
main()
|