artifacts/accuracy_eval.py

"""Discriminating eval: does REAM's -0.19 PPL reflect capability or smoothing?

PPL rewards a flatter output distribution; it cannot tell a real gain from a
harmless smoothing effect. This scores EXACT-ANSWER tasks right/wrong -- tasks
where there is one correct answer and smoothing cannot help. Same scorer for
both models, so the comparison is fair regardless of scorer strictness.

Prediction if -0.19 is smoothing: REAM accuracy ~= shared8-head8 accuracy,
especially on math (PPL already showed reasoning_math did NOT improve, +0.007).
If REAM is genuinely better, math accuracy rises.

Usage: python accuracy_eval.py <model-id> <out.json>
"""
import json, re, sys, urllib.request

MODEL = sys.argv[1]
OUT = sys.argv[2] if len(sys.argv) > 2 else None

# (category, prompt, expected). Hand-written to avoid benchmark contamination;
# multi-step so smoothing cannot luck into them, but within a strong model's reach.
ITEMS = [
    ("math", "A tank holds 480 liters. It drains at 12 liters per minute for 15 minutes, then is refilled by 30 liters. How many liters are in it now? Give only the number.", "330"),
    ("math", "A book has 342 pages. Maria reads 18 pages a day for 9 days, then 24 pages a day for 4 days. How many pages are left? Number only.", "84"),
    ("math", "There are 7 boxes with 23 apples each. 41 apples are rotten and removed. How many good apples remain? Number only.", "120"),
    ("math", "A car travels 65 km/h for 3 hours, then 80 km/h for 2 hours. Total distance in km? Number only.", "355"),
    ("math", "A store sells pens at 3 for $2. How much do 27 pens cost, in dollars? Number only.", "18"),
    ("math", "Compute 144 divided by 8, then multiply the result by 15. Number only.", "270"),
    ("math", "A rectangle is 14 by 9. A square of side 5 is cut out. Remaining area? Number only.", "101"),
    ("math", "Sarah has $250. She buys 6 shirts at $18 each and 2 hats at $14 each. How much money is left? Number only.", "114"),
    ("math", "A train departs at 09:45 and arrives at 13:20. Journey length in minutes? Number only.", "215"),
    ("math", "If 5 machines make 5 widgets in 5 minutes, how many widgets do 5 machines make in 60 minutes? Number only.", "60"),
    ("math", "A recipe needs 3 eggs per cake. You have 40 eggs. After making as many whole cakes as possible, how many eggs are left over? Number only.", "1"),
    ("math", "The sum of three consecutive integers is 72. What is the largest of them? Number only.", "25"),
    ("math", "A phone costs $600. It is discounted 20%, then 8% sales tax is added. Final price in dollars? Number only.", "518.4"),
    ("math", "A garden is 12 m by 8 m. A path 1 m wide runs around the inside edge. Area of the path in square metres? Number only.", "36"),
    ("factual", "What is the chemical symbol for gold? Symbol only.", "Au"),
    ("factual", "In what year did the first human land on the Moon? Year only.", "1969"),
    ("factual", "What is the capital city of Canada? One word.", "Ottawa"),
    ("factual", "How many sides does a hexagon have? Number only.", "6"),
    ("factual", "What planet is known as the Red Planet? One word.", "Mars"),
    ("factual", "What is the largest ocean on Earth? One word.", "Pacific"),
    ("factual", "Who wrote the play 'Romeo and Juliet'? Last name only.", "Shakespeare"),
    ("factual", "What is the freezing point of water in Celsius? Number only.", "0"),
    ("factual", "What gas do plants primarily absorb from the air for photosynthesis? Two words.", "carbon dioxide"),
    ("factual", "How many degrees are in a right angle? Number only.", "90"),
]


def ask(prompt, mt=3500):
    body = json.dumps({"model": MODEL, "messages": [{"role": "user", "content": prompt}],
                       "temperature": 0.0, "top_p": 1.0, "top_k": 0, "min_p": 0.0,
                       "repetition_penalty": 1.0, "max_tokens": mt, "stream": False}).encode()
    r = urllib.request.Request("http://localhost:1234/v1/chat/completions", data=body,
                               headers={"Content-Type": "application/json"})
    d = json.load(urllib.request.urlopen(r, timeout=600))["choices"][0]
    m = d["message"]
    return (m.get("content") or "").strip(), d.get("finish_reason")


def norm_num(s):
    # strip digit-internal separators (handles the corruption AND legit commas);
    # both models scored identically so this is fair.
    return re.sub(r"(?<=\d)[ ,_](?=\d)", "", s)


def check(ans, expected, cat):
    a = norm_num(ans)
    if cat == "factual" and not expected.replace(".", "").isdigit():
        return expected.lower() in a.lower()
    # numeric: compare the set of numbers present; credit if expected appears
    want = expected
    nums = re.findall(r"-?\d+\.?\d*", a)
    # exact match, or match ignoring trailing .0
    for n in nums:
        if n == want or n.rstrip("0").rstrip(".") == want.rstrip("0").rstrip("."):
            return True
    return False


def self_test():
    assert check("330", "330", "math")
    assert check("The answer is 518.40 dollars.", "518.4", "math")
    assert check("3 3 0", "330", "math")            # space-corrupted
    assert check("**Au**", "Au", "factual")
    assert not check("The answer is 331.", "330", "math")
    assert check("Ottawa is the capital.", "Ottawa", "factual")
    print("[selftest] answer-extraction ok", flush=True)


if __name__ == "__main__":
    self_test()
    if MODEL == "SELFTEST":
        sys.exit(0)
    from collections import defaultdict
    tally = defaultdict(lambda: [0, 0]); results = []
    for i, (cat, q, exp) in enumerate(ITEMS):
        try:
            ans, fin = ask(q)
        except Exception as e:
            print(f"  item {i} ERROR {e}", flush=True); continue
        ok = check(ans, exp, cat)
        tally[cat][0] += ok; tally[cat][1] += 1
        results.append({"cat": cat, "expected": exp, "ok": ok, "answer": ans[:80], "finish": fin})
        print(f"  [{'OK ' if ok else 'XX '}] {cat:8} want={exp:14} got={ans[:44]!r}", flush=True)
    tot_ok = sum(v[0] for v in tally.values()); tot_n = sum(v[1] for v in tally.values())
    print(f"\n{MODEL}", flush=True)
    for cat, (ok, n) in sorted(tally.items()):
        print(f"  {cat:10} {ok}/{n}", flush=True)
    print(f"  OVERALL   {tot_ok}/{tot_n}  ({100*tot_ok/tot_n:.0f}%)", flush=True)
    if OUT:
        json.dump({"model": MODEL, "overall": [tot_ok, tot_n],
                   "by_cat": {k: v for k, v in tally.items()}, "results": results},
                  open(OUT, "w"), indent=2)