File size: 4,804 Bytes
1ba301a | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 | """Stage 4 — evaluate NPC-Reason-SFT on the FROZEN eval, vs the frozen baseline + PREREG.
Same EVAL.lock, same greedy decoding (temp 0, seed 0, max_tokens 12288) as the baseline =
apples-to-apples. Scores with the frozen verifier. Grades the pre-registered bars honestly.
"""
import json
import sys
import time
from pathlib import Path
from transformers import AutoTokenizer
from vllm import LLM, SamplingParams
HERE = Path(__file__).resolve().parent
ROOT = HERE.parent
sys.path.insert(0, str(ROOT))
from verifier.step_verifier import verify_chain # FROZEN
MERGED = str(ROOT / "sft" / "merged")
MAX_TOKENS = 12288
MAX_MODEL_LEN = 16384
PLAIN_INSTRUCTION = (
"Solve this math problem. Show your work step by step, then give the final answer "
"on the last line as \\boxed{{ANSWER}}.\n\nProblem: {problem}"
)
FORMAT_INSTRUCTION = (
"Solve this math problem. For EVERY load-bearing arithmetic step, write the "
"computation as an inline checkable assertion in the exact form <<EXPR = RESULT>>, "
"where EXPR is the arithmetic expression and RESULT is its value (for example "
"<<3*8 = 24>>). If a quantity is reused, you may name it, e.g. "
"let total = <<3*8 = 24>>, and reference it later as <<total + 6 = 30>>. Do not "
"assert any number that drives the answer without wrapping it in <<...>>. End with "
"the final answer as \\boxed{{ANSWER}}, and make sure it equals the result of your "
"last <<...>> step.\n\nProblem: {problem}"
)
def load_eval():
return [json.loads(l) for l in (ROOT / "eval" / "eval_set.jsonl").open()]
def rate(recs, k):
return round(100 * sum(1 for r in recs if r[k]) / len(recs), 2) if recs else 0.0
def main():
rows = load_eval()
tok = AutoTokenizer.from_pretrained(MERGED)
variants = {"plain": PLAIN_INSTRUCTION, "format": FORMAT_INSTRUCTION}
jobs = []
for v, tmpl in variants.items():
for r in rows:
p = tok.apply_chat_template([{"role": "user", "content": tmpl.format(problem=r["problem"])}],
tokenize=False, add_generation_prompt=True)
jobs.append((v, r, p))
llm = LLM(model=MERGED, dtype="bfloat16", max_model_len=MAX_MODEL_LEN,
gpu_memory_utilization=0.85, seed=0)
sp = SamplingParams(temperature=0.0, max_tokens=MAX_TOKENS, seed=0)
t0 = time.time()
outs = llm.generate([j[2] for j in jobs], sp)
gen_s = round(time.time() - t0, 1)
per = {"plain": [], "format": []}
raw = HERE / "eval_raw.jsonl"
with raw.open("w") as f:
for (v, r, _), o in zip(jobs, outs):
text = o.outputs[0].text
rec = verify_chain(text, gold_answer=r["gold"])
slim = {"verifiable": rec["verifiable"], "correct": bool(rec["correct"]),
"verified_and_correct": bool(rec["verified_and_correct"]),
"n_assertions": rec["n_assertions"]}
per[v].append(slim)
f.write(json.dumps({"id": r["id"], "variant": v, "gold": r["gold"],
"output": text, "verifier": slim}, ensure_ascii=False) + "\n")
summary = {v: {"n": len(rs), "verifiable_rate": rate(rs, "verifiable"),
"accuracy": rate(rs, "correct"),
"verified_and_correct": rate(rs, "verified_and_correct"),
"mean_assertions": round(sum(x["n_assertions"] for x in rs) / len(rs), 2)}
for v, rs in per.items()}
base = json.load(open(ROOT / "baseline" / "BASELINE.lock"))["summary"]
bf = base["format"]
sf = summary["format"]
# PREREG bars (b5a49437): +15pp v&c over format baseline; verifiable >=90%; acc regress <=5pp.
lift = round(sf["verified_and_correct"] - bf["verified_and_correct"], 2)
acc_delta = round(sf["accuracy"] - bf["accuracy"], 2)
grade = {
"primary_lift_vc_pp": lift,
"primary_SUPPORTED": lift >= 15.0,
"verifiable_rate": sf["verifiable_rate"],
"verifiable_SUPPORTED": sf["verifiable_rate"] >= 90.0,
"accuracy_delta_pp": acc_delta,
"accuracy_guard_SUPPORTED": acc_delta >= -5.0,
}
grade["ALL_THREE"] = all([grade["primary_SUPPORTED"], grade["verifiable_SUPPORTED"],
grade["accuracy_guard_SUPPORTED"]])
rl_gate = grade["ALL_THREE"] or sf["verifiable_rate"] >= 90.0
result = {"model": MERGED, "decoding": {"greedy": True, "max_tokens": MAX_TOKENS, "seed": 0},
"gen_seconds": gen_s, "summary": summary, "baseline_format": bf,
"prereg_grade": grade, "rl_gate_cleared": rl_gate}
(HERE / "eval_result.json").write_text(json.dumps(result, indent=2))
print(json.dumps({"summary": summary, "grade": grade, "rl_gate": rl_gate}, indent=2))
if __name__ == "__main__":
main()
|