File size: 2,805 Bytes
62dbf09 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 |
import json, sys, importlib.util
# Direct file import to avoid verl.__init__
spec = importlib.util.spec_from_file_location(
"physics_reward",
"/workspace/rl4phyx/RL4Phyx/ZeroSearch/One-Shot-RLVR/verl/utils/reward_score/physics_reward.py"
)
mod = importlib.util.module_from_spec(spec)
spec.loader.exec_module(mod)
rule_based_score = mod.rule_based_score
sympy_fallback_score = mod.sympy_fallback_score
f = "/workspace/rl4phyx/RL4Phyx/SFT/sft_eval_footprint/inference_results_base.jsonl"
with open(f) as fh:
lines = [json.loads(l) for l in fh if l.strip()]
results = {"rule_ok": 0, "sympy_ok": 0, "sympy_fail": 0, "rule_wrong": 0}
categories = {}
fail_examples = []
for r in lines:
gt = str(r.get("ground_truth_value", "")).strip()
cat = r.get("category", "unknown")
score = rule_based_score(gt, gt)
if score == 1.0:
status = "rule_ok"
results["rule_ok"] += 1
elif score is None:
try:
s = sympy_fallback_score(gt, gt)
if s == 1.0:
status = "sympy_ok"
results["sympy_ok"] += 1
else:
status = "sympy_fail"
results["sympy_fail"] += 1
if len(fail_examples) < 20:
fail_examples.append({"gt": gt[:80], "cat": cat})
except Exception as e:
status = "sympy_fail"
results["sympy_fail"] += 1
if len(fail_examples) < 20:
fail_examples.append({"gt": gt[:80], "cat": cat})
else:
status = "rule_wrong"
results["rule_wrong"] += 1
if len(fail_examples) < 20:
fail_examples.append({"gt": gt[:80], "cat": cat})
if cat not in categories:
categories[cat] = {"rule_ok": 0, "sympy_ok": 0, "fail": 0}
if "ok" in status:
categories[cat][status] += 1
else:
categories[cat]["fail"] += 1
total = len(lines)
covered = results["rule_ok"] + results["sympy_ok"]
print(f"=== Hybrid Reward Coverage ({total} questions) ===")
print(f" Rule-based: {results['rule_ok']:4d} ({results['rule_ok']/total*100:.1f}%)")
print(f" SymPy: {results['sympy_ok']:4d} ({results['sympy_ok']/total*100:.1f}%)")
print(f" TOTAL OK: {covered:4d} ({covered/total*100:.1f}%)")
print(f" Failed: {results['sympy_fail']+results['rule_wrong']:4d} ({(results['sympy_fail']+results['rule_wrong'])/total*100:.1f}%)")
print(f"\n=== By Category ===")
for cat, v in sorted(categories.items()):
t = v["rule_ok"] + v["sympy_ok"] + v["fail"]
ok = v["rule_ok"] + v["sympy_ok"]
print(f" {cat:25s}: {ok}/{t} = {ok/t*100:.0f}% [rule={v['rule_ok']}, sympy={v['sympy_ok']}, fail={v['fail']}]")
print(f"\n=== Fail Examples ===")
for ex in fail_examples:
print(f" [{ex['cat']:20s}] {ex['gt']}")
|