|
|
| import json, sys, importlib.util |
|
|
| |
| spec = importlib.util.spec_from_file_location( |
| "physics_reward", |
| "/workspace/rl4phyx/RL4Phyx/ZeroSearch/One-Shot-RLVR/verl/utils/reward_score/physics_reward.py" |
| ) |
| mod = importlib.util.module_from_spec(spec) |
| spec.loader.exec_module(mod) |
|
|
| rule_based_score = mod.rule_based_score |
| sympy_fallback_score = mod.sympy_fallback_score |
|
|
| f = "/workspace/rl4phyx/RL4Phyx/SFT/sft_eval_footprint/inference_results_base.jsonl" |
| with open(f) as fh: |
| lines = [json.loads(l) for l in fh if l.strip()] |
|
|
| results = {"rule_ok": 0, "sympy_ok": 0, "sympy_fail": 0, "rule_wrong": 0} |
| categories = {} |
| fail_examples = [] |
|
|
| for r in lines: |
| gt = str(r.get("ground_truth_value", "")).strip() |
| cat = r.get("category", "unknown") |
| |
| score = rule_based_score(gt, gt) |
| |
| if score == 1.0: |
| status = "rule_ok" |
| results["rule_ok"] += 1 |
| elif score is None: |
| try: |
| s = sympy_fallback_score(gt, gt) |
| if s == 1.0: |
| status = "sympy_ok" |
| results["sympy_ok"] += 1 |
| else: |
| status = "sympy_fail" |
| results["sympy_fail"] += 1 |
| if len(fail_examples) < 20: |
| fail_examples.append({"gt": gt[:80], "cat": cat}) |
| except Exception as e: |
| status = "sympy_fail" |
| results["sympy_fail"] += 1 |
| if len(fail_examples) < 20: |
| fail_examples.append({"gt": gt[:80], "cat": cat}) |
| else: |
| status = "rule_wrong" |
| results["rule_wrong"] += 1 |
| if len(fail_examples) < 20: |
| fail_examples.append({"gt": gt[:80], "cat": cat}) |
| |
| if cat not in categories: |
| categories[cat] = {"rule_ok": 0, "sympy_ok": 0, "fail": 0} |
| if "ok" in status: |
| categories[cat][status] += 1 |
| else: |
| categories[cat]["fail"] += 1 |
|
|
| total = len(lines) |
| covered = results["rule_ok"] + results["sympy_ok"] |
|
|
| print(f"=== Hybrid Reward Coverage ({total} questions) ===") |
| print(f" Rule-based: {results['rule_ok']:4d} ({results['rule_ok']/total*100:.1f}%)") |
| print(f" SymPy: {results['sympy_ok']:4d} ({results['sympy_ok']/total*100:.1f}%)") |
| print(f" TOTAL OK: {covered:4d} ({covered/total*100:.1f}%)") |
| print(f" Failed: {results['sympy_fail']+results['rule_wrong']:4d} ({(results['sympy_fail']+results['rule_wrong'])/total*100:.1f}%)") |
|
|
| print(f"\n=== By Category ===") |
| for cat, v in sorted(categories.items()): |
| t = v["rule_ok"] + v["sympy_ok"] + v["fail"] |
| ok = v["rule_ok"] + v["sympy_ok"] |
| print(f" {cat:25s}: {ok}/{t} = {ok/t*100:.0f}% [rule={v['rule_ok']}, sympy={v['sympy_ok']}, fail={v['fail']}]") |
|
|
| print(f"\n=== Fail Examples ===") |
| for ex in fail_examples: |
| print(f" [{ex['cat']:20s}] {ex['gt']}") |
|
|
