Upload root_scripts/coverage_test.py with huggingface_hub

root_scripts/coverage_test.py

@@ -0,0 +1,78 @@
+
+import json, sys, importlib.util
+
+# Direct file import to avoid verl.__init__
+spec = importlib.util.spec_from_file_location(
+    "physics_reward",
+    "/workspace/rl4phyx/RL4Phyx/ZeroSearch/One-Shot-RLVR/verl/utils/reward_score/physics_reward.py"
+)
+mod = importlib.util.module_from_spec(spec)
+spec.loader.exec_module(mod)
+
+rule_based_score = mod.rule_based_score
+sympy_fallback_score = mod.sympy_fallback_score
+
+f = "/workspace/rl4phyx/RL4Phyx/SFT/sft_eval_footprint/inference_results_base.jsonl"
+with open(f) as fh:
+    lines = [json.loads(l) for l in fh if l.strip()]
+
+results = {"rule_ok": 0, "sympy_ok": 0, "sympy_fail": 0, "rule_wrong": 0}
+categories = {}
+fail_examples = []
+
+for r in lines:
+    gt = str(r.get("ground_truth_value", "")).strip()
+    cat = r.get("category", "unknown")
+    
+    score = rule_based_score(gt, gt)
+    
+    if score == 1.0:
+        status = "rule_ok"
+        results["rule_ok"] += 1
+    elif score is None:
+        try:
+            s = sympy_fallback_score(gt, gt)
+            if s == 1.0:
+                status = "sympy_ok"
+                results["sympy_ok"] += 1
+            else:
+                status = "sympy_fail"
+                results["sympy_fail"] += 1
+                if len(fail_examples) < 20:
+                    fail_examples.append({"gt": gt[:80], "cat": cat})
+        except Exception as e:
+            status = "sympy_fail"
+            results["sympy_fail"] += 1
+            if len(fail_examples) < 20:
+                fail_examples.append({"gt": gt[:80], "cat": cat})
+    else:
+        status = "rule_wrong"
+        results["rule_wrong"] += 1
+        if len(fail_examples) < 20:
+            fail_examples.append({"gt": gt[:80], "cat": cat})
+    
+    if cat not in categories:
+        categories[cat] = {"rule_ok": 0, "sympy_ok": 0, "fail": 0}
+    if "ok" in status:
+        categories[cat][status] += 1
+    else:
+        categories[cat]["fail"] += 1
+
+total = len(lines)
+covered = results["rule_ok"] + results["sympy_ok"]
+
+print(f"=== Hybrid Reward Coverage ({total} questions) ===")
+print(f"  Rule-based:  {results['rule_ok']:4d} ({results['rule_ok']/total*100:.1f}%)")
+print(f"  SymPy:       {results['sympy_ok']:4d} ({results['sympy_ok']/total*100:.1f}%)")
+print(f"  TOTAL OK:    {covered:4d} ({covered/total*100:.1f}%)")
+print(f"  Failed:      {results['sympy_fail']+results['rule_wrong']:4d} ({(results['sympy_fail']+results['rule_wrong'])/total*100:.1f}%)")
+
+print(f"\n=== By Category ===")
+for cat, v in sorted(categories.items()):
+    t = v["rule_ok"] + v["sympy_ok"] + v["fail"]
+    ok = v["rule_ok"] + v["sympy_ok"]
+    print(f"  {cat:25s}: {ok}/{t} = {ok/t*100:.0f}%  [rule={v['rule_ok']}, sympy={v['sympy_ok']}, fail={v['fail']}]")
+
+print(f"\n=== Fail Examples ===")
+for ex in fail_examples:
+    print(f"  [{ex['cat']:20s}] {ex['gt']}")