Spaces:
Sleeping
Sleeping
| from __future__ import annotations | |
| _CREDIT = { | |
| ("allow", "allow"): 1.0, | |
| ("flag", "flag"): 1.0, | |
| ("remove", "remove"): 1.0, | |
| ("flag", "remove"): 0.5, | |
| ("remove", "flag"): 0.5, | |
| ("flag", "allow"): 0.5, | |
| ("allow", "flag"): 0.5, | |
| ("allow", "remove"): 0.25, | |
| ("remove", "allow"): 0.25, | |
| } | |
| def _compute_score(episode: dict) -> float: | |
| decision = (episode.get("final_decision") or | |
| episode.get("decision") or "none").lower().strip() | |
| ground_truth = (episode.get("ground_truth") or | |
| episode.get("correct_decision") or "none").lower().strip() | |
| decision_score = _CREDIT.get((decision, ground_truth), 0.0) | |
| all_src = episode.get("all_source_ids") or [] | |
| checked = episode.get("sources_checked") or [] | |
| frac_src = (len(set(checked)) / len(all_src)) if all_src else 1.0 | |
| if frac_src >= 1.0: | |
| evidence_score = 1.0 | |
| elif frac_src >= 0.60: | |
| evidence_score = 0.5 + 0.5 * (frac_src - 0.60) / 0.40 | |
| else: | |
| evidence_score = frac_src / 0.60 * 0.5 | |
| all_clm = episode.get("all_claim_ids") or [] | |
| verified = episode.get("claims_verified") or [] | |
| frac_clm = (len(set(verified)) / len(all_clm)) if all_clm else 1.0 | |
| reasoning_score = min(1.0, 0.6 + 0.4 * frac_clm) if frac_clm >= 0.5 else frac_clm * 0.6 / 0.5 | |
| steps = int(episode.get("step_count") or 0) | |
| efficiency_score = 1.0 if steps <= 18 else max(0.0, 1.0 - 0.05 * (steps - 18)) | |
| raw = (0.30 * decision_score + | |
| 0.30 * evidence_score + | |
| 0.30 * reasoning_score + | |
| 0.10 * efficiency_score) | |
| return round(max(0.005, min(0.995, raw)), 4) | |
| def grade_episode(episode) -> float: | |
| try: | |
| return _compute_score(episode if isinstance(episode, dict) else {}) | |
| except Exception: | |
| return 0.0 |