Spaces:
Sleeping
Sleeping
| from __future__ import annotations | |
| _CREDIT = { | |
| ("allow", "allow"): 1.0, | |
| ("flag", "flag"): 1.0, | |
| ("remove", "remove"): 1.0, | |
| ("flag", "remove"): 0.5, | |
| ("remove", "flag"): 0.5, | |
| ("flag", "allow"): 0.5, | |
| ("allow", "flag"): 0.5, | |
| ("allow", "remove"): 0.0, | |
| ("remove", "allow"): 0.0, | |
| } | |
| def _compute_score(episode: dict) -> float: | |
| decision = (episode.get("final_decision") or | |
| episode.get("decision") or "none").lower().strip() | |
| ground_truth = (episode.get("ground_truth") or | |
| episode.get("correct_decision") or "none").lower().strip() | |
| decision_score = _CREDIT.get((decision, ground_truth), 0.0) | |
| all_src = episode.get("all_source_ids") or [] | |
| checked = episode.get("sources_checked") or [] | |
| frac_src = (len(set(checked)) / len(all_src)) if all_src else 1.0 | |
| if frac_src >= 0.75: | |
| evidence_score = 1.0 | |
| elif frac_src >= 0.50: | |
| evidence_score = 0.6 + 0.4 * (frac_src - 0.50) / 0.25 | |
| elif frac_src >= 0.33: | |
| evidence_score = 0.3 + 0.3 * (frac_src - 0.33) / 0.17 | |
| else: | |
| evidence_score = frac_src / 0.33 * 0.3 | |
| all_clm = episode.get("all_claim_ids") or [] | |
| verified = episode.get("claims_verified") or [] | |
| frac_clm = (len(set(verified)) / len(all_clm)) if all_clm else 1.0 | |
| reasoning_score = min(1.0, frac_clm / 0.75) if frac_clm < 0.75 else 1.0 | |
| steps = int(episode.get("step_count") or 0) | |
| efficiency_score = 1.0 if steps <= 12 else max(0.0, 1.0 - 0.1 * (steps - 12)) | |
| raw = (0.35 * decision_score + | |
| 0.35 * evidence_score + | |
| 0.20 * reasoning_score + | |
| 0.10 * efficiency_score) | |
| return round(max(0.005, min(0.995, raw)), 4) | |
| def grade_episode(episode) -> float: | |
| try: | |
| return _compute_score(episode if isinstance(episode, dict) else {}) | |
| except Exception: | |
| return 0.0 |