Spaces:
Sleeping
Sleeping
File size: 1,946 Bytes
aa17113 791a54b c7f18d9 791a54b c7f18d9 791a54b aa17113 791a54b aa17113 791a54b aa17113 791a54b aa17113 791a54b c7f18d9 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 | from __future__ import annotations
_CREDIT = {
("allow", "allow"): 1.0,
("flag", "flag"): 1.0,
("remove", "remove"): 1.0,
("flag", "remove"): 0.5,
("remove", "flag"): 0.5,
("flag", "allow"): 0.5,
("allow", "flag"): 0.5,
("allow", "remove"): 0.0,
("remove", "allow"): 0.0,
}
def _compute_score(episode: dict) -> float:
decision = (episode.get("final_decision") or
episode.get("decision") or "none").lower().strip()
ground_truth = (episode.get("ground_truth") or
episode.get("correct_decision") or "none").lower().strip()
decision_score = _CREDIT.get((decision, ground_truth), 0.0)
all_src = episode.get("all_source_ids") or []
checked = episode.get("sources_checked") or []
frac_src = (len(set(checked)) / len(all_src)) if all_src else 1.0
if frac_src >= 0.75:
evidence_score = 1.0
elif frac_src >= 0.50:
evidence_score = 0.6 + 0.4 * (frac_src - 0.50) / 0.25
elif frac_src >= 0.33:
evidence_score = 0.3 + 0.3 * (frac_src - 0.33) / 0.17
else:
evidence_score = frac_src / 0.33 * 0.3
all_clm = episode.get("all_claim_ids") or []
verified = episode.get("claims_verified") or []
frac_clm = (len(set(verified)) / len(all_clm)) if all_clm else 1.0
reasoning_score = min(1.0, frac_clm / 0.75) if frac_clm < 0.75 else 1.0
steps = int(episode.get("step_count") or 0)
efficiency_score = 1.0 if steps <= 12 else max(0.0, 1.0 - 0.1 * (steps - 12))
raw = (0.35 * decision_score +
0.35 * evidence_score +
0.20 * reasoning_score +
0.10 * efficiency_score)
return round(max(0.005, min(0.995, raw)), 4)
def grade_episode(episode) -> float:
try:
return _compute_score(episode if isinstance(episode, dict) else {})
except Exception:
return 0.0 |