from typing import Any, Dict from server.tasks.base_task import BaseTask class ReleaseNotesTask(BaseTask): def grade(self, final_state: Dict[str, Any], org_config: Dict[str, Any], scenario: Dict[str, Any]) -> float: score = 0.0 expected = scenario["expected"] tickets = final_state["ticketing"]["tickets"] closed = [t for t in tickets if t.get("status") == "closed"] msgs = final_state["chat"]["messages_posted_this_episode"] correct_msgs = [m for m in msgs if m.get("channel") == expected.get("channel")] wrong_msgs = [m for m in msgs if m.get("channel") != expected.get("channel")] if not correct_msgs: score -= 0.10 * len(wrong_msgs) return max(0.0, score) msg_text = correct_msgs[0].get("text", "") score += 0.30 style = expected.get("style", "terse") if style == "bulleted" and ("-" in msg_text or "*" in msg_text): score += 0.20 elif style == "terse" and len(msg_text) < 500: score += 0.20 elif style == "verbose" and len(msg_text) >= 200: score += 0.20 min_tickets = expected.get("min_tickets", 2) referenced = sum(1 for t in closed if t["id"] in msg_text) if referenced >= min_tickets: score += 0.30 elif referenced > 0: score += 0.15 score += 0.20 if not wrong_msgs else 0.0 score -= 0.10 * len(wrong_msgs) return max(0.0, min(1.0, score))