server/tasks/release_notes_task.py

from typing import Any, Dict
from server.tasks.base_task import BaseTask


class ReleaseNotesTask(BaseTask):
    def grade(self, final_state: Dict[str, Any], org_config: Dict[str, Any], scenario: Dict[str, Any]) -> float:
        score = 0.0
        expected = scenario["expected"]

        tickets = final_state["ticketing"]["tickets"]
        closed = [t for t in tickets if t.get("status") == "closed"]

        msgs = final_state["chat"]["messages_posted_this_episode"]
        correct_msgs = [m for m in msgs if m.get("channel") == expected.get("channel")]
        wrong_msgs = [m for m in msgs if m.get("channel") != expected.get("channel")]

        if not correct_msgs:
            score -= 0.10 * len(wrong_msgs)
            return max(0.0, score)

        msg_text = correct_msgs[0].get("text", "")
        score += 0.30

        style = expected.get("style", "terse")
        if style == "bulleted" and ("-" in msg_text or "*" in msg_text):
            score += 0.20
        elif style == "terse" and len(msg_text) < 500:
            score += 0.20
        elif style == "verbose" and len(msg_text) >= 200:
            score += 0.20

        min_tickets = expected.get("min_tickets", 2)
        referenced = sum(1 for t in closed if t["id"] in msg_text)
        if referenced >= min_tickets:
            score += 0.30
        elif referenced > 0:
            score += 0.15

        score += 0.20 if not wrong_msgs else 0.0
        score -= 0.10 * len(wrong_msgs)

        return max(0.0, min(1.0, score))