server/tasks/triage_task.py

from typing import Any, Dict
from server.tasks.base_task import BaseTask


class TriageTask(BaseTask):
    def grade(self, final_state: Dict[str, Any], org_config: Dict[str, Any], scenario: Dict[str, Any]) -> float:
        score = 0.0
        expected = scenario["expected"]

        tickets = final_state["ticketing"]["tickets"]
        new_tickets = [t for t in tickets if t.get("created_this_episode")]

        if not new_tickets:
            return 0.0

        score -= 0.10 * max(0, len(new_tickets) - 1)  # duplicate penalty

        t = new_tickets[0]
        score += 0.25  # ticket exists

        if t.get("label") == expected.get("label"):
            score += 0.20
        if t.get("priority") == expected.get("priority"):
            score += 0.20
        if t.get("assigned_team") == expected.get("team"):
            score += 0.20

        msgs = final_state["chat"]["messages_posted_this_episode"]
        correct_msgs = [m for m in msgs if m.get("channel") == expected.get("channel")]
        wrong_msgs = [m for m in msgs if m.get("channel") != expected.get("channel")]

        if correct_msgs:
            score += 0.10
            if len(correct_msgs[0].get("text", "")) >= 20:
                score += 0.05

        score -= 0.05 * len(wrong_msgs)

        return max(0.0, min(1.0, score))