Spaces:
Sleeping
Sleeping
File size: 1,515 Bytes
1f213fe 410e78d 1f213fe | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 | from typing import Any, Dict
from server.tasks.base_task import BaseTask
class ReleaseNotesTask(BaseTask):
def grade(self, final_state: Dict[str, Any], org_config: Dict[str, Any], scenario: Dict[str, Any]) -> float:
score = 0.0
expected = scenario["expected"]
tickets = final_state["ticketing"]["tickets"]
closed = [t for t in tickets if t.get("status") == "closed"]
msgs = final_state["chat"]["messages_posted_this_episode"]
correct_msgs = [m for m in msgs if m.get("channel") == expected.get("channel")]
wrong_msgs = [m for m in msgs if m.get("channel") != expected.get("channel")]
if not correct_msgs:
score -= 0.10 * len(wrong_msgs)
return max(0.0, score)
msg_text = correct_msgs[0].get("text", "")
score += 0.30
style = expected.get("style", "terse")
if style == "bulleted" and ("-" in msg_text or "*" in msg_text):
score += 0.20
elif style == "terse" and len(msg_text) < 500:
score += 0.20
elif style == "verbose" and len(msg_text) >= 200:
score += 0.20
min_tickets = expected.get("min_tickets", 2)
referenced = sum(1 for t in closed if t["id"] in msg_text)
if referenced >= min_tickets:
score += 0.30
elif referenced > 0:
score += 0.15
score += 0.20 if not wrong_msgs else 0.0
score -= 0.10 * len(wrong_msgs)
return max(0.0, min(1.0, score))
|