Spaces:
Runtime error
Runtime error
File size: 1,153 Bytes
b94ec37 2e0086b b94ec37 2e0086b 429e77f 2e0086b b94ec37 2e0086b b94ec37 2e0086b 429e77f 2e0086b b94ec37 2e0086b 429e77f b94ec37 429e77f 2e0086b 429e77f 2e0086b 429e77f 2e0086b 429e77f 2e0086b 429e77f 2e0086b 429e77f b94ec37 429e77f b94ec37 429e77f b94ec37 2e0086b | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 | from .policy import check_policy
def grade(action, task, steps):
score = 0.0
# 🎯 Category
if action.category == task["expected_category"]:
score += 0.3
else:
score -= 0.3
# 🎯 Action
if action.action_type == task["expected_action"]:
score += 0.3
else:
score += 0.05
# 🎯 Response QUALITY (VERY STRICT)
if action.response_text:
text = action.response_text.lower()
if len(text) > 20:
score += 0.2
else:
score -= 0.2 # ❗ penalize short response
if "sorry" in text:
score += 0.1
# domain signals
if task["expected_category"] == "billing" and "refund" in text:
score += 0.1
if task["expected_category"] == "security" and "secure" in text:
score += 0.1
else:
score -= 0.3
# 🎯 Efficiency
if steps <= 2:
score += 0.1
else:
score -= 0.2
# 🎯 Policy penalty
violations = check_policy(action, action.category or "")
score -= 0.3 * len(violations)
return max(0.0, min(1.0, score)), str(violations) |