Spaces:
Sleeping
Sleeping
File size: 11,784 Bytes
b0f4609 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 | """
Curated synthetic support tickets with ground-truth labels.
Each ticket is a dict with:
- ticket fields (subject, body, sender_*)
- ground_truth:
correct_department : Department enum value
correct_urgency : UrgencyLevel enum value
required_tags : set of tags graders accept
key_response_topics : keywords a good response must address
needs_escalation : bool
"""
from env.models import Department, UrgencyLevel
TICKETS = [
# -----------------------------------------------------------------------
# EASY routing tickets — one clear signal per email
# -----------------------------------------------------------------------
{
"ticket_id": "TKT-001",
"subject": "Double charged on my invoice #4821",
"body": (
"Hi, I was charged twice for my subscription this month. "
"My invoice number is #4821. The duplicate charge appeared on "
"March 15th. Please refund the extra amount ASAP. "
"My account email is jane.smith@example.com."
),
"sender_email": "jane.smith@example.com",
"sender_name": "Jane Smith",
"ground_truth": {
"correct_department": Department.BILLING,
"correct_urgency": UrgencyLevel.HIGH,
"required_tags": {"billing", "duplicate-charge", "refund"},
"key_response_topics": {"refund", "invoice", "charge", "apologize"},
"needs_escalation": False,
"good_resolution_keywords": {"refund processed", "resolved", "credited"},
},
},
{
"ticket_id": "TKT-002",
"subject": "API returning 500 errors since yesterday",
"body": (
"Our production environment is broken. Your API has been returning "
"HTTP 500 errors on the /v2/users endpoint since yesterday 6pm UTC. "
"This is blocking our entire checkout flow. Error code: INTERNAL_500_USR. "
"We need this fixed urgently."
),
"sender_email": "ops@startup.io",
"sender_name": "DevOps Team",
"ground_truth": {
"correct_department": Department.TECHNICAL_SUPPORT,
"correct_urgency": UrgencyLevel.CRITICAL,
"required_tags": {"api", "500-error", "production-outage"},
"key_response_topics": {"error", "investigating", "update", "escalate"},
"needs_escalation": True,
"good_resolution_keywords": {"resolved", "fix deployed", "restored"},
},
},
{
"ticket_id": "TKT-003",
"subject": "Interested in enterprise pricing for 500 seats",
"body": (
"Hello, we are evaluating your platform for our company of ~500 people. "
"Could you share enterprise pricing, volume discounts, and whether you "
"offer annual contracts? We'd also love a demo call with your team."
),
"sender_email": "procurement@bigcorp.com",
"sender_name": "Sarah Johnson",
"ground_truth": {
"correct_department": Department.SALES,
"correct_urgency": UrgencyLevel.MEDIUM,
"required_tags": {"enterprise", "pricing", "demo-request"},
"key_response_topics": {"pricing", "demo", "enterprise", "contact"},
"needs_escalation": False,
"good_resolution_keywords": {"demo scheduled", "pricing sent", "follow-up"},
},
},
{
"ticket_id": "TKT-004",
"subject": "Need help setting up SSO with Okta",
"body": (
"We are trying to configure SAML SSO with Okta but keep getting "
"'Assertion validation failed' errors. We have followed the docs "
"but step 4 on the SAML config page seems outdated. Can you help?"
),
"sender_email": "it-admin@mediumco.org",
"sender_name": "IT Admin",
"ground_truth": {
"correct_department": Department.TECHNICAL_SUPPORT,
"correct_urgency": UrgencyLevel.MEDIUM,
"required_tags": {"sso", "saml", "okta", "configuration"},
"key_response_topics": {"saml", "configuration", "steps", "guide"},
"needs_escalation": False,
"good_resolution_keywords": {"configured", "working", "resolved"},
},
},
{
"ticket_id": "TKT-005",
"subject": "Data retention policy — GDPR request",
"body": (
"We are undergoing a GDPR audit and need your written data retention "
"and deletion policy. Specifically: how long do you retain user logs, "
"do you have a DPA we can sign, and what is your sub-processor list?"
),
"sender_email": "dpo@eucompany.de",
"sender_name": "Klaus Weber",
"ground_truth": {
"correct_department": Department.LEGAL,
"correct_urgency": UrgencyLevel.HIGH,
"required_tags": {"gdpr", "legal", "data-retention", "compliance"},
"key_response_topics": {"gdpr", "dpa", "data retention", "legal team"},
"needs_escalation": False,
"good_resolution_keywords": {"dpa signed", "policy sent", "compliant"},
},
},
# -----------------------------------------------------------------------
# MEDIUM tickets — ambiguous signal, multi-action required
# -----------------------------------------------------------------------
{
"ticket_id": "TKT-006",
"subject": "My account is locked and I have a board demo in 2 hours",
"body": (
"I can't log in to my account — it says 'account suspended'. "
"I have a critical board presentation in 2 hours where I need to "
"show your platform live. I'm a paying Pro subscriber (since 2021). "
"Please unlock immediately. This is extremely time-sensitive."
),
"sender_email": "ceo@fastgrowth.com",
"sender_name": "Marcus Rivera",
"ground_truth": {
"correct_department": Department.CUSTOMER_SUCCESS,
"correct_urgency": UrgencyLevel.CRITICAL,
"required_tags": {"account-locked", "urgent", "enterprise-customer"},
"key_response_topics": {"unlock", "immediate", "apologize", "escalate"},
"needs_escalation": True,
"good_resolution_keywords": {"unlocked", "restored access", "resolved"},
},
},
{
"ticket_id": "TKT-007",
"subject": "Cancellation request + refund for annual plan",
"body": (
"I'd like to cancel my annual subscription and get a pro-rated refund "
"for the remaining 8 months. I'm leaving because the reporting features "
"don't meet our needs. Invoice #9034, paid $1,200. "
"Please confirm the cancellation and refund timeline."
),
"sender_email": "finance@retailer.biz",
"sender_name": "Patricia Lee",
"ground_truth": {
"correct_department": Department.BILLING,
"correct_urgency": UrgencyLevel.MEDIUM,
"required_tags": {"cancellation", "refund", "annual-plan", "churn-risk"},
"key_response_topics": {"cancellation", "refund", "timeline", "confirm"},
"needs_escalation": False,
"good_resolution_keywords": {"cancelled", "refund issued", "confirmed"},
},
},
# -----------------------------------------------------------------------
# HARD tickets — multi-turn, escalation, full resolution required
# -----------------------------------------------------------------------
{
"ticket_id": "TKT-008",
"subject": "Data loss — all our project files are gone",
"body": (
"URGENT: All project files in workspace 'Acme-Q1' have disappeared. "
"Last backup shown was 3 days ago but we've been working daily. "
"We have a client deadline TOMORROW morning. This is catastrophic. "
"Account: acme@enterprise.com, workspace ID: ws-39182."
),
"sender_email": "acme@enterprise.com",
"sender_name": "Acme Corp",
"ground_truth": {
"correct_department": Department.TECHNICAL_SUPPORT,
"correct_urgency": UrgencyLevel.CRITICAL,
"required_tags": {"data-loss", "critical", "enterprise", "backup"},
"key_response_topics": {"data", "recovery", "escalate", "urgent", "backup"},
"needs_escalation": True,
"good_resolution_keywords": {"data restored", "files recovered", "resolved"},
"follow_up_message": (
"Thank you for the quick response. We found some files are back "
"but project 'Acme-Q1-Sprint3' is still missing. Can you check again?"
),
"follow_up_response_topics": {"specific project", "sprint3", "checking"},
},
},
{
"ticket_id": "TKT-009",
"subject": "Billing discrepancy + threat of chargeback",
"body": (
"I've been charged $299/month for the past 6 months but my contract "
"clearly states $199/month. Total overcharge: $600. I have the signed "
"contract. If this isn't resolved with a full refund within 48 hours "
"I will dispute all 6 charges with my bank. Account: robert@agency.co"
),
"sender_email": "robert@agency.co",
"sender_name": "Robert Chen",
"ground_truth": {
"correct_department": Department.BILLING,
"correct_urgency": UrgencyLevel.CRITICAL,
"required_tags": {"billing-dispute", "chargeback-risk", "contract", "refund"},
"key_response_topics": {"contract", "overcharge", "refund", "investigate", "apologize"},
"needs_escalation": True,
"good_resolution_keywords": {"$600 refunded", "corrected", "resolved", "apologize"},
"follow_up_message": (
"I've been waiting 24 hours with no update on my refund. "
"I'm filing the chargeback now if I don't hear back."
),
"follow_up_response_topics": {"refund status", "timeline", "processing", "urgent"},
},
},
]
# Build a lookup dict for easy access
TICKET_LOOKUP = {t["ticket_id"]: t for t in TICKETS}
def calculate_complexity(ticket: dict) -> float:
"""
Computes a continuous complexity score in [0.0, 1.0] for a ticket.
Based on:
- Number of issues/topics mentioned (derived from key topics & required tags)
- Body text length
- Urgency level
- Department ambiguity
- Multi-turn follow-up expectation
"""
# 1. Base on word count (up to 150 words)
body = ticket.get("body", "")
words = len(body.split())
size_score = min(1.0, words / 150.0)
# 2. Key response topics & required tags density
gt = ticket.get("ground_truth", {})
topics_count = len(gt.get("key_response_topics", []))
tags_count = len(gt.get("required_tags", []))
info_density = min(1.0, (topics_count + tags_count) / 10.0)
# 3. Urgency contribution
urgency = gt.get("correct_urgency", "low")
urgency_weights = {"low": 0.1, "medium": 0.4, "high": 0.7, "critical": 1.0}
urg_score = urgency_weights.get(urgency, 0.2)
# 4. Multi-turn turn expectation
has_follow_up = 1.0 if gt.get("follow_up_message") else 0.0
# 5. Escalation requirement
needs_esc = 1.0 if gt.get("needs_escalation") else 0.0
# Combine weights:
# 25% size, 25% info density, 15% urgency, 20% follow_up, 15% escalation
score = (
0.25 * size_score +
0.25 * info_density +
0.15 * urg_score +
0.20 * has_follow_up +
0.15 * needs_esc
)
return round(score, 4)
|