File size: 11,784 Bytes
b0f4609
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
"""
Curated synthetic support tickets with ground-truth labels.

Each ticket is a dict with:
  - ticket fields (subject, body, sender_*)
  - ground_truth:
      correct_department   : Department enum value
      correct_urgency      : UrgencyLevel enum value
      required_tags        : set of tags graders accept
      key_response_topics  : keywords a good response must address
      needs_escalation     : bool
"""

from env.models import Department, UrgencyLevel

TICKETS = [
    # -----------------------------------------------------------------------
    # EASY routing tickets — one clear signal per email
    # -----------------------------------------------------------------------
    {
        "ticket_id": "TKT-001",
        "subject": "Double charged on my invoice #4821",
        "body": (
            "Hi, I was charged twice for my subscription this month. "
            "My invoice number is #4821. The duplicate charge appeared on "
            "March 15th. Please refund the extra amount ASAP. "
            "My account email is jane.smith@example.com."
        ),
        "sender_email": "jane.smith@example.com",
        "sender_name": "Jane Smith",
        "ground_truth": {
            "correct_department": Department.BILLING,
            "correct_urgency": UrgencyLevel.HIGH,
            "required_tags": {"billing", "duplicate-charge", "refund"},
            "key_response_topics": {"refund", "invoice", "charge", "apologize"},
            "needs_escalation": False,
            "good_resolution_keywords": {"refund processed", "resolved", "credited"},
        },
    },
    {
        "ticket_id": "TKT-002",
        "subject": "API returning 500 errors since yesterday",
        "body": (
            "Our production environment is broken. Your API has been returning "
            "HTTP 500 errors on the /v2/users endpoint since yesterday 6pm UTC. "
            "This is blocking our entire checkout flow. Error code: INTERNAL_500_USR. "
            "We need this fixed urgently."
        ),
        "sender_email": "ops@startup.io",
        "sender_name": "DevOps Team",
        "ground_truth": {
            "correct_department": Department.TECHNICAL_SUPPORT,
            "correct_urgency": UrgencyLevel.CRITICAL,
            "required_tags": {"api", "500-error", "production-outage"},
            "key_response_topics": {"error", "investigating", "update", "escalate"},
            "needs_escalation": True,
            "good_resolution_keywords": {"resolved", "fix deployed", "restored"},
        },
    },
    {
        "ticket_id": "TKT-003",
        "subject": "Interested in enterprise pricing for 500 seats",
        "body": (
            "Hello, we are evaluating your platform for our company of ~500 people. "
            "Could you share enterprise pricing, volume discounts, and whether you "
            "offer annual contracts? We'd also love a demo call with your team."
        ),
        "sender_email": "procurement@bigcorp.com",
        "sender_name": "Sarah Johnson",
        "ground_truth": {
            "correct_department": Department.SALES,
            "correct_urgency": UrgencyLevel.MEDIUM,
            "required_tags": {"enterprise", "pricing", "demo-request"},
            "key_response_topics": {"pricing", "demo", "enterprise", "contact"},
            "needs_escalation": False,
            "good_resolution_keywords": {"demo scheduled", "pricing sent", "follow-up"},
        },
    },
    {
        "ticket_id": "TKT-004",
        "subject": "Need help setting up SSO with Okta",
        "body": (
            "We are trying to configure SAML SSO with Okta but keep getting "
            "'Assertion validation failed' errors. We have followed the docs "
            "but step 4 on the SAML config page seems outdated. Can you help?"
        ),
        "sender_email": "it-admin@mediumco.org",
        "sender_name": "IT Admin",
        "ground_truth": {
            "correct_department": Department.TECHNICAL_SUPPORT,
            "correct_urgency": UrgencyLevel.MEDIUM,
            "required_tags": {"sso", "saml", "okta", "configuration"},
            "key_response_topics": {"saml", "configuration", "steps", "guide"},
            "needs_escalation": False,
            "good_resolution_keywords": {"configured", "working", "resolved"},
        },
    },
    {
        "ticket_id": "TKT-005",
        "subject": "Data retention policy — GDPR request",
        "body": (
            "We are undergoing a GDPR audit and need your written data retention "
            "and deletion policy. Specifically: how long do you retain user logs, "
            "do you have a DPA we can sign, and what is your sub-processor list?"
        ),
        "sender_email": "dpo@eucompany.de",
        "sender_name": "Klaus Weber",
        "ground_truth": {
            "correct_department": Department.LEGAL,
            "correct_urgency": UrgencyLevel.HIGH,
            "required_tags": {"gdpr", "legal", "data-retention", "compliance"},
            "key_response_topics": {"gdpr", "dpa", "data retention", "legal team"},
            "needs_escalation": False,
            "good_resolution_keywords": {"dpa signed", "policy sent", "compliant"},
        },
    },
    # -----------------------------------------------------------------------
    # MEDIUM tickets — ambiguous signal, multi-action required
    # -----------------------------------------------------------------------
    {
        "ticket_id": "TKT-006",
        "subject": "My account is locked and I have a board demo in 2 hours",
        "body": (
            "I can't log in to my account — it says 'account suspended'. "
            "I have a critical board presentation in 2 hours where I need to "
            "show your platform live. I'm a paying Pro subscriber (since 2021). "
            "Please unlock immediately. This is extremely time-sensitive."
        ),
        "sender_email": "ceo@fastgrowth.com",
        "sender_name": "Marcus Rivera",
        "ground_truth": {
            "correct_department": Department.CUSTOMER_SUCCESS,
            "correct_urgency": UrgencyLevel.CRITICAL,
            "required_tags": {"account-locked", "urgent", "enterprise-customer"},
            "key_response_topics": {"unlock", "immediate", "apologize", "escalate"},
            "needs_escalation": True,
            "good_resolution_keywords": {"unlocked", "restored access", "resolved"},
        },
    },
    {
        "ticket_id": "TKT-007",
        "subject": "Cancellation request + refund for annual plan",
        "body": (
            "I'd like to cancel my annual subscription and get a pro-rated refund "
            "for the remaining 8 months. I'm leaving because the reporting features "
            "don't meet our needs. Invoice #9034, paid $1,200. "
            "Please confirm the cancellation and refund timeline."
        ),
        "sender_email": "finance@retailer.biz",
        "sender_name": "Patricia Lee",
        "ground_truth": {
            "correct_department": Department.BILLING,
            "correct_urgency": UrgencyLevel.MEDIUM,
            "required_tags": {"cancellation", "refund", "annual-plan", "churn-risk"},
            "key_response_topics": {"cancellation", "refund", "timeline", "confirm"},
            "needs_escalation": False,
            "good_resolution_keywords": {"cancelled", "refund issued", "confirmed"},
        },
    },
    # -----------------------------------------------------------------------
    # HARD tickets — multi-turn, escalation, full resolution required
    # -----------------------------------------------------------------------
    {
        "ticket_id": "TKT-008",
        "subject": "Data loss — all our project files are gone",
        "body": (
            "URGENT: All project files in workspace 'Acme-Q1' have disappeared. "
            "Last backup shown was 3 days ago but we've been working daily. "
            "We have a client deadline TOMORROW morning. This is catastrophic. "
            "Account: acme@enterprise.com, workspace ID: ws-39182."
        ),
        "sender_email": "acme@enterprise.com",
        "sender_name": "Acme Corp",
        "ground_truth": {
            "correct_department": Department.TECHNICAL_SUPPORT,
            "correct_urgency": UrgencyLevel.CRITICAL,
            "required_tags": {"data-loss", "critical", "enterprise", "backup"},
            "key_response_topics": {"data", "recovery", "escalate", "urgent", "backup"},
            "needs_escalation": True,
            "good_resolution_keywords": {"data restored", "files recovered", "resolved"},
            "follow_up_message": (
                "Thank you for the quick response. We found some files are back "
                "but project 'Acme-Q1-Sprint3' is still missing. Can you check again?"
            ),
            "follow_up_response_topics": {"specific project", "sprint3", "checking"},
        },
    },
    {
        "ticket_id": "TKT-009",
        "subject": "Billing discrepancy + threat of chargeback",
        "body": (
            "I've been charged $299/month for the past 6 months but my contract "
            "clearly states $199/month. Total overcharge: $600. I have the signed "
            "contract. If this isn't resolved with a full refund within 48 hours "
            "I will dispute all 6 charges with my bank. Account: robert@agency.co"
        ),
        "sender_email": "robert@agency.co",
        "sender_name": "Robert Chen",
        "ground_truth": {
            "correct_department": Department.BILLING,
            "correct_urgency": UrgencyLevel.CRITICAL,
            "required_tags": {"billing-dispute", "chargeback-risk", "contract", "refund"},
            "key_response_topics": {"contract", "overcharge", "refund", "investigate", "apologize"},
            "needs_escalation": True,
            "good_resolution_keywords": {"$600 refunded", "corrected", "resolved", "apologize"},
            "follow_up_message": (
                "I've been waiting 24 hours with no update on my refund. "
                "I'm filing the chargeback now if I don't hear back."
            ),
            "follow_up_response_topics": {"refund status", "timeline", "processing", "urgent"},
        },
    },
]

# Build a lookup dict for easy access
TICKET_LOOKUP = {t["ticket_id"]: t for t in TICKETS}


def calculate_complexity(ticket: dict) -> float:
    """
    Computes a continuous complexity score in [0.0, 1.0] for a ticket.
    Based on:
    - Number of issues/topics mentioned (derived from key topics & required tags)
    - Body text length
    - Urgency level
    - Department ambiguity
    - Multi-turn follow-up expectation
    """
    # 1. Base on word count (up to 150 words)
    body = ticket.get("body", "")
    words = len(body.split())
    size_score = min(1.0, words / 150.0)

    # 2. Key response topics & required tags density
    gt = ticket.get("ground_truth", {})
    topics_count = len(gt.get("key_response_topics", []))
    tags_count = len(gt.get("required_tags", []))
    info_density = min(1.0, (topics_count + tags_count) / 10.0)

    # 3. Urgency contribution
    urgency = gt.get("correct_urgency", "low")
    urgency_weights = {"low": 0.1, "medium": 0.4, "high": 0.7, "critical": 1.0}
    urg_score = urgency_weights.get(urgency, 0.2)

    # 4. Multi-turn turn expectation
    has_follow_up = 1.0 if gt.get("follow_up_message") else 0.0

    # 5. Escalation requirement
    needs_esc = 1.0 if gt.get("needs_escalation") else 0.0

    # Combine weights:
    # 25% size, 25% info density, 15% urgency, 20% follow_up, 15% escalation
    score = (
        0.25 * size_score +
        0.25 * info_density +
        0.15 * urg_score +
        0.20 * has_follow_up +
        0.15 * needs_esc
    )
    return round(score, 4)