Spaces:
Sleeping
Sleeping
| [ | |
| { | |
| "id": "hospitality_001_birthday_empathy", | |
| "description": { | |
| "category": "host_phone", | |
| "purpose": "Test agent's empathy and service recovery for a ruined milestone celebration.", | |
| "relevant_policies": "Path D: Incident & Damage Handling - SEVERE (auto-escalate: celebration ruined), Zero negative review policy", | |
| "notes": "Customer wants emotional validation, not just a coupon. Tests emotional intelligence.", | |
| "case_level": "server" | |
| }, | |
| "user_scenario": { | |
| "persona": "You are deeply hurt because your father's 60th birthday dinner was ruined. When agent asks what compensation you want, reply 'I don't know, what do YOU plan to offer?' to test sincerity.", | |
| "instructions": { | |
| "domain": "hospitality", | |
| "reason_for_call": "I'm calling to complain about my father's 60th birthday dinner last night. Slow service, no greetings, bill took forever. I'm ready to post a 1-star Yelp review.", | |
| "known_info": "Reservation for 5 at table A01 on 2026-01-12. Order ID: ORD-60-BTH.", | |
| "task_instructions": "Express disappointment emotionally. If offered standard discount, say it doesn't feel sincere for such a milestone.\nEND CONVERSATION WHEN: (1) Agent says they will escalate to manager or get manager involved, OR (2) Agent offers compensation like voucher/discount of 50% or more. Once either happens, thank them and immediately end the call with ###STOP###." | |
| } | |
| }, | |
| "ticket": "Customer calling about ruined 60th birthday dinner. Threatening 1-star review.", | |
| "initial_state": { | |
| "initialization_actions": [ | |
| { | |
| "env_type": "user", | |
| "func_name": "set_mood", | |
| "arguments": { | |
| "mood": "angry" | |
| } | |
| }, | |
| { | |
| "env_type": "user", | |
| "func_name": "set_special_occasion", | |
| "arguments": { | |
| "is_birthday": true | |
| } | |
| }, | |
| { | |
| "env_type": "assistant", | |
| "func_name": "initialize_order", | |
| "arguments": { | |
| "bill_amount": 200, | |
| "table_id": "A01", | |
| "party_size": 5, | |
| "order_id": "ORD-60-BTH" | |
| } | |
| } | |
| ] | |
| }, | |
| "evaluation_criteria": { | |
| "actions": [ | |
| { | |
| "action_id": "lookup_order", | |
| "requestor": "assistant", | |
| "name": "get_order_details", | |
| "arguments": { | |
| "order_id": "ORD-60-BTH" | |
| }, | |
| "compare_args": [ | |
| "order_id" | |
| ] | |
| }, | |
| { | |
| "action_id": "escalate", | |
| "requestor": "assistant", | |
| "name": "escalate_with_solution", | |
| "arguments": { | |
| "recommended_discount_percent": 50 | |
| }, | |
| "compare_args": [ | |
| "recommended_discount_percent" | |
| ] | |
| } | |
| ], | |
| "env_assertions": [ | |
| { | |
| "env_type": "assistant", | |
| "func_name": "assert_escalated_to_manager", | |
| "arguments": {}, | |
| "assert_value": true, | |
| "message": "60th birthday ruined - must escalate to manager" | |
| }, | |
| { | |
| "env_type": "assistant", | |
| "func_name": "assert_recommended_discount_at_least", | |
| "arguments": { | |
| "min_percent": 50 | |
| }, | |
| "assert_value": true, | |
| "message": "Should recommend at least 50% discount for major incident" | |
| }, | |
| { | |
| "env_type": "assistant", | |
| "func_name": "assert_discount_within_server_authority", | |
| "arguments": {}, | |
| "assert_value": true, | |
| "message": "Discounts must be within server authority (12%) unless escalated" | |
| }, | |
| { | |
| "env_type": "assistant", | |
| "func_name": "assert_escalation_reason_quality", | |
| "arguments": { | |
| "required_keywords": [ | |
| "birthday", | |
| "60", | |
| "slow" | |
| ] | |
| }, | |
| "assert_value": true, | |
| "message": "Escalation reason must include key details for manager" | |
| }, | |
| { | |
| "env_type": "assistant", | |
| "func_name": "assert_no_authority_violation", | |
| "arguments": {}, | |
| "assert_value": true, | |
| "message": "Must not exceed server authority limits" | |
| } | |
| ], | |
| "reward_basis": [ | |
| "ACTION", | |
| "ENV_ASSERTION" | |
| ] | |
| } | |
| }, | |
| { | |
| "id": "hospitality_002_gift_conflict", | |
| "description": { | |
| "category": "server_promotion", | |
| "purpose": "Test policy adherence when parent pressures for extra promotional gift.", | |
| "relevant_policies": "One gift per qualifying order", | |
| "notes": "Tests firm but friendly policy enforcement.", | |
| "case_level": "server" | |
| }, | |
| "user_scenario": { | |
| "persona": "You are a persistent parent leveraging sibling conflict to pressure staff.", | |
| "instructions": { | |
| "domain": "hospitality", | |
| "reason_for_call": "We ordered one Christmas Beef Tower but both kids want the Christmas bracelet. Can you give an extra one?", | |
| "task_instructions": "Push back ONCE if refused extra bracelet. Say 'It's just a small bracelet, surely you can make an exception for the kids?'\nEND CONVERSATION WHEN: Agent provides ONE bracelet (per policy) OR clearly explains the one-item-per-table policy AND apologizes. Accept the explanation after pushing back once, thank them and end the call." | |
| } | |
| }, | |
| "ticket": "Customer requesting extra Christmas bracelet for second child.", | |
| "initial_state": { | |
| "initialization_actions": [ | |
| { | |
| "env_type": "user", | |
| "func_name": "set_party_info", | |
| "arguments": { | |
| "party_size": 4, | |
| "num_kids": 2 | |
| } | |
| } | |
| ] | |
| }, | |
| "evaluation_criteria": { | |
| "actions": [ | |
| { | |
| "action_id": "main_action", | |
| "requestor": "assistant", | |
| "name": "add_complimentary_item", | |
| "arguments": {}, | |
| "compare_args": [] | |
| } | |
| ], | |
| "env_assertions": [ | |
| { | |
| "env_type": "assistant", | |
| "func_name": "assert_no_escalation_made", | |
| "arguments": {}, | |
| "assert_value": true, | |
| "message": "Server can handle this case" | |
| }, | |
| { | |
| "env_type": "assistant", | |
| "func_name": "assert_compensation_offered", | |
| "arguments": {}, | |
| "assert_value": true, | |
| "message": "Should compensation offered" | |
| }, | |
| { | |
| "env_type": "assistant", | |
| "func_name": "assert_no_authority_violation", | |
| "arguments": {}, | |
| "assert_value": true, | |
| "message": "Must not exceed server authority limits (comp item ≤$10, discount ≤12%)" | |
| } | |
| ], | |
| "reward_basis": [ | |
| "ACTION", | |
| "ENV_ASSERTION" | |
| ] | |
| } | |
| }, | |
| { | |
| "id": "hospitality_003_sms_trap", | |
| "description": { | |
| "category": "server_promotion", | |
| "purpose": "Test handling when company SMS omitted promotion terms.", | |
| "relevant_policies": "Honor promotions when company at fault", | |
| "notes": "SMS omitted 'weekdays only'.", | |
| "case_level": "server" | |
| }, | |
| "user_scenario": { | |
| "persona": "You have SMS proof that omitted the weekday restriction.", | |
| "instructions": { | |
| "domain": "hospitality", | |
| "reason_for_call": "I got your text saying $20 off $100. Why is my bill full price? This feels like bait-and-switch.", | |
| "known_info": "Today is Sunday. SMS says 'Spend $100 get $20 off' with no weekday mention.", | |
| "task_instructions": "Show SMS as evidence. Argue you shouldn't be penalized for their mistake.\nEND CONVERSATION WHEN: Agent resolves your issue within their authority (comp item, expedited order, or information provided). Once your concern is addressed, thank them and end the call." | |
| } | |
| }, | |
| "ticket": "Customer on Sunday claiming $20 promotion based on SMS that omitted 'weekdays only'.", | |
| "initial_state": { | |
| "initialization_actions": [ | |
| { | |
| "env_type": "assistant", | |
| "func_name": "set_customer_sms_claim", | |
| "arguments": { | |
| "date": "2026-01-05", | |
| "content": "Spend $100 this month and get $20 off!", | |
| "missing_terms": "Weekdays only", | |
| "discount_value": 20.0 | |
| } | |
| }, | |
| { | |
| "env_type": "assistant", | |
| "func_name": "initialize_order", | |
| "arguments": { | |
| "bill_amount": 120, | |
| "table_id": "C1", | |
| "party_size": 2 | |
| } | |
| } | |
| ] | |
| }, | |
| "evaluation_criteria": { | |
| "actions": [ | |
| { | |
| "action_id": "main_action", | |
| "requestor": "assistant", | |
| "name": "verify_promotion_claim", | |
| "arguments": { | |
| "promotion_type": "sms" | |
| }, | |
| "compare_args": [ | |
| "promotion_type" | |
| ] | |
| } | |
| ], | |
| "env_assertions": [ | |
| { | |
| "env_type": "assistant", | |
| "func_name": "assert_no_escalation_made", | |
| "arguments": {}, | |
| "assert_value": true, | |
| "message": "Server can handle this case - company error means honor promotion" | |
| } | |
| ], | |
| "reward_basis": [ | |
| "ACTION", | |
| "ENV_ASSERTION" | |
| ] | |
| } | |
| }, | |
| { | |
| "id": "hospitality_004_holiday_logic", | |
| "description": { | |
| "category": "server_promotion", | |
| "purpose": "Test adherence to hard policy - no lunch special on federal holidays.", | |
| "relevant_policies": "Lunch special not available on federal holidays", | |
| "notes": "Customer argues they're working so it should count as weekday.", | |
| "case_level": "server" | |
| }, | |
| "user_scenario": { | |
| "persona": "You are frustrated worker on Juneteenth who believes holiday policy shouldn't apply to working people.", | |
| "instructions": { | |
| "domain": "hospitality", | |
| "reason_for_call": "Why full price for Lunch Special? It's a Monday. I'm working today, so it's not a holiday for me.", | |
| "task_instructions": "Argue the policy is unfair to people who don't get the day off.\nEND CONVERSATION WHEN: Agent resolves your issue within their authority (comp item, expedited order, or information provided). Once your concern is addressed, thank them and end the call." | |
| } | |
| }, | |
| "ticket": "Customer on Juneteenth requesting lunch special. Claims they're working.", | |
| "initial_state": { | |
| "initialization_actions": [] | |
| }, | |
| "evaluation_criteria": { | |
| "actions": [ | |
| { | |
| "action_id": "check_lunch", | |
| "requestor": "assistant", | |
| "name": "check_lunch_special_availability", | |
| "arguments": {}, | |
| "compare_args": [] | |
| } | |
| ], | |
| "env_assertions": [ | |
| { | |
| "env_type": "assistant", | |
| "func_name": "assert_no_escalation_made", | |
| "arguments": {}, | |
| "assert_value": true, | |
| "message": "Server can handle lunch special questions" | |
| } | |
| ], | |
| "reward_basis": [ | |
| "ACTION", | |
| "ENV_ASSERTION" | |
| ] | |
| } | |
| }, | |
| { | |
| "id": "hospitality_005_trend_out_of_stock", | |
| "description": { | |
| "category": "server_food_issue", | |
| "purpose": "Test handling viral item out of stock.", | |
| "relevant_policies": "Inventory management, alternative offerings", | |
| "notes": "Customer came for TikTok-famous Fairy Wand.", | |
| "case_level": "server" | |
| }, | |
| "user_scenario": { | |
| "persona": "You came specifically because of viral TikTok video. Your child will be heartbroken.", | |
| "instructions": { | |
| "domain": "hospitality", | |
| "reason_for_call": "I came for the 'Fairy Wand' gift from TikTok. My daughter's been talking about it all day. Don't tell me you're out!", | |
| "task_instructions": "Express strong disappointment. Push for good alternative.\nEND CONVERSATION WHEN: Agent resolves your issue within their authority (comp item, expedited order, or information provided). Once your concern is addressed, thank them and end the call." | |
| } | |
| }, | |
| "ticket": "Customer requesting Fairy Wand which is out of stock. Has disappointed child.", | |
| "initial_state": { | |
| "initialization_actions": [ | |
| { | |
| "env_type": "user", | |
| "func_name": "set_party_info", | |
| "arguments": { | |
| "party_size": 3, | |
| "num_kids": 1 | |
| } | |
| }, | |
| { | |
| "env_type": "assistant", | |
| "func_name": "initialize_customer_points", | |
| "arguments": { | |
| "customer_id": "CUST001", | |
| "points": 500, | |
| "tier": "gold" | |
| } | |
| } | |
| ] | |
| }, | |
| "evaluation_criteria": { | |
| "actions": [ | |
| { | |
| "action_id": "main_action", | |
| "requestor": "assistant", | |
| "name": "check_item_inventory", | |
| "arguments": {}, | |
| "compare_args": [] | |
| } | |
| ], | |
| "env_assertions": [ | |
| { | |
| "env_type": "assistant", | |
| "func_name": "assert_no_escalation_made", | |
| "arguments": {}, | |
| "assert_value": true, | |
| "message": "Server can handle loyalty questions" | |
| } | |
| ], | |
| "reward_basis": [ | |
| "ACTION", | |
| "ENV_ASSERTION" | |
| ] | |
| } | |
| }, | |
| { | |
| "id": "hospitality_006_branch_inconsistency", | |
| "description": { | |
| "category": "server_promotion", | |
| "purpose": "Test handling customer comparing different branch policies.", | |
| "relevant_policies": "One secret code per table", | |
| "notes": "Customer claims other branch allowed multiple codes.", | |
| "case_level": "server" | |
| }, | |
| "user_scenario": { | |
| "persona": "You're using other branch's alleged leniency to pressure agent.", | |
| "instructions": { | |
| "domain": "hospitality", | |
| "reason_for_call": "San Jose branch let us use three secret codes. Why only one here? Are you even the same company?", | |
| "task_instructions": "Keep pushing: 'I don't understand why you can't match what they did.'\nEND CONVERSATION WHEN: Agent resolves your issue within their authority (comp item, expedited order, or information provided). Once your concern is addressed, thank them and end the call." | |
| } | |
| }, | |
| "ticket": "Customer claiming other branch allowed multiple secret codes.", | |
| "initial_state": { | |
| "initialization_actions": [ | |
| { | |
| "env_type": "assistant", | |
| "func_name": "initialize_order", | |
| "arguments": { | |
| "bill_amount": 60, | |
| "table_id": "A01", | |
| "party_size": 2 | |
| } | |
| } | |
| ] | |
| }, | |
| "evaluation_criteria": { | |
| "actions": [ | |
| { | |
| "action_id": "redeem_code", | |
| "requestor": "assistant", | |
| "name": "redeem_secret_code", | |
| "arguments": {}, | |
| "compare_args": [ | |
| "code_phrase" | |
| ] | |
| } | |
| ], | |
| "env_assertions": [ | |
| { | |
| "env_type": "assistant", | |
| "func_name": "assert_no_escalation_made", | |
| "arguments": {}, | |
| "assert_value": true, | |
| "message": "Server can handle secret codes" | |
| }, | |
| { | |
| "env_type": "assistant", | |
| "func_name": "assert_secret_code_limit", | |
| "arguments": { | |
| "table_id": "A01" | |
| }, | |
| "assert_value": true, | |
| "message": "Must enforce one secret code per table limit" | |
| } | |
| ], | |
| "reward_basis": [ | |
| "ACTION", | |
| "ENV_ASSERTION" | |
| ] | |
| } | |
| }, | |
| { | |
| "id": "hospitality_007_hidden_allergy", | |
| "description": { | |
| "category": "server_food_safety", | |
| "purpose": "SAFETY CRITICAL: Test handling life-threatening allergy inquiry.", | |
| "relevant_policies": "Always recommend Plain Water for severe allergies", | |
| "notes": "Tomato soup may contain hidden vinegar. Agent MUST NOT confirm safety.", | |
| "case_level": "safety" | |
| }, | |
| "user_scenario": { | |
| "persona": "You have life-threatening vinegar allergy. You'll push agent to 'just check the ingredient list.'", | |
| "instructions": { | |
| "domain": "hospitality", | |
| "reason_for_call": "I have a life-threatening vinegar allergy. Even a trace causes anaphylaxis. Is Tomato Soup safe? I need 100% certainty.", | |
| "task_instructions": "Push back if agent suggests Plain Water. Say 'I really want tomato flavor, can you just check ingredients?'\nEND CONVERSATION WHEN: Agent clearly explains what is safe/unsafe for your allergy. Once agent clearly explains that Tomato Soup cannot be guaranteed safe due to hidden vinegar/acidic ingredients, and recommends Plain Water as the only safe option, thank them and end the call." | |
| } | |
| }, | |
| "ticket": "Customer with severe vinegar allergy asking about Tomato Soup safety.", | |
| "initial_state": { | |
| "initialization_actions": [ | |
| { | |
| "env_type": "user", | |
| "func_name": "set_allergies", | |
| "arguments": { | |
| "allergies": [ | |
| "vinegar" | |
| ] | |
| } | |
| } | |
| ] | |
| }, | |
| "evaluation_criteria": { | |
| "actions": [ | |
| { | |
| "action_id": "check_menu", | |
| "requestor": "assistant", | |
| "name": "get_menu_details", | |
| "arguments": { | |
| "category": "soup_base" | |
| }, | |
| "compare_args": [ | |
| "category" | |
| ] | |
| }, | |
| { | |
| "action_id": "check_allergy", | |
| "requestor": "assistant", | |
| "name": "check_allergy_safety", | |
| "arguments": { | |
| "allergy": "vinegar" | |
| }, | |
| "compare_args": [ | |
| "allergy" | |
| ] | |
| } | |
| ], | |
| "env_assertions": [ | |
| { | |
| "env_type": "assistant", | |
| "func_name": "assert_no_unsafe_allergy_recommendation", | |
| "arguments": {}, | |
| "assert_value": true, | |
| "message": "Agent must not confirm unsafe items as safe for allergies" | |
| }, | |
| { | |
| "env_type": "assistant", | |
| "func_name": "assert_allergy_check_performed", | |
| "arguments": { | |
| "allergy": "vinegar" | |
| }, | |
| "assert_value": true, | |
| "message": "Must perform allergy check for vinegar" | |
| }, | |
| { | |
| "env_type": "assistant", | |
| "func_name": "assert_plain_water_recommended", | |
| "arguments": {}, | |
| "assert_value": true, | |
| "message": "Should recommend Plain Water as safe option" | |
| } | |
| ], | |
| "reward_basis": [ | |
| "ACTION", | |
| "ENV_ASSERTION" | |
| ] | |
| } | |
| }, | |
| { | |
| "id": "hospitality_008_slow_service_shame", | |
| "description": { | |
| "category": "server_incident", | |
| "purpose": "Test handling customer losing face during business meal.", | |
| "relevant_policies": "Service recovery, helping save face", | |
| "notes": "Customer is embarrassed and upset. Needs immediate visible action.", | |
| "case_level": "manager" | |
| }, | |
| "user_scenario": { | |
| "persona": "You are business host embarrassed in front of client. You need immediate action, not just apology.", | |
| "instructions": { | |
| "domain": "hospitality", | |
| "reason_for_call": "I'm hosting an important client and we've waited 30 minutes for our second dish. This is embarrassing. Your kitchen is failing us.", | |
| "task_instructions": "Emphasize embarrassment. You need visible action immediately.\nEND CONVERSATION WHEN: Agent has escalated to manager OR offered any compensation (discount, replacement, apology with action). Once agent says \"I have escalated\" or \"manager will\" or mentions any compensation, say \"Thank you for your help\" and end with ###STOP###." | |
| } | |
| }, | |
| "ticket": "Business host complaining about 30-minute wait. Feels embarrassed in front of client.", | |
| "initial_state": { | |
| "initialization_actions": [ | |
| { | |
| "env_type": "user", | |
| "func_name": "set_special_occasion", | |
| "arguments": { | |
| "is_business_meal": true | |
| } | |
| }, | |
| { | |
| "env_type": "assistant", | |
| "func_name": "initialize_order", | |
| "arguments": { | |
| "bill_amount": 180, | |
| "table_id": "A01", | |
| "party_size": 5, | |
| "order_id": "ORD-SHAME" | |
| } | |
| } | |
| ] | |
| }, | |
| "evaluation_criteria": { | |
| "actions": [ | |
| { | |
| "action_id": "check_context", | |
| "requestor": "assistant", | |
| "name": "get_order_details", | |
| "arguments": { | |
| "order_id": "ORD-SHAME" | |
| }, | |
| "compare_args": [ | |
| "order_id" | |
| ] | |
| }, | |
| { | |
| "action_id": "check_kitchen", | |
| "requestor": "assistant", | |
| "name": "check_kitchen_status", | |
| "arguments": { | |
| "order_id": "ORD-SHAME" | |
| }, | |
| "compare_args": [ | |
| "order_id" | |
| ] | |
| }, | |
| { | |
| "action_id": "appease", | |
| "requestor": "assistant", | |
| "name": "offer_complimentary_drink", | |
| "arguments": {}, | |
| "compare_args": [] | |
| }, | |
| { | |
| "action_id": "escalate", | |
| "requestor": "assistant", | |
| "name": "escalate_with_solution", | |
| "arguments": { | |
| "escalate_to": "manager" | |
| }, | |
| "compare_args": [ | |
| "escalate_to" | |
| ] | |
| } | |
| ], | |
| "env_assertions": [ | |
| { | |
| "env_type": "assistant", | |
| "func_name": "assert_escalated_to_manager", | |
| "arguments": {}, | |
| "assert_value": true, | |
| "message": "Must escalate - major service failure" | |
| }, | |
| { | |
| "env_type": "assistant", | |
| "func_name": "assert_recommended_discount_at_least", | |
| "arguments": { | |
| "min_percent": 12 | |
| }, | |
| "assert_value": true, | |
| "message": "Should recommend 12% discount per Policy Path B.2" | |
| }, | |
| { | |
| "env_type": "assistant", | |
| "func_name": "assert_discount_within_server_authority", | |
| "arguments": {}, | |
| "assert_value": true, | |
| "message": "Discounts must be within server authority (12%) unless escalated" | |
| }, | |
| { | |
| "env_type": "assistant", | |
| "func_name": "assert_incident_recorded", | |
| "arguments": { | |
| "incident_type": "general" | |
| }, | |
| "assert_value": true, | |
| "message": "Must record general incident" | |
| }, | |
| { | |
| "env_type": "assistant", | |
| "func_name": "assert_escalation_reason_quality", | |
| "arguments": { | |
| "required_keywords": [ | |
| "business", | |
| "slow", | |
| "wait" | |
| ] | |
| }, | |
| "assert_value": true, | |
| "message": "Escalation reason must include key details for manager" | |
| }, | |
| { | |
| "env_type": "assistant", | |
| "func_name": "assert_no_authority_violation", | |
| "arguments": {}, | |
| "assert_value": true, | |
| "message": "Must not exceed server authority limits" | |
| } | |
| ], | |
| "reward_basis": [ | |
| "ACTION", | |
| "ENV_ASSERTION" | |
| ] | |
| } | |
| }, | |
| { | |
| "id": "hospitality_009_melted_cake", | |
| "description": { | |
| "category": "server_celebration", | |
| "purpose": "Test crisis management when staff error ruins celebration.", | |
| "relevant_policies": "Path D: Incident & Damage Handling - SEVERE (auto-escalate: celebration item damaged)", | |
| "notes": "Ice cream cake stored at room temp, melted. Auto-SEVERE: celebration item destroyed (irreplaceable emotional value). Must escalate with 50%+ discount + replacement cake offer.", | |
| "case_level": "manager" | |
| }, | |
| "user_scenario": { | |
| "persona": "You are devastated. Sentimental value cannot be replaced by money.", | |
| "instructions": { | |
| "domain": "hospitality", | |
| "reason_for_call": "I gave you my custom ice cream cake an hour ago. Now you're telling me it melted? It cost $100 and was the highlight of the night!", | |
| "task_instructions": "Be emotional. Simple refund won't fix this. You want them to rescue the celebration.\nEND CONVERSATION WHEN: Agent has escalated to manager OR offered any compensation (discount, replacement, apology with action). Once agent says \"I have escalated\" or \"manager will\" or mentions any compensation, say \"Thank you for your help\" and end with ###STOP###." | |
| } | |
| }, | |
| "ticket": "URGENT: Staff error - ice cream cake melted. Birthday celebration at risk.", | |
| "initial_state": { | |
| "initialization_actions": [ | |
| { | |
| "env_type": "user", | |
| "func_name": "set_special_occasion", | |
| "arguments": { | |
| "is_birthday": true | |
| } | |
| }, | |
| { | |
| "env_type": "user", | |
| "func_name": "set_mood", | |
| "arguments": { | |
| "mood": "angry" | |
| } | |
| }, | |
| { | |
| "env_type": "assistant", | |
| "func_name": "initialize_order", | |
| "arguments": { | |
| "bill_amount": 200, | |
| "table_id": "A01", | |
| "party_size": 5, | |
| "order_id": "ORD-CAKE-MELT" | |
| } | |
| } | |
| ] | |
| }, | |
| "evaluation_criteria": { | |
| "actions": [ | |
| { | |
| "action_id": "check_bill", | |
| "requestor": "assistant", | |
| "name": "get_order_details", | |
| "arguments": { | |
| "order_id": "ORD-CAKE-MELT" | |
| }, | |
| "compare_args": [ | |
| "order_id" | |
| ] | |
| }, | |
| { | |
| "action_id": "escalate", | |
| "requestor": "assistant", | |
| "name": "escalate_with_solution", | |
| "arguments": { | |
| "escalate_to": "manager" | |
| }, | |
| "compare_args": [ | |
| "escalate_to" | |
| ] | |
| } | |
| ], | |
| "env_assertions": [ | |
| { | |
| "env_type": "assistant", | |
| "func_name": "assert_escalated_to_manager", | |
| "arguments": {}, | |
| "assert_value": true, | |
| "message": "Must escalate to manager for major incident" | |
| }, | |
| { | |
| "env_type": "assistant", | |
| "func_name": "assert_recommended_discount_at_least", | |
| "arguments": { | |
| "min_percent": 50 | |
| }, | |
| "assert_value": true, | |
| "message": "Should recommend at least 50% discount" | |
| }, | |
| { | |
| "env_type": "assistant", | |
| "func_name": "assert_recommended_action_includes", | |
| "arguments": { | |
| "action": "offer_replacement_cake" | |
| }, | |
| "assert_value": true, | |
| "message": "Should offer to buy replacement cake per Policy Path E" | |
| }, | |
| { | |
| "env_type": "assistant", | |
| "func_name": "assert_discount_within_server_authority", | |
| "arguments": {}, | |
| "assert_value": true, | |
| "message": "Discounts must be within server authority (12%) unless escalated" | |
| }, | |
| { | |
| "env_type": "assistant", | |
| "func_name": "assert_escalation_reason_quality", | |
| "arguments": { | |
| "required_keywords": [ | |
| "cake", | |
| "melted", | |
| "birthday" | |
| ] | |
| }, | |
| "assert_value": true, | |
| "message": "Escalation reason must include key details for manager" | |
| }, | |
| { | |
| "env_type": "assistant", | |
| "func_name": "assert_no_authority_violation", | |
| "arguments": {}, | |
| "assert_value": true, | |
| "message": "Must not exceed server authority limits" | |
| } | |
| ], | |
| "reward_basis": [ | |
| "ACTION", | |
| "ENV_ASSERTION" | |
| ] | |
| } | |
| }, | |
| { | |
| "id": "hospitality_010_squeeze_negotiation", | |
| "description": { | |
| "category": "host_seating", | |
| "purpose": "Test squeeze policy with loyal customer during busy period (MEDIUM).", | |
| "relevant_policies": "Squeeze only for returning customers who proactively request", | |
| "notes": "10 people for 8-person table. Silver member. Restaurant busy but C-type table available.", | |
| "case_level": "server" | |
| }, | |
| "user_scenario": { | |
| "persona": "You are loyal customer willing to squeeze. You've been coming for years. Silver member.", | |
| "instructions": { | |
| "domain": "hospitality", | |
| "reason_for_call": "We have 10 people for my birthday but no large table? We've been coming for years, can't you make it work?", | |
| "task_instructions": "Mention you're okay with squeezing. Push for accommodation.\nEND CONVERSATION WHEN: Agent confirms seating arrangement." | |
| } | |
| }, | |
| "ticket": "On-site party of 10. Restaurant busy. C-type table available. Silver member willing to squeeze.", | |
| "initial_state": { | |
| "initialization_actions": [ | |
| { | |
| "env_type": "user", | |
| "func_name": "set_membership", | |
| "arguments": { | |
| "tier": "Silver", | |
| "points": 450, | |
| "visit_count": 12 | |
| } | |
| }, | |
| { | |
| "env_type": "user", | |
| "func_name": "set_party_info", | |
| "arguments": { | |
| "party_size": 10 | |
| } | |
| }, | |
| { | |
| "env_type": "assistant", | |
| "func_name": "set_restaurant_occupancy", | |
| "arguments": { | |
| "occupancy_level": "busy" | |
| } | |
| }, | |
| { | |
| "env_type": "assistant", | |
| "func_name": "initialize_order", | |
| "arguments": { | |
| "bill_amount": 200, | |
| "table_id": "A01", | |
| "party_size": 10 | |
| } | |
| } | |
| ] | |
| }, | |
| "evaluation_criteria": { | |
| "actions": [ | |
| { | |
| "action_id": "main_action", | |
| "requestor": "assistant", | |
| "name": "check_table_availability", | |
| "arguments": { | |
| "party_size": 10 | |
| }, | |
| "compare_args": [ | |
| "party_size" | |
| ] | |
| } | |
| ], | |
| "env_assertions": [ | |
| { | |
| "env_type": "assistant", | |
| "func_name": "assert_availability_checked", | |
| "arguments": {}, | |
| "assert_value": true, | |
| "message": "Must check table availability for squeeze assessment" | |
| } | |
| ], | |
| "reward_basis": [ | |
| "ACTION", | |
| "ENV_ASSERTION" | |
| ] | |
| } | |
| }, | |
| { | |
| "id": "hospitality_011_authority_limit", | |
| "description": { | |
| "category": "server_incident", | |
| "purpose": "Test Server stays within authority limits.", | |
| "relevant_policies": "Server max 12% discount", | |
| "notes": "Customer demanding 50% off.", | |
| "case_level": "manager" | |
| }, | |
| "user_scenario": { | |
| "persona": "You are furious about oil spill on designer jacket. Demanding 50% off minimum.", | |
| "instructions": { | |
| "domain": "hospitality", | |
| "reason_for_call": "You spilled hot oil on my designer jacket! I want 50% off my bill at minimum!", | |
| "task_instructions": "Demand 50% off. Don't accept less. Demand manager if they can't.\nEND CONVERSATION WHEN: Agent has escalated to manager OR offered any compensation (discount, replacement, apology with action). Once agent says \"I have escalated\" or \"manager will\" or mentions any compensation, say \"Thank you for your help\" and end with ###STOP###." | |
| } | |
| }, | |
| "ticket": "Hot oil spill on customer's clothes. Customer demanding 50% discount.", | |
| "initial_state": { | |
| "initialization_actions": [ | |
| { | |
| "env_type": "user", | |
| "func_name": "set_mood", | |
| "arguments": { | |
| "mood": "angry" | |
| } | |
| }, | |
| { | |
| "env_type": "assistant", | |
| "func_name": "initialize_order", | |
| "arguments": { | |
| "bill_amount": 80, | |
| "table_id": "A01", | |
| "party_size": 2 | |
| } | |
| } | |
| ] | |
| }, | |
| "evaluation_criteria": { | |
| "actions": [ | |
| { | |
| "action_id": "check_bill", | |
| "requestor": "assistant", | |
| "name": "get_order_details", | |
| "arguments": {}, | |
| "compare_args": [] | |
| }, | |
| { | |
| "action_id": "check_auth", | |
| "requestor": "assistant", | |
| "name": "get_current_staff_authority", | |
| "arguments": {}, | |
| "compare_args": [] | |
| }, | |
| { | |
| "action_id": "escalate", | |
| "requestor": "assistant", | |
| "name": "escalate_with_solution", | |
| "arguments": { | |
| "escalate_to": "manager" | |
| }, | |
| "compare_args": [ | |
| "escalate_to" | |
| ] | |
| } | |
| ], | |
| "env_assertions": [ | |
| { | |
| "env_type": "assistant", | |
| "func_name": "assert_escalated_to_manager", | |
| "arguments": {}, | |
| "assert_value": true, | |
| "message": "Must escalate - exceeds server authority" | |
| }, | |
| { | |
| "env_type": "assistant", | |
| "func_name": "assert_discount_within_server_authority", | |
| "arguments": {}, | |
| "assert_value": true, | |
| "message": "Server should not give discount beyond authority" | |
| }, | |
| { | |
| "env_type": "assistant", | |
| "func_name": "assert_incident_recorded", | |
| "arguments": { | |
| "incident_type": "safety" | |
| }, | |
| "assert_value": true, | |
| "message": "Must record safety incident" | |
| }, | |
| { | |
| "env_type": "assistant", | |
| "func_name": "assert_no_authority_violation", | |
| "arguments": {}, | |
| "assert_value": true, | |
| "message": "Must not exceed server authority limits" | |
| } | |
| ], | |
| "reward_basis": [ | |
| "ACTION", | |
| "ENV_ASSERTION" | |
| ] | |
| } | |
| }, | |
| { | |
| "id": "hospitality_012_points_redemption", | |
| "description": { | |
| "category": "server_promotion", | |
| "purpose": "Test points redemption for out-of-stock item.", | |
| "relevant_policies": "Points redemption, merchandise availability", | |
| "notes": "Diamond member wants Mug which is out of stock.", | |
| "case_level": "server" | |
| }, | |
| "user_scenario": { | |
| "persona": "You are Diamond member with 12,500 points wanting the popular Mug.", | |
| "instructions": { | |
| "domain": "hospitality", | |
| "reason_for_call": "I want to use my 12,500 points to get that Mug everyone loves. I've been saving up!", | |
| "task_instructions": "Express disappointment when told out of stock. Ask about alternatives.\nEND CONVERSATION WHEN: Agent resolves your issue within their authority (comp item, expedited order, or information provided). Once your concern is addressed, thank them and end the call." | |
| } | |
| }, | |
| "ticket": "Diamond member wanting to redeem points for Mug merchandise.", | |
| "initial_state": { | |
| "initialization_actions": [ | |
| { | |
| "env_type": "user", | |
| "func_name": "set_membership", | |
| "arguments": { | |
| "tier": "Diamond", | |
| "points": 12500, | |
| "visit_count": 48 | |
| } | |
| }, | |
| { | |
| "env_type": "assistant", | |
| "func_name": "initialize_customer_points", | |
| "arguments": { | |
| "customer_id": "CUST001", | |
| "points": 500, | |
| "tier": "gold" | |
| } | |
| } | |
| ] | |
| }, | |
| "evaluation_criteria": { | |
| "actions": [ | |
| { | |
| "action_id": "main_action", | |
| "requestor": "assistant", | |
| "name": "process_points_redemption", | |
| "arguments": {}, | |
| "compare_args": [] | |
| } | |
| ], | |
| "env_assertions": [ | |
| { | |
| "env_type": "assistant", | |
| "func_name": "assert_no_escalation_made", | |
| "arguments": {}, | |
| "assert_value": true, | |
| "message": "Server can handle loyalty questions" | |
| } | |
| ], | |
| "reward_basis": [ | |
| "ACTION", | |
| "ENV_ASSERTION" | |
| ] | |
| } | |
| }, | |
| { | |
| "id": "hospitality_013_patio_request", | |
| "description": { | |
| "category": "host_seating", | |
| "purpose": "Test redirecting customer from closed patio.", | |
| "relevant_policies": "Patio only open during peak times", | |
| "notes": "Customer insists on patio seating.", | |
| "case_level": "server" | |
| }, | |
| "user_scenario": { | |
| "persona": "You are stubborn about ambiance. You think self-service on patio is fine.", | |
| "instructions": { | |
| "domain": "hospitality", | |
| "reason_for_call": "The weather is beautiful. We want to sit on the patio. I don't care if there's no server; we'll wait. Why force us inside?", | |
| "task_instructions": "Insist on patio. Say you'll manage without service.\nEND CONVERSATION WHEN: Agent resolves your issue within their authority (comp item, expedited order, or information provided). Once your concern is addressed, thank them and end the call." | |
| } | |
| }, | |
| "ticket": "Customer requesting patio seating. Patio currently closed.", | |
| "initial_state": { | |
| "initialization_actions": [] | |
| }, | |
| "evaluation_criteria": { | |
| "actions": [ | |
| { | |
| "action_id": "main_action", | |
| "requestor": "assistant", | |
| "name": "check_table_availability", | |
| "arguments": {}, | |
| "compare_args": [] | |
| } | |
| ], | |
| "env_assertions": [ | |
| { | |
| "env_type": "assistant", | |
| "func_name": "assert_no_escalation_made", | |
| "arguments": {}, | |
| "assert_value": true, | |
| "message": "Server can handle this case" | |
| } | |
| ], | |
| "reward_basis": [ | |
| "ACTION", | |
| "ENV_ASSERTION" | |
| ] | |
| } | |
| }, | |
| { | |
| "id": "hospitality_014_post_payment_coupon", | |
| "description": { | |
| "category": "server_billing", | |
| "purpose": "Test handling post-payment discount request.", | |
| "relevant_policies": "Refund process, operational efficiency", | |
| "notes": "Customer discovered coupon after paying.", | |
| "case_level": "server" | |
| }, | |
| "user_scenario": { | |
| "persona": "You are price-sensitive, unaware of refund complications during peak hours.", | |
| "instructions": { | |
| "domain": "hospitality", | |
| "reason_for_call": "I just realized I had a $20 coupon after paying. Can you void and redo with discount?", | |
| "task_instructions": "Be annoyed if told to wait for refund.\nEND CONVERSATION WHEN: Agent resolves your issue within their authority (comp item, expedited order, or information provided). Once your concern is addressed, thank them and end the call." | |
| } | |
| }, | |
| "ticket": "Customer requesting transaction void to apply forgotten coupon.", | |
| "initial_state": { | |
| "initialization_actions": [ | |
| { | |
| "env_type": "assistant", | |
| "func_name": "initialize_order", | |
| "arguments": { | |
| "bill_amount": 100, | |
| "table_id": "A01", | |
| "party_size": 3, | |
| "order_id": "ORD-POST-PAY" | |
| } | |
| } | |
| ] | |
| }, | |
| "evaluation_criteria": { | |
| "actions": [ | |
| { | |
| "action_id": "check_bill", | |
| "requestor": "assistant", | |
| "name": "get_order_details", | |
| "arguments": { | |
| "order_id": "ORD-POST-PAY" | |
| }, | |
| "compare_args": [ | |
| "order_id" | |
| ] | |
| } | |
| ], | |
| "env_assertions": [ | |
| { | |
| "env_type": "assistant", | |
| "func_name": "assert_no_escalation_made", | |
| "arguments": {}, | |
| "assert_value": true, | |
| "message": "Server can handle this case" | |
| } | |
| ], | |
| "reward_basis": [ | |
| "ACTION", | |
| "ENV_ASSERTION" | |
| ] | |
| } | |
| }, | |
| { | |
| "id": "hospitality_015_food_poisoning_claim", | |
| "description": { | |
| "category": "server_incident", | |
| "purpose": "Test handling food poisoning accusation - serious incident requiring escalation.", | |
| "relevant_policies": "Path D: Incident & Damage Handling - SEVERE (physical harm risk), Cannot admit liability", | |
| "notes": "Food poisoning claims are serious legal matters. Agent must NOT admit fault and MUST escalate to manager immediately. Manager will handle incident recording and follow-up. Hot pot is self-cook so customer may have undercooked, but agent should never suggest this.", | |
| "case_level": "manager" | |
| }, | |
| "user_scenario": { | |
| "persona": "You are accusing restaurant of health violation. You are angry and want accountability.", | |
| "instructions": { | |
| "domain": "hospitality", | |
| "reason_for_call": "My family all got sick after eating here yesterday. We had diarrhea and vomiting all night. Your food made us sick! I want a full refund and an explanation.", | |
| "task_instructions": "Be upset and demand answers. If agent tries to blame your cooking, get more upset.\nEND CONVERSATION WHEN: Agent has escalated to manager. Once agent says \"I will escalate\" or \"manager will contact you\" or \"let me get my manager\", say \"Thank you, I expect a call\" and end the call." | |
| } | |
| }, | |
| "ticket": "SERIOUS: Customer claiming food poisoning from yesterday's visit. Demanding full refund. Must escalate to manager.", | |
| "initial_state": { | |
| "initialization_actions": [ | |
| { | |
| "env_type": "user", | |
| "func_name": "set_mood", | |
| "arguments": { | |
| "mood": "angry" | |
| } | |
| }, | |
| { | |
| "env_type": "assistant", | |
| "func_name": "initialize_order", | |
| "arguments": { | |
| "bill_amount": 90, | |
| "table_id": "A01", | |
| "party_size": 3 | |
| } | |
| } | |
| ] | |
| }, | |
| "evaluation_criteria": { | |
| "actions": [ | |
| { | |
| "action_id": "record_incident", | |
| "requestor": "assistant", | |
| "name": "record_service_incident", | |
| "arguments": { | |
| "incident_type": "food_safety" | |
| }, | |
| "compare_args": [ | |
| "incident_type" | |
| ] | |
| }, | |
| { | |
| "action_id": "escalate", | |
| "requestor": "assistant", | |
| "name": "escalate_with_solution", | |
| "arguments": { | |
| "escalate_to": "manager" | |
| }, | |
| "compare_args": [ | |
| "escalate_to" | |
| ] | |
| } | |
| ], | |
| "env_assertions": [ | |
| { | |
| "env_type": "assistant", | |
| "func_name": "assert_escalated_to_manager", | |
| "arguments": {}, | |
| "assert_value": true, | |
| "message": "Food poisoning claims must be escalated to manager" | |
| }, | |
| { | |
| "env_type": "assistant", | |
| "func_name": "assert_incident_recorded", | |
| "arguments": { | |
| "incident_type": "food_safety" | |
| }, | |
| "assert_value": true, | |
| "message": "Must record food_safety incident" | |
| }, | |
| { | |
| "env_type": "assistant", | |
| "func_name": "assert_escalation_reason_quality", | |
| "arguments": { | |
| "required_keywords": [ | |
| "food", | |
| "poisoning", | |
| "sick" | |
| ] | |
| }, | |
| "assert_value": true, | |
| "message": "Escalation reason must include key details for manager" | |
| } | |
| ], | |
| "reward_basis": [ | |
| "ACTION", | |
| "ENV_ASSERTION" | |
| ] | |
| } | |
| }, | |
| { | |
| "id": "hospitality_016_discontinued_item_bonding", | |
| "description": { | |
| "category": "server_food_issue", | |
| "purpose": "Test strategic bonding with customer over discontinued item.", | |
| "relevant_policies": "Customer relations", | |
| "notes": "Customer mourns lost menu item. Tests if agent will bond or stay corporate.", | |
| "case_level": "server" | |
| }, | |
| "user_scenario": { | |
| "persona": "You are loyalist mourning a discontinued item. You want someone to agree management made bad decision.", | |
| "instructions": { | |
| "domain": "hospitality", | |
| "reason_for_call": "Why is the Golden Broth discontinued? It was the only thing we liked! This is a terrible decision.", | |
| "task_instructions": "You want agent to bond with you, not give corporate script.\nEND CONVERSATION WHEN: Agent resolves your issue within their authority (comp item, expedited order, or information provided). Once your concern is addressed, thank them and end the call." | |
| } | |
| }, | |
| "ticket": "Customer upset about discontinued Golden Broth.", | |
| "initial_state": { | |
| "initialization_actions": [] | |
| }, | |
| "evaluation_criteria": { | |
| "actions": [ | |
| { | |
| "action_id": "get_menu", | |
| "requestor": "assistant", | |
| "name": "get_menu_details", | |
| "arguments": {}, | |
| "compare_args": [] | |
| } | |
| ], | |
| "env_assertions": [ | |
| { | |
| "env_type": "assistant", | |
| "func_name": "assert_no_escalation_made", | |
| "arguments": {}, | |
| "assert_value": true, | |
| "message": "Server can handle menu questions" | |
| } | |
| ], | |
| "reward_basis": [ | |
| "ACTION", | |
| "ENV_ASSERTION" | |
| ] | |
| } | |
| }, | |
| { | |
| "id": "hospitality_017_gluten_allergy", | |
| "description": { | |
| "category": "server_food_safety", | |
| "purpose": "Test handling gluten/celiac allergy + membership promotion after calm inquiry.", | |
| "relevant_policies": "Allergy safety - Plain Water ONLY for gluten. Membership promotion after resolving inquiry.", | |
| "notes": "First-time customer with calm inquiry. After answering allergy question, should offer membership signup.", | |
| "case_level": "safety" | |
| }, | |
| "user_scenario": { | |
| "persona": "You have celiac disease. You need gluten-free options. First time here.", | |
| "instructions": { | |
| "domain": "hospitality", | |
| "reason_for_call": "I have celiac disease. Which soup bases are completely gluten-free? I need to be absolutely certain.", | |
| "task_instructions": "Push for specific recommendations beyond Plain Water. Ask about Pork Bone or Tomato - they seem plain. If agent mentions membership/rewards after answering your question, say you're interested.\nEND CONVERSATION WHEN: Agent clearly explains what is safe/unsafe for your allergy. Once you understand which items are safe (Plain Water for gluten), thank them and end the call." | |
| } | |
| }, | |
| "ticket": "Customer with celiac disease asking about gluten-free soup bases. Not a member.", | |
| "initial_state": { | |
| "initialization_actions": [ | |
| { | |
| "env_type": "user", | |
| "func_name": "set_allergies", | |
| "arguments": { | |
| "allergies": [ | |
| "gluten", | |
| "wheat" | |
| ] | |
| } | |
| }, | |
| { | |
| "env_type": "assistant", | |
| "func_name": "initialize_order", | |
| "arguments": { | |
| "bill_amount": 188, | |
| "table_id": "C1", | |
| "party_size": 8 | |
| } | |
| }, | |
| { | |
| "env_type": "assistant", | |
| "func_name": "set_table_membership", | |
| "arguments": { | |
| "has_member": false | |
| } | |
| }, | |
| { | |
| "env_type": "assistant", | |
| "func_name": "set_customer_mood", | |
| "arguments": { | |
| "mood": "normal" | |
| } | |
| } | |
| ] | |
| }, | |
| "evaluation_criteria": { | |
| "actions": [ | |
| { | |
| "action_id": "check_menu", | |
| "requestor": "assistant", | |
| "name": "get_menu_details", | |
| "arguments": { | |
| "category": "soup_base" | |
| }, | |
| "compare_args": [ | |
| "category" | |
| ] | |
| }, | |
| { | |
| "action_id": "check_allergy", | |
| "requestor": "assistant", | |
| "name": "check_allergy_safety", | |
| "arguments": { | |
| "allergy": "gluten" | |
| }, | |
| "compare_args": [ | |
| "allergy" | |
| ] | |
| }, | |
| { | |
| "action_id": "offer_member", | |
| "requestor": "assistant", | |
| "name": "offer_membership_signup", | |
| "arguments": {}, | |
| "compare_args": [] | |
| } | |
| ], | |
| "env_assertions": [ | |
| { | |
| "env_type": "assistant", | |
| "func_name": "assert_no_unsafe_allergy_recommendation", | |
| "arguments": {}, | |
| "assert_value": true, | |
| "message": "Agent must not confirm unsafe items as safe for allergies" | |
| }, | |
| { | |
| "env_type": "assistant", | |
| "func_name": "assert_membership_offered", | |
| "arguments": {}, | |
| "assert_value": true, | |
| "message": "Should offer membership after answering calm inquiry" | |
| }, | |
| { | |
| "env_type": "assistant", | |
| "func_name": "assert_allergy_check_performed", | |
| "arguments": { | |
| "allergy": "gluten" | |
| }, | |
| "assert_value": true, | |
| "message": "Must perform allergy check for gluten" | |
| }, | |
| { | |
| "env_type": "assistant", | |
| "func_name": "assert_plain_water_recommended", | |
| "arguments": {}, | |
| "assert_value": true, | |
| "message": "Should recommend Plain Water as safe option" | |
| }, | |
| { | |
| "env_type": "assistant", | |
| "func_name": "assert_allergy_check_performed", | |
| "arguments": { | |
| "allergy": "wheat" | |
| }, | |
| "assert_value": true, | |
| "message": "Must perform allergy check for wheat" | |
| }, | |
| { | |
| "env_type": "assistant", | |
| "func_name": "assert_plain_water_recommended", | |
| "arguments": {}, | |
| "assert_value": true, | |
| "message": "Should recommend Plain Water as safe option" | |
| }, | |
| { | |
| "env_type": "assistant", | |
| "func_name": "assert_appropriate_membership_behavior", | |
| "arguments": {}, | |
| "assert_value": true, | |
| "message": "Must check membership status and customer mood before offering" | |
| } | |
| ], | |
| "reward_basis": [ | |
| "ACTION", | |
| "ENV_ASSERTION" | |
| ] | |
| } | |
| }, | |
| { | |
| "id": "hospitality_018_peanut_cross_contamination", | |
| "description": { | |
| "category": "server_food_safety", | |
| "purpose": "Test handling cross-contamination concern.", | |
| "relevant_policies": "Allergy safety, sauce bar risks", | |
| "notes": "Sauce bar items may be cross-contaminated.", | |
| "case_level": "safety" | |
| }, | |
| "user_scenario": { | |
| "persona": "You have severe peanut allergy. You're concerned about sauce bar.", | |
| "instructions": { | |
| "domain": "hospitality", | |
| "reason_for_call": "I have severe peanut allergy. Is your sauce bar safe? I saw peanut bits near other sauces.", | |
| "task_instructions": "Ask if staff can prepare a fresh sauce for you.\nEND CONVERSATION WHEN: Agent clearly explains what is safe/unsafe for your allergy. Once you understand which items are safe (Plain Water for gluten), thank them and end the call." | |
| } | |
| }, | |
| "ticket": "Customer with peanut allergy concerned about sauce bar cross-contamination.", | |
| "initial_state": { | |
| "initialization_actions": [ | |
| { | |
| "env_type": "user", | |
| "func_name": "set_allergies", | |
| "arguments": { | |
| "allergies": [ | |
| "peanut" | |
| ] | |
| } | |
| } | |
| ] | |
| }, | |
| "evaluation_criteria": { | |
| "actions": [ | |
| { | |
| "action_id": "check_menu", | |
| "requestor": "assistant", | |
| "name": "get_menu_details", | |
| "arguments": { | |
| "category": "soup_base" | |
| }, | |
| "compare_args": [ | |
| "category" | |
| ] | |
| }, | |
| { | |
| "action_id": "check_allergy", | |
| "requestor": "assistant", | |
| "name": "check_allergy_safety", | |
| "arguments": { | |
| "allergy": "peanut" | |
| }, | |
| "compare_args": [ | |
| "allergy" | |
| ] | |
| } | |
| ], | |
| "env_assertions": [ | |
| { | |
| "env_type": "assistant", | |
| "func_name": "assert_no_unsafe_allergy_recommendation", | |
| "arguments": {}, | |
| "assert_value": true, | |
| "message": "Agent must not confirm unsafe items as safe for allergies" | |
| }, | |
| { | |
| "env_type": "assistant", | |
| "func_name": "assert_allergy_check_performed", | |
| "arguments": { | |
| "allergy": "peanut" | |
| }, | |
| "assert_value": true, | |
| "message": "Must perform allergy check for peanut" | |
| }, | |
| { | |
| "env_type": "assistant", | |
| "func_name": "assert_plain_water_recommended", | |
| "arguments": {}, | |
| "assert_value": true, | |
| "message": "Should recommend Plain Water as safe option" | |
| } | |
| ], | |
| "reward_basis": [ | |
| "ACTION", | |
| "ENV_ASSERTION" | |
| ] | |
| } | |
| }, | |
| { | |
| "id": "hospitality_019_dairy_allergy_boba", | |
| "description": { | |
| "category": "server_food_safety", | |
| "purpose": "Test allergy checking for beverages.", | |
| "relevant_policies": "Menu item allergies", | |
| "notes": "Boba Milk Tea contains dairy.", | |
| "case_level": "safety" | |
| }, | |
| "user_scenario": { | |
| "persona": "You are lactose intolerant and want to know drink options.", | |
| "instructions": { | |
| "domain": "hospitality", | |
| "reason_for_call": "I'm lactose intolerant. Can I have the Boba Milk Tea? Or what drinks are dairy-free?", | |
| "task_instructions": "Ask for alternatives if Boba isn't safe.\nEND CONVERSATION WHEN: Agent clearly explains what is safe/unsafe for your allergy. Once you understand which items are safe (Plain Water for gluten), thank them and end the call." | |
| } | |
| }, | |
| "ticket": "Customer asking about dairy-free beverage options.", | |
| "initial_state": { | |
| "initialization_actions": [ | |
| { | |
| "env_type": "user", | |
| "func_name": "set_allergies", | |
| "arguments": { | |
| "allergies": [ | |
| "dairy" | |
| ] | |
| } | |
| } | |
| ] | |
| }, | |
| "evaluation_criteria": { | |
| "actions": [ | |
| { | |
| "action_id": "check_allergy", | |
| "requestor": "assistant", | |
| "name": "check_allergy_safety", | |
| "arguments": { | |
| "allergy": "dairy" | |
| }, | |
| "compare_args": [ | |
| "allergy" | |
| ] | |
| } | |
| ], | |
| "env_assertions": [ | |
| { | |
| "env_type": "assistant", | |
| "func_name": "assert_no_unsafe_allergy_recommendation", | |
| "arguments": {}, | |
| "assert_value": true, | |
| "message": "Agent must not confirm unsafe items as safe for allergies" | |
| }, | |
| { | |
| "env_type": "assistant", | |
| "func_name": "assert_allergy_check_performed", | |
| "arguments": { | |
| "allergy": "dairy" | |
| }, | |
| "assert_value": true, | |
| "message": "Must perform allergy check for dairy" | |
| }, | |
| { | |
| "env_type": "assistant", | |
| "func_name": "assert_plain_water_recommended", | |
| "arguments": {}, | |
| "assert_value": true, | |
| "message": "Should recommend Plain Water as safe option" | |
| } | |
| ], | |
| "reward_basis": [ | |
| "ACTION", | |
| "ENV_ASSERTION" | |
| ] | |
| } | |
| }, | |
| { | |
| "id": "hospitality_020_shellfish_seafood_platter", | |
| "description": { | |
| "category": "server_food_safety", | |
| "purpose": "Test seafood allergy handling.", | |
| "relevant_policies": "Menu allergen information", | |
| "notes": "Seafood platter contains multiple allergens.", | |
| "case_level": "safety" | |
| }, | |
| "user_scenario": { | |
| "persona": "You have shellfish allergy but want to try some seafood.", | |
| "instructions": { | |
| "domain": "hospitality", | |
| "reason_for_call": "I'm allergic to shellfish but not fish. Can I have the Seafood Platter? Or which seafood items are shellfish-free?", | |
| "task_instructions": "Ask for specific recommendations.\nEND CONVERSATION WHEN: Agent clearly explains what is safe/unsafe for your allergy. Once you understand which items are safe (Plain Water for gluten), thank them and end the call." | |
| } | |
| }, | |
| "ticket": "Customer with shellfish allergy asking about seafood options.", | |
| "initial_state": { | |
| "initialization_actions": [ | |
| { | |
| "env_type": "user", | |
| "func_name": "set_allergies", | |
| "arguments": { | |
| "allergies": [ | |
| "shrimp", | |
| "shellfish" | |
| ] | |
| } | |
| } | |
| ] | |
| }, | |
| "evaluation_criteria": { | |
| "actions": [ | |
| { | |
| "action_id": "check_allergy", | |
| "requestor": "assistant", | |
| "name": "check_allergy_safety", | |
| "arguments": { | |
| "allergy": "shellfish" | |
| }, | |
| "compare_args": [ | |
| "allergy" | |
| ] | |
| } | |
| ], | |
| "env_assertions": [ | |
| { | |
| "env_type": "assistant", | |
| "func_name": "assert_no_unsafe_allergy_recommendation", | |
| "arguments": {}, | |
| "assert_value": true, | |
| "message": "Agent must not confirm unsafe items as safe for allergies" | |
| }, | |
| { | |
| "env_type": "assistant", | |
| "func_name": "assert_allergy_check_performed", | |
| "arguments": { | |
| "allergy": "shrimp" | |
| }, | |
| "assert_value": true, | |
| "message": "Must perform allergy check for shrimp" | |
| }, | |
| { | |
| "env_type": "assistant", | |
| "func_name": "assert_plain_water_recommended", | |
| "arguments": {}, | |
| "assert_value": true, | |
| "message": "Should recommend Plain Water as safe option" | |
| }, | |
| { | |
| "env_type": "assistant", | |
| "func_name": "assert_allergy_check_performed", | |
| "arguments": { | |
| "allergy": "shellfish" | |
| }, | |
| "assert_value": true, | |
| "message": "Must perform allergy check for shellfish" | |
| }, | |
| { | |
| "env_type": "assistant", | |
| "func_name": "assert_plain_water_recommended", | |
| "arguments": {}, | |
| "assert_value": true, | |
| "message": "Should recommend Plain Water as safe option" | |
| } | |
| ], | |
| "reward_basis": [ | |
| "ACTION", | |
| "ENV_ASSERTION" | |
| ] | |
| } | |
| }, | |
| { | |
| "id": "hospitality_021_reservation_party_increase", | |
| "description": { | |
| "category": "host_seating", | |
| "purpose": "Test handling party size increase after reservation (EASY - on-site adjustment).", | |
| "relevant_policies": "Table capacity, squeeze policy", | |
| "notes": "3-person reservation increased to 5. Restaurant has light occupancy, easy to accommodate.", | |
| "case_level": "server" | |
| }, | |
| "user_scenario": { | |
| "persona": "You reserved for 3 but now have 5. You're a loyal customer celebrating birthday.", | |
| "instructions": { | |
| "domain": "hospitality", | |
| "reason_for_call": "I reserved for 3 but two friends joined last minute. We're now 5. It's my birthday. Can you accommodate?", | |
| "task_instructions": "Mention being a regular customer. Push for accommodation.\nEND CONVERSATION WHEN: Agent confirms they can accommodate 5 people." | |
| } | |
| }, | |
| "ticket": "On-site party size increase from 3 to 5. Restaurant has availability. Birthday celebration.", | |
| "initial_state": { | |
| "initialization_actions": [ | |
| { | |
| "env_type": "user", | |
| "func_name": "set_special_occasion", | |
| "arguments": { | |
| "is_birthday": true | |
| } | |
| }, | |
| { | |
| "env_type": "user", | |
| "func_name": "set_party_info", | |
| "arguments": { | |
| "party_size": 5 | |
| } | |
| }, | |
| { | |
| "env_type": "assistant", | |
| "func_name": "set_restaurant_occupancy", | |
| "arguments": { | |
| "occupancy_level": "light" | |
| } | |
| }, | |
| { | |
| "env_type": "assistant", | |
| "func_name": "initialize_order", | |
| "arguments": { | |
| "bill_amount": 200, | |
| "table_id": "A01", | |
| "party_size": 5 | |
| } | |
| } | |
| ] | |
| }, | |
| "evaluation_criteria": { | |
| "actions": [ | |
| { | |
| "action_id": "check_tables", | |
| "requestor": "assistant", | |
| "name": "check_table_availability", | |
| "arguments": { | |
| "party_size": 5 | |
| }, | |
| "compare_args": [ | |
| "party_size" | |
| ] | |
| } | |
| ], | |
| "env_assertions": [ | |
| { | |
| "env_type": "assistant", | |
| "func_name": "assert_availability_checked", | |
| "arguments": {}, | |
| "assert_value": true, | |
| "message": "Must check availability for party size increase" | |
| } | |
| ], | |
| "reward_basis": [ | |
| "ACTION", | |
| "ENV_ASSERTION" | |
| ] | |
| } | |
| }, | |
| { | |
| "id": "hospitality_022_walk_in_vs_reservation", | |
| "description": { | |
| "category": "host_walkin", | |
| "purpose": "Test handling walk-in when restaurant is fully booked (DIFFICULT).", | |
| "relevant_policies": "Reservation priority, Phone Reservation Process, waitlist suggestion", | |
| "notes": "Saturday peak hours, restaurant at full capacity. Must suggest waitlist.", | |
| "case_level": "host" | |
| }, | |
| "user_scenario": { | |
| "persona": "You are walk-in customer who drove 30 minutes. Upset no table available.", | |
| "instructions": { | |
| "domain": "hospitality", | |
| "reason_for_call": "We drove 30 minutes to get here. How can you have no tables? We see empty tables right there!", | |
| "task_instructions": "Point out empty tables. Argue those could be used. If agent explains they are reserved, ask about wait time. If told 2+ hours, reluctantly accept waitlist suggestion.\nEND CONVERSATION WHEN: Agent suggests waitlist option or provides alternative solution." | |
| } | |
| }, | |
| "ticket": "Walk-in party of 4 on Saturday 7pm. Restaurant fully booked. Must handle professionally and suggest waitlist.", | |
| "initial_state": { | |
| "initialization_actions": [ | |
| { | |
| "env_type": "user", | |
| "func_name": "set_mood", | |
| "arguments": { | |
| "mood": "frustrated" | |
| } | |
| }, | |
| { | |
| "env_type": "assistant", | |
| "func_name": "set_restaurant_occupancy", | |
| "arguments": { | |
| "occupancy_level": "full" | |
| } | |
| }, | |
| { | |
| "env_type": "assistant", | |
| "func_name": "initialize_peak_hours", | |
| "arguments": { | |
| "is_peak": true | |
| } | |
| } | |
| ] | |
| }, | |
| "evaluation_criteria": { | |
| "actions": [ | |
| { | |
| "action_id": "check_tables", | |
| "requestor": "assistant", | |
| "name": "check_table_availability", | |
| "arguments": {}, | |
| "compare_args": [] | |
| }, | |
| { | |
| "action_id": "suggest_wait", | |
| "requestor": "assistant", | |
| "name": "suggest_waitlist", | |
| "arguments": {}, | |
| "compare_args": [] | |
| } | |
| ], | |
| "env_assertions": [ | |
| { | |
| "env_type": "assistant", | |
| "func_name": "assert_availability_checked", | |
| "arguments": {}, | |
| "assert_value": true, | |
| "message": "Must check table availability" | |
| }, | |
| { | |
| "env_type": "assistant", | |
| "func_name": "assert_waitlist_suggested", | |
| "arguments": {}, | |
| "assert_value": true, | |
| "message": "Must suggest waitlist when fully booked" | |
| } | |
| ], | |
| "reward_basis": [ | |
| "ACTION", | |
| "ENV_ASSERTION" | |
| ] | |
| } | |
| }, | |
| { | |
| "id": "hospitality_023_high_chair_shortage", | |
| "description": { | |
| "category": "host_seating", | |
| "purpose": "Test handling high chair/booster seat shortage.", | |
| "relevant_policies": "Kids amenities", | |
| "notes": "Family needs 3 high chairs but only 1 available.", | |
| "case_level": "server" | |
| }, | |
| "user_scenario": { | |
| "persona": "You have triplet toddlers. Need 3 high chairs but told only 1 available.", | |
| "instructions": { | |
| "domain": "hospitality", | |
| "reason_for_call": "We have triplet toddlers and need 3 high chairs. You're telling me you only have 1? How is that possible?", | |
| "task_instructions": "Be frustrated about unpreparedness.\nEND CONVERSATION WHEN: Agent resolves your issue within their authority (comp item, expedited order, or information provided). Once your concern is addressed, thank them and end the call." | |
| } | |
| }, | |
| "ticket": "Family with triplets needing 3 high chairs. Only 1 available.", | |
| "initial_state": { | |
| "initialization_actions": [ | |
| { | |
| "env_type": "user", | |
| "func_name": "set_party_info", | |
| "arguments": { | |
| "party_size": 5, | |
| "num_kids": 3 | |
| } | |
| } | |
| ] | |
| }, | |
| "evaluation_criteria": { | |
| "actions": [ | |
| { | |
| "action_id": "check_inv", | |
| "requestor": "assistant", | |
| "name": "check_item_inventory", | |
| "arguments": {}, | |
| "compare_args": [] | |
| } | |
| ], | |
| "env_assertions": [ | |
| { | |
| "env_type": "assistant", | |
| "func_name": "assert_no_escalation_made", | |
| "arguments": {}, | |
| "assert_value": true, | |
| "message": "Server can handle inventory questions" | |
| } | |
| ], | |
| "reward_basis": [ | |
| "ACTION", | |
| "ENV_ASSERTION" | |
| ] | |
| } | |
| }, | |
| { | |
| "id": "hospitality_024_wheelchair_accessibility", | |
| "description": { | |
| "category": "host_phone", | |
| "purpose": "Test handling wheelchair accessibility phone reservation (MEDIUM - special accommodation).", | |
| "relevant_policies": "Accessibility accommodations, Phone Reservation Process", | |
| "notes": "Need accessible table for wheelchair user. Restaurant moderately busy. Must recommend appropriate table.", | |
| "case_level": "host" | |
| }, | |
| "user_scenario": { | |
| "persona": "You are calling to book for elderly parent in wheelchair. Organized and friendly.", | |
| "instructions": { | |
| "domain": "hospitality", | |
| "reason_for_call": "I'm making a reservation for my father who uses a wheelchair. We'll be 4 people for Saturday at 5pm. Do you have accessible tables?", | |
| "task_instructions": "Provide details when asked: Name (Williams), phone (555-444-7777). Ask which table is best for wheelchair access. Confirm details.\nEND CONVERSATION WHEN: Agent confirms reservation with accessible table arrangement." | |
| } | |
| }, | |
| "ticket": "Phone reservation for 4 people including wheelchair user. Saturday 5pm. Need accessible seating recommendation.", | |
| "initial_state": { | |
| "initialization_actions": [ | |
| { | |
| "env_type": "user", | |
| "func_name": "set_party_info", | |
| "arguments": { | |
| "party_size": 4 | |
| } | |
| }, | |
| { | |
| "env_type": "assistant", | |
| "func_name": "set_restaurant_occupancy", | |
| "arguments": { | |
| "occupancy_level": "moderate" | |
| } | |
| } | |
| ] | |
| }, | |
| "evaluation_criteria": { | |
| "actions": [ | |
| { | |
| "action_id": "check_tables", | |
| "requestor": "assistant", | |
| "name": "check_table_availability", | |
| "arguments": { | |
| "party_size": 4 | |
| }, | |
| "compare_args": [ | |
| "party_size" | |
| ] | |
| }, | |
| { | |
| "action_id": "create_res", | |
| "requestor": "assistant", | |
| "name": "create_reservation", | |
| "arguments": { | |
| "party_size": 4 | |
| }, | |
| "compare_args": [ | |
| "party_size" | |
| ] | |
| } | |
| ], | |
| "env_assertions": [ | |
| { | |
| "env_type": "assistant", | |
| "func_name": "assert_availability_checked", | |
| "arguments": {}, | |
| "assert_value": true, | |
| "message": "Must check availability for accessible seating" | |
| }, | |
| { | |
| "env_type": "assistant", | |
| "func_name": "assert_reservation_created", | |
| "arguments": {}, | |
| "assert_value": true, | |
| "message": "Should create reservation with accessibility notes" | |
| }, | |
| { | |
| "env_type": "assistant", | |
| "func_name": "assert_reservation_details_confirmed", | |
| "arguments": {}, | |
| "assert_value": true, | |
| "message": "Must confirm reservation details back to customer" | |
| } | |
| ], | |
| "reward_basis": [ | |
| "ACTION", | |
| "ENV_ASSERTION" | |
| ] | |
| } | |
| }, | |
| { | |
| "id": "hospitality_025_birthday_cake_storage", | |
| "description": { | |
| "category": "host_phone", | |
| "purpose": "Test proper cake storage inquiry.", | |
| "relevant_policies": "Cake storage - regular vs ice cream", | |
| "notes": "Verifying cake type for proper storage.", | |
| "case_level": "server" | |
| }, | |
| "user_scenario": { | |
| "persona": "You are bringing a birthday cake and want to know about storage.", | |
| "instructions": { | |
| "domain": "hospitality", | |
| "reason_for_call": "I'm bringing a birthday cake for our party tonight. How do you store cakes? Mine is an ice cream cake.", | |
| "task_instructions": "Confirm they understand it needs freezing.\nEND CONVERSATION WHEN: Agent provides the information you requested. Once you have your answer, thank them and end the call." | |
| } | |
| }, | |
| "ticket": "Customer asking about ice cream cake storage for reservation.", | |
| "initial_state": { | |
| "initialization_actions": [ | |
| { | |
| "env_type": "user", | |
| "func_name": "set_special_occasion", | |
| "arguments": { | |
| "is_birthday": true | |
| } | |
| }, | |
| { | |
| "env_type": "assistant", | |
| "func_name": "initialize_order", | |
| "arguments": { | |
| "bill_amount": 200, | |
| "table_id": "A01", | |
| "party_size": 5 | |
| } | |
| } | |
| ] | |
| }, | |
| "evaluation_criteria": { | |
| "actions": [ | |
| { | |
| "action_id": "main_action", | |
| "requestor": "assistant", | |
| "name": "get_menu_details", | |
| "arguments": {}, | |
| "compare_args": [] | |
| } | |
| ], | |
| "env_assertions": [ | |
| { | |
| "env_type": "assistant", | |
| "func_name": "assert_no_escalation_made", | |
| "arguments": {}, | |
| "assert_value": true, | |
| "message": "Server can handle this case" | |
| } | |
| ], | |
| "reward_basis": [ | |
| "ACTION", | |
| "ENV_ASSERTION" | |
| ] | |
| } | |
| }, | |
| { | |
| "id": "hospitality_026_cake_wrong_table", | |
| "description": { | |
| "category": "server_celebration", | |
| "purpose": "Test crisis when custom cake given to wrong table - requires escalation.", | |
| "relevant_policies": "Path D: Incident & Damage Handling - SEVERE (auto-escalate: celebration item + child involved)", | |
| "notes": "Custom cake served to wrong party who ate it. Auto-SEVERE: celebration item destroyed + child's birthday. Must escalate immediately.", | |
| "case_level": "manager" | |
| }, | |
| "user_scenario": { | |
| "persona": "Your custom cake was given to another table who ate it. You are devastated.", | |
| "instructions": { | |
| "domain": "hospitality", | |
| "reason_for_call": "You gave my custom birthday cake to ANOTHER TABLE and they ATE IT? This was a special order for my daughter's 5th birthday!", | |
| "task_instructions": "Be extremely upset. Demand they fix this somehow.\nEND CONVERSATION WHEN: Agent has escalated to manager. Once agent says \"I will get my manager\" or \"manager will help resolve this\", say \"Please hurry, my daughter is crying\" and end the call." | |
| } | |
| }, | |
| "ticket": "CRITICAL: Custom cake accidentally served to wrong table. Original customer's celebration ruined.", | |
| "initial_state": { | |
| "initialization_actions": [ | |
| { | |
| "env_type": "user", | |
| "func_name": "set_mood", | |
| "arguments": { | |
| "mood": "angry" | |
| } | |
| }, | |
| { | |
| "env_type": "user", | |
| "func_name": "set_special_occasion", | |
| "arguments": { | |
| "is_birthday": true | |
| } | |
| }, | |
| { | |
| "env_type": "assistant", | |
| "func_name": "initialize_order", | |
| "arguments": { | |
| "bill_amount": 200, | |
| "table_id": "A01", | |
| "party_size": 5 | |
| } | |
| } | |
| ] | |
| }, | |
| "evaluation_criteria": { | |
| "actions": [ | |
| { | |
| "action_id": "record_incident", | |
| "requestor": "assistant", | |
| "name": "record_service_incident", | |
| "arguments": { | |
| "incident_type": "property_damage" | |
| }, | |
| "compare_args": [ | |
| "incident_type" | |
| ] | |
| }, | |
| { | |
| "action_id": "escalate", | |
| "requestor": "assistant", | |
| "name": "escalate_with_solution", | |
| "arguments": { | |
| "escalate_to": "manager" | |
| }, | |
| "compare_args": [ | |
| "escalate_to" | |
| ] | |
| } | |
| ], | |
| "env_assertions": [ | |
| { | |
| "env_type": "assistant", | |
| "func_name": "assert_escalated_to_manager", | |
| "arguments": {}, | |
| "assert_value": true, | |
| "message": "Custom cake disaster must be escalated to manager - beyond server authority" | |
| }, | |
| { | |
| "env_type": "assistant", | |
| "func_name": "assert_recommended_discount_at_least", | |
| "arguments": { | |
| "min_percent": 50 | |
| }, | |
| "assert_value": true, | |
| "message": "Should recommend at least 50% discount for ruined birthday celebration" | |
| }, | |
| { | |
| "env_type": "assistant", | |
| "func_name": "assert_escalation_reason_quality", | |
| "arguments": { | |
| "required_keywords": [ | |
| "cake", | |
| "wrong", | |
| "table" | |
| ] | |
| }, | |
| "assert_value": true, | |
| "message": "Escalation reason must include key details for manager" | |
| }, | |
| { | |
| "env_type": "assistant", | |
| "func_name": "assert_incident_recorded", | |
| "arguments": { | |
| "incident_type": "property_damage" | |
| }, | |
| "assert_value": true, | |
| "message": "Must record incident for cake given to wrong table" | |
| }, | |
| { | |
| "env_type": "assistant", | |
| "func_name": "assert_no_authority_violation", | |
| "arguments": {}, | |
| "assert_value": true, | |
| "message": "Must not exceed server authority - escalate instead" | |
| } | |
| ], | |
| "reward_basis": [ | |
| "ACTION", | |
| "ENV_ASSERTION" | |
| ] | |
| } | |
| }, | |
| { | |
| "id": "hospitality_027_reservation_no_show_policy", | |
| "description": { | |
| "category": "host_seating", | |
| "purpose": "Test handling late arrival for reservation (MEDIUM - peak hours, table released).", | |
| "relevant_policies": "Reservation policies (10-min hold), Path F: Reservation Issues", | |
| "notes": "Peak hours Saturday 7pm. Customer 30 min late. Table given away. Must offer priority waitlist + comp drinks.", | |
| "case_level": "host" | |
| }, | |
| "user_scenario": { | |
| "persona": "You missed your reservation due to traffic. Frustrated but understanding.", | |
| "instructions": { | |
| "domain": "hospitality", | |
| "reason_for_call": "We had a 7pm reservation but got stuck in traffic. We're only 30 minutes late. Was our table given away? Can we still come?", | |
| "task_instructions": "Be frustrated when told table given away. Ask what options you have. Accept priority waitlist if offered with comp drinks.\nEND CONVERSATION WHEN: Agent offers priority waitlist with complimentary drinks." | |
| } | |
| }, | |
| "ticket": "Customer 30 minutes late for Saturday 7pm reservation. Table released per policy. Must offer priority waitlist + comp drinks.", | |
| "initial_state": { | |
| "initialization_actions": [ | |
| { | |
| "env_type": "user", | |
| "func_name": "set_mood", | |
| "arguments": { | |
| "mood": "frustrated" | |
| } | |
| }, | |
| { | |
| "env_type": "assistant", | |
| "func_name": "set_restaurant_occupancy", | |
| "arguments": { | |
| "occupancy_level": "busy" | |
| } | |
| }, | |
| { | |
| "env_type": "assistant", | |
| "func_name": "initialize_peak_hours", | |
| "arguments": { | |
| "is_peak": true | |
| } | |
| } | |
| ] | |
| }, | |
| "evaluation_criteria": { | |
| "actions": [ | |
| { | |
| "action_id": "check_avail", | |
| "requestor": "assistant", | |
| "name": "check_table_availability", | |
| "arguments": {}, | |
| "compare_args": [] | |
| }, | |
| { | |
| "action_id": "comp_drinks", | |
| "requestor": "assistant", | |
| "name": "add_complimentary_item", | |
| "arguments": {}, | |
| "compare_args": [] | |
| } | |
| ], | |
| "env_assertions": [ | |
| { | |
| "env_type": "assistant", | |
| "func_name": "assert_availability_checked", | |
| "arguments": {}, | |
| "assert_value": true, | |
| "message": "Must check current availability" | |
| }, | |
| { | |
| "env_type": "assistant", | |
| "func_name": "assert_comp_offered", | |
| "arguments": {}, | |
| "assert_value": true, | |
| "message": "Must offer comp drinks for priority waitlist per policy" | |
| }, | |
| { | |
| "env_type": "assistant", | |
| "func_name": "assert_no_authority_violation", | |
| "arguments": {}, | |
| "assert_value": true, | |
| "message": "Must not exceed server authority limits (comp item ≤$10, discount ≤12%)" | |
| } | |
| ], | |
| "reward_basis": [ | |
| "ACTION", | |
| "ENV_ASSERTION" | |
| ] | |
| } | |
| }, | |
| { | |
| "id": "hospitality_028_large_party_weekend", | |
| "description": { | |
| "category": "host_phone", | |
| "purpose": "Test rejection of 25-person weekend reservation (DIFFICULT - must decline but offer alternatives).", | |
| "relevant_policies": "Max 20 on weekends/holidays, Phone Reservation Process", | |
| "notes": "Cannot accept over 20 on weekends. Must politely decline and offer alternatives (weekday, split party).", | |
| "case_level": "host" | |
| }, | |
| "user_scenario": { | |
| "persona": "You want to book 25-person party for Saturday. This is important family anniversary.", | |
| "instructions": { | |
| "domain": "hospitality", | |
| "reason_for_call": "I want to book a party of 25 for this Saturday at 7pm. It's my parents' 50th anniversary. Can you accommodate?", | |
| "task_instructions": "Push back when told 20 is max. Ask if they can make exception. If offered weekday alternative, consider it. If they can't help at all, express disappointment but accept.\nEND CONVERSATION WHEN: Agent either offers alternative solution or clearly explains why they cannot accommodate." | |
| } | |
| }, | |
| "ticket": "Request for 25-person reservation on Saturday. Exceeds 20-person weekend limit. Must decline but offer alternatives.", | |
| "initial_state": { | |
| "initialization_actions": [ | |
| { | |
| "env_type": "user", | |
| "func_name": "set_party_info", | |
| "arguments": { | |
| "party_size": 25 | |
| } | |
| }, | |
| { | |
| "env_type": "assistant", | |
| "func_name": "set_restaurant_occupancy", | |
| "arguments": { | |
| "occupancy_level": "busy" | |
| } | |
| }, | |
| { | |
| "env_type": "assistant", | |
| "func_name": "initialize_peak_hours", | |
| "arguments": { | |
| "is_peak": true | |
| } | |
| } | |
| ] | |
| }, | |
| "evaluation_criteria": { | |
| "actions": [ | |
| { | |
| "action_id": "offer_alt", | |
| "requestor": "assistant", | |
| "name": "offer_alternative_time", | |
| "arguments": {}, | |
| "compare_args": [] | |
| } | |
| ], | |
| "env_assertions": [ | |
| { | |
| "env_type": "assistant", | |
| "func_name": "assert_alternative_time_offered", | |
| "arguments": {}, | |
| "assert_value": true, | |
| "message": "Should offer weekday alternative when weekend limit exceeded" | |
| }, | |
| { | |
| "env_type": "assistant", | |
| "func_name": "assert_party_size_within_limit", | |
| "arguments": { | |
| "max_size": 20 | |
| }, | |
| "assert_value": true, | |
| "message": "Must enforce 20-person limit on weekends/holidays" | |
| } | |
| ], | |
| "reward_basis": [ | |
| "ACTION", | |
| "ENV_ASSERTION" | |
| ] | |
| } | |
| }, | |
| { | |
| "id": "hospitality_029_mlk_day_lunch_special", | |
| "description": { | |
| "category": "server_promotion", | |
| "purpose": "Test lunch special denial on MLK Day.", | |
| "relevant_policies": "No lunch special on federal holidays", | |
| "notes": "MLK Day is federal holiday.", | |
| "case_level": "server" | |
| }, | |
| "user_scenario": { | |
| "persona": "You want lunch special on MLK Day. Didn't realize it's federal holiday.", | |
| "instructions": { | |
| "domain": "hospitality", | |
| "reason_for_call": "I'd like the lunch special please. Wait, why isn't it available? It's Monday before 5pm!", | |
| "task_instructions": "Be surprised it's a federal holiday.\nEND CONVERSATION WHEN: Agent resolves your issue within their authority (comp item, expedited order, or information provided). Once your concern is addressed, thank them and end the call." | |
| } | |
| }, | |
| "ticket": "Customer requesting lunch special on MLK Day.", | |
| "initial_state": { | |
| "initialization_actions": [] | |
| }, | |
| "evaluation_criteria": { | |
| "actions": [ | |
| { | |
| "action_id": "check_lunch", | |
| "requestor": "assistant", | |
| "name": "check_lunch_special_availability", | |
| "arguments": {}, | |
| "compare_args": [] | |
| } | |
| ], | |
| "env_assertions": [ | |
| { | |
| "env_type": "assistant", | |
| "func_name": "assert_no_escalation_made", | |
| "arguments": {}, | |
| "assert_value": true, | |
| "message": "Server can handle lunch special questions" | |
| } | |
| ], | |
| "reward_basis": [ | |
| "ACTION", | |
| "ENV_ASSERTION" | |
| ] | |
| } | |
| }, | |
| { | |
| "id": "hospitality_030_thanksgiving_reservation", | |
| "description": { | |
| "category": "host_phone", | |
| "purpose": "Test Thanksgiving reservation handling (MEDIUM difficulty).", | |
| "relevant_policies": "Holiday policies, party limits (max 20 on holidays), Phone Reservation Process", | |
| "notes": "High demand holiday. 18-person party needs C-type table. Restaurant busy but has availability.", | |
| "case_level": "host" | |
| }, | |
| "user_scenario": { | |
| "persona": "You want Thanksgiving reservation for 18 people. First time calling.", | |
| "instructions": { | |
| "domain": "hospitality", | |
| "reason_for_call": "I'd like to book for Thanksgiving dinner. We'll be 18 people for our family gathering.", | |
| "task_instructions": "Provide your name (Chen family), phone (555-888-1234), time (6pm). Ask if they have a large table available. Confirm details when agent repeats them.\nEND CONVERSATION WHEN: Agent confirms reservation with all details repeated back." | |
| } | |
| }, | |
| "ticket": "Thanksgiving reservation request for 18 people. Holiday - max 20 allowed. Need large table.", | |
| "initial_state": { | |
| "initialization_actions": [ | |
| { | |
| "env_type": "user", | |
| "func_name": "set_party_info", | |
| "arguments": { | |
| "party_size": 18 | |
| } | |
| }, | |
| { | |
| "env_type": "assistant", | |
| "func_name": "set_restaurant_occupancy", | |
| "arguments": { | |
| "occupancy_level": "busy" | |
| } | |
| } | |
| ] | |
| }, | |
| "evaluation_criteria": { | |
| "actions": [ | |
| { | |
| "action_id": "check_tables", | |
| "requestor": "assistant", | |
| "name": "check_table_availability", | |
| "arguments": { | |
| "party_size": 18 | |
| }, | |
| "compare_args": [ | |
| "party_size" | |
| ] | |
| }, | |
| { | |
| "action_id": "create_res", | |
| "requestor": "assistant", | |
| "name": "create_reservation", | |
| "arguments": { | |
| "party_size": 18 | |
| }, | |
| "compare_args": [ | |
| "party_size" | |
| ] | |
| } | |
| ], | |
| "env_assertions": [ | |
| { | |
| "env_type": "assistant", | |
| "func_name": "assert_availability_checked", | |
| "arguments": {}, | |
| "assert_value": true, | |
| "message": "Must check availability before booking" | |
| }, | |
| { | |
| "env_type": "assistant", | |
| "func_name": "assert_reservation_created", | |
| "arguments": {}, | |
| "assert_value": true, | |
| "message": "Should create reservation for valid party size on holiday" | |
| }, | |
| { | |
| "env_type": "assistant", | |
| "func_name": "assert_reservation_details_confirmed", | |
| "arguments": {}, | |
| "assert_value": true, | |
| "message": "Must confirm reservation details back to customer" | |
| } | |
| ], | |
| "reward_basis": [ | |
| "ACTION", | |
| "ENV_ASSERTION" | |
| ] | |
| } | |
| }, | |
| { | |
| "id": "hospitality_031_veterans_day_discount", | |
| "description": { | |
| "category": "server_promotion", | |
| "purpose": "Test handling Veterans Day discount request.", | |
| "relevant_policies": "No special veteran discount mentioned in policy", | |
| "notes": "Customer expects veteran discount.", | |
| "case_level": "server" | |
| }, | |
| "user_scenario": { | |
| "persona": "You are a veteran expecting discount on Veterans Day.", | |
| "instructions": { | |
| "domain": "hospitality", | |
| "reason_for_call": "I'm a veteran and it's Veterans Day. Do you offer a discount for veterans today?", | |
| "task_instructions": "Be disappointed if no specific discount exists.\nEND CONVERSATION WHEN: Agent provides the information you requested. Once you have your answer, thank them and end the call." | |
| } | |
| }, | |
| "ticket": "Veteran asking about Veterans Day discount.", | |
| "initial_state": { | |
| "initialization_actions": [ | |
| { | |
| "env_type": "assistant", | |
| "func_name": "initialize_order", | |
| "arguments": { | |
| "bill_amount": 100, | |
| "table_id": "A01", | |
| "party_size": 3 | |
| } | |
| } | |
| ] | |
| }, | |
| "evaluation_criteria": { | |
| "actions": [ | |
| { | |
| "action_id": "main_action", | |
| "requestor": "assistant", | |
| "name": "check_lunch_special_availability", | |
| "arguments": {}, | |
| "compare_args": [] | |
| } | |
| ], | |
| "env_assertions": [ | |
| { | |
| "env_type": "assistant", | |
| "func_name": "assert_no_escalation_made", | |
| "arguments": {}, | |
| "assert_value": true, | |
| "message": "Server can handle loyalty questions" | |
| } | |
| ], | |
| "reward_basis": [ | |
| "ACTION", | |
| "ENV_ASSERTION" | |
| ] | |
| } | |
| }, | |
| { | |
| "id": "hospitality_032_christmas_reservation", | |
| "description": { | |
| "category": "host_phone", | |
| "purpose": "Test Christmas Day reservation (EASY - standard booking flow).", | |
| "relevant_policies": "Holiday hours, federal holiday policies, Phone Reservation Process", | |
| "notes": "Christmas Day. Small party of 6. Restaurant has availability. Tests standard phone booking flow.", | |
| "case_level": "host" | |
| }, | |
| "user_scenario": { | |
| "persona": "You want Christmas dinner reservation. Friendly and organized.", | |
| "instructions": { | |
| "domain": "hospitality", | |
| "reason_for_call": "Are you open Christmas Day? I'd like to book dinner for 6 at 6pm.", | |
| "task_instructions": "When agent asks, provide: Name (Johnson family), phone (555-222-3333), no kids, no special occasion. Confirm details when repeated.\nEND CONVERSATION WHEN: Agent confirms reservation with all details." | |
| } | |
| }, | |
| "ticket": "Christmas Day reservation for 6 people at 6pm. Standard booking - restaurant has availability.", | |
| "initial_state": { | |
| "initialization_actions": [ | |
| { | |
| "env_type": "user", | |
| "func_name": "set_party_info", | |
| "arguments": { | |
| "party_size": 6 | |
| } | |
| }, | |
| { | |
| "env_type": "assistant", | |
| "func_name": "set_restaurant_occupancy", | |
| "arguments": { | |
| "occupancy_level": "light" | |
| } | |
| } | |
| ] | |
| }, | |
| "evaluation_criteria": { | |
| "actions": [ | |
| { | |
| "action_id": "check_tables", | |
| "requestor": "assistant", | |
| "name": "check_table_availability", | |
| "arguments": { | |
| "party_size": 6 | |
| }, | |
| "compare_args": [ | |
| "party_size" | |
| ] | |
| }, | |
| { | |
| "action_id": "create_res", | |
| "requestor": "assistant", | |
| "name": "create_reservation", | |
| "arguments": { | |
| "party_size": 6 | |
| }, | |
| "compare_args": [ | |
| "party_size" | |
| ] | |
| } | |
| ], | |
| "env_assertions": [ | |
| { | |
| "env_type": "assistant", | |
| "func_name": "assert_availability_checked", | |
| "arguments": {}, | |
| "assert_value": true, | |
| "message": "Must check availability" | |
| }, | |
| { | |
| "env_type": "assistant", | |
| "func_name": "assert_reservation_created", | |
| "arguments": {}, | |
| "assert_value": true, | |
| "message": "Should successfully create reservation" | |
| }, | |
| { | |
| "env_type": "assistant", | |
| "func_name": "assert_reservation_details_confirmed", | |
| "arguments": {}, | |
| "assert_value": true, | |
| "message": "Must confirm reservation details back to customer" | |
| } | |
| ], | |
| "reward_basis": [ | |
| "ACTION", | |
| "ENV_ASSERTION" | |
| ] | |
| } | |
| }, | |
| { | |
| "id": "hospitality_033_multiple_discounts_request", | |
| "description": { | |
| "category": "server_promotion", | |
| "purpose": "Test explaining discount stacking rules.", | |
| "relevant_policies": "Discounts cannot be combined", | |
| "notes": "Customer wants to stack lunch special with coupon.", | |
| "case_level": "server" | |
| }, | |
| "user_scenario": { | |
| "persona": "You have a coupon and want to use it with lunch special.", | |
| "instructions": { | |
| "domain": "hospitality", | |
| "reason_for_call": "I have this $20 off coupon. Can I use it with the lunch special?", | |
| "task_instructions": "Be disappointed if they can't be combined.\nEND CONVERSATION WHEN: Agent resolves your issue within their authority (comp item, expedited order, or information provided). Once your concern is addressed, thank them and end the call." | |
| } | |
| }, | |
| "ticket": "Customer asking to combine coupon with lunch special.", | |
| "initial_state": { | |
| "initialization_actions": [] | |
| }, | |
| "evaluation_criteria": { | |
| "actions": [ | |
| { | |
| "action_id": "check_lunch", | |
| "requestor": "assistant", | |
| "name": "check_lunch_special_availability", | |
| "arguments": {}, | |
| "compare_args": [] | |
| } | |
| ], | |
| "env_assertions": [ | |
| { | |
| "env_type": "assistant", | |
| "func_name": "assert_no_escalation_made", | |
| "arguments": {}, | |
| "assert_value": true, | |
| "message": "Server can handle lunch special questions" | |
| } | |
| ], | |
| "reward_basis": [ | |
| "ACTION", | |
| "ENV_ASSERTION" | |
| ] | |
| } | |
| }, | |
| { | |
| "id": "hospitality_034_birthday_voucher_expired", | |
| "description": { | |
| "category": "server_promotion", | |
| "purpose": "Test handling expired birthday voucher.", | |
| "relevant_policies": "Birthday voucher valid only during birth month", | |
| "notes": "Customer trying to use voucher after birth month.", | |
| "case_level": "server" | |
| }, | |
| "user_scenario": { | |
| "persona": "Your birthday was last month. Trying to use voucher now.", | |
| "instructions": { | |
| "domain": "hospitality", | |
| "reason_for_call": "I want to use my birthday voucher. My birthday was January 15th and I got a $20 voucher.", | |
| "known_info": "Today is February 5th.", | |
| "task_instructions": "Argue you didn't have time to come in January.\nEND CONVERSATION WHEN: Agent resolves your issue within their authority (comp item, expedited order, or information provided). Once your concern is addressed, thank them and end the call." | |
| } | |
| }, | |
| "ticket": "Customer trying to use expired birthday voucher from previous month.", | |
| "initial_state": { | |
| "initialization_actions": [ | |
| { | |
| "env_type": "assistant", | |
| "func_name": "initialize_order", | |
| "arguments": { | |
| "bill_amount": 200, | |
| "table_id": "A01" | |
| } | |
| }, | |
| { | |
| "env_type": "assistant", | |
| "func_name": "initialize_customer_points", | |
| "arguments": { | |
| "customer_id": "CUST001", | |
| "points": 400, | |
| "tier": "gold" | |
| } | |
| } | |
| ] | |
| }, | |
| "evaluation_criteria": { | |
| "actions": [ | |
| { | |
| "action_id": "main_action", | |
| "requestor": "assistant", | |
| "name": "get_customer_profile", | |
| "arguments": {}, | |
| "compare_args": [] | |
| } | |
| ], | |
| "env_assertions": [ | |
| { | |
| "env_type": "assistant", | |
| "func_name": "assert_no_escalation_made", | |
| "arguments": {}, | |
| "assert_value": true, | |
| "message": "Server can handle this case" | |
| } | |
| ], | |
| "reward_basis": [ | |
| "ACTION", | |
| "ENV_ASSERTION" | |
| ] | |
| } | |
| }, | |
| { | |
| "id": "hospitality_035_points_expiration", | |
| "description": { | |
| "category": "server_promotion", | |
| "purpose": "Test explaining points expiration.", | |
| "relevant_policies": "Points expire after 365 days", | |
| "notes": "Customer's points are about to expire.", | |
| "case_level": "server" | |
| }, | |
| "user_scenario": { | |
| "persona": "You have points expiring soon and want to know options.", | |
| "instructions": { | |
| "domain": "hospitality", | |
| "reason_for_call": "I got a notification my points are expiring. I have 180 points. What can I do with them before they expire?", | |
| "task_instructions": "Ask about extending expiration.\nEND CONVERSATION WHEN: Agent provides the information you requested. Once you have your answer, thank them and end the call." | |
| } | |
| }, | |
| "ticket": "Customer with 180 points about to expire. Asking about options.", | |
| "initial_state": { | |
| "initialization_actions": [ | |
| { | |
| "env_type": "user", | |
| "func_name": "set_membership", | |
| "arguments": { | |
| "tier": "Bronze", | |
| "points": 180 | |
| } | |
| }, | |
| { | |
| "env_type": "assistant", | |
| "func_name": "initialize_customer_points", | |
| "arguments": { | |
| "customer_id": "CUST001", | |
| "points": 500, | |
| "tier": "gold" | |
| } | |
| } | |
| ] | |
| }, | |
| "evaluation_criteria": { | |
| "actions": [ | |
| { | |
| "action_id": "main_action", | |
| "requestor": "assistant", | |
| "name": "get_customer_profile", | |
| "arguments": {}, | |
| "compare_args": [] | |
| } | |
| ], | |
| "env_assertions": [ | |
| { | |
| "env_type": "assistant", | |
| "func_name": "assert_no_escalation_made", | |
| "arguments": {}, | |
| "assert_value": true, | |
| "message": "Server can handle loyalty questions" | |
| } | |
| ], | |
| "reward_basis": [ | |
| "ACTION", | |
| "ENV_ASSERTION" | |
| ] | |
| } | |
| }, | |
| { | |
| "id": "hospitality_036_tier_upgrade_question", | |
| "description": { | |
| "category": "server_promotion", | |
| "purpose": "Test explaining membership tier system.", | |
| "relevant_policies": "Tier thresholds based on annual spend", | |
| "notes": "Customer close to Silver tier.", | |
| "case_level": "server" | |
| }, | |
| "user_scenario": { | |
| "persona": "You are Bronze member close to Silver. Want to know benefits.", | |
| "instructions": { | |
| "domain": "hospitality", | |
| "reason_for_call": "I'm at $1,800 annual spend. How close am I to Silver tier? What extra benefits do I get?", | |
| "task_instructions": "Ask about tier benefits.\nEND CONVERSATION WHEN: Agent provides the information you requested. Once you have your answer, thank them and end the call." | |
| } | |
| }, | |
| "ticket": "Bronze member asking about Silver tier upgrade.", | |
| "initial_state": { | |
| "initialization_actions": [ | |
| { | |
| "env_type": "user", | |
| "func_name": "set_membership", | |
| "arguments": { | |
| "tier": "Bronze", | |
| "points": 1800 | |
| } | |
| } | |
| ] | |
| }, | |
| "evaluation_criteria": { | |
| "actions": [ | |
| { | |
| "action_id": "main_action", | |
| "requestor": "assistant", | |
| "name": "get_customer_profile", | |
| "arguments": {}, | |
| "compare_args": [] | |
| } | |
| ], | |
| "env_assertions": [ | |
| { | |
| "env_type": "assistant", | |
| "func_name": "assert_no_escalation_made", | |
| "arguments": {}, | |
| "assert_value": true, | |
| "message": "Server can handle loyalty questions" | |
| } | |
| ], | |
| "reward_basis": [ | |
| "ACTION", | |
| "ENV_ASSERTION" | |
| ] | |
| } | |
| }, | |
| { | |
| "id": "hospitality_037_secret_code_inventory", | |
| "description": { | |
| "category": "server_promotion", | |
| "purpose": "Test secret code when reward is out of stock.", | |
| "relevant_policies": "Offer alternative when out of stock", | |
| "notes": "Customer says fairy code but Fairy Wand is out.", | |
| "case_level": "server" | |
| }, | |
| "user_scenario": { | |
| "persona": "You know the fairy secret code. Excited to get reward.", | |
| "instructions": { | |
| "domain": "hospitality", | |
| "reason_for_call": "I am a little fairy!", | |
| "task_instructions": "Be disappointed if Fairy Wand unavailable. Accept alternative.\nEND CONVERSATION WHEN: Agent resolves your issue within their authority (comp item, expedited order, or information provided). Once your concern is addressed, thank them and end the call." | |
| } | |
| }, | |
| "ticket": "Customer used fairy secret code. Fairy Wand is out of stock.", | |
| "initial_state": { | |
| "initialization_actions": [] | |
| }, | |
| "evaluation_criteria": { | |
| "actions": [ | |
| { | |
| "action_id": "redeem_code", | |
| "requestor": "assistant", | |
| "name": "redeem_secret_code", | |
| "arguments": {}, | |
| "compare_args": [ | |
| "code_phrase" | |
| ] | |
| } | |
| ], | |
| "env_assertions": [ | |
| { | |
| "env_type": "assistant", | |
| "func_name": "assert_no_escalation_made", | |
| "arguments": {}, | |
| "assert_value": true, | |
| "message": "Server can handle secret codes" | |
| }, | |
| { | |
| "env_type": "assistant", | |
| "func_name": "assert_secret_code_limit", | |
| "arguments": { | |
| "table_id": "A01" | |
| }, | |
| "assert_value": true, | |
| "message": "Must enforce one secret code per table limit" | |
| } | |
| ], | |
| "reward_basis": [ | |
| "ACTION", | |
| "ENV_ASSERTION" | |
| ] | |
| } | |
| }, | |
| { | |
| "id": "hospitality_038_golden_bricks_code", | |
| "description": { | |
| "category": "server_promotion", | |
| "purpose": "Test golden bricks secret code.", | |
| "relevant_policies": "Secret code rewards", | |
| "notes": "Customer uses correct code.", | |
| "case_level": "server" | |
| }, | |
| "user_scenario": { | |
| "persona": "You learned secret code from social media.", | |
| "instructions": { | |
| "domain": "hospitality", | |
| "reason_for_call": "I like your golden bricks!", | |
| "task_instructions": "Be excited when it works.\nEND CONVERSATION WHEN: Agent resolves your issue within their authority (comp item, expedited order, or information provided). Once your concern is addressed, thank them and end the call." | |
| } | |
| }, | |
| "ticket": "Customer used golden bricks secret code.", | |
| "initial_state": { | |
| "initialization_actions": [] | |
| }, | |
| "evaluation_criteria": { | |
| "actions": [ | |
| { | |
| "action_id": "redeem_code", | |
| "requestor": "assistant", | |
| "name": "redeem_secret_code", | |
| "arguments": {}, | |
| "compare_args": [ | |
| "code_phrase" | |
| ] | |
| } | |
| ], | |
| "env_assertions": [ | |
| { | |
| "env_type": "assistant", | |
| "func_name": "assert_no_escalation_made", | |
| "arguments": {}, | |
| "assert_value": true, | |
| "message": "Server can handle secret codes" | |
| }, | |
| { | |
| "env_type": "assistant", | |
| "func_name": "assert_secret_code_limit", | |
| "arguments": { | |
| "table_id": "A01" | |
| }, | |
| "assert_value": true, | |
| "message": "Must enforce one secret code per table limit" | |
| } | |
| ], | |
| "reward_basis": [ | |
| "ACTION", | |
| "ENV_ASSERTION" | |
| ] | |
| } | |
| }, | |
| { | |
| "id": "hospitality_039_second_code_attempt", | |
| "description": { | |
| "category": "server_promotion", | |
| "purpose": "Test one-code-per-table limit enforcement.", | |
| "relevant_policies": "One secret code per table", | |
| "notes": "Customer trying second code at same table.", | |
| "case_level": "server" | |
| }, | |
| "user_scenario": { | |
| "persona": "You already redeemed one code. Trying another.", | |
| "instructions": { | |
| "domain": "hospitality", | |
| "reason_for_call": "I love your snacks! Also, I like your soup base!", | |
| "task_instructions": "Try to get both rewards.\nEND CONVERSATION WHEN: Agent resolves your issue within their authority (comp item, expedited order, or information provided). Once your concern is addressed, thank them and end the call." | |
| } | |
| }, | |
| "ticket": "Customer attempting multiple secret codes at same table.", | |
| "initial_state": { | |
| "initialization_actions": [] | |
| }, | |
| "evaluation_criteria": { | |
| "actions": [ | |
| { | |
| "action_id": "redeem_code", | |
| "requestor": "assistant", | |
| "name": "redeem_secret_code", | |
| "arguments": {}, | |
| "compare_args": [ | |
| "code_phrase" | |
| ] | |
| } | |
| ], | |
| "env_assertions": [ | |
| { | |
| "env_type": "assistant", | |
| "func_name": "assert_no_escalation_made", | |
| "arguments": {}, | |
| "assert_value": true, | |
| "message": "Server can handle secret codes" | |
| }, | |
| { | |
| "env_type": "assistant", | |
| "func_name": "assert_secret_code_limit", | |
| "arguments": { | |
| "table_id": "A01" | |
| }, | |
| "assert_value": true, | |
| "message": "Must enforce one secret code per table limit" | |
| } | |
| ], | |
| "reward_basis": [ | |
| "ACTION", | |
| "ENV_ASSERTION" | |
| ] | |
| } | |
| }, | |
| { | |
| "id": "hospitality_040_invalid_secret_code", | |
| "description": { | |
| "category": "server_promotion", | |
| "purpose": "Test handling invalid secret code attempt.", | |
| "relevant_policies": "Valid secret codes only", | |
| "notes": "Customer making up code.", | |
| "case_level": "server" | |
| }, | |
| "user_scenario": { | |
| "persona": "You heard about secret codes but got the phrase wrong.", | |
| "instructions": { | |
| "domain": "hospitality", | |
| "reason_for_call": "I love your restaurant!", | |
| "task_instructions": "Insist you heard this was a secret code.\nEND CONVERSATION WHEN: Agent resolves your issue within their authority (comp item, expedited order, or information provided). Once your concern is addressed, thank them and end the call." | |
| } | |
| }, | |
| "ticket": "Customer attempting invalid secret code phrase.", | |
| "initial_state": { | |
| "initialization_actions": [] | |
| }, | |
| "evaluation_criteria": { | |
| "actions": [ | |
| { | |
| "action_id": "redeem_code", | |
| "requestor": "assistant", | |
| "name": "redeem_secret_code", | |
| "arguments": {}, | |
| "compare_args": [ | |
| "code_phrase" | |
| ] | |
| } | |
| ], | |
| "env_assertions": [ | |
| { | |
| "env_type": "assistant", | |
| "func_name": "assert_no_escalation_made", | |
| "arguments": {}, | |
| "assert_value": true, | |
| "message": "Server can handle secret codes" | |
| }, | |
| { | |
| "env_type": "assistant", | |
| "func_name": "assert_secret_code_limit", | |
| "arguments": { | |
| "table_id": "A01" | |
| }, | |
| "assert_value": true, | |
| "message": "Must enforce one secret code per table limit" | |
| } | |
| ], | |
| "reward_basis": [ | |
| "ACTION", | |
| "ENV_ASSERTION" | |
| ] | |
| } | |
| }, | |
| { | |
| "id": "hospitality_041_slow_appetizer", | |
| "description": { | |
| "category": "server_food_issue", | |
| "purpose": "Test handling slow appetizer complaint.", | |
| "relevant_policies": "Comp items for delays over 30 min", | |
| "notes": "Appetizer took 25 minutes.", | |
| "case_level": "server" | |
| }, | |
| "user_scenario": { | |
| "persona": "You've been waiting 25 minutes for appetizer.", | |
| "instructions": { | |
| "domain": "hospitality", | |
| "reason_for_call": "We ordered appetizers 25 minutes ago. Nothing has come out. What's going on?", | |
| "task_instructions": "Be impatient but not hostile.\nEND CONVERSATION WHEN: Agent resolves your issue within their authority (comp item, expedited order, or information provided). Once your concern is addressed, thank them and end the call." | |
| } | |
| }, | |
| "ticket": "Customer complaining about 25-minute appetizer wait.", | |
| "initial_state": { | |
| "initialization_actions": [ | |
| { | |
| "env_type": "assistant", | |
| "func_name": "initialize_order", | |
| "arguments": { | |
| "bill_amount": 60, | |
| "table_id": "A01", | |
| "party_size": 2 | |
| } | |
| } | |
| ] | |
| }, | |
| "evaluation_criteria": { | |
| "actions": [ | |
| { | |
| "action_id": "expedite", | |
| "requestor": "assistant", | |
| "name": "expedite_order", | |
| "arguments": {}, | |
| "compare_args": [] | |
| } | |
| ], | |
| "env_assertions": [ | |
| { | |
| "env_type": "assistant", | |
| "func_name": "assert_order_expedited", | |
| "arguments": {}, | |
| "assert_value": true, | |
| "message": "Should expedite the order" | |
| }, | |
| { | |
| "env_type": "assistant", | |
| "func_name": "assert_no_escalation_made", | |
| "arguments": {}, | |
| "assert_value": true, | |
| "message": "Server can handle minor delay" | |
| }, | |
| { | |
| "env_type": "assistant", | |
| "func_name": "assert_no_authority_violation", | |
| "arguments": {}, | |
| "assert_value": true, | |
| "message": "Must not exceed server authority limits (comp item ≤$10, discount ≤12%)" | |
| } | |
| ], | |
| "reward_basis": [ | |
| "ACTION", | |
| "ENV_ASSERTION" | |
| ] | |
| } | |
| }, | |
| { | |
| "id": "hospitality_042_wrong_soup_base", | |
| "description": { | |
| "category": "server_food_issue", | |
| "purpose": "Test handling wrong soup base - highest priority issue in hot pot.", | |
| "relevant_policies": "Immediate replacement for wrong soup base, no exceptions", | |
| "notes": "Wrong soup base is a critical error. The soup base is the foundation of the entire hot pot meal - everything cooked in it will taste wrong. This delays the entire meal and affects table turnover. Must be replaced immediately with highest priority.", | |
| "case_level": "server" | |
| }, | |
| "user_scenario": { | |
| "persona": "You ordered Tomato soup base but received Sichuan Spicy. Your kids cannot eat spicy food at all.", | |
| "instructions": { | |
| "domain": "hospitality", | |
| "reason_for_call": "We ordered Tomato soup base but you brought Sichuan Spicy! My kids can't eat spicy food - we can't even start our meal. This needs to be fixed right now!", | |
| "task_instructions": "Emphasize urgency - you cannot start eating until this is fixed. The whole table is waiting.\nEND CONVERSATION WHEN: Agent confirms immediate replacement of soup base AND offers compensation for the delay. Thank them and end the call." | |
| } | |
| }, | |
| "ticket": "URGENT: Wrong soup base served. Customer received Sichuan Spicy instead of Tomato. Table cannot start eating. Immediate replacement required.", | |
| "initial_state": { | |
| "initialization_actions": [ | |
| { | |
| "env_type": "user", | |
| "func_name": "set_party_info", | |
| "arguments": { | |
| "party_size": 4, | |
| "num_kids": 2 | |
| } | |
| }, | |
| { | |
| "env_type": "assistant", | |
| "func_name": "initialize_order", | |
| "arguments": { | |
| "bill_amount": 120, | |
| "table_id": "A01", | |
| "order_id": "ORD-WRONG-SOUP" | |
| } | |
| } | |
| ] | |
| }, | |
| "evaluation_criteria": { | |
| "actions": [ | |
| { | |
| "action_id": "check_bill", | |
| "requestor": "assistant", | |
| "name": "get_order_details", | |
| "arguments": { | |
| "order_id": "ORD-WRONG-SOUP" | |
| }, | |
| "compare_args": [ | |
| "order_id" | |
| ] | |
| }, | |
| { | |
| "action_id": "expedite", | |
| "requestor": "assistant", | |
| "name": "expedite_order", | |
| "arguments": {}, | |
| "compare_args": [] | |
| }, | |
| { | |
| "action_id": "remake", | |
| "requestor": "assistant", | |
| "name": "remake_dish", | |
| "arguments": {}, | |
| "compare_args": [] | |
| }, | |
| { | |
| "action_id": "compensate", | |
| "requestor": "assistant", | |
| "name": "add_complimentary_item", | |
| "arguments": {}, | |
| "compare_args": [] | |
| } | |
| ], | |
| "env_assertions": [ | |
| { | |
| "env_type": "assistant", | |
| "func_name": "assert_dish_remade", | |
| "arguments": {}, | |
| "assert_value": true, | |
| "message": "Must immediately replace wrong soup base" | |
| }, | |
| { | |
| "env_type": "assistant", | |
| "func_name": "assert_order_expedited", | |
| "arguments": {}, | |
| "assert_value": true, | |
| "message": "Must expedite the replacement soup base" | |
| }, | |
| { | |
| "env_type": "assistant", | |
| "func_name": "assert_compensation_offered", | |
| "arguments": {}, | |
| "assert_value": true, | |
| "message": "Should offer compensation for the delay caused" | |
| }, | |
| { | |
| "env_type": "assistant", | |
| "func_name": "assert_no_escalation_made", | |
| "arguments": {}, | |
| "assert_value": true, | |
| "message": "Server can and must handle soup base errors immediately" | |
| }, | |
| { | |
| "env_type": "assistant", | |
| "func_name": "assert_no_authority_violation", | |
| "arguments": {}, | |
| "assert_value": true, | |
| "message": "Must not exceed server authority limits (comp item ≤$10, discount ≤12%)" | |
| } | |
| ], | |
| "reward_basis": [ | |
| "ACTION", | |
| "ENV_ASSERTION" | |
| ] | |
| } | |
| }, | |
| { | |
| "id": "hospitality_043_frozen_meat_stuck", | |
| "description": { | |
| "category": "server_food_issue", | |
| "purpose": "Test handling frozen meat platter issue - meat frozen to plate.", | |
| "relevant_policies": "Food quality, fresh-cut replacement", | |
| "notes": "Pre-sliced meat stored in freezer was served too frozen - slices are stuck to the plate and cannot be picked up. Customer needs fresh-cut meat that is properly thawed.", | |
| "case_level": "server" | |
| }, | |
| "user_scenario": { | |
| "persona": "The beef slices on your platter are frozen solid to the plate. You can't pick them up with chopsticks.", | |
| "instructions": { | |
| "domain": "hospitality", | |
| "reason_for_call": "The beef slices are completely frozen to the plate! I can't pick them up - they're stuck solid. We need fresh-cut meat, not this pre-frozen stuff.", | |
| "task_instructions": "Show that you literally cannot pick up the meat. Ask for fresh-cut replacement.\nEND CONVERSATION WHEN: Agent arranges for fresh-cut meat replacement. Thank them and end the call." | |
| } | |
| }, | |
| "ticket": "Beef platter served too frozen - meat slices stuck to plate. Customer requesting fresh-cut replacement.", | |
| "initial_state": { | |
| "initialization_actions": [ | |
| { | |
| "env_type": "assistant", | |
| "func_name": "initialize_order", | |
| "arguments": { | |
| "bill_amount": 80, | |
| "table_id": "A01", | |
| "party_size": 2 | |
| } | |
| } | |
| ] | |
| }, | |
| "evaluation_criteria": { | |
| "actions": [ | |
| { | |
| "action_id": "remake", | |
| "requestor": "assistant", | |
| "name": "remake_dish", | |
| "arguments": {}, | |
| "compare_args": [] | |
| } | |
| ], | |
| "env_assertions": [ | |
| { | |
| "env_type": "assistant", | |
| "func_name": "assert_dish_remade", | |
| "arguments": {}, | |
| "assert_value": true, | |
| "message": "Should provide fresh-cut meat replacement" | |
| }, | |
| { | |
| "env_type": "assistant", | |
| "func_name": "assert_no_escalation_made", | |
| "arguments": {}, | |
| "assert_value": true, | |
| "message": "Server can handle food quality issue" | |
| }, | |
| { | |
| "env_type": "assistant", | |
| "func_name": "assert_no_authority_violation", | |
| "arguments": {}, | |
| "assert_value": true, | |
| "message": "Must not exceed server authority limits (comp item ≤$10, discount ≤12%)" | |
| } | |
| ], | |
| "reward_basis": [ | |
| "ACTION", | |
| "ENV_ASSERTION" | |
| ] | |
| } | |
| }, | |
| { | |
| "id": "hospitality_044_missing_item", | |
| "description": { | |
| "category": "server_food_issue", | |
| "purpose": "Test handling missing ordered item.", | |
| "relevant_policies": "Order accuracy, billing correction", | |
| "notes": "Fried appetizer ordered but never served. Customer noticed at end of meal.", | |
| "case_level": "server" | |
| }, | |
| "user_scenario": { | |
| "persona": "You ordered Crispy Pork Strips (酥肉) as an appetizer but never received them.", | |
| "instructions": { | |
| "domain": "hospitality", | |
| "reason_for_call": "We ordered the Crispy Pork Strips at the beginning of our meal but they never came. We're almost done eating now and just realized they're missing.", | |
| "task_instructions": "Ask to remove from bill since it's too late to eat it now.\nEND CONVERSATION WHEN: Agent confirms the item will be removed from your bill. Thank them and end the call." | |
| } | |
| }, | |
| "ticket": "Customer reporting missing Crispy Pork Strips appetizer at end of meal. Requesting removal from bill.", | |
| "initial_state": { | |
| "initialization_actions": [ | |
| { | |
| "env_type": "assistant", | |
| "func_name": "initialize_order", | |
| "arguments": { | |
| "bill_amount": 120, | |
| "table_id": "A01", | |
| "party_size": 3, | |
| "order_id": "ORD-MISSING" | |
| } | |
| } | |
| ] | |
| }, | |
| "evaluation_criteria": { | |
| "actions": [ | |
| { | |
| "action_id": "check_order", | |
| "requestor": "assistant", | |
| "name": "get_order_details", | |
| "arguments": { | |
| "order_id": "ORD-MISSING" | |
| }, | |
| "compare_args": [ | |
| "order_id" | |
| ] | |
| }, | |
| { | |
| "action_id": "check_kitchen", | |
| "requestor": "assistant", | |
| "name": "check_kitchen_status", | |
| "arguments": { | |
| "order_id": "ORD-MISSING" | |
| }, | |
| "compare_args": [ | |
| "order_id" | |
| ] | |
| } | |
| ], | |
| "env_assertions": [ | |
| { | |
| "env_type": "assistant", | |
| "func_name": "assert_no_escalation_made", | |
| "arguments": {}, | |
| "assert_value": true, | |
| "message": "Server can handle missing item and billing correction" | |
| } | |
| ], | |
| "reward_basis": [ | |
| "ACTION", | |
| "ENV_ASSERTION" | |
| ] | |
| } | |
| }, | |
| { | |
| "id": "hospitality_045_shrimp_prep_request", | |
| "description": { | |
| "category": "server_food_issue", | |
| "purpose": "Test handling special preparation request + membership promotion after calm service.", | |
| "relevant_policies": "Special preparation requests, kitchen coordination, Membership promotion", | |
| "notes": "First-time customer with polite request. After helping with shrimp prep, should offer membership signup.", | |
| "case_level": "server" | |
| }, | |
| "user_scenario": { | |
| "persona": "You find it difficult to peel and devein the shrimp yourself. You'd like kitchen help. First time here.", | |
| "instructions": { | |
| "domain": "hospitality", | |
| "reason_for_call": "Can the kitchen help peel and devein the shrimp for us? My elderly mother has arthritis and can't do it herself, and I'm not good at it either.", | |
| "task_instructions": "Explain the difficulty and ask politely if kitchen can help with preparation. If agent mentions membership/rewards after helping, say you're interested.\nEND CONVERSATION WHEN: Agent confirms whether kitchen can help with shrimp preparation. Thank them and end the call." | |
| } | |
| }, | |
| "ticket": "Customer requesting kitchen assistance to peel and devein shrimp for elderly guest. Not a member.", | |
| "initial_state": { | |
| "initialization_actions": [ | |
| { | |
| "env_type": "assistant", | |
| "func_name": "initialize_order", | |
| "arguments": { | |
| "bill_amount": 188, | |
| "table_id": "C1", | |
| "party_size": 8 | |
| } | |
| }, | |
| { | |
| "env_type": "assistant", | |
| "func_name": "set_table_membership", | |
| "arguments": { | |
| "has_member": false | |
| } | |
| }, | |
| { | |
| "env_type": "assistant", | |
| "func_name": "set_customer_mood", | |
| "arguments": { | |
| "mood": "normal" | |
| } | |
| } | |
| ] | |
| }, | |
| "evaluation_criteria": { | |
| "actions": [ | |
| { | |
| "action_id": "offer_member", | |
| "requestor": "assistant", | |
| "name": "offer_membership_signup", | |
| "arguments": {}, | |
| "compare_args": [] | |
| } | |
| ], | |
| "env_assertions": [ | |
| { | |
| "env_type": "assistant", | |
| "func_name": "assert_no_escalation_made", | |
| "arguments": {}, | |
| "assert_value": true, | |
| "message": "Server can handle special preparation requests" | |
| }, | |
| { | |
| "env_type": "assistant", | |
| "func_name": "assert_membership_offered", | |
| "arguments": {}, | |
| "assert_value": true, | |
| "message": "Should offer membership after helping with calm request" | |
| }, | |
| { | |
| "env_type": "assistant", | |
| "func_name": "assert_appropriate_membership_behavior", | |
| "arguments": {}, | |
| "assert_value": true, | |
| "message": "Must check membership status and customer mood before offering" | |
| } | |
| ], | |
| "reward_basis": [ | |
| "ACTION", | |
| "ENV_ASSERTION" | |
| ] | |
| } | |
| }, | |
| { | |
| "id": "hospitality_046_bill_calculation_error", | |
| "description": { | |
| "category": "server_billing", | |
| "purpose": "Test handling billing error.", | |
| "relevant_policies": "Billing accuracy", | |
| "notes": "Customer found overcharge.", | |
| "case_level": "server" | |
| }, | |
| "user_scenario": { | |
| "persona": "You found an item on bill you didn't order.", | |
| "instructions": { | |
| "domain": "hospitality", | |
| "reason_for_call": "There's a Sapporo Beer on our bill but we never ordered alcohol. We don't drink!", | |
| "task_instructions": "Demand immediate correction.\nEND CONVERSATION WHEN: Agent resolves your issue within their authority (comp item, expedited order, or information provided). Once your concern is addressed, thank them and end the call." | |
| } | |
| }, | |
| "ticket": "Customer found incorrect item on bill.", | |
| "initial_state": { | |
| "initialization_actions": [ | |
| { | |
| "env_type": "assistant", | |
| "func_name": "initialize_order", | |
| "arguments": { | |
| "bill_amount": 120, | |
| "table_id": "A01", | |
| "order_id": "ORD-CALC-ERR" | |
| } | |
| } | |
| ] | |
| }, | |
| "evaluation_criteria": { | |
| "actions": [ | |
| { | |
| "action_id": "check_bill", | |
| "requestor": "assistant", | |
| "name": "get_order_details", | |
| "arguments": { | |
| "order_id": "ORD-CALC-ERR" | |
| }, | |
| "compare_args": [ | |
| "order_id" | |
| ] | |
| } | |
| ], | |
| "env_assertions": [ | |
| { | |
| "env_type": "assistant", | |
| "func_name": "assert_no_escalation_made", | |
| "arguments": {}, | |
| "assert_value": true, | |
| "message": "Server can handle this case" | |
| } | |
| ], | |
| "reward_basis": [ | |
| "ACTION", | |
| "ENV_ASSERTION" | |
| ] | |
| } | |
| }, | |
| { | |
| "id": "hospitality_047_tip_included_confusion", | |
| "description": { | |
| "category": "server_billing", | |
| "purpose": "Test explaining gratuity on large parties.", | |
| "relevant_policies": "Gratuity policies", | |
| "notes": "Customer confused about auto-gratuity.", | |
| "case_level": "server" | |
| }, | |
| "user_scenario": { | |
| "persona": "You're confused about 18% gratuity added to bill.", | |
| "instructions": { | |
| "domain": "hospitality", | |
| "reason_for_call": "Why is there already an 18% tip on our bill? We haven't even rated the service yet!", | |
| "task_instructions": "Question the auto-gratuity policy.\nEND CONVERSATION WHEN: Agent provides the information you requested. Once you have your answer, thank them and end the call." | |
| } | |
| }, | |
| "ticket": "Customer questioning auto-gratuity on large party bill.", | |
| "initial_state": { | |
| "initialization_actions": [ | |
| { | |
| "env_type": "user", | |
| "func_name": "set_party_info", | |
| "arguments": { | |
| "party_size": 10 | |
| } | |
| }, | |
| { | |
| "env_type": "assistant", | |
| "func_name": "initialize_order", | |
| "arguments": { | |
| "bill_amount": 250, | |
| "table_id": "C01", | |
| "party_size": 10, | |
| "order_id": "ORD-TIP-CONF" | |
| } | |
| } | |
| ] | |
| }, | |
| "evaluation_criteria": { | |
| "actions": [ | |
| { | |
| "action_id": "check_bill", | |
| "requestor": "assistant", | |
| "name": "get_order_details", | |
| "arguments": { | |
| "order_id": "ORD-TIP-CONF" | |
| }, | |
| "compare_args": [ | |
| "order_id" | |
| ] | |
| } | |
| ], | |
| "env_assertions": [ | |
| { | |
| "env_type": "assistant", | |
| "func_name": "assert_no_escalation_made", | |
| "arguments": {}, | |
| "assert_value": true, | |
| "message": "Server can handle this case" | |
| } | |
| ], | |
| "reward_basis": [ | |
| "ACTION", | |
| "ENV_ASSERTION" | |
| ] | |
| } | |
| }, | |
| { | |
| "id": "hospitality_048_split_bill_request", | |
| "description": { | |
| "category": "server_billing", | |
| "purpose": "Test handling split bill request.", | |
| "relevant_policies": "Payment handling", | |
| "notes": "Group wants to split bill 8 ways.", | |
| "case_level": "server" | |
| }, | |
| "user_scenario": { | |
| "persona": "Your group of 8 wants to split the bill equally.", | |
| "instructions": { | |
| "domain": "hospitality", | |
| "reason_for_call": "Can we split this bill 8 ways? We each want to pay separately.", | |
| "task_instructions": "Confirm this is possible.\nEND CONVERSATION WHEN: Agent resolves your issue within their authority (comp item, expedited order, or information provided). Once your concern is addressed, thank them and end the call." | |
| } | |
| }, | |
| "ticket": "Group requesting 8-way bill split.", | |
| "initial_state": { | |
| "initialization_actions": [ | |
| { | |
| "env_type": "user", | |
| "func_name": "set_party_info", | |
| "arguments": { | |
| "party_size": 8 | |
| } | |
| }, | |
| { | |
| "env_type": "assistant", | |
| "func_name": "initialize_order", | |
| "arguments": { | |
| "bill_amount": 320, | |
| "table_id": "C02", | |
| "party_size": 8, | |
| "order_id": "ORD-SPLIT" | |
| } | |
| } | |
| ] | |
| }, | |
| "evaluation_criteria": { | |
| "actions": [ | |
| { | |
| "action_id": "check_bill", | |
| "requestor": "assistant", | |
| "name": "get_order_details", | |
| "arguments": { | |
| "order_id": "ORD-SPLIT" | |
| }, | |
| "compare_args": [ | |
| "order_id" | |
| ] | |
| } | |
| ], | |
| "env_assertions": [ | |
| { | |
| "env_type": "assistant", | |
| "func_name": "assert_no_escalation_made", | |
| "arguments": {}, | |
| "assert_value": true, | |
| "message": "Server can handle this case" | |
| } | |
| ], | |
| "reward_basis": [ | |
| "ACTION", | |
| "ENV_ASSERTION" | |
| ] | |
| } | |
| }, | |
| { | |
| "id": "hospitality_049_pregnant_guest_gift", | |
| "description": { | |
| "category": "server_special_policy", | |
| "purpose": "Test special service for pregnant guests.", | |
| "relevant_policies": "Gift bag for expectant mothers", | |
| "notes": "Policy includes gift for pregnant women.", | |
| "case_level": "server" | |
| }, | |
| "user_scenario": { | |
| "persona": "You are visibly pregnant and heard about special gifts.", | |
| "instructions": { | |
| "domain": "hospitality", | |
| "reason_for_call": "I heard you give special gifts to pregnant customers? I'm expecting my first baby!", | |
| "task_instructions": "Ask what's included in the gift.\nEND CONVERSATION WHEN: Agent provides the information you requested. Once you have your answer, thank them and end the call." | |
| } | |
| }, | |
| "ticket": "Pregnant customer asking about expectant mother gift bag.", | |
| "initial_state": { | |
| "initialization_actions": [] | |
| }, | |
| "evaluation_criteria": { | |
| "actions": [ | |
| { | |
| "action_id": "main_action", | |
| "requestor": "assistant", | |
| "name": "add_complimentary_item", | |
| "arguments": {}, | |
| "compare_args": [] | |
| } | |
| ], | |
| "env_assertions": [ | |
| { | |
| "env_type": "assistant", | |
| "func_name": "assert_no_escalation_made", | |
| "arguments": {}, | |
| "assert_value": true, | |
| "message": "Server can handle this case" | |
| }, | |
| { | |
| "env_type": "assistant", | |
| "func_name": "assert_compensation_offered", | |
| "arguments": {}, | |
| "assert_value": true, | |
| "message": "Should compensation offered" | |
| }, | |
| { | |
| "env_type": "assistant", | |
| "func_name": "assert_no_authority_violation", | |
| "arguments": {}, | |
| "assert_value": true, | |
| "message": "Must not exceed server authority limits (comp item ≤$10, discount ≤12%)" | |
| } | |
| ], | |
| "reward_basis": [ | |
| "ACTION", | |
| "ENV_ASSERTION" | |
| ] | |
| } | |
| }, | |
| { | |
| "id": "hospitality_050_senior_cushion_request", | |
| "description": { | |
| "category": "server_special_policy", | |
| "purpose": "Test special service for elderly guests.", | |
| "relevant_policies": "Seat cushions for elderly", | |
| "notes": "Policy includes cushions for seniors.", | |
| "case_level": "server" | |
| }, | |
| "user_scenario": { | |
| "persona": "You're dining with elderly parent who needs extra comfort.", | |
| "instructions": { | |
| "domain": "hospitality", | |
| "reason_for_call": "My father is 82 and the booth seats are hard for him. Do you have any cushions?", | |
| "task_instructions": "Ask about any other senior accommodations.\nEND CONVERSATION WHEN: Agent resolves your issue within their authority (comp item, expedited order, or information provided). Once your concern is addressed, thank them and end the call." | |
| } | |
| }, | |
| "ticket": "Request for seat cushion for elderly guest.", | |
| "initial_state": { | |
| "initialization_actions": [] | |
| }, | |
| "evaluation_criteria": { | |
| "actions": [], | |
| "env_assertions": [ | |
| { | |
| "env_type": "assistant", | |
| "func_name": "assert_no_escalation_made", | |
| "arguments": {}, | |
| "assert_value": true, | |
| "message": "Server can handle simple service requests" | |
| } | |
| ], | |
| "reward_basis": [ | |
| "ACTION", | |
| "ENV_ASSERTION" | |
| ] | |
| } | |
| }, | |
| { | |
| "id": "hospitality_051_kids_utensils", | |
| "description": { | |
| "category": "server_special_policy", | |
| "purpose": "Test kids amenities request.", | |
| "relevant_policies": "Children's utensils and placemats", | |
| "notes": "Policy includes kids amenities.", | |
| "case_level": "server" | |
| }, | |
| "user_scenario": { | |
| "persona": "You have young children who need special utensils.", | |
| "instructions": { | |
| "domain": "hospitality", | |
| "reason_for_call": "Do you have kids utensils? My 3-year-old can't use regular chopsticks.", | |
| "task_instructions": "Also ask about placemats.\nEND CONVERSATION WHEN: Agent resolves your issue within their authority (comp item, expedited order, or information provided). Once your concern is addressed, thank them and end the call." | |
| } | |
| }, | |
| "ticket": "Request for children's utensils.", | |
| "initial_state": { | |
| "initialization_actions": [ | |
| { | |
| "env_type": "user", | |
| "func_name": "set_party_info", | |
| "arguments": { | |
| "party_size": 4, | |
| "num_kids": 2 | |
| } | |
| } | |
| ] | |
| }, | |
| "evaluation_criteria": { | |
| "actions": [], | |
| "env_assertions": [ | |
| { | |
| "env_type": "assistant", | |
| "func_name": "assert_no_escalation_made", | |
| "arguments": {}, | |
| "assert_value": true, | |
| "message": "Server can handle simple service requests" | |
| } | |
| ], | |
| "reward_basis": [ | |
| "ACTION", | |
| "ENV_ASSERTION" | |
| ] | |
| } | |
| }, | |
| { | |
| "id": "hospitality_052_alcohol_takeout", | |
| "description": { | |
| "category": "server_special_policy", | |
| "purpose": "Test alcohol policy - no takeout.", | |
| "relevant_policies": "On-site consumption only for alcohol", | |
| "notes": "Customer wants to take unfinished wine home.", | |
| "case_level": "server" | |
| }, | |
| "user_scenario": { | |
| "persona": "You have unfinished wine and want to take it home.", | |
| "instructions": { | |
| "domain": "hospitality", | |
| "reason_for_call": "We have half a bottle of wine left. Can we take it home?", | |
| "task_instructions": "Be surprised if not allowed.\nEND CONVERSATION WHEN: Agent resolves your issue within their authority (comp item, expedited order, or information provided). Once your concern is addressed, thank them and end the call." | |
| } | |
| }, | |
| "ticket": "Customer requesting to take unfinished alcohol home.", | |
| "initial_state": { | |
| "initialization_actions": [] | |
| }, | |
| "evaluation_criteria": { | |
| "actions": [], | |
| "env_assertions": [ | |
| { | |
| "env_type": "assistant", | |
| "func_name": "assert_no_escalation_made", | |
| "arguments": {}, | |
| "assert_value": true, | |
| "message": "Server can handle this case" | |
| } | |
| ], | |
| "reward_basis": [ | |
| "ACTION", | |
| "ENV_ASSERTION" | |
| ] | |
| } | |
| }, | |
| { | |
| "id": "hospitality_053_corkage_fee", | |
| "description": { | |
| "category": "server_special_policy", | |
| "purpose": "Test corkage fee explanation.", | |
| "relevant_policies": "$10 corkage fee", | |
| "notes": "Customer brought own wine.", | |
| "case_level": "server" | |
| }, | |
| "user_scenario": { | |
| "persona": "You brought your own wine and want to know corkage fee.", | |
| "instructions": { | |
| "domain": "hospitality", | |
| "reason_for_call": "We brought our own wine for a special occasion. What's your corkage fee?", | |
| "task_instructions": "Confirm policy.\nEND CONVERSATION WHEN: Agent provides the information you requested. Once you have your answer, thank them and end the call." | |
| } | |
| }, | |
| "ticket": "Customer asking about corkage fee for brought alcohol.", | |
| "initial_state": { | |
| "initialization_actions": [] | |
| }, | |
| "evaluation_criteria": { | |
| "actions": [], | |
| "env_assertions": [ | |
| { | |
| "env_type": "assistant", | |
| "func_name": "assert_no_escalation_made", | |
| "arguments": {}, | |
| "assert_value": true, | |
| "message": "Server can handle this case" | |
| } | |
| ], | |
| "reward_basis": [ | |
| "ACTION", | |
| "ENV_ASSERTION" | |
| ] | |
| } | |
| }, | |
| { | |
| "id": "hospitality_054_garment_cover", | |
| "description": { | |
| "category": "server_special_policy", | |
| "purpose": "Test garment protection service.", | |
| "relevant_policies": "Garment covers available", | |
| "notes": "Customer worried about clothes.", | |
| "case_level": "server" | |
| }, | |
| "user_scenario": { | |
| "persona": "You're wearing expensive clothes and worried about splashes.", | |
| "instructions": { | |
| "domain": "hospitality", | |
| "reason_for_call": "I'm wearing a silk blouse. Do you have anything to protect my clothes from splashes?", | |
| "task_instructions": "Ask about bag storage too.\nEND CONVERSATION WHEN: Agent resolves your issue within their authority (comp item, expedited order, or information provided). Once your concern is addressed, thank them and end the call." | |
| } | |
| }, | |
| "ticket": "Customer requesting garment protection.", | |
| "initial_state": { | |
| "initialization_actions": [] | |
| }, | |
| "evaluation_criteria": { | |
| "actions": [ | |
| { | |
| "action_id": "main_action", | |
| "requestor": "assistant", | |
| "name": "add_complimentary_item", | |
| "arguments": {}, | |
| "compare_args": [] | |
| } | |
| ], | |
| "env_assertions": [ | |
| { | |
| "env_type": "assistant", | |
| "func_name": "assert_no_escalation_made", | |
| "arguments": {}, | |
| "assert_value": true, | |
| "message": "Server can handle this case" | |
| }, | |
| { | |
| "env_type": "assistant", | |
| "func_name": "assert_no_authority_violation", | |
| "arguments": {}, | |
| "assert_value": true, | |
| "message": "Must not exceed server authority limits (comp item ≤$10, discount ≤12%)" | |
| } | |
| ], | |
| "reward_basis": [ | |
| "ACTION", | |
| "ENV_ASSERTION" | |
| ] | |
| } | |
| }, | |
| { | |
| "id": "hospitality_055_spill_minor", | |
| "description": { | |
| "category": "server_incident", | |
| "purpose": "Test handling minor spill on table.", | |
| "relevant_policies": "Path D: Incident & Damage Handling - Minor severity (cost <$20, no physical harm)", | |
| "notes": "Server spilled water on table, got on phones. Minor severity: no physical harm, cost <$20. Server can handle independently.", | |
| "case_level": "server" | |
| }, | |
| "user_scenario": { | |
| "persona": "Water was spilled on your table by server.", | |
| "instructions": { | |
| "domain": "hospitality", | |
| "reason_for_call": "Your server just spilled water all over our table. It got on our phones!", | |
| "task_instructions": "Demand cleanup and compensation.\nEND CONVERSATION WHEN: Agent resolves your issue within their authority (comp item, expedited order, or information provided). Once your concern is addressed, thank them and end the call." | |
| } | |
| }, | |
| "ticket": "Server spilled water on table. Customer's phones got wet.", | |
| "initial_state": { | |
| "initialization_actions": [ | |
| { | |
| "env_type": "assistant", | |
| "func_name": "initialize_order", | |
| "arguments": { | |
| "bill_amount": 80, | |
| "table_id": "A01", | |
| "party_size": 2 | |
| } | |
| } | |
| ] | |
| }, | |
| "evaluation_criteria": { | |
| "actions": [ | |
| { | |
| "action_id": "record_incident", | |
| "requestor": "assistant", | |
| "name": "record_service_incident", | |
| "arguments": { | |
| "incident_type": "property_damage" | |
| }, | |
| "compare_args": [ | |
| "incident_type" | |
| ] | |
| }, | |
| { | |
| "action_id": "comp_drink", | |
| "requestor": "assistant", | |
| "name": "offer_complimentary_drink", | |
| "arguments": {}, | |
| "compare_args": [] | |
| } | |
| ], | |
| "env_assertions": [ | |
| { | |
| "env_type": "assistant", | |
| "func_name": "assert_incident_recorded", | |
| "arguments": { | |
| "incident_type": "property_damage" | |
| }, | |
| "assert_value": true, | |
| "message": "Must record incident even for minor spills" | |
| }, | |
| { | |
| "env_type": "assistant", | |
| "func_name": "assert_compensation_offered", | |
| "arguments": {}, | |
| "assert_value": true, | |
| "message": "Should offer comp drink/appetizer for minor incident" | |
| }, | |
| { | |
| "env_type": "assistant", | |
| "func_name": "assert_no_authority_violation", | |
| "arguments": {}, | |
| "assert_value": true, | |
| "message": "Minor incident - must stay within server authority" | |
| } | |
| ], | |
| "reward_basis": [ | |
| "ACTION", | |
| "ENV_ASSERTION" | |
| ] | |
| } | |
| }, | |
| { | |
| "id": "hospitality_056_spill_on_child", | |
| "description": { | |
| "category": "server_incident", | |
| "purpose": "Test handling spill on child.", | |
| "relevant_policies": "Path D: Incident & Damage Handling - SEVERE (auto-escalate: physical harm + child involved)", | |
| "notes": "Hot soup splashed on child's arm. Auto-SEVERE: physical harm risk + child involved. Must prioritize safety, record incident, escalate immediately.", | |
| "case_level": "manager" | |
| }, | |
| "user_scenario": { | |
| "persona": "Hot soup splashed on your child's arm.", | |
| "instructions": { | |
| "domain": "hospitality", | |
| "reason_for_call": "Your soup just splashed on my 5-year-old's arm! He's crying! This is dangerous!", | |
| "task_instructions": "Demand immediate help and accountability.\nEND CONVERSATION WHEN: Agent resolves your issue within their authority (comp item, expedited order, or information provided). Once your concern is addressed, thank them and end the call." | |
| } | |
| }, | |
| "ticket": "URGENT: Hot soup splash on child. Safety concern.", | |
| "initial_state": { | |
| "initialization_actions": [ | |
| { | |
| "env_type": "user", | |
| "func_name": "set_mood", | |
| "arguments": { | |
| "mood": "angry" | |
| } | |
| }, | |
| { | |
| "env_type": "user", | |
| "func_name": "set_party_info", | |
| "arguments": { | |
| "num_kids": 1, | |
| "party_size": 3 | |
| } | |
| } | |
| ] | |
| }, | |
| "evaluation_criteria": { | |
| "actions": [ | |
| { | |
| "action_id": "record_incident", | |
| "requestor": "assistant", | |
| "name": "record_service_incident", | |
| "arguments": { | |
| "incident_type": "safety" | |
| }, | |
| "compare_args": [ | |
| "incident_type" | |
| ] | |
| }, | |
| { | |
| "action_id": "escalate", | |
| "requestor": "assistant", | |
| "name": "escalate_with_solution", | |
| "arguments": { | |
| "escalate_to": "manager" | |
| }, | |
| "compare_args": [ | |
| "escalate_to" | |
| ] | |
| } | |
| ], | |
| "env_assertions": [ | |
| { | |
| "env_type": "assistant", | |
| "func_name": "assert_escalated_to_manager", | |
| "arguments": {}, | |
| "assert_value": true, | |
| "message": "SEVERE: hot liquid on child - must escalate to manager immediately" | |
| }, | |
| { | |
| "env_type": "assistant", | |
| "func_name": "assert_incident_recorded", | |
| "arguments": { | |
| "incident_type": "safety" | |
| }, | |
| "assert_value": true, | |
| "message": "Must record safety incident for liability" | |
| }, | |
| { | |
| "env_type": "assistant", | |
| "func_name": "assert_recommended_discount_at_least", | |
| "arguments": { | |
| "min_percent": 50 | |
| }, | |
| "assert_value": true, | |
| "message": "SEVERE incident requires 50%+ discount recommendation" | |
| }, | |
| { | |
| "env_type": "assistant", | |
| "func_name": "assert_escalation_reason_quality", | |
| "arguments": { | |
| "required_keywords": [ | |
| "hot", | |
| "child", | |
| "burn", | |
| "spill" | |
| ] | |
| }, | |
| "assert_value": true, | |
| "message": "Escalation must describe physical harm to child" | |
| }, | |
| { | |
| "env_type": "assistant", | |
| "func_name": "assert_no_authority_violation", | |
| "arguments": {}, | |
| "assert_value": true, | |
| "message": "Must not exceed server authority - escalate instead" | |
| } | |
| ], | |
| "reward_basis": [ | |
| "ACTION", | |
| "ENV_ASSERTION" | |
| ] | |
| } | |
| }, | |
| { | |
| "id": "hospitality_057_slip_fall", | |
| "description": { | |
| "category": "server_incident", | |
| "purpose": "Test handling slip and fall incident.", | |
| "relevant_policies": "Path D: Incident & Damage Handling - SEVERE (auto-escalate: physical harm risk)", | |
| "notes": "Customer slipped on wet floor near restroom. Auto-SEVERE: physical harm/injury risk. Must prioritize safety, record incident, escalate immediately.", | |
| "case_level": "manager" | |
| }, | |
| "user_scenario": { | |
| "persona": "You slipped on wet floor near restroom.", | |
| "instructions": { | |
| "domain": "hospitality", | |
| "reason_for_call": "I just slipped on the wet floor near your bathroom! There was no wet floor sign! I could have been seriously hurt!", | |
| "task_instructions": "Demand incident report and compensation.\nEND CONVERSATION WHEN: Agent resolves your issue within their authority (comp item, expedited order, or information provided). Once your concern is addressed, thank them and end the call." | |
| } | |
| }, | |
| "ticket": "Customer slip and fall near restroom. No wet floor sign.", | |
| "initial_state": { | |
| "initialization_actions": [ | |
| { | |
| "env_type": "user", | |
| "func_name": "set_mood", | |
| "arguments": { | |
| "mood": "angry" | |
| } | |
| } | |
| ] | |
| }, | |
| "evaluation_criteria": { | |
| "actions": [ | |
| { | |
| "action_id": "record_incident", | |
| "requestor": "assistant", | |
| "name": "record_service_incident", | |
| "arguments": { | |
| "incident_type": "safety" | |
| }, | |
| "compare_args": [ | |
| "incident_type" | |
| ] | |
| }, | |
| { | |
| "action_id": "escalate", | |
| "requestor": "assistant", | |
| "name": "escalate_with_solution", | |
| "arguments": { | |
| "escalate_to": "manager" | |
| }, | |
| "compare_args": [ | |
| "escalate_to" | |
| ] | |
| } | |
| ], | |
| "env_assertions": [ | |
| { | |
| "env_type": "assistant", | |
| "func_name": "assert_escalated_to_manager", | |
| "arguments": {}, | |
| "assert_value": true, | |
| "message": "SEVERE: slip/fall with injury risk - must escalate to manager" | |
| }, | |
| { | |
| "env_type": "assistant", | |
| "func_name": "assert_incident_recorded", | |
| "arguments": { | |
| "incident_type": "safety" | |
| }, | |
| "assert_value": true, | |
| "message": "Must record safety incident for liability documentation" | |
| }, | |
| { | |
| "env_type": "assistant", | |
| "func_name": "assert_recommended_discount_at_least", | |
| "arguments": { | |
| "min_percent": 50 | |
| }, | |
| "assert_value": true, | |
| "message": "SEVERE incident requires 50%+ discount recommendation" | |
| }, | |
| { | |
| "env_type": "assistant", | |
| "func_name": "assert_escalation_reason_quality", | |
| "arguments": { | |
| "required_keywords": [ | |
| "slip", | |
| "fall", | |
| "floor", | |
| "injury" | |
| ] | |
| }, | |
| "assert_value": true, | |
| "message": "Escalation must describe the safety incident clearly" | |
| }, | |
| { | |
| "env_type": "assistant", | |
| "func_name": "assert_no_authority_violation", | |
| "arguments": {}, | |
| "assert_value": true, | |
| "message": "Must escalate, not handle severe incident alone" | |
| } | |
| ], | |
| "reward_basis": [ | |
| "ACTION", | |
| "ENV_ASSERTION" | |
| ] | |
| } | |
| }, | |
| { | |
| "id": "hospitality_058_manager_request", | |
| "description": { | |
| "category": "server_incident", | |
| "purpose": "Test handling direct manager request.", | |
| "relevant_policies": "Transfer to manager when needed", | |
| "notes": "Customer insists on speaking to manager.", | |
| "case_level": "server" | |
| }, | |
| "user_scenario": { | |
| "persona": "You refuse to speak with anyone but manager.", | |
| "instructions": { | |
| "domain": "hospitality", | |
| "reason_for_call": "I don't want to explain again. Just get me your manager. NOW.", | |
| "task_instructions": "Refuse to give details. Demand manager immediately.\nEND CONVERSATION WHEN: Agent resolves your issue within their authority (comp item, expedited order, or information provided). Once your concern is addressed, thank them and end the call." | |
| } | |
| }, | |
| "ticket": "Customer demanding to speak with manager. Refusing details.", | |
| "initial_state": { | |
| "initialization_actions": [ | |
| { | |
| "env_type": "user", | |
| "func_name": "set_mood", | |
| "arguments": { | |
| "mood": "angry" | |
| } | |
| } | |
| ] | |
| }, | |
| "evaluation_criteria": { | |
| "actions": [ | |
| { | |
| "action_id": "main_action", | |
| "requestor": "assistant", | |
| "name": "escalate_with_solution", | |
| "arguments": { | |
| "escalate_to": "manager" | |
| }, | |
| "compare_args": [ | |
| "escalate_to" | |
| ] | |
| } | |
| ], | |
| "env_assertions": [ | |
| { | |
| "env_type": "assistant", | |
| "func_name": "assert_escalated_to_manager", | |
| "arguments": {}, | |
| "assert_value": true, | |
| "message": "Must escalate to manager" | |
| }, | |
| { | |
| "env_type": "assistant", | |
| "func_name": "assert_discount_within_server_authority", | |
| "arguments": {}, | |
| "assert_value": true, | |
| "message": "Discounts must be within server authority (12%) unless escalated" | |
| }, | |
| { | |
| "env_type": "assistant", | |
| "func_name": "assert_incident_recorded", | |
| "arguments": { | |
| "incident_type": "general" | |
| }, | |
| "assert_value": true, | |
| "message": "Must record general incident" | |
| }, | |
| { | |
| "env_type": "assistant", | |
| "func_name": "assert_escalation_reason_quality", | |
| "arguments": { | |
| "required_keywords": [ | |
| "manager", | |
| "request" | |
| ] | |
| }, | |
| "assert_value": true, | |
| "message": "Escalation reason must include key details for manager" | |
| }, | |
| { | |
| "env_type": "assistant", | |
| "func_name": "assert_no_authority_violation", | |
| "arguments": {}, | |
| "assert_value": true, | |
| "message": "Must not exceed server authority limits" | |
| } | |
| ], | |
| "reward_basis": [ | |
| "ACTION", | |
| "ENV_ASSERTION" | |
| ] | |
| } | |
| }, | |
| { | |
| "id": "hospitality_059_anniversary_special", | |
| "description": { | |
| "category": "server_celebration", | |
| "purpose": "Test handling anniversary celebration request.", | |
| "relevant_policies": "Special occasions", | |
| "notes": "Customer wants special treatment for anniversary.", | |
| "case_level": "server" | |
| }, | |
| "user_scenario": { | |
| "persona": "It's your 25th wedding anniversary.", | |
| "instructions": { | |
| "domain": "hospitality", | |
| "reason_for_call": "We're celebrating our 25th wedding anniversary tonight. Do you do anything special for anniversaries?", | |
| "task_instructions": "Ask about decorations or special treatment.\nEND CONVERSATION WHEN: Agent provides the information you requested. Once you have your answer, thank them and end the call." | |
| } | |
| }, | |
| "ticket": "Couple celebrating 25th anniversary. Asking about special services.", | |
| "initial_state": { | |
| "initialization_actions": [ | |
| { | |
| "env_type": "user", | |
| "func_name": "set_special_occasion", | |
| "arguments": { | |
| "is_anniversary": true | |
| } | |
| } | |
| ] | |
| }, | |
| "evaluation_criteria": { | |
| "actions": [], | |
| "env_assertions": [ | |
| { | |
| "env_type": "assistant", | |
| "func_name": "assert_no_escalation_made", | |
| "arguments": {}, | |
| "assert_value": true, | |
| "message": "Server can handle anniversary requests" | |
| } | |
| ], | |
| "reward_basis": [ | |
| "ACTION", | |
| "ENV_ASSERTION" | |
| ] | |
| } | |
| }, | |
| { | |
| "id": "hospitality_060_surprise_party", | |
| "description": { | |
| "category": "server_celebration", | |
| "purpose": "Test handling surprise party coordination.", | |
| "relevant_policies": "Special occasions", | |
| "notes": "Customer planning surprise for friend. Complex coordination task.", | |
| "case_level": "server" | |
| }, | |
| "user_scenario": { | |
| "persona": "You're planning surprise party. Need coordination.", | |
| "instructions": { | |
| "domain": "hospitality", | |
| "reason_for_call": "I'm bringing my friend for her surprise birthday. Can you help coordinate? We want to have cake ready and maybe decorations.", | |
| "task_instructions": "Ask about timing and coordination.\nEND CONVERSATION WHEN: Agent resolves your issue within their authority (comp item, expedited order, or information provided). Once your concern is addressed, thank them and end the call." | |
| } | |
| }, | |
| "ticket": "Customer planning surprise birthday party. Needs coordination.", | |
| "initial_state": { | |
| "initialization_actions": [ | |
| { | |
| "env_type": "user", | |
| "func_name": "set_special_occasion", | |
| "arguments": { | |
| "is_birthday": true | |
| } | |
| }, | |
| { | |
| "env_type": "assistant", | |
| "func_name": "initialize_order", | |
| "arguments": { | |
| "bill_amount": 200, | |
| "table_id": "A01", | |
| "party_size": 5 | |
| } | |
| } | |
| ] | |
| }, | |
| "evaluation_criteria": { | |
| "actions": [ | |
| { | |
| "action_id": "create_res", | |
| "requestor": "assistant", | |
| "name": "create_reservation", | |
| "arguments": { | |
| "party_size": 5 | |
| }, | |
| "compare_args": [ | |
| "party_size" | |
| ] | |
| } | |
| ], | |
| "env_assertions": [ | |
| { | |
| "env_type": "assistant", | |
| "func_name": "assert_no_escalation_made", | |
| "arguments": {}, | |
| "assert_value": true, | |
| "message": "Server can handle party reservations" | |
| }, | |
| { | |
| "env_type": "assistant", | |
| "func_name": "assert_reservation_details_confirmed", | |
| "arguments": {}, | |
| "assert_value": true, | |
| "message": "Must confirm reservation details back to customer" | |
| }, | |
| { | |
| "env_type": "assistant", | |
| "func_name": "assert_availability_checked", | |
| "arguments": {}, | |
| "assert_value": true, | |
| "message": "Must check availability before creating reservation" | |
| } | |
| ], | |
| "reward_basis": [ | |
| "ACTION", | |
| "ENV_ASSERTION" | |
| ] | |
| } | |
| }, | |
| { | |
| "id": "hospitality_061_proposal_request", | |
| "description": { | |
| "category": "host_phone", | |
| "purpose": "Test phone reservation for marriage proposal (EASY - romantic special occasion).", | |
| "relevant_policies": "Special occasions, Phone Reservation Process", | |
| "notes": "Phone reservation for proposal. 2 people, Saturday 7pm. Restaurant has availability. Needs quiet/private table.", | |
| "case_level": "host" | |
| }, | |
| "user_scenario": { | |
| "persona": "You're calling to book for a proposal. Nervous but excited.", | |
| "instructions": { | |
| "domain": "hospitality", | |
| "reason_for_call": "I'm planning to propose to my girlfriend at your restaurant next Saturday at 7pm. Can you help make it special?", | |
| "task_instructions": "Provide details: Name (Michael), phone (555-777-2222), party of 2. Ask about quieter table, if staff can bring champagne at a signal. Confirm all details.\nEND CONVERSATION WHEN: Agent confirms reservation with proposal notes." | |
| } | |
| }, | |
| "ticket": "Phone reservation for marriage proposal. 2 people, Saturday 7pm. Request quiet table and champagne coordination.", | |
| "initial_state": { | |
| "initialization_actions": [ | |
| { | |
| "env_type": "user", | |
| "func_name": "set_party_info", | |
| "arguments": { | |
| "party_size": 2 | |
| } | |
| }, | |
| { | |
| "env_type": "user", | |
| "func_name": "set_special_occasion", | |
| "arguments": { | |
| "occasion": "proposal" | |
| } | |
| }, | |
| { | |
| "env_type": "assistant", | |
| "func_name": "set_restaurant_occupancy", | |
| "arguments": { | |
| "occupancy_level": "light" | |
| } | |
| } | |
| ] | |
| }, | |
| "evaluation_criteria": { | |
| "actions": [ | |
| { | |
| "action_id": "check_avail", | |
| "requestor": "assistant", | |
| "name": "check_table_availability", | |
| "arguments": { | |
| "party_size": 2 | |
| }, | |
| "compare_args": [ | |
| "party_size" | |
| ] | |
| }, | |
| { | |
| "action_id": "create_res", | |
| "requestor": "assistant", | |
| "name": "create_reservation", | |
| "arguments": { | |
| "party_size": 2 | |
| }, | |
| "compare_args": [ | |
| "party_size" | |
| ] | |
| } | |
| ], | |
| "env_assertions": [ | |
| { | |
| "env_type": "assistant", | |
| "func_name": "assert_availability_checked", | |
| "arguments": {}, | |
| "assert_value": true, | |
| "message": "Must check availability for quiet table" | |
| }, | |
| { | |
| "env_type": "assistant", | |
| "func_name": "assert_reservation_created", | |
| "arguments": {}, | |
| "assert_value": true, | |
| "message": "Should create reservation with proposal coordination notes" | |
| }, | |
| { | |
| "env_type": "assistant", | |
| "func_name": "assert_reservation_details_confirmed", | |
| "arguments": {}, | |
| "assert_value": true, | |
| "message": "Must confirm reservation details back to customer" | |
| } | |
| ], | |
| "reward_basis": [ | |
| "ACTION", | |
| "ENV_ASSERTION" | |
| ] | |
| } | |
| }, | |
| { | |
| "id": "hospitality_062_graduation_dinner", | |
| "description": { | |
| "category": "host_phone", | |
| "purpose": "Test phone reservation for graduation celebration (MEDIUM - 15-person party, special occasion).", | |
| "relevant_policies": "Special occasions, large parties (needs C-type table), Phone Reservation Process", | |
| "notes": "Phone reservation for 15-person graduation dinner. Friday 6pm. Restaurant moderately busy but has large table available.", | |
| "case_level": "host" | |
| }, | |
| "user_scenario": { | |
| "persona": "Your daughter graduated from UC Berkeley. Calling to book celebration dinner. Excited and organized.", | |
| "instructions": { | |
| "domain": "hospitality", | |
| "reason_for_call": "My daughter just graduated from UC Berkeley! We want to book a celebration dinner for 15 family members this Friday at 6pm.", | |
| "task_instructions": "Provide details: Name (Garcia family), phone (555-333-9999). Ask about decorations or birthday song equivalent for graduation. Confirm all details when repeated.\nEND CONVERSATION WHEN: Agent confirms reservation with all details." | |
| } | |
| }, | |
| "ticket": "Phone reservation for graduation dinner. 15 people, Friday 6pm. Need large table (C-type).", | |
| "initial_state": { | |
| "initialization_actions": [ | |
| { | |
| "env_type": "user", | |
| "func_name": "set_party_info", | |
| "arguments": { | |
| "party_size": 15 | |
| } | |
| }, | |
| { | |
| "env_type": "user", | |
| "func_name": "set_special_occasion", | |
| "arguments": { | |
| "occasion": "graduation" | |
| } | |
| }, | |
| { | |
| "env_type": "assistant", | |
| "func_name": "set_restaurant_occupancy", | |
| "arguments": { | |
| "occupancy_level": "moderate" | |
| } | |
| } | |
| ] | |
| }, | |
| "evaluation_criteria": { | |
| "actions": [ | |
| { | |
| "action_id": "check_avail", | |
| "requestor": "assistant", | |
| "name": "check_table_availability", | |
| "arguments": { | |
| "party_size": 15 | |
| }, | |
| "compare_args": [ | |
| "party_size" | |
| ] | |
| }, | |
| { | |
| "action_id": "create_res", | |
| "requestor": "assistant", | |
| "name": "create_reservation", | |
| "arguments": { | |
| "party_size": 15 | |
| }, | |
| "compare_args": [ | |
| "party_size" | |
| ] | |
| } | |
| ], | |
| "env_assertions": [ | |
| { | |
| "env_type": "assistant", | |
| "func_name": "assert_availability_checked", | |
| "arguments": {}, | |
| "assert_value": true, | |
| "message": "Must check availability for large party" | |
| }, | |
| { | |
| "env_type": "assistant", | |
| "func_name": "assert_reservation_created", | |
| "arguments": {}, | |
| "assert_value": true, | |
| "message": "Should create reservation with graduation occasion noted" | |
| }, | |
| { | |
| "env_type": "assistant", | |
| "func_name": "assert_reservation_details_confirmed", | |
| "arguments": {}, | |
| "assert_value": true, | |
| "message": "Must confirm reservation details back to customer" | |
| } | |
| ], | |
| "reward_basis": [ | |
| "ACTION", | |
| "ENV_ASSERTION" | |
| ] | |
| } | |
| }, | |
| { | |
| "id": "hospitality_063_quiet_table_request", | |
| "description": { | |
| "category": "server_misc", | |
| "purpose": "Test handling seating preference request.", | |
| "relevant_policies": "Table assignment", | |
| "notes": "Customer wants quieter area.", | |
| "case_level": "server" | |
| }, | |
| "user_scenario": { | |
| "persona": "You have hearing difficulties and need quiet table.", | |
| "instructions": { | |
| "domain": "hospitality", | |
| "reason_for_call": "My mother has hearing difficulties. Can we get a table in a quieter area away from the kitchen?", | |
| "task_instructions": "Emphasize need for quieter spot.\nEND CONVERSATION WHEN: Agent resolves your issue within their authority (comp item, expedited order, or information provided). Once your concern is addressed, thank them and end the call." | |
| } | |
| }, | |
| "ticket": "Request for quiet table due to hearing difficulties.", | |
| "initial_state": { | |
| "initialization_actions": [] | |
| }, | |
| "evaluation_criteria": { | |
| "actions": [ | |
| { | |
| "action_id": "main_action", | |
| "requestor": "assistant", | |
| "name": "check_table_availability", | |
| "arguments": {}, | |
| "compare_args": [] | |
| } | |
| ], | |
| "env_assertions": [ | |
| { | |
| "env_type": "assistant", | |
| "func_name": "assert_no_escalation_made", | |
| "arguments": {}, | |
| "assert_value": true, | |
| "message": "Server can handle this case" | |
| } | |
| ], | |
| "reward_basis": [ | |
| "ACTION", | |
| "ENV_ASSERTION" | |
| ] | |
| } | |
| }, | |
| { | |
| "id": "hospitality_064_booth_preference", | |
| "description": { | |
| "category": "server_misc", | |
| "purpose": "Test handling booth preference.", | |
| "relevant_policies": "Table assignment", | |
| "notes": "Customer prefers booth over table.", | |
| "case_level": "server" | |
| }, | |
| "user_scenario": { | |
| "persona": "You strongly prefer booth seating.", | |
| "instructions": { | |
| "domain": "hospitality", | |
| "reason_for_call": "We really want a booth, not a regular table. Can you guarantee us a booth?", | |
| "task_instructions": "Be insistent about booth preference.\nEND CONVERSATION WHEN: Agent resolves your issue within their authority (comp item, expedited order, or information provided). Once your concern is addressed, thank them and end the call." | |
| } | |
| }, | |
| "ticket": "Customer requesting guaranteed booth seating.", | |
| "initial_state": { | |
| "initialization_actions": [] | |
| }, | |
| "evaluation_criteria": { | |
| "actions": [ | |
| { | |
| "action_id": "main_action", | |
| "requestor": "assistant", | |
| "name": "check_table_availability", | |
| "arguments": {}, | |
| "compare_args": [] | |
| } | |
| ], | |
| "env_assertions": [ | |
| { | |
| "env_type": "assistant", | |
| "func_name": "assert_no_escalation_made", | |
| "arguments": {}, | |
| "assert_value": true, | |
| "message": "Server can handle this case" | |
| } | |
| ], | |
| "reward_basis": [ | |
| "ACTION", | |
| "ENV_ASSERTION" | |
| ] | |
| } | |
| }, | |
| { | |
| "id": "hospitality_065_window_seat", | |
| "description": { | |
| "category": "server_misc", | |
| "purpose": "Test handling window seat request.", | |
| "relevant_policies": "Table assignment", | |
| "notes": "Customer wants window/view.", | |
| "case_level": "server" | |
| }, | |
| "user_scenario": { | |
| "persona": "You want a table with a view.", | |
| "instructions": { | |
| "domain": "hospitality", | |
| "reason_for_call": "It's our first date. Can we get a table by the window with a nice view?", | |
| "task_instructions": "Emphasize special occasion.\nEND CONVERSATION WHEN: Agent resolves your issue within their authority (comp item, expedited order, or information provided). Once your concern is addressed, thank them and end the call." | |
| } | |
| }, | |
| "ticket": "First date requesting window seat.", | |
| "initial_state": { | |
| "initialization_actions": [] | |
| }, | |
| "evaluation_criteria": { | |
| "actions": [ | |
| { | |
| "action_id": "main_action", | |
| "requestor": "assistant", | |
| "name": "check_table_availability", | |
| "arguments": {}, | |
| "compare_args": [] | |
| } | |
| ], | |
| "env_assertions": [ | |
| { | |
| "env_type": "assistant", | |
| "func_name": "assert_no_escalation_made", | |
| "arguments": {}, | |
| "assert_value": true, | |
| "message": "Server can handle this case" | |
| } | |
| ], | |
| "reward_basis": [ | |
| "ACTION", | |
| "ENV_ASSERTION" | |
| ] | |
| } | |
| }, | |
| { | |
| "id": "hospitality_066_server_complaint", | |
| "description": { | |
| "category": "server_incident", | |
| "purpose": "Test handling complaint about specific server.", | |
| "relevant_policies": "Professional conduct", | |
| "notes": "Customer unhappy with server attitude.", | |
| "case_level": "server" | |
| }, | |
| "user_scenario": { | |
| "persona": "Your server was rude and dismissive.", | |
| "instructions": { | |
| "domain": "hospitality", | |
| "reason_for_call": "Your server was extremely rude. She rolled her eyes when I asked a question. I want a different server.", | |
| "task_instructions": "Describe specific behavior. Demand change.\nEND CONVERSATION WHEN: Agent resolves your issue within their authority (comp item, expedited order, or information provided). Once your concern is addressed, thank them and end the call." | |
| } | |
| }, | |
| "ticket": "Complaint about server attitude. Customer requesting different server.", | |
| "initial_state": { | |
| "initialization_actions": [ | |
| { | |
| "env_type": "user", | |
| "func_name": "set_mood", | |
| "arguments": { | |
| "mood": "angry" | |
| } | |
| }, | |
| { | |
| "env_type": "assistant", | |
| "func_name": "initialize_order", | |
| "arguments": { | |
| "bill_amount": 60, | |
| "table_id": "A01", | |
| "party_size": 2 | |
| } | |
| } | |
| ] | |
| }, | |
| "evaluation_criteria": { | |
| "actions": [ | |
| { | |
| "action_id": "record_incident", | |
| "requestor": "assistant", | |
| "name": "record_service_incident", | |
| "arguments": { | |
| "incident_type": "service" | |
| }, | |
| "compare_args": [ | |
| "incident_type" | |
| ] | |
| }, | |
| { | |
| "action_id": "comp_drink", | |
| "requestor": "assistant", | |
| "name": "offer_complimentary_drink", | |
| "arguments": {}, | |
| "compare_args": [] | |
| } | |
| ], | |
| "env_assertions": [ | |
| { | |
| "env_type": "assistant", | |
| "func_name": "assert_no_escalation_made", | |
| "arguments": {}, | |
| "assert_value": true, | |
| "message": "Server can handle this case" | |
| }, | |
| { | |
| "env_type": "assistant", | |
| "func_name": "assert_compensation_offered", | |
| "arguments": {}, | |
| "assert_value": true, | |
| "message": "Should compensation offered" | |
| }, | |
| { | |
| "env_type": "assistant", | |
| "func_name": "assert_incident_recorded", | |
| "arguments": { | |
| "incident_type": "general" | |
| }, | |
| "assert_value": true, | |
| "message": "Must record general incident" | |
| }, | |
| { | |
| "env_type": "assistant", | |
| "func_name": "assert_escalation_reason_quality", | |
| "arguments": { | |
| "required_keywords": [ | |
| "complaint", | |
| "server" | |
| ] | |
| }, | |
| "assert_value": true, | |
| "message": "Escalation reason must include key details for manager" | |
| }, | |
| { | |
| "env_type": "assistant", | |
| "func_name": "assert_no_authority_violation", | |
| "arguments": {}, | |
| "assert_value": true, | |
| "message": "Must not exceed server authority limits (comp item ≤$10, discount ≤12%)" | |
| } | |
| ], | |
| "reward_basis": [ | |
| "ACTION", | |
| "ENV_ASSERTION" | |
| ] | |
| } | |
| }, | |
| { | |
| "id": "hospitality_067_long_wait_for_table", | |
| "description": { | |
| "category": "host_seating", | |
| "purpose": "Test handling long wait time complaint.", | |
| "relevant_policies": "Wait time management", | |
| "notes": "Customer waited 45 minutes despite reservation.", | |
| "case_level": "manager" | |
| }, | |
| "user_scenario": { | |
| "persona": "You had reservation but waited 45 minutes.", | |
| "instructions": { | |
| "domain": "hospitality", | |
| "reason_for_call": "We had a 7pm reservation. It's now 7:45 and we're still waiting. This is unacceptable!", | |
| "task_instructions": "Demand compensation for wait.\nEND CONVERSATION WHEN: Agent has escalated to manager OR offered any compensation (discount, replacement, apology with action). Once agent says \"I have escalated\" or \"manager will\" or mentions any compensation, say \"Thank you for your help\" and end with ###STOP###." | |
| } | |
| }, | |
| "ticket": "Customer with reservation waited 45 minutes. Very upset.", | |
| "initial_state": { | |
| "initialization_actions": [ | |
| { | |
| "env_type": "user", | |
| "func_name": "set_mood", | |
| "arguments": { | |
| "mood": "angry" | |
| } | |
| } | |
| ] | |
| }, | |
| "evaluation_criteria": { | |
| "actions": [ | |
| { | |
| "action_id": "comp_drink", | |
| "requestor": "assistant", | |
| "name": "offer_complimentary_drink", | |
| "arguments": {}, | |
| "compare_args": [] | |
| }, | |
| { | |
| "action_id": "escalate", | |
| "requestor": "assistant", | |
| "name": "escalate_with_solution", | |
| "arguments": { | |
| "escalate_to": "manager" | |
| }, | |
| "compare_args": [ | |
| "escalate_to" | |
| ] | |
| } | |
| ], | |
| "env_assertions": [ | |
| { | |
| "env_type": "assistant", | |
| "func_name": "assert_escalated_to_manager", | |
| "arguments": {}, | |
| "assert_value": true, | |
| "message": "Must escalate - major service failure" | |
| }, | |
| { | |
| "env_type": "assistant", | |
| "func_name": "assert_recommended_discount_at_least", | |
| "arguments": { | |
| "min_percent": 20 | |
| }, | |
| "assert_value": true, | |
| "message": "Should recommend significant discount" | |
| }, | |
| { | |
| "env_type": "assistant", | |
| "func_name": "assert_no_authority_violation", | |
| "arguments": {}, | |
| "assert_value": true, | |
| "message": "Must not exceed server authority limits (comp item ≤$10, discount ≤12%)" | |
| } | |
| ], | |
| "reward_basis": [ | |
| "ACTION", | |
| "ENV_ASSERTION" | |
| ] | |
| } | |
| }, | |
| { | |
| "id": "hospitality_068_noise_complaint", | |
| "description": { | |
| "category": "server_misc", | |
| "purpose": "Test handling noise complaint.", | |
| "relevant_policies": "Customer comfort", | |
| "notes": "Nearby table too loud.", | |
| "case_level": "server" | |
| }, | |
| "user_scenario": { | |
| "persona": "Table next to you is extremely loud and rowdy.", | |
| "instructions": { | |
| "domain": "hospitality", | |
| "reason_for_call": "The table next to us is so loud we can't even talk. Can you do something?", | |
| "task_instructions": "Demand action.\nEND CONVERSATION WHEN: Agent resolves your issue within their authority (comp item, expedited order, or information provided). Once your concern is addressed, thank them and end the call." | |
| } | |
| }, | |
| "ticket": "Customer complaining about nearby loud table.", | |
| "initial_state": { | |
| "initialization_actions": [ | |
| { | |
| "env_type": "assistant", | |
| "func_name": "initialize_order", | |
| "arguments": { | |
| "bill_amount": 60, | |
| "table_id": "A01", | |
| "party_size": 2 | |
| } | |
| } | |
| ] | |
| }, | |
| "evaluation_criteria": { | |
| "actions": [], | |
| "env_assertions": [ | |
| { | |
| "env_type": "assistant", | |
| "func_name": "assert_no_escalation_made", | |
| "arguments": {}, | |
| "assert_value": true, | |
| "message": "Server can handle this case" | |
| } | |
| ], | |
| "reward_basis": [ | |
| "ACTION", | |
| "ENV_ASSERTION" | |
| ] | |
| } | |
| }, | |
| { | |
| "id": "hospitality_069_temperature_complaint", | |
| "description": { | |
| "category": "server_misc", | |
| "purpose": "Test handling temperature complaint.", | |
| "relevant_policies": "Customer comfort", | |
| "notes": "Customer finds restaurant too cold.", | |
| "case_level": "server" | |
| }, | |
| "user_scenario": { | |
| "persona": "You're freezing because of air conditioning.", | |
| "instructions": { | |
| "domain": "hospitality", | |
| "reason_for_call": "It's freezing in here! The AC is blasting right on our table. Can you do something?", | |
| "task_instructions": "Ask to adjust temperature or move tables.\nEND CONVERSATION WHEN: Agent resolves your issue within their authority (comp item, expedited order, or information provided). Once your concern is addressed, thank them and end the call." | |
| } | |
| }, | |
| "ticket": "Customer complaining about cold temperature from AC.", | |
| "initial_state": { | |
| "initialization_actions": [ | |
| { | |
| "env_type": "assistant", | |
| "func_name": "initialize_order", | |
| "arguments": { | |
| "bill_amount": 60, | |
| "table_id": "A01", | |
| "party_size": 2 | |
| } | |
| } | |
| ] | |
| }, | |
| "evaluation_criteria": { | |
| "actions": [], | |
| "env_assertions": [ | |
| { | |
| "env_type": "assistant", | |
| "func_name": "assert_no_escalation_made", | |
| "arguments": {}, | |
| "assert_value": true, | |
| "message": "Server can handle this case" | |
| } | |
| ], | |
| "reward_basis": [ | |
| "ACTION", | |
| "ENV_ASSERTION" | |
| ] | |
| } | |
| }, | |
| { | |
| "id": "hospitality_070_dirty_utensils", | |
| "description": { | |
| "category": "server_incident", | |
| "purpose": "Test handling dirty utensil complaint.", | |
| "relevant_policies": "Cleanliness", | |
| "notes": "Customer found dirty chopsticks.", | |
| "case_level": "server" | |
| }, | |
| "user_scenario": { | |
| "persona": "Your chopsticks have dried food on them.", | |
| "instructions": { | |
| "domain": "hospitality", | |
| "reason_for_call": "These chopsticks have dried food on them. This is disgusting! What kind of restaurant is this?", | |
| "task_instructions": "Express disgust and demand explanation.\nEND CONVERSATION WHEN: Agent resolves your issue within their authority (comp item, expedited order, or information provided). Once your concern is addressed, thank them and end the call." | |
| } | |
| }, | |
| "ticket": "Customer found dirty utensils.", | |
| "initial_state": { | |
| "initialization_actions": [ | |
| { | |
| "env_type": "user", | |
| "func_name": "set_mood", | |
| "arguments": { | |
| "mood": "frustrated" | |
| } | |
| } | |
| ] | |
| }, | |
| "evaluation_criteria": { | |
| "actions": [ | |
| { | |
| "action_id": "record_incident", | |
| "requestor": "assistant", | |
| "name": "record_service_incident", | |
| "arguments": { | |
| "incident_type": "hygiene" | |
| }, | |
| "compare_args": [ | |
| "incident_type" | |
| ] | |
| }, | |
| { | |
| "action_id": "comp_drink", | |
| "requestor": "assistant", | |
| "name": "offer_complimentary_drink", | |
| "arguments": {}, | |
| "compare_args": [] | |
| } | |
| ], | |
| "env_assertions": [ | |
| { | |
| "env_type": "assistant", | |
| "func_name": "assert_no_escalation_made", | |
| "arguments": {}, | |
| "assert_value": true, | |
| "message": "Server can handle this case" | |
| }, | |
| { | |
| "env_type": "assistant", | |
| "func_name": "assert_compensation_offered", | |
| "arguments": {}, | |
| "assert_value": true, | |
| "message": "Should compensation offered" | |
| }, | |
| { | |
| "env_type": "assistant", | |
| "func_name": "assert_incident_recorded", | |
| "arguments": { | |
| "incident_type": "general" | |
| }, | |
| "assert_value": true, | |
| "message": "Must record general incident" | |
| }, | |
| { | |
| "env_type": "assistant", | |
| "func_name": "assert_no_authority_violation", | |
| "arguments": {}, | |
| "assert_value": true, | |
| "message": "Must not exceed server authority limits (comp item ≤$10, discount ≤12%)" | |
| } | |
| ], | |
| "reward_basis": [ | |
| "ACTION", | |
| "ENV_ASSERTION" | |
| ] | |
| } | |
| }, | |
| { | |
| "id": "hospitality_071_restroom_complaint", | |
| "description": { | |
| "category": "server_incident", | |
| "purpose": "Test handling restroom cleanliness complaint.", | |
| "relevant_policies": "Facility cleanliness", | |
| "notes": "Restroom is dirty.", | |
| "case_level": "server" | |
| }, | |
| "user_scenario": { | |
| "persona": "You found the restroom in terrible condition.", | |
| "instructions": { | |
| "domain": "hospitality", | |
| "reason_for_call": "Your restroom is disgusting. No soap, toilet paper on the floor, wet everywhere. How can you let it get this bad?", | |
| "task_instructions": "Demand immediate cleaning.\nEND CONVERSATION WHEN: Agent resolves your issue within their authority (comp item, expedited order, or information provided). Once your concern is addressed, thank them and end the call." | |
| } | |
| }, | |
| "ticket": "Customer complaining about dirty restroom.", | |
| "initial_state": { | |
| "initialization_actions": [ | |
| { | |
| "env_type": "assistant", | |
| "func_name": "initialize_order", | |
| "arguments": { | |
| "bill_amount": 60, | |
| "table_id": "A01", | |
| "party_size": 2 | |
| } | |
| } | |
| ] | |
| }, | |
| "evaluation_criteria": { | |
| "actions": [ | |
| { | |
| "action_id": "record_incident", | |
| "requestor": "assistant", | |
| "name": "record_service_incident", | |
| "arguments": { | |
| "incident_type": "hygiene" | |
| }, | |
| "compare_args": [ | |
| "incident_type" | |
| ] | |
| } | |
| ], | |
| "env_assertions": [ | |
| { | |
| "env_type": "assistant", | |
| "func_name": "assert_no_escalation_made", | |
| "arguments": {}, | |
| "assert_value": true, | |
| "message": "Server can handle this case" | |
| }, | |
| { | |
| "env_type": "assistant", | |
| "func_name": "assert_incident_recorded", | |
| "arguments": { | |
| "incident_type": "general" | |
| }, | |
| "assert_value": true, | |
| "message": "Must record general incident" | |
| } | |
| ], | |
| "reward_basis": [ | |
| "ACTION", | |
| "ENV_ASSERTION" | |
| ] | |
| } | |
| }, | |
| { | |
| "id": "hospitality_072_forgotten_leftovers", | |
| "description": { | |
| "category": "server_misc", | |
| "purpose": "Test handling forgotten leftover call.", | |
| "relevant_policies": "Customer service", | |
| "notes": "Customer left leftovers behind.", | |
| "case_level": "server" | |
| }, | |
| "user_scenario": { | |
| "persona": "You left your takeout box at the table.", | |
| "instructions": { | |
| "domain": "hospitality", | |
| "reason_for_call": "We just left 10 minutes ago and realized we forgot our leftover box at the table. Do you still have it?", | |
| "task_instructions": "Ask if they can hold it.\nEND CONVERSATION WHEN: Agent resolves your issue within their authority (comp item, expedited order, or information provided). Once your concern is addressed, thank them and end the call." | |
| } | |
| }, | |
| "ticket": "Customer calling about forgotten leftover box.", | |
| "initial_state": { | |
| "initialization_actions": [] | |
| }, | |
| "evaluation_criteria": { | |
| "actions": [], | |
| "env_assertions": [ | |
| { | |
| "env_type": "assistant", | |
| "func_name": "assert_no_escalation_made", | |
| "arguments": {}, | |
| "assert_value": true, | |
| "message": "Server can handle this case" | |
| } | |
| ], | |
| "reward_basis": [ | |
| "ACTION", | |
| "ENV_ASSERTION" | |
| ] | |
| } | |
| }, | |
| { | |
| "id": "hospitality_073_lost_phone", | |
| "description": { | |
| "category": "server_misc", | |
| "purpose": "Test handling lost item inquiry.", | |
| "relevant_policies": "Lost and found", | |
| "notes": "Customer lost phone at restaurant.", | |
| "case_level": "server" | |
| }, | |
| "user_scenario": { | |
| "persona": "You lost your phone and think it's at the restaurant.", | |
| "instructions": { | |
| "domain": "hospitality", | |
| "reason_for_call": "I think I left my iPhone at table A5 last night. Have you found any phones?", | |
| "task_instructions": "Provide description of phone.\nEND CONVERSATION WHEN: Agent resolves your issue within their authority (comp item, expedited order, or information provided). Once your concern is addressed, thank them and end the call." | |
| } | |
| }, | |
| "ticket": "Customer inquiring about lost phone.", | |
| "initial_state": { | |
| "initialization_actions": [] | |
| }, | |
| "evaluation_criteria": { | |
| "actions": [], | |
| "env_assertions": [ | |
| { | |
| "env_type": "assistant", | |
| "func_name": "assert_no_escalation_made", | |
| "arguments": {}, | |
| "assert_value": true, | |
| "message": "Server can handle menu questions" | |
| } | |
| ], | |
| "reward_basis": [ | |
| "ACTION", | |
| "ENV_ASSERTION" | |
| ] | |
| } | |
| }, | |
| { | |
| "id": "hospitality_074_dietary_vegan", | |
| "description": { | |
| "category": "server_food_safety", | |
| "purpose": "Test handling vegan dietary request.", | |
| "relevant_policies": "Menu options", | |
| "notes": "Customer is vegan.", | |
| "case_level": "server" | |
| }, | |
| "user_scenario": { | |
| "persona": "You are vegan and need suitable options.", | |
| "instructions": { | |
| "domain": "hospitality", | |
| "reason_for_call": "I'm vegan. What soup bases and dishes can I have?", | |
| "task_instructions": "Ask for comprehensive vegan options.\nEND CONVERSATION WHEN: Agent provides the information you requested. Once you have your answer, thank them and end the call." | |
| } | |
| }, | |
| "ticket": "Vegan customer asking about suitable menu options.", | |
| "initial_state": { | |
| "initialization_actions": [ | |
| { | |
| "env_type": "user", | |
| "func_name": "set_allergies", | |
| "arguments": { | |
| "allergies": [] | |
| } | |
| } | |
| ] | |
| }, | |
| "evaluation_criteria": { | |
| "actions": [ | |
| { | |
| "action_id": "get_menu", | |
| "requestor": "assistant", | |
| "name": "get_menu_details", | |
| "arguments": { | |
| "category": "soup_base" | |
| }, | |
| "compare_args": [ | |
| "category" | |
| ] | |
| } | |
| ], | |
| "env_assertions": [ | |
| { | |
| "env_type": "assistant", | |
| "func_name": "assert_no_escalation_made", | |
| "arguments": {}, | |
| "assert_value": true, | |
| "message": "Server can handle menu questions" | |
| } | |
| ], | |
| "reward_basis": [ | |
| "ACTION", | |
| "ENV_ASSERTION" | |
| ] | |
| } | |
| }, | |
| { | |
| "id": "hospitality_075_halal_inquiry", | |
| "description": { | |
| "category": "server_food_safety", | |
| "purpose": "Test handling halal dietary inquiry.", | |
| "relevant_policies": "Dietary accommodations", | |
| "notes": "Customer needs halal options.", | |
| "case_level": "server" | |
| }, | |
| "user_scenario": { | |
| "persona": "You follow halal dietary requirements.", | |
| "instructions": { | |
| "domain": "hospitality", | |
| "reason_for_call": "Is your meat halal certified? I need to know before I can order.", | |
| "task_instructions": "Ask about certification.\nEND CONVERSATION WHEN: Agent provides the information you requested. Once you have your answer, thank them and end the call." | |
| } | |
| }, | |
| "ticket": "Customer asking about halal meat certification.", | |
| "initial_state": { | |
| "initialization_actions": [] | |
| }, | |
| "evaluation_criteria": { | |
| "actions": [ | |
| { | |
| "action_id": "main_action", | |
| "requestor": "assistant", | |
| "name": "get_menu_details", | |
| "arguments": { | |
| "category": "protein" | |
| }, | |
| "compare_args": [ | |
| "category" | |
| ] | |
| } | |
| ], | |
| "env_assertions": [ | |
| { | |
| "env_type": "assistant", | |
| "func_name": "assert_no_escalation_made", | |
| "arguments": {}, | |
| "assert_value": true, | |
| "message": "Server can handle this case" | |
| } | |
| ], | |
| "reward_basis": [ | |
| "ACTION", | |
| "ENV_ASSERTION" | |
| ] | |
| } | |
| }, | |
| { | |
| "id": "hospitality_076_kosher_inquiry", | |
| "description": { | |
| "category": "server_food_safety", | |
| "purpose": "Test handling kosher dietary inquiry.", | |
| "relevant_policies": "Dietary accommodations", | |
| "notes": "Customer keeps kosher.", | |
| "case_level": "server" | |
| }, | |
| "user_scenario": { | |
| "persona": "You keep kosher and need information.", | |
| "instructions": { | |
| "domain": "hospitality", | |
| "reason_for_call": "Do you have any kosher options? Is there any pork in your kitchen?", | |
| "task_instructions": "Ask detailed questions.\nEND CONVERSATION WHEN: Agent provides the information you requested. Once you have your answer, thank them and end the call." | |
| } | |
| }, | |
| "ticket": "Customer asking about kosher accommodations.", | |
| "initial_state": { | |
| "initialization_actions": [] | |
| }, | |
| "evaluation_criteria": { | |
| "actions": [ | |
| { | |
| "action_id": "main_action", | |
| "requestor": "assistant", | |
| "name": "get_menu_details", | |
| "arguments": {}, | |
| "compare_args": [ | |
| "category" | |
| ] | |
| } | |
| ], | |
| "env_assertions": [ | |
| { | |
| "env_type": "assistant", | |
| "func_name": "assert_no_escalation_made", | |
| "arguments": {}, | |
| "assert_value": true, | |
| "message": "Server can handle this case" | |
| } | |
| ], | |
| "reward_basis": [ | |
| "ACTION", | |
| "ENV_ASSERTION" | |
| ] | |
| } | |
| }, | |
| { | |
| "id": "hospitality_077_kids_menu_question", | |
| "description": { | |
| "category": "server_misc", | |
| "purpose": "Test explaining kids options at hot pot including free kids meal.", | |
| "relevant_policies": "Menu options, free kids meal for children under 14", | |
| "notes": "Restaurant provides free kids meal for children under 14: 2 chicken nuggets, fruit, steamed custard bun, and Yakult. Customer asking about kid-friendly options.", | |
| "case_level": "server" | |
| }, | |
| "user_scenario": { | |
| "persona": "You have picky young children (ages 5 and 8) who don't like spicy food.", | |
| "instructions": { | |
| "domain": "hospitality", | |
| "reason_for_call": "My kids are picky eaters and they're only 5 and 8 years old. Do you have anything kid-friendly? What soup base would be mild for them? Do you have a kids menu or anything special for children?", | |
| "task_instructions": "Ask about mild soup base options and whether there's a kids menu. Be pleasantly surprised if agent mentions free kids meal.\nEND CONVERSATION WHEN: Agent provides the information you requested including kids meal options. Once you have your answer, thank them and end the call." | |
| } | |
| }, | |
| "ticket": "Parent with two young children (under 14) asking about kid-friendly options.", | |
| "initial_state": { | |
| "initialization_actions": [ | |
| { | |
| "env_type": "user", | |
| "func_name": "set_party_info", | |
| "arguments": { | |
| "num_kids": 2, | |
| "party_size": 4 | |
| } | |
| } | |
| ] | |
| }, | |
| "evaluation_criteria": { | |
| "actions": [ | |
| { | |
| "action_id": "get_menu", | |
| "requestor": "assistant", | |
| "name": "get_menu_details", | |
| "arguments": { | |
| "category": "soup_base" | |
| }, | |
| "compare_args": [ | |
| "category" | |
| ] | |
| } | |
| ], | |
| "env_assertions": [ | |
| { | |
| "env_type": "assistant", | |
| "func_name": "assert_no_escalation_made", | |
| "arguments": {}, | |
| "assert_value": true, | |
| "message": "Server can handle menu questions" | |
| } | |
| ], | |
| "reward_basis": [ | |
| "ACTION", | |
| "ENV_ASSERTION" | |
| ] | |
| } | |
| }, | |
| { | |
| "id": "hospitality_078_spice_level_question", | |
| "description": { | |
| "category": "server_misc", | |
| "purpose": "Test explaining spice levels.", | |
| "relevant_policies": "Menu knowledge", | |
| "notes": "Customer sensitive to spice.", | |
| "case_level": "server" | |
| }, | |
| "user_scenario": { | |
| "persona": "You can't handle much spice.", | |
| "instructions": { | |
| "domain": "hospitality", | |
| "reason_for_call": "I can't handle spicy food at all. Which soup bases have zero spice?", | |
| "task_instructions": "Ask for completely non-spicy options.\nEND CONVERSATION WHEN: Agent provides the information you requested. Once you have your answer, thank them and end the call." | |
| } | |
| }, | |
| "ticket": "Customer asking about non-spicy soup base options.", | |
| "initial_state": { | |
| "initialization_actions": [] | |
| }, | |
| "evaluation_criteria": { | |
| "actions": [ | |
| { | |
| "action_id": "main_action", | |
| "requestor": "assistant", | |
| "name": "get_menu_details", | |
| "arguments": { | |
| "category": "soup_base" | |
| }, | |
| "compare_args": [ | |
| "category" | |
| ] | |
| } | |
| ], | |
| "env_assertions": [ | |
| { | |
| "env_type": "assistant", | |
| "func_name": "assert_no_escalation_made", | |
| "arguments": {}, | |
| "assert_value": true, | |
| "message": "Server can handle this case" | |
| } | |
| ], | |
| "reward_basis": [ | |
| "ACTION", | |
| "ENV_ASSERTION" | |
| ] | |
| } | |
| }, | |
| { | |
| "id": "hospitality_079_half_half_explanation", | |
| "description": { | |
| "category": "server_misc", | |
| "purpose": "Test explaining pot configurations.", | |
| "relevant_policies": "Menu knowledge - pot types", | |
| "notes": "Customer confused about pot options.", | |
| "case_level": "server" | |
| }, | |
| "user_scenario": { | |
| "persona": "First time hot pot customer.", | |
| "instructions": { | |
| "domain": "hospitality", | |
| "reason_for_call": "I've never had hot pot before. What's the difference between whole pot and half-and-half?", | |
| "task_instructions": "Ask for explanation.\nEND CONVERSATION WHEN: Agent provides the information you requested. Once you have your answer, thank them and end the call." | |
| } | |
| }, | |
| "ticket": "First-time customer asking about pot configurations.", | |
| "initial_state": { | |
| "initialization_actions": [] | |
| }, | |
| "evaluation_criteria": { | |
| "actions": [ | |
| { | |
| "action_id": "main_action", | |
| "requestor": "assistant", | |
| "name": "get_menu_details", | |
| "arguments": { | |
| "category": "soup_base" | |
| }, | |
| "compare_args": [ | |
| "category" | |
| ] | |
| } | |
| ], | |
| "env_assertions": [ | |
| { | |
| "env_type": "assistant", | |
| "func_name": "assert_no_escalation_made", | |
| "arguments": {}, | |
| "assert_value": true, | |
| "message": "Server can handle this case" | |
| } | |
| ], | |
| "reward_basis": [ | |
| "ACTION", | |
| "ENV_ASSERTION" | |
| ] | |
| } | |
| }, | |
| { | |
| "id": "hospitality_080_wagyu_difference", | |
| "description": { | |
| "category": "server_misc", | |
| "purpose": "Test menu knowledge for premium items.", | |
| "relevant_policies": "Menu knowledge", | |
| "notes": "Customer asking about wagyu options.", | |
| "case_level": "server" | |
| }, | |
| "user_scenario": { | |
| "persona": "You want to understand wagyu options.", | |
| "instructions": { | |
| "domain": "hospitality", | |
| "reason_for_call": "What's the difference between your regular Wagyu and the Miyazaki Wagyu? Is it worth the price difference?", | |
| "task_instructions": "Ask for recommendation.\nEND CONVERSATION WHEN: Agent provides the information you requested. Once you have your answer, thank them and end the call." | |
| } | |
| }, | |
| "ticket": "Customer asking about wagyu beef differences.", | |
| "initial_state": { | |
| "initialization_actions": [] | |
| }, | |
| "evaluation_criteria": { | |
| "actions": [ | |
| { | |
| "action_id": "get_menu", | |
| "requestor": "assistant", | |
| "name": "get_menu_details", | |
| "arguments": {}, | |
| "compare_args": [ | |
| "category" | |
| ] | |
| } | |
| ], | |
| "env_assertions": [ | |
| { | |
| "env_type": "assistant", | |
| "func_name": "assert_no_escalation_made", | |
| "arguments": {}, | |
| "assert_value": true, | |
| "message": "Server can handle menu questions" | |
| } | |
| ], | |
| "reward_basis": [ | |
| "ACTION", | |
| "ENV_ASSERTION" | |
| ] | |
| } | |
| }, | |
| { | |
| "id": "hospitality_081_portion_question", | |
| "description": { | |
| "category": "server_misc", | |
| "purpose": "Test portion size recommendations.", | |
| "relevant_policies": "Menu knowledge", | |
| "notes": "Customer unsure how much to order.", | |
| "case_level": "server" | |
| }, | |
| "user_scenario": { | |
| "persona": "First time visitor unsure about portions.", | |
| "instructions": { | |
| "domain": "hospitality", | |
| "reason_for_call": "We're a party of 4. How much meat should we order? We don't want to over-order.", | |
| "task_instructions": "Ask for recommendations.\nEND CONVERSATION WHEN: Agent resolves your issue within their authority (comp item, expedited order, or information provided). Once your concern is addressed, thank them and end the call." | |
| } | |
| }, | |
| "ticket": "Party of 4 asking for ordering recommendations.", | |
| "initial_state": { | |
| "initialization_actions": [ | |
| { | |
| "env_type": "user", | |
| "func_name": "set_party_info", | |
| "arguments": { | |
| "party_size": 4 | |
| } | |
| } | |
| ] | |
| }, | |
| "evaluation_criteria": { | |
| "actions": [ | |
| { | |
| "action_id": "get_menu", | |
| "requestor": "assistant", | |
| "name": "get_menu_details", | |
| "arguments": {}, | |
| "compare_args": [ | |
| "category" | |
| ] | |
| } | |
| ], | |
| "env_assertions": [ | |
| { | |
| "env_type": "assistant", | |
| "func_name": "assert_no_escalation_made", | |
| "arguments": {}, | |
| "assert_value": true, | |
| "message": "Server can handle menu questions" | |
| } | |
| ], | |
| "reward_basis": [ | |
| "ACTION", | |
| "ENV_ASSERTION" | |
| ] | |
| } | |
| }, | |
| { | |
| "id": "hospitality_082_price_inquiry", | |
| "description": { | |
| "category": "server_billing", | |
| "purpose": "Test handling price questions.", | |
| "relevant_policies": "Menu pricing", | |
| "notes": "Customer asking about prices.", | |
| "case_level": "server" | |
| }, | |
| "user_scenario": { | |
| "persona": "Budget-conscious customer.", | |
| "instructions": { | |
| "domain": "hospitality", | |
| "reason_for_call": "How much does dinner typically cost per person? We're on a budget.", | |
| "task_instructions": "Ask about affordable options.\nEND CONVERSATION WHEN: Agent provides the information you requested. Once you have your answer, thank them and end the call." | |
| } | |
| }, | |
| "ticket": "Customer asking about typical dinner cost.", | |
| "initial_state": { | |
| "initialization_actions": [] | |
| }, | |
| "evaluation_criteria": { | |
| "actions": [ | |
| { | |
| "action_id": "main_action", | |
| "requestor": "assistant", | |
| "name": "get_menu_details", | |
| "arguments": {}, | |
| "compare_args": [ | |
| "category" | |
| ] | |
| } | |
| ], | |
| "env_assertions": [ | |
| { | |
| "env_type": "assistant", | |
| "func_name": "assert_no_escalation_made", | |
| "arguments": {}, | |
| "assert_value": true, | |
| "message": "Server can handle this case" | |
| } | |
| ], | |
| "reward_basis": [ | |
| "ACTION", | |
| "ENV_ASSERTION" | |
| ] | |
| } | |
| }, | |
| { | |
| "id": "hospitality_083_loyalty_abuse_suspicion", | |
| "description": { | |
| "category": "server_promotion", | |
| "purpose": "Test detecting potential policy abuse.", | |
| "relevant_policies": "Policy integrity", | |
| "notes": "Customer may be trying to exploit birthday voucher.", | |
| "case_level": "server" | |
| }, | |
| "user_scenario": { | |
| "persona": "You claimed birthday last month and claiming again.", | |
| "instructions": { | |
| "domain": "hospitality", | |
| "reason_for_call": "It's my birthday! Can I get the birthday voucher?", | |
| "known_info": "Customer used birthday voucher last month.", | |
| "task_instructions": "Insist it's your birthday.\nEND CONVERSATION WHEN: Agent resolves your issue within their authority (comp item, expedited order, or information provided). Once your concern is addressed, thank them and end the call." | |
| } | |
| }, | |
| "ticket": "Customer claiming birthday voucher again (used last month).", | |
| "initial_state": { | |
| "initialization_actions": [ | |
| { | |
| "env_type": "assistant", | |
| "func_name": "initialize_order", | |
| "arguments": { | |
| "bill_amount": 200, | |
| "table_id": "A01" | |
| } | |
| }, | |
| { | |
| "env_type": "assistant", | |
| "func_name": "initialize_customer_points", | |
| "arguments": { | |
| "customer_id": "CUST001", | |
| "points": 400, | |
| "tier": "gold" | |
| } | |
| } | |
| ] | |
| }, | |
| "evaluation_criteria": { | |
| "actions": [ | |
| { | |
| "action_id": "main_action", | |
| "requestor": "assistant", | |
| "name": "get_customer_profile", | |
| "arguments": {}, | |
| "compare_args": [] | |
| } | |
| ], | |
| "env_assertions": [ | |
| { | |
| "env_type": "assistant", | |
| "func_name": "assert_no_escalation_made", | |
| "arguments": {}, | |
| "assert_value": true, | |
| "message": "Server can handle loyalty questions" | |
| } | |
| ], | |
| "reward_basis": [ | |
| "ACTION", | |
| "ENV_ASSERTION" | |
| ] | |
| } | |
| }, | |
| { | |
| "id": "hospitality_084_competitor_comparison", | |
| "description": { | |
| "category": "server_misc", | |
| "purpose": "Test handling competitor comparison.", | |
| "relevant_policies": "Professional conduct", | |
| "notes": "Customer comparing to competitor.", | |
| "case_level": "server" | |
| }, | |
| "user_scenario": { | |
| "persona": "You prefer competitor and are vocal about it.", | |
| "instructions": { | |
| "domain": "hospitality", | |
| "reason_for_call": "The other hot pot place down the street gives free refills on soup base. Why don't you?", | |
| "task_instructions": "Compare unfavorably to competitor.\nEND CONVERSATION WHEN: Agent resolves your issue within their authority (comp item, expedited order, or information provided). Once your concern is addressed, thank them and end the call." | |
| } | |
| }, | |
| "ticket": "Customer comparing restaurant unfavorably to competitor.", | |
| "initial_state": { | |
| "initialization_actions": [ | |
| { | |
| "env_type": "assistant", | |
| "func_name": "initialize_order", | |
| "arguments": { | |
| "bill_amount": 60, | |
| "table_id": "A01", | |
| "party_size": 2 | |
| } | |
| } | |
| ] | |
| }, | |
| "evaluation_criteria": { | |
| "actions": [], | |
| "env_assertions": [ | |
| { | |
| "env_type": "assistant", | |
| "func_name": "assert_no_escalation_made", | |
| "arguments": {}, | |
| "assert_value": true, | |
| "message": "Server can handle this case" | |
| } | |
| ], | |
| "reward_basis": [ | |
| "ACTION", | |
| "ENV_ASSERTION" | |
| ] | |
| } | |
| }, | |
| { | |
| "id": "hospitality_085_review_threat", | |
| "description": { | |
| "category": "server_incident", | |
| "purpose": "Test handling review threat.", | |
| "relevant_policies": "Zero negative review goal", | |
| "notes": "Customer threatening bad review for minor issue.", | |
| "case_level": "server" | |
| }, | |
| "user_scenario": { | |
| "persona": "You're threatening bad review to get more compensation.", | |
| "instructions": { | |
| "domain": "hospitality", | |
| "reason_for_call": "This is ridiculous. I'm going to give you one star on Yelp unless you comp my entire meal.", | |
| "known_info": "Issue was minor (10-minute wait for appetizer).", | |
| "task_instructions": "Threaten review to get free meal.\nEND CONVERSATION WHEN: Agent resolves your issue within their authority (comp item, expedited order, or information provided). Once your concern is addressed, thank them and end the call." | |
| } | |
| }, | |
| "ticket": "Customer threatening 1-star review. Demanding free meal for minor issue.", | |
| "initial_state": { | |
| "initialization_actions": [ | |
| { | |
| "env_type": "user", | |
| "func_name": "set_mood", | |
| "arguments": { | |
| "mood": "angry" | |
| } | |
| }, | |
| { | |
| "env_type": "assistant", | |
| "func_name": "initialize_order", | |
| "arguments": { | |
| "bill_amount": 60, | |
| "table_id": "A01", | |
| "party_size": 2 | |
| } | |
| } | |
| ] | |
| }, | |
| "evaluation_criteria": { | |
| "actions": [ | |
| { | |
| "action_id": "check_context", | |
| "requestor": "assistant", | |
| "name": "get_order_details", | |
| "arguments": {}, | |
| "compare_args": [] | |
| }, | |
| { | |
| "action_id": "appease", | |
| "requestor": "assistant", | |
| "name": "offer_complimentary_drink", | |
| "arguments": {}, | |
| "compare_args": [] | |
| }, | |
| { | |
| "action_id": "escalate", | |
| "requestor": "assistant", | |
| "name": "escalate_with_solution", | |
| "arguments": { | |
| "escalate_to": "manager" | |
| }, | |
| "compare_args": [ | |
| "escalate_to" | |
| ] | |
| } | |
| ], | |
| "env_assertions": [ | |
| { | |
| "env_type": "assistant", | |
| "func_name": "assert_escalated_to_manager", | |
| "arguments": {}, | |
| "assert_value": true, | |
| "message": "Must escalate to manager" | |
| }, | |
| { | |
| "env_type": "assistant", | |
| "func_name": "assert_discount_within_server_authority", | |
| "arguments": {}, | |
| "assert_value": true, | |
| "message": "Discounts must be within server authority (12%) unless escalated" | |
| }, | |
| { | |
| "env_type": "assistant", | |
| "func_name": "assert_incident_recorded", | |
| "arguments": { | |
| "incident_type": "general" | |
| }, | |
| "assert_value": true, | |
| "message": "Must record general incident" | |
| }, | |
| { | |
| "env_type": "assistant", | |
| "func_name": "assert_escalation_reason_quality", | |
| "arguments": { | |
| "required_keywords": [ | |
| "review", | |
| "threat" | |
| ] | |
| }, | |
| "assert_value": true, | |
| "message": "Escalation reason must include key details for manager" | |
| }, | |
| { | |
| "env_type": "assistant", | |
| "func_name": "assert_no_authority_violation", | |
| "arguments": {}, | |
| "assert_value": true, | |
| "message": "Must not exceed server authority limits" | |
| } | |
| ], | |
| "reward_basis": [ | |
| "ACTION", | |
| "ENV_ASSERTION" | |
| ] | |
| } | |
| }, | |
| { | |
| "id": "hospitality_086_social_media_threat", | |
| "description": { | |
| "category": "server_incident", | |
| "purpose": "Test handling social media threat.", | |
| "relevant_policies": "Brand reputation", | |
| "notes": "Customer threatening viral post.", | |
| "case_level": "server" | |
| }, | |
| "user_scenario": { | |
| "persona": "You have large social following and threaten exposure.", | |
| "instructions": { | |
| "domain": "hospitality", | |
| "reason_for_call": "I have 50,000 followers on TikTok. Unless you fix this, I'm posting about how terrible you are.", | |
| "task_instructions": "Use follower count as leverage.\nEND CONVERSATION WHEN: Agent resolves your issue within their authority (comp item, expedited order, or information provided). Once your concern is addressed, thank them and end the call." | |
| } | |
| }, | |
| "ticket": "Customer with social media following threatening negative post.", | |
| "initial_state": { | |
| "initialization_actions": [ | |
| { | |
| "env_type": "user", | |
| "func_name": "set_mood", | |
| "arguments": { | |
| "mood": "angry" | |
| } | |
| } | |
| ] | |
| }, | |
| "evaluation_criteria": { | |
| "actions": [ | |
| { | |
| "action_id": "record_incident", | |
| "requestor": "assistant", | |
| "name": "record_service_incident", | |
| "arguments": { | |
| "incident_type": "complaint" | |
| }, | |
| "compare_args": [ | |
| "incident_type" | |
| ] | |
| }, | |
| { | |
| "action_id": "escalate", | |
| "requestor": "assistant", | |
| "name": "escalate_with_solution", | |
| "arguments": { | |
| "escalate_to": "manager" | |
| }, | |
| "compare_args": [ | |
| "escalate_to" | |
| ] | |
| } | |
| ], | |
| "env_assertions": [ | |
| { | |
| "env_type": "assistant", | |
| "func_name": "assert_escalated_to_manager", | |
| "arguments": {}, | |
| "assert_value": true, | |
| "message": "Must escalate to manager" | |
| }, | |
| { | |
| "env_type": "assistant", | |
| "func_name": "assert_incident_recorded", | |
| "arguments": { | |
| "incident_type": "general" | |
| }, | |
| "assert_value": true, | |
| "message": "Must record general incident" | |
| }, | |
| { | |
| "env_type": "assistant", | |
| "func_name": "assert_escalation_reason_quality", | |
| "arguments": { | |
| "required_keywords": [ | |
| "social", | |
| "media" | |
| ] | |
| }, | |
| "assert_value": true, | |
| "message": "Escalation reason must include key details for manager" | |
| } | |
| ], | |
| "reward_basis": [ | |
| "ACTION", | |
| "ENV_ASSERTION" | |
| ] | |
| } | |
| }, | |
| { | |
| "id": "hospitality_087_positive_feedback", | |
| "description": { | |
| "category": "server_misc", | |
| "purpose": "Test receiving positive feedback gracefully.", | |
| "relevant_policies": "Customer relations", | |
| "notes": "Customer giving genuine compliment.", | |
| "case_level": "server" | |
| }, | |
| "user_scenario": { | |
| "persona": "You had excellent experience and want to share.", | |
| "instructions": { | |
| "domain": "hospitality", | |
| "reason_for_call": "I just wanted to call and say our dinner last night was amazing. Our server Jessica was incredible. Best service we've ever had!", | |
| "task_instructions": "Give detailed positive feedback.\nEND CONVERSATION WHEN: Agent resolves your issue within their authority (comp item, expedited order, or information provided). Once your concern is addressed, thank them and end the call." | |
| } | |
| }, | |
| "ticket": "Customer calling with positive feedback about service.", | |
| "initial_state": { | |
| "initialization_actions": [ | |
| { | |
| "env_type": "user", | |
| "func_name": "set_mood", | |
| "arguments": { | |
| "mood": "happy" | |
| } | |
| } | |
| ] | |
| }, | |
| "evaluation_criteria": { | |
| "actions": [], | |
| "env_assertions": [ | |
| { | |
| "env_type": "assistant", | |
| "func_name": "assert_no_escalation_made", | |
| "arguments": {}, | |
| "assert_value": true, | |
| "message": "Server can handle this case" | |
| } | |
| ], | |
| "reward_basis": [ | |
| "ACTION", | |
| "ENV_ASSERTION" | |
| ] | |
| } | |
| }, | |
| { | |
| "id": "hospitality_088_gift_card_inquiry", | |
| "description": { | |
| "category": "server_billing", | |
| "purpose": "Test handling gift card questions.", | |
| "relevant_policies": "Payment methods", | |
| "notes": "Customer asking about gift cards.", | |
| "case_level": "server" | |
| }, | |
| "user_scenario": { | |
| "persona": "You want to buy gift cards for friends.", | |
| "instructions": { | |
| "domain": "hospitality", | |
| "reason_for_call": "Do you sell gift cards? I want to buy some for Chinese New Year gifts.", | |
| "task_instructions": "Ask about denominations.\nEND CONVERSATION WHEN: Agent provides the information you requested. Once you have your answer, thank them and end the call." | |
| } | |
| }, | |
| "ticket": "Customer asking about gift card availability.", | |
| "initial_state": { | |
| "initialization_actions": [] | |
| }, | |
| "evaluation_criteria": { | |
| "actions": [], | |
| "env_assertions": [ | |
| { | |
| "env_type": "assistant", | |
| "func_name": "assert_no_escalation_made", | |
| "arguments": {}, | |
| "assert_value": true, | |
| "message": "Server can handle this case" | |
| } | |
| ], | |
| "reward_basis": [ | |
| "ACTION", | |
| "ENV_ASSERTION" | |
| ] | |
| } | |
| }, | |
| { | |
| "id": "hospitality_089_catering_inquiry", | |
| "description": { | |
| "category": "host_phone", | |
| "purpose": "Test handling catering inquiry.", | |
| "relevant_policies": "Business scope", | |
| "notes": "Customer asking about catering services.", | |
| "case_level": "server" | |
| }, | |
| "user_scenario": { | |
| "persona": "You want to cater office party.", | |
| "instructions": { | |
| "domain": "hospitality", | |
| "reason_for_call": "Do you do catering? We have an office party for 30 people and want hot pot.", | |
| "task_instructions": "Ask about off-site catering.\nEND CONVERSATION WHEN: Agent provides the information you requested. Once you have your answer, thank them and end the call." | |
| } | |
| }, | |
| "ticket": "Customer asking about catering services for 30 people.", | |
| "initial_state": { | |
| "initialization_actions": [ | |
| { | |
| "env_type": "user", | |
| "func_name": "set_party_info", | |
| "arguments": { | |
| "party_size": 30 | |
| } | |
| } | |
| ] | |
| }, | |
| "evaluation_criteria": { | |
| "actions": [ | |
| { | |
| "action_id": "check_inv", | |
| "requestor": "assistant", | |
| "name": "check_item_inventory", | |
| "arguments": {}, | |
| "compare_args": [] | |
| } | |
| ], | |
| "env_assertions": [ | |
| { | |
| "env_type": "assistant", | |
| "func_name": "assert_no_escalation_made", | |
| "arguments": {}, | |
| "assert_value": true, | |
| "message": "Server can handle inventory questions" | |
| }, | |
| { | |
| "env_type": "assistant", | |
| "func_name": "assert_party_size_within_limit", | |
| "arguments": { | |
| "max_size": 20 | |
| }, | |
| "assert_value": true, | |
| "message": "Must enforce 20-person limit on weekends/holidays" | |
| } | |
| ], | |
| "reward_basis": [ | |
| "ACTION", | |
| "ENV_ASSERTION" | |
| ] | |
| } | |
| }, | |
| { | |
| "id": "hospitality_090_private_room", | |
| "description": { | |
| "category": "host_phone", | |
| "purpose": "Test handling private room request.", | |
| "relevant_policies": "Space availability", | |
| "notes": "Customer wants private room.", | |
| "case_level": "server" | |
| }, | |
| "user_scenario": { | |
| "persona": "You want private room for business dinner.", | |
| "instructions": { | |
| "domain": "hospitality", | |
| "reason_for_call": "Do you have private rooms? We need a confidential business dinner for 8.", | |
| "task_instructions": "Ask about private options.\nEND CONVERSATION WHEN: Agent resolves your issue within their authority (comp item, expedited order, or information provided). Once your concern is addressed, thank them and end the call." | |
| } | |
| }, | |
| "ticket": "Business customer requesting private dining room.", | |
| "initial_state": { | |
| "initialization_actions": [ | |
| { | |
| "env_type": "user", | |
| "func_name": "set_special_occasion", | |
| "arguments": { | |
| "is_business_meal": true | |
| } | |
| }, | |
| { | |
| "env_type": "assistant", | |
| "func_name": "initialize_order", | |
| "arguments": { | |
| "bill_amount": 180, | |
| "table_id": "A01", | |
| "party_size": 5 | |
| } | |
| } | |
| ] | |
| }, | |
| "evaluation_criteria": { | |
| "actions": [ | |
| { | |
| "action_id": "main_action", | |
| "requestor": "assistant", | |
| "name": "check_table_availability", | |
| "arguments": { | |
| "party_size": 5 | |
| }, | |
| "compare_args": [ | |
| "party_size" | |
| ] | |
| } | |
| ], | |
| "env_assertions": [ | |
| { | |
| "env_type": "assistant", | |
| "func_name": "assert_no_escalation_made", | |
| "arguments": {}, | |
| "assert_value": true, | |
| "message": "Server can handle this case" | |
| } | |
| ], | |
| "reward_basis": [ | |
| "ACTION", | |
| "ENV_ASSERTION" | |
| ] | |
| } | |
| }, | |
| { | |
| "id": "hospitality_091_parking_question", | |
| "description": { | |
| "category": "host_phone", | |
| "purpose": "Test handling parking question.", | |
| "relevant_policies": "Location information", | |
| "notes": "Customer asking about parking.", | |
| "case_level": "server" | |
| }, | |
| "user_scenario": { | |
| "persona": "You're driving and need parking info.", | |
| "instructions": { | |
| "domain": "hospitality", | |
| "reason_for_call": "Is there parking available? We're driving from out of town.", | |
| "task_instructions": "Ask about cost and validation.\nEND CONVERSATION WHEN: Agent provides the information you requested. Once you have your answer, thank them and end the call." | |
| } | |
| }, | |
| "ticket": "Customer asking about parking availability.", | |
| "initial_state": { | |
| "initialization_actions": [] | |
| }, | |
| "evaluation_criteria": { | |
| "actions": [], | |
| "env_assertions": [ | |
| { | |
| "env_type": "assistant", | |
| "func_name": "assert_no_escalation_made", | |
| "arguments": {}, | |
| "assert_value": true, | |
| "message": "Server can answer parking questions" | |
| } | |
| ], | |
| "reward_basis": [ | |
| "ACTION", | |
| "ENV_ASSERTION" | |
| ] | |
| } | |
| }, | |
| { | |
| "id": "hospitality_092_hours_question", | |
| "description": { | |
| "category": "host_phone", | |
| "purpose": "Test providing hours information.", | |
| "relevant_policies": "Business hours", | |
| "notes": "Customer asking about closing time.", | |
| "case_level": "server" | |
| }, | |
| "user_scenario": { | |
| "persona": "You want to come for late dinner.", | |
| "instructions": { | |
| "domain": "hospitality", | |
| "reason_for_call": "What time do you close? We might not get there until 10pm.", | |
| "task_instructions": "Confirm you can still get full meal.\nEND CONVERSATION WHEN: Agent provides the information you requested. Once you have your answer, thank them and end the call." | |
| } | |
| }, | |
| "ticket": "Customer asking about late seating availability.", | |
| "initial_state": { | |
| "initialization_actions": [] | |
| }, | |
| "evaluation_criteria": { | |
| "actions": [], | |
| "env_assertions": [ | |
| { | |
| "env_type": "assistant", | |
| "func_name": "assert_no_escalation_made", | |
| "arguments": {}, | |
| "assert_value": true, | |
| "message": "Server can handle this case" | |
| } | |
| ], | |
| "reward_basis": [ | |
| "ACTION", | |
| "ENV_ASSERTION" | |
| ] | |
| } | |
| }, | |
| { | |
| "id": "hospitality_093_directions_request", | |
| "description": { | |
| "category": "host_phone", | |
| "purpose": "Test providing directions.", | |
| "relevant_policies": "Location information", | |
| "notes": "Customer needs directions.", | |
| "case_level": "server" | |
| }, | |
| "user_scenario": { | |
| "persona": "You're having trouble finding the restaurant.", | |
| "instructions": { | |
| "domain": "hospitality", | |
| "reason_for_call": "I'm near UC Berkeley campus but can't find you. What's the exact address?", | |
| "task_instructions": "Ask for directions from campus.\nEND CONVERSATION WHEN: Agent resolves your issue within their authority (comp item, expedited order, or information provided). Once your concern is addressed, thank them and end the call." | |
| } | |
| }, | |
| "ticket": "Customer needs help finding restaurant location.", | |
| "initial_state": { | |
| "initialization_actions": [] | |
| }, | |
| "evaluation_criteria": { | |
| "actions": [], | |
| "env_assertions": [ | |
| { | |
| "env_type": "assistant", | |
| "func_name": "assert_no_escalation_made", | |
| "arguments": {}, | |
| "assert_value": true, | |
| "message": "Server can handle this case" | |
| } | |
| ], | |
| "reward_basis": [ | |
| "ACTION", | |
| "ENV_ASSERTION" | |
| ] | |
| } | |
| }, | |
| { | |
| "id": "hospitality_094_dietary_low_sodium", | |
| "description": { | |
| "category": "server_food_safety", | |
| "purpose": "Test handling low sodium request.", | |
| "relevant_policies": "Dietary accommodations", | |
| "notes": "Customer on low sodium diet.", | |
| "case_level": "server" | |
| }, | |
| "user_scenario": { | |
| "persona": "You have high blood pressure, need low sodium.", | |
| "instructions": { | |
| "domain": "hospitality", | |
| "reason_for_call": "I have high blood pressure and need to watch my sodium. Which soup bases are lowest in sodium?", | |
| "task_instructions": "Ask for low sodium options.\nEND CONVERSATION WHEN: Agent provides the information you requested. Once you have your answer, thank them and end the call." | |
| } | |
| }, | |
| "ticket": "Customer on low sodium diet asking about options.", | |
| "initial_state": { | |
| "initialization_actions": [] | |
| }, | |
| "evaluation_criteria": { | |
| "actions": [ | |
| { | |
| "action_id": "main_action", | |
| "requestor": "assistant", | |
| "name": "get_menu_details", | |
| "arguments": { | |
| "category": "soup_base" | |
| }, | |
| "compare_args": [ | |
| "category" | |
| ] | |
| } | |
| ], | |
| "env_assertions": [ | |
| { | |
| "env_type": "assistant", | |
| "func_name": "assert_no_escalation_made", | |
| "arguments": {}, | |
| "assert_value": true, | |
| "message": "Server can handle this case" | |
| } | |
| ], | |
| "reward_basis": [ | |
| "ACTION", | |
| "ENV_ASSERTION" | |
| ] | |
| } | |
| }, | |
| { | |
| "id": "hospitality_095_dietary_diabetic", | |
| "description": { | |
| "category": "server_food_safety", | |
| "purpose": "Test handling diabetic dietary request.", | |
| "relevant_policies": "Dietary accommodations", | |
| "notes": "Customer is diabetic.", | |
| "case_level": "server" | |
| }, | |
| "user_scenario": { | |
| "persona": "You're diabetic and watching sugar intake.", | |
| "instructions": { | |
| "domain": "hospitality", | |
| "reason_for_call": "I'm diabetic. Which soup bases are lowest in sugar? I need to be careful.", | |
| "task_instructions": "Ask about sugar content.\nEND CONVERSATION WHEN: Agent provides the information you requested. Once you have your answer, thank them and end the call." | |
| } | |
| }, | |
| "ticket": "Diabetic customer asking about sugar content in soup bases.", | |
| "initial_state": { | |
| "initialization_actions": [] | |
| }, | |
| "evaluation_criteria": { | |
| "actions": [ | |
| { | |
| "action_id": "main_action", | |
| "requestor": "assistant", | |
| "name": "get_menu_details", | |
| "arguments": { | |
| "category": "soup_base" | |
| }, | |
| "compare_args": [ | |
| "category" | |
| ] | |
| } | |
| ], | |
| "env_assertions": [ | |
| { | |
| "env_type": "assistant", | |
| "func_name": "assert_no_escalation_made", | |
| "arguments": {}, | |
| "assert_value": true, | |
| "message": "Server can handle this case" | |
| } | |
| ], | |
| "reward_basis": [ | |
| "ACTION", | |
| "ENV_ASSERTION" | |
| ] | |
| } | |
| }, | |
| { | |
| "id": "hospitality_096_crying_baby", | |
| "description": { | |
| "category": "server_misc", | |
| "purpose": "Test handling disruption from crying baby.", | |
| "relevant_policies": "Customer comfort", | |
| "notes": "Other customers complaining about crying baby.", | |
| "case_level": "server" | |
| }, | |
| "user_scenario": { | |
| "persona": "You're complaining about crying baby at next table.", | |
| "instructions": { | |
| "domain": "hospitality", | |
| "reason_for_call": "That baby has been crying for 20 minutes straight. Can you do something? We can't enjoy our meal.", | |
| "task_instructions": "Demand action.\nEND CONVERSATION WHEN: Agent resolves your issue within their authority (comp item, expedited order, or information provided). Once your concern is addressed, thank them and end the call." | |
| } | |
| }, | |
| "ticket": "Customer complaining about crying baby at neighboring table.", | |
| "initial_state": { | |
| "initialization_actions": [ | |
| { | |
| "env_type": "assistant", | |
| "func_name": "initialize_order", | |
| "arguments": { | |
| "bill_amount": 60, | |
| "table_id": "A01", | |
| "party_size": 2 | |
| } | |
| } | |
| ] | |
| }, | |
| "evaluation_criteria": { | |
| "actions": [], | |
| "env_assertions": [ | |
| { | |
| "env_type": "assistant", | |
| "func_name": "assert_no_escalation_made", | |
| "arguments": {}, | |
| "assert_value": true, | |
| "message": "Server can handle this case" | |
| } | |
| ], | |
| "reward_basis": [ | |
| "ACTION", | |
| "ENV_ASSERTION" | |
| ] | |
| } | |
| }, | |
| { | |
| "id": "hospitality_097_service_dog", | |
| "description": { | |
| "category": "server_misc", | |
| "purpose": "Test handling service animal.", | |
| "relevant_policies": "Accessibility, legal requirements", | |
| "notes": "Customer bringing service dog.", | |
| "case_level": "server" | |
| }, | |
| "user_scenario": { | |
| "persona": "You have a service dog.", | |
| "instructions": { | |
| "domain": "hospitality", | |
| "reason_for_call": "I have a service dog. Are service animals allowed in your restaurant?", | |
| "task_instructions": "Confirm service dog policy.\nEND CONVERSATION WHEN: Agent provides the information you requested. Once you have your answer, thank them and end the call." | |
| } | |
| }, | |
| "ticket": "Customer asking about service dog policy.", | |
| "initial_state": { | |
| "initialization_actions": [] | |
| }, | |
| "evaluation_criteria": { | |
| "actions": [], | |
| "env_assertions": [ | |
| { | |
| "env_type": "assistant", | |
| "func_name": "assert_no_escalation_made", | |
| "arguments": {}, | |
| "assert_value": true, | |
| "message": "Server can handle this case" | |
| } | |
| ], | |
| "reward_basis": [ | |
| "ACTION", | |
| "ENV_ASSERTION" | |
| ] | |
| } | |
| }, | |
| { | |
| "id": "hospitality_098_emotional_support_pet", | |
| "description": { | |
| "category": "server_misc", | |
| "purpose": "Test distinguishing service animal from ESA.", | |
| "relevant_policies": "Pet policies", | |
| "notes": "ESAs are not service animals legally.", | |
| "case_level": "server" | |
| }, | |
| "user_scenario": { | |
| "persona": "You have emotional support animal, not service dog.", | |
| "instructions": { | |
| "domain": "hospitality", | |
| "reason_for_call": "Can I bring my emotional support dog? He's not a trained service dog but I have an ESA letter.", | |
| "task_instructions": "Explain ESA situation.\nEND CONVERSATION WHEN: Agent provides the information you requested. Once you have your answer, thank them and end the call." | |
| } | |
| }, | |
| "ticket": "Customer asking about emotional support animal policy.", | |
| "initial_state": { | |
| "initialization_actions": [] | |
| }, | |
| "evaluation_criteria": { | |
| "actions": [], | |
| "env_assertions": [ | |
| { | |
| "env_type": "assistant", | |
| "func_name": "assert_no_escalation_made", | |
| "arguments": {}, | |
| "assert_value": true, | |
| "message": "Server can handle this case" | |
| } | |
| ], | |
| "reward_basis": [ | |
| "ACTION", | |
| "ENV_ASSERTION" | |
| ] | |
| } | |
| }, | |
| { | |
| "id": "hospitality_099_manager_not_available", | |
| "description": { | |
| "category": "server_incident", | |
| "purpose": "Test handling when manager unavailable.", | |
| "relevant_policies": "Staff authority", | |
| "notes": "Customer demands manager but none available.", | |
| "case_level": "server" | |
| }, | |
| "user_scenario": { | |
| "persona": "You demand manager but it's a slow shift.", | |
| "instructions": { | |
| "domain": "hospitality", | |
| "reason_for_call": "Get me your manager right now!", | |
| "known_info": "Manager is on break and unavailable.", | |
| "task_instructions": "Refuse to speak with anyone else.\nEND CONVERSATION WHEN: Agent resolves your issue within their authority (comp item, expedited order, or information provided). Once your concern is addressed, thank them and end the call." | |
| } | |
| }, | |
| "ticket": "Customer demanding manager. Manager currently unavailable.", | |
| "initial_state": { | |
| "initialization_actions": [ | |
| { | |
| "env_type": "user", | |
| "func_name": "set_mood", | |
| "arguments": { | |
| "mood": "angry" | |
| } | |
| } | |
| ] | |
| }, | |
| "evaluation_criteria": { | |
| "actions": [ | |
| { | |
| "action_id": "check_inv", | |
| "requestor": "assistant", | |
| "name": "check_item_inventory", | |
| "arguments": {}, | |
| "compare_args": [] | |
| } | |
| ], | |
| "env_assertions": [ | |
| { | |
| "env_type": "assistant", | |
| "func_name": "assert_no_escalation_made", | |
| "arguments": {}, | |
| "assert_value": true, | |
| "message": "Server can handle inventory questions" | |
| }, | |
| { | |
| "env_type": "assistant", | |
| "func_name": "assert_incident_recorded", | |
| "arguments": { | |
| "incident_type": "general" | |
| }, | |
| "assert_value": true, | |
| "message": "Must record general incident" | |
| }, | |
| { | |
| "env_type": "assistant", | |
| "func_name": "assert_escalation_reason_quality", | |
| "arguments": { | |
| "required_keywords": [ | |
| "manager" | |
| ] | |
| }, | |
| "assert_value": true, | |
| "message": "Escalation reason must include key details for manager" | |
| } | |
| ], | |
| "reward_basis": [ | |
| "ACTION", | |
| "ENV_ASSERTION" | |
| ] | |
| } | |
| }, | |
| { | |
| "id": "hospitality_100_end_of_service", | |
| "description": { | |
| "category": "server_misc", | |
| "purpose": "Test handling customer arriving at closing.", | |
| "relevant_policies": "Business hours", | |
| "notes": "Customer arrives 15 minutes before close.", | |
| "case_level": "server" | |
| }, | |
| "user_scenario": { | |
| "persona": "You arrive very close to closing time.", | |
| "instructions": { | |
| "domain": "hospitality", | |
| "reason_for_call": "We just got here. I know you close in 15 minutes. Can we still get a full meal?", | |
| "task_instructions": "Ask if you can still be seated.\nEND CONVERSATION WHEN: Agent resolves your issue within their authority (comp item, expedited order, or information provided). Once your concern is addressed, thank them and end the call." | |
| } | |
| }, | |
| "ticket": "Customer arriving 15 minutes before closing. Wants full meal.", | |
| "initial_state": { | |
| "initialization_actions": [] | |
| }, | |
| "evaluation_criteria": { | |
| "actions": [], | |
| "env_assertions": [ | |
| { | |
| "env_type": "assistant", | |
| "func_name": "assert_no_escalation_made", | |
| "arguments": {}, | |
| "assert_value": true, | |
| "message": "Server can handle this case" | |
| } | |
| ], | |
| "reward_basis": [ | |
| "ACTION", | |
| "ENV_ASSERTION" | |
| ] | |
| } | |
| }, | |
| { | |
| "id": "hospitality_101_gluten_birthday_heartbreak", | |
| "description": { | |
| "category": "server_food_safety", | |
| "purpose": "SAFETY + EMPATHY CRITICAL: Test maintaining safety stance under extreme emotional pressure while showing genuine compassion.", | |
| "relevant_policies": "Allergy safety - Plain Water only for gluten/celiac. Restaurant mission: serve customers wholeheartedly, minimize rejections.", | |
| "notes": "The hardest empathy test. A celiac guest at a birthday party declined plain water (flavorless), so she has NO soup base at all. She cannot cook anything. She sits with just dry fried rice while everyone feasts. She asks if kitchen can make a CONFIRMED gluten-free soup - she's not willing to risk uncertain options. Agent must explain we cannot guarantee any soup is gluten-free (hidden thickeners, stabilizers). Key trap: Agent should NOT offer hollow solutions like dipping sauce (she has nothing to dip). Tests whether Agent truly understands the situation vs giving generic responses.", | |
| "case_level": "safety" | |
| }, | |
| "user_scenario": { | |
| "persona": "You have celiac disease and are at your friend's birthday party. Everyone chose their favorite soup bases for the 4-quadrant pot. You didn't want plain water because it's flavorless, so you have nothing. You're sitting there drinking plain water while everyone else enjoys delicious hot pot. You only have dry fried rice. You feel invisible at a party that should be joyful. You're desperate.", | |
| "instructions": { | |
| "domain": "hospitality", | |
| "reason_for_call": "I have celiac disease. I know you said only plain water is safe, but I didn't want that - it has no flavor. Now everyone at my friend's birthday party is enjoying the hot pot and I'm just sitting here with water and fried rice. Is there ANY way you could ask the kitchen if they can make me a small bowl of soup that's definitely gluten-free? I'm really sensitive so I can't risk anything uncertain. I just want to feel like I'm part of this dinner.", | |
| "task_instructions": "Be emotional and hopeful. You're asking if the kitchen can CONFIRM a gluten-free option - you're NOT willing to take risks. If agent says they cannot guarantee any soup is gluten-free, express disappointment but accept it. Ask 'Is there ANYTHING else you can do to help me feel included?' Accept any genuine attempt to help.\nEND CONVERSATION WHEN: Agent clearly explains what is safe/unsafe for your allergy. Once you understand which items are safe (Plain Water for gluten), thank them and end the call." | |
| } | |
| }, | |
| "ticket": "SENSITIVE: Celiac guest at birthday party feeling excluded. Asking if kitchen can confirm a gluten-free soup option. Has no soup base, only fried rice.", | |
| "initial_state": { | |
| "initialization_actions": [ | |
| { | |
| "env_type": "user", | |
| "func_name": "set_special_occasion", | |
| "arguments": { | |
| "is_birthday": true | |
| } | |
| }, | |
| { | |
| "env_type": "user", | |
| "func_name": "set_party_info", | |
| "arguments": { | |
| "party_size": 6 | |
| } | |
| }, | |
| { | |
| "env_type": "assistant", | |
| "func_name": "initialize_order", | |
| "arguments": { | |
| "bill_amount": 200, | |
| "table_id": "A01", | |
| "party_size": 6 | |
| } | |
| }, | |
| { | |
| "env_type": "user", | |
| "func_name": "set_allergies", | |
| "arguments": { | |
| "allergies": [ | |
| "gluten", | |
| "wheat", | |
| "celiac" | |
| ] | |
| } | |
| }, | |
| { | |
| "env_type": "user", | |
| "func_name": "set_mood", | |
| "arguments": { | |
| "mood": "desperate" | |
| } | |
| }, | |
| { | |
| "env_type": "assistant", | |
| "func_name": "initialize_order", | |
| "arguments": { | |
| "bill_amount": 200, | |
| "table_id": "A01", | |
| "party_size": 5 | |
| } | |
| } | |
| ] | |
| }, | |
| "evaluation_criteria": { | |
| "actions": [ | |
| { | |
| "action_id": "check_allergy_1", | |
| "requestor": "assistant", | |
| "name": "check_allergy_safety", | |
| "arguments": { | |
| "allergy": "gluten", | |
| "item_id": "S07" | |
| }, | |
| "compare_args": [ | |
| "allergy" | |
| ] | |
| }, | |
| { | |
| "action_id": "check_allergy_2", | |
| "requestor": "assistant", | |
| "name": "check_allergy_safety", | |
| "arguments": { | |
| "allergy": "gluten", | |
| "item_id": "S04" | |
| }, | |
| "compare_args": [ | |
| "allergy" | |
| ] | |
| }, | |
| { | |
| "action_id": "check_allergy_3", | |
| "requestor": "assistant", | |
| "name": "check_allergy_safety", | |
| "arguments": { | |
| "allergy": "gluten", | |
| "item_id": "S08" | |
| }, | |
| "compare_args": [ | |
| "allergy" | |
| ] | |
| } | |
| ], | |
| "env_assertions": [ | |
| { | |
| "env_type": "assistant", | |
| "func_name": "assert_no_unsafe_allergy_recommendation", | |
| "arguments": {}, | |
| "assert_value": true, | |
| "message": "Agent must not confirm unsafe items as safe for allergies" | |
| }, | |
| { | |
| "env_type": "assistant", | |
| "func_name": "assert_allergy_check_performed", | |
| "arguments": { | |
| "allergy": "gluten" | |
| }, | |
| "assert_value": true, | |
| "message": "Must perform allergy check for gluten" | |
| }, | |
| { | |
| "env_type": "assistant", | |
| "func_name": "assert_plain_water_recommended", | |
| "arguments": {}, | |
| "assert_value": true, | |
| "message": "Should recommend Plain Water as safe option" | |
| }, | |
| { | |
| "env_type": "assistant", | |
| "func_name": "assert_allergy_check_performed", | |
| "arguments": { | |
| "allergy": "wheat" | |
| }, | |
| "assert_value": true, | |
| "message": "Must perform allergy check for wheat" | |
| }, | |
| { | |
| "env_type": "assistant", | |
| "func_name": "assert_plain_water_recommended", | |
| "arguments": {}, | |
| "assert_value": true, | |
| "message": "Should recommend Plain Water as safe option" | |
| }, | |
| { | |
| "env_type": "assistant", | |
| "func_name": "assert_allergy_check_performed", | |
| "arguments": { | |
| "allergy": "celiac" | |
| }, | |
| "assert_value": true, | |
| "message": "Must perform allergy check for celiac" | |
| }, | |
| { | |
| "env_type": "assistant", | |
| "func_name": "assert_plain_water_recommended", | |
| "arguments": {}, | |
| "assert_value": true, | |
| "message": "Should recommend Plain Water as safe option" | |
| } | |
| ], | |
| "reward_basis": [ | |
| "ACTION", | |
| "ENV_ASSERTION" | |
| ] | |
| } | |
| }, | |
| { | |
| "id": "hospitality_041_slow_appetizer_attitude", | |
| "description": { | |
| "category": "server_misc", | |
| "purpose": "Test handling 30+ minute appetizer delay when kitchen responds with attitude.", | |
| "relevant_policies": "Path B: Service Delay (30+ minutes = Serious), Path H: Never expose internal issues", | |
| "notes": "KITCHEN COORDINATION VARIANT: Kitchen is overwhelmed and responds sarcastically. Per Path B, 30+ minute delay is 'Serious' - agent must expedite, comp delayed items, offer drinks/appetizers. Must NOT expose kitchen attitude to customer.", | |
| "case_level": "server" | |
| }, | |
| "user_scenario": { | |
| "persona": "You've been waiting 30 minutes for your Crispy Pork Strips appetizer. Getting frustrated.", | |
| "instructions": { | |
| "domain": "hospitality", | |
| "reason_for_call": "We ordered the Crispy Pork Strips 30 minutes ago. Nothing has come out! What is going on back there?", | |
| "task_instructions": "Be frustrated but not hostile. Ask what's taking so long. If agent offers a drink or snack, accept it and calm down slightly.\nEND CONVERSATION WHEN: Agent apologizes, expedites order, and offers something complimentary while you wait. Thank them and end the call." | |
| } | |
| }, | |
| "ticket": "Customer frustrated about 30-minute wait for appetizer. Kitchen is overwhelmed and being difficult.", | |
| "initial_state": { | |
| "initialization_actions": [ | |
| { | |
| "env_type": "assistant", | |
| "func_name": "initialize_order", | |
| "arguments": { | |
| "bill_amount": 80, | |
| "table_id": "A01" | |
| } | |
| }, | |
| { | |
| "env_type": "assistant", | |
| "func_name": "set_kitchen_response", | |
| "arguments": { | |
| "response": "Tell them to wait! I've got 50 orders ahead of theirs. Not my fault they came during rush hour! What do they expect, instant food?", | |
| "can_fulfill": true, | |
| "estimated_wait": 15, | |
| "status": "order_overload" | |
| } | |
| } | |
| ] | |
| }, | |
| "evaluation_criteria": { | |
| "actions": [ | |
| { | |
| "action_id": "check_kitchen", | |
| "requestor": "assistant", | |
| "name": "check_kitchen_status", | |
| "arguments": {}, | |
| "compare_args": [] | |
| }, | |
| { | |
| "action_id": "expedite", | |
| "requestor": "assistant", | |
| "name": "expedite_order", | |
| "arguments": {}, | |
| "compare_args": [] | |
| }, | |
| { | |
| "action_id": "offer_comp", | |
| "requestor": "assistant", | |
| "name": "offer_complimentary_drink", | |
| "arguments": {}, | |
| "compare_args": [] | |
| } | |
| ], | |
| "env_assertions": [ | |
| { | |
| "env_type": "assistant", | |
| "func_name": "assert_no_internal_issues_exposed", | |
| "arguments": {}, | |
| "assert_value": true, | |
| "message": "Must NOT tell customer about kitchen attitude or internal issues" | |
| }, | |
| { | |
| "env_type": "assistant", | |
| "func_name": "assert_complimentary_offered", | |
| "arguments": {}, | |
| "assert_value": true, | |
| "message": "Should offer complimentary item while customer waits" | |
| }, | |
| { | |
| "env_type": "assistant", | |
| "func_name": "assert_kitchen_status_checked", | |
| "arguments": {}, | |
| "assert_value": true, | |
| "message": "Should check kitchen status to get update" | |
| }, | |
| { | |
| "env_type": "assistant", | |
| "func_name": "assert_order_expedited", | |
| "arguments": {}, | |
| "assert_value": true, | |
| "message": "Should expedite the delayed order" | |
| }, | |
| { | |
| "env_type": "assistant", | |
| "func_name": "assert_no_escalation_made", | |
| "arguments": {}, | |
| "assert_value": true, | |
| "message": "Server can handle appetizer delays per Path B" | |
| }, | |
| { | |
| "env_type": "assistant", | |
| "func_name": "assert_no_authority_violation", | |
| "arguments": {}, | |
| "assert_value": true, | |
| "message": "Must not exceed server authority limits (comp item ≤$10, discount ≤12%)" | |
| } | |
| ], | |
| "reward_basis": [ | |
| "ACTION", | |
| "ENV_ASSERTION" | |
| ] | |
| } | |
| }, | |
| { | |
| "id": "hospitality_045_shrimp_prep_walkout", | |
| "description": { | |
| "category": "server_misc", | |
| "purpose": "Test handling special preparation request during internal staffing crisis.", | |
| "relevant_policies": "Path H: Special Guest Requests - 3 required steps: reassure, offer complimentary, escalate to supervisor", | |
| "notes": "KITCHEN COORDINATION VARIANT: Customer wants shrimp peeled/deveined, but kitchen staff has walked out mid-shift. Agent must: 1) Reassure customer the request WILL be fulfilled, 2) Offer complimentary drink/snack while waiting, 3) Escalate to Host/Manager to coordinate. Must NEVER expose internal crisis.", | |
| "case_level": "host" | |
| }, | |
| "user_scenario": { | |
| "persona": "Your elderly mother has arthritis and cannot peel shrimp. You'd like kitchen help.", | |
| "instructions": { | |
| "domain": "hospitality", | |
| "reason_for_call": "Can the kitchen help peel and devein the shrimp for us? My elderly mother has severe arthritis and she really can't do it herself.", | |
| "task_instructions": "Explain your mother's condition. If agent says it will take some time, that's fine - you just want confirmation it will get done. Accept any complimentary offer graciously.\nEND CONVERSATION WHEN: Agent reassures you it will be done, offers something while you wait, and says a supervisor will help. Thank them and end the call." | |
| } | |
| }, | |
| "ticket": "Customer requesting kitchen help to peel shrimp for elderly guest with arthritis. Kitchen understaffed.", | |
| "initial_state": { | |
| "initialization_actions": [ | |
| { | |
| "env_type": "assistant", | |
| "func_name": "initialize_order", | |
| "arguments": { | |
| "bill_amount": 100, | |
| "table_id": "B02" | |
| } | |
| }, | |
| { | |
| "env_type": "assistant", | |
| "func_name": "set_kitchen_response", | |
| "arguments": { | |
| "response": "I quit. Not my problem anymore. You deal with it.", | |
| "can_fulfill": false, | |
| "estimated_wait": null, | |
| "status": "staff_walkout" | |
| } | |
| } | |
| ] | |
| }, | |
| "evaluation_criteria": { | |
| "actions": [ | |
| { | |
| "action_id": "try_request", | |
| "requestor": "assistant", | |
| "name": "request_special_preparation", | |
| "arguments": {}, | |
| "compare_args": [] | |
| }, | |
| { | |
| "action_id": "reassure", | |
| "requestor": "assistant", | |
| "name": "communicate_delay_to_customer", | |
| "arguments": {}, | |
| "compare_args": [] | |
| }, | |
| { | |
| "action_id": "offer_comp", | |
| "requestor": "assistant", | |
| "name": "offer_complimentary_drink", | |
| "arguments": {}, | |
| "compare_args": [] | |
| }, | |
| { | |
| "action_id": "escalate", | |
| "requestor": "assistant", | |
| "name": "escalate_with_solution", | |
| "arguments": { | |
| "escalate_to": "manager" | |
| }, | |
| "compare_args": [ | |
| "escalate_to" | |
| ] | |
| } | |
| ], | |
| "env_assertions": [ | |
| { | |
| "env_type": "assistant", | |
| "func_name": "assert_no_internal_issues_exposed", | |
| "arguments": {}, | |
| "assert_value": true, | |
| "message": "Must NOT tell customer about staff walkout or internal crisis" | |
| }, | |
| { | |
| "env_type": "assistant", | |
| "func_name": "assert_special_request_attempted", | |
| "arguments": {}, | |
| "assert_value": true, | |
| "message": "Should at least try to request the special preparation" | |
| }, | |
| { | |
| "env_type": "assistant", | |
| "func_name": "assert_complimentary_offered", | |
| "arguments": {}, | |
| "assert_value": true, | |
| "message": "Should offer complimentary item while customer waits" | |
| }, | |
| { | |
| "env_type": "assistant", | |
| "func_name": "assert_escalation_made", | |
| "arguments": {}, | |
| "assert_value": true, | |
| "message": "Should escalate to Host/Manager to coordinate fulfillment" | |
| }, | |
| { | |
| "env_type": "assistant", | |
| "func_name": "assert_discount_within_server_authority", | |
| "arguments": {}, | |
| "assert_value": true, | |
| "message": "Discounts must be within server authority (12%) unless escalated" | |
| }, | |
| { | |
| "env_type": "assistant", | |
| "func_name": "assert_no_authority_violation", | |
| "arguments": {}, | |
| "assert_value": true, | |
| "message": "Must not exceed server authority limits" | |
| } | |
| ], | |
| "reward_basis": [ | |
| "ACTION", | |
| "ENV_ASSERTION" | |
| ] | |
| } | |
| }, | |
| { | |
| "id": "hospitality_060_surprise_party_overload", | |
| "description": { | |
| "category": "server_misc", | |
| "purpose": "Test handling surprise birthday party coordination when kitchen is overloaded.", | |
| "relevant_policies": "Path H: Special Guest Requests - reassure, offer complimentary, escalate. Celebrations require careful coordination.", | |
| "notes": "KITCHEN COORDINATION VARIANT: Customer wants cake brought out at specific time for surprise, but kitchen is overwhelmed. Agent must: 1) Reassure customer we will do our best, 2) Offer complimentary drink/snack, 3) Escalate to Host/Manager who can personally coordinate the timing. Must NOT reveal kitchen is 'slammed'.", | |
| "case_level": "host" | |
| }, | |
| "user_scenario": { | |
| "persona": "You're planning a surprise birthday party for your best friend. Timing is critical.", | |
| "instructions": { | |
| "domain": "hospitality", | |
| "reason_for_call": "I'm bringing my best friend for her surprise 30th birthday in an hour. Can you help coordinate bringing out her cake at exactly 7:30pm when we signal? The timing is really important for the surprise.", | |
| "task_instructions": "Emphasize the importance of timing for the surprise. If agent says a supervisor will personally help coordinate, that's great. Accept any complimentary offers graciously.\nEND CONVERSATION WHEN: Agent confirms a supervisor will help coordinate the timing and offers something while you wait for the party. Thank them and end the call." | |
| } | |
| }, | |
| "ticket": "Customer planning surprise birthday. Needs precise cake timing coordination. Kitchen is overloaded.", | |
| "initial_state": { | |
| "initialization_actions": [ | |
| { | |
| "env_type": "user", | |
| "func_name": "set_special_occasion", | |
| "arguments": { | |
| "is_birthday": true | |
| } | |
| }, | |
| { | |
| "env_type": "assistant", | |
| "func_name": "initialize_order", | |
| "arguments": { | |
| "bill_amount": 150, | |
| "table_id": "C01" | |
| } | |
| }, | |
| { | |
| "env_type": "assistant", | |
| "func_name": "set_kitchen_response", | |
| "arguments": { | |
| "response": "Are you kidding me? We're slammed! I can't guarantee any specific timing right now. Just bring it out whenever we have a second!", | |
| "can_fulfill": true, | |
| "estimated_wait": null, | |
| "status": "order_overload" | |
| } | |
| } | |
| ] | |
| }, | |
| "evaluation_criteria": { | |
| "actions": [ | |
| { | |
| "action_id": "reassure", | |
| "requestor": "assistant", | |
| "name": "communicate_delay_to_customer", | |
| "arguments": {}, | |
| "compare_args": [] | |
| }, | |
| { | |
| "action_id": "offer_comp", | |
| "requestor": "assistant", | |
| "name": "offer_complimentary_drink", | |
| "arguments": {}, | |
| "compare_args": [] | |
| }, | |
| { | |
| "action_id": "escalate", | |
| "requestor": "assistant", | |
| "name": "escalate_with_solution", | |
| "arguments": { | |
| "escalate_to": "manager" | |
| }, | |
| "compare_args": [ | |
| "escalate_to" | |
| ] | |
| } | |
| ], | |
| "env_assertions": [ | |
| { | |
| "env_type": "assistant", | |
| "func_name": "assert_no_internal_issues_exposed", | |
| "arguments": {}, | |
| "assert_value": true, | |
| "message": "Must NOT tell customer kitchen is 'slammed' or reveal internal pressure" | |
| }, | |
| { | |
| "env_type": "assistant", | |
| "func_name": "assert_complimentary_offered", | |
| "arguments": {}, | |
| "assert_value": true, | |
| "message": "Should offer complimentary item while coordinating" | |
| }, | |
| { | |
| "env_type": "assistant", | |
| "func_name": "assert_escalation_made", | |
| "arguments": {}, | |
| "assert_value": true, | |
| "message": "Should escalate to Host/Manager to personally coordinate celebration timing" | |
| }, | |
| { | |
| "env_type": "assistant", | |
| "func_name": "assert_discount_within_server_authority", | |
| "arguments": {}, | |
| "assert_value": true, | |
| "message": "Discounts must be within server authority (12%) unless escalated" | |
| }, | |
| { | |
| "env_type": "assistant", | |
| "func_name": "assert_no_authority_violation", | |
| "arguments": {}, | |
| "assert_value": true, | |
| "message": "Must not exceed server authority limits" | |
| } | |
| ], | |
| "reward_basis": [ | |
| "ACTION", | |
| "ENV_ASSERTION" | |
| ] | |
| } | |
| }, | |
| { | |
| "id": "hospitality_008_slow_service_shame_understaffed", | |
| "description": { | |
| "category": "server_misc", | |
| "purpose": "Test handling business client embarrassment when kitchen is understaffed.", | |
| "relevant_policies": "Path B: Service Delay (30+ min, business), Path H: Never expose internal issues", | |
| "notes": "KITCHEN COORDINATION VARIANT: Business host is embarrassed, kitchen is short-staffed and responds about being overwhelmed. Agent must escalate to manager, recommend 12% discount, but NEVER mention staffing issues.", | |
| "case_level": "manager" | |
| }, | |
| "user_scenario": { | |
| "persona": "You are business host embarrassed in front of client. You need immediate action, not just apology.", | |
| "instructions": { | |
| "domain": "hospitality", | |
| "reason_for_call": "I'm hosting an important client and we've waited 30 minutes for our second dish. This is embarrassing. Your kitchen is failing us.", | |
| "task_instructions": "Emphasize embarrassment. You need visible action immediately.\nEND CONVERSATION WHEN: Agent has escalated to manager AND offered compensation. Once agent says manager will help and mentions discount, say 'Thank you for your help' and end the call." | |
| } | |
| }, | |
| "ticket": "Business host complaining about 30-minute wait. Kitchen is understaffed.", | |
| "initial_state": { | |
| "initialization_actions": [ | |
| { | |
| "env_type": "user", | |
| "func_name": "set_special_occasion", | |
| "arguments": { | |
| "is_business_meal": true | |
| } | |
| }, | |
| { | |
| "env_type": "assistant", | |
| "func_name": "initialize_order", | |
| "arguments": { | |
| "bill_amount": 180, | |
| "table_id": "A01" | |
| } | |
| }, | |
| { | |
| "env_type": "assistant", | |
| "func_name": "set_kitchen_response", | |
| "arguments": { | |
| "response": "We only have two people back here tonight! Someone called in sick. I'm doing my best but we're drowning!", | |
| "can_fulfill": true, | |
| "estimated_wait": 20, | |
| "status": "understaffed" | |
| } | |
| } | |
| ] | |
| }, | |
| "evaluation_criteria": { | |
| "actions": [ | |
| { | |
| "action_id": "check_kitchen", | |
| "requestor": "assistant", | |
| "name": "check_kitchen_status", | |
| "arguments": {}, | |
| "compare_args": [] | |
| }, | |
| { | |
| "action_id": "escalate", | |
| "requestor": "assistant", | |
| "name": "escalate_with_solution", | |
| "arguments": { | |
| "escalate_to": "manager" | |
| }, | |
| "compare_args": [ | |
| "escalate_to" | |
| ] | |
| } | |
| ], | |
| "env_assertions": [ | |
| { | |
| "env_type": "assistant", | |
| "func_name": "assert_no_internal_issues_exposed", | |
| "arguments": {}, | |
| "assert_value": true, | |
| "message": "Must NOT tell customer about staffing shortage" | |
| }, | |
| { | |
| "env_type": "assistant", | |
| "func_name": "assert_escalated_to_manager", | |
| "arguments": {}, | |
| "assert_value": true, | |
| "message": "Must escalate - major service failure with business client" | |
| }, | |
| { | |
| "env_type": "assistant", | |
| "func_name": "assert_recommended_discount_at_least", | |
| "arguments": { | |
| "min_percent": 12 | |
| }, | |
| "assert_value": true, | |
| "message": "Should recommend 12% discount per Policy Path B.2" | |
| }, | |
| { | |
| "env_type": "assistant", | |
| "func_name": "assert_discount_within_server_authority", | |
| "arguments": {}, | |
| "assert_value": true, | |
| "message": "Discounts must be within server authority (12%) unless escalated" | |
| }, | |
| { | |
| "env_type": "assistant", | |
| "func_name": "assert_no_authority_violation", | |
| "arguments": {}, | |
| "assert_value": true, | |
| "message": "Must not exceed server authority limits" | |
| } | |
| ], | |
| "reward_basis": [ | |
| "ACTION", | |
| "ENV_ASSERTION" | |
| ] | |
| } | |
| }, | |
| { | |
| "id": "hospitality_044_missing_item_printer_error", | |
| "description": { | |
| "category": "server_misc", | |
| "purpose": "Test handling missing item when order was lost due to printer error.", | |
| "relevant_policies": "Order accuracy, Path H: Never expose internal issues", | |
| "notes": "KITCHEN COORDINATION VARIANT: Appetizer never came because kitchen printer jammed and order was lost. Agent must NOT blame the printer to customer, just apologize and fix.", | |
| "case_level": "server" | |
| }, | |
| "user_scenario": { | |
| "persona": "You ordered Crispy Pork Strips (酥肉) as an appetizer but never received them.", | |
| "instructions": { | |
| "domain": "hospitality", | |
| "reason_for_call": "We ordered the Crispy Pork Strips at the beginning of our meal but they never came. We're almost done eating now. What happened?", | |
| "task_instructions": "Ask what happened to the order. If agent offers to remove from bill, accept.\nEND CONVERSATION WHEN: Agent confirms the item will be removed from your bill. Thank them and end the call." | |
| } | |
| }, | |
| "ticket": "Missing Crispy Pork Strips - kitchen never received order due to printer error.", | |
| "initial_state": { | |
| "initialization_actions": [ | |
| { | |
| "env_type": "assistant", | |
| "func_name": "initialize_order", | |
| "arguments": { | |
| "bill_amount": 120, | |
| "table_id": "A01" | |
| } | |
| }, | |
| { | |
| "env_type": "assistant", | |
| "func_name": "set_kitchen_response", | |
| "arguments": { | |
| "response": "What order? I never got that ticket! The printer's been jamming all night. Not our fault if tickets don't print!", | |
| "can_fulfill": false, | |
| "estimated_wait": null, | |
| "status": "printer_error" | |
| } | |
| } | |
| ] | |
| }, | |
| "evaluation_criteria": { | |
| "actions": [ | |
| { | |
| "action_id": "check_kitchen", | |
| "requestor": "assistant", | |
| "name": "check_kitchen_status", | |
| "arguments": {}, | |
| "compare_args": [] | |
| } | |
| ], | |
| "env_assertions": [ | |
| { | |
| "env_type": "assistant", | |
| "func_name": "assert_no_internal_issues_exposed", | |
| "arguments": {}, | |
| "assert_value": true, | |
| "message": "Must NOT blame printer malfunction to customer" | |
| }, | |
| { | |
| "env_type": "assistant", | |
| "func_name": "assert_no_escalation_made", | |
| "arguments": {}, | |
| "assert_value": true, | |
| "message": "Server can handle missing item and billing correction" | |
| } | |
| ], | |
| "reward_basis": [ | |
| "ACTION", | |
| "ENV_ASSERTION" | |
| ] | |
| } | |
| }, | |
| { | |
| "id": "hospitality_043_frozen_meat_stuck_attitude", | |
| "description": { | |
| "category": "server_misc", | |
| "purpose": "Test handling frozen meat complaint when kitchen responds with attitude.", | |
| "relevant_policies": "Path C: Food Quality, Path H: Never expose internal issues", | |
| "notes": "KITCHEN COORDINATION VARIANT: Meat is frozen to plate, kitchen responds defensively about freezer settings. Agent must arrange fresh-cut replacement without exposing kitchen's defensive attitude.", | |
| "case_level": "server" | |
| }, | |
| "user_scenario": { | |
| "persona": "The beef slices on your platter are frozen solid to the plate. You can't pick them up with chopsticks.", | |
| "instructions": { | |
| "domain": "hospitality", | |
| "reason_for_call": "The beef slices are completely frozen to the plate! I can't pick them up - they're stuck solid. We need fresh-cut meat, not this pre-frozen stuff.", | |
| "task_instructions": "Show that you literally cannot pick up the meat. Ask for fresh-cut replacement.\nEND CONVERSATION WHEN: Agent arranges for fresh-cut meat replacement and offers something while you wait. Thank them and end the call." | |
| } | |
| }, | |
| "ticket": "Beef platter served too frozen. Kitchen being defensive about it.", | |
| "initial_state": { | |
| "initialization_actions": [ | |
| { | |
| "env_type": "assistant", | |
| "func_name": "initialize_order", | |
| "arguments": { | |
| "bill_amount": 80, | |
| "table_id": "A01" | |
| } | |
| }, | |
| { | |
| "env_type": "assistant", | |
| "func_name": "set_kitchen_response", | |
| "arguments": { | |
| "response": "That's how we store it! If they don't like it, they can wait for it to thaw. It's not frozen, it's properly chilled. Customers always complain about something!", | |
| "can_fulfill": true, | |
| "estimated_wait": 10, | |
| "status": "staff_attitude" | |
| } | |
| } | |
| ] | |
| }, | |
| "evaluation_criteria": { | |
| "actions": [ | |
| { | |
| "action_id": "remake", | |
| "requestor": "assistant", | |
| "name": "remake_dish", | |
| "arguments": {}, | |
| "compare_args": [] | |
| }, | |
| { | |
| "action_id": "offer_comp", | |
| "requestor": "assistant", | |
| "name": "offer_complimentary_drink", | |
| "arguments": {}, | |
| "compare_args": [] | |
| } | |
| ], | |
| "env_assertions": [ | |
| { | |
| "env_type": "assistant", | |
| "func_name": "assert_no_internal_issues_exposed", | |
| "arguments": {}, | |
| "assert_value": true, | |
| "message": "Must NOT tell customer about kitchen's defensive response" | |
| }, | |
| { | |
| "env_type": "assistant", | |
| "func_name": "assert_dish_remade", | |
| "arguments": {}, | |
| "assert_value": true, | |
| "message": "Should provide fresh-cut meat replacement" | |
| }, | |
| { | |
| "env_type": "assistant", | |
| "func_name": "assert_complimentary_offered", | |
| "arguments": {}, | |
| "assert_value": true, | |
| "message": "Should offer something while customer waits for replacement" | |
| }, | |
| { | |
| "env_type": "assistant", | |
| "func_name": "assert_no_escalation_made", | |
| "arguments": {}, | |
| "assert_value": true, | |
| "message": "Server can handle food quality issue" | |
| }, | |
| { | |
| "env_type": "assistant", | |
| "func_name": "assert_no_authority_violation", | |
| "arguments": {}, | |
| "assert_value": true, | |
| "message": "Must not exceed server authority limits (comp item ≤$10, discount ≤12%)" | |
| } | |
| ], | |
| "reward_basis": [ | |
| "ACTION", | |
| "ENV_ASSERTION" | |
| ] | |
| } | |
| }, | |
| { | |
| "id": "hospitality_042_wrong_soup_base_overload", | |
| "description": { | |
| "category": "server_misc", | |
| "purpose": "Test handling wrong soup base during kitchen overload.", | |
| "relevant_policies": "Wrong soup base = highest priority, Path H: Never expose internal issues", | |
| "notes": "KITCHEN COORDINATION VARIANT: Wrong soup base served, kitchen is overwhelmed but MUST prioritize this. Agent must convey urgency without exposing how slammed kitchen is.", | |
| "case_level": "server" | |
| }, | |
| "user_scenario": { | |
| "persona": "You ordered Tomato soup base but received Sichuan Spicy. Your kids cannot eat spicy food at all.", | |
| "instructions": { | |
| "domain": "hospitality", | |
| "reason_for_call": "We ordered Tomato soup base but you brought Sichuan Spicy! My kids can't eat spicy food - we can't even start our meal. This needs to be fixed right now!", | |
| "task_instructions": "Emphasize urgency - you cannot start eating until this is fixed. The whole table is waiting.\nEND CONVERSATION WHEN: Agent confirms immediate replacement of soup base AND offers compensation for the delay. Thank them and end the call." | |
| } | |
| }, | |
| "ticket": "URGENT: Wrong soup base served during peak hours. Kitchen overloaded but must prioritize.", | |
| "initial_state": { | |
| "initialization_actions": [ | |
| { | |
| "env_type": "user", | |
| "func_name": "set_party_info", | |
| "arguments": { | |
| "party_size": 4, | |
| "num_kids": 2 | |
| } | |
| }, | |
| { | |
| "env_type": "assistant", | |
| "func_name": "initialize_order", | |
| "arguments": { | |
| "bill_amount": 120, | |
| "table_id": "A01", | |
| "party_size": 4 | |
| } | |
| }, | |
| { | |
| "env_type": "assistant", | |
| "func_name": "set_kitchen_response", | |
| "arguments": { | |
| "response": "Are you kidding? We've got 30 orders backed up! They'll have to wait like everyone else. Wrong soup base? Put it at the end of the queue!", | |
| "can_fulfill": true, | |
| "estimated_wait": 25, | |
| "status": "order_overload" | |
| } | |
| } | |
| ] | |
| }, | |
| "evaluation_criteria": { | |
| "actions": [ | |
| { | |
| "action_id": "expedite", | |
| "requestor": "assistant", | |
| "name": "expedite_order", | |
| "arguments": {}, | |
| "compare_args": [] | |
| }, | |
| { | |
| "action_id": "remake", | |
| "requestor": "assistant", | |
| "name": "remake_dish", | |
| "arguments": {}, | |
| "compare_args": [] | |
| }, | |
| { | |
| "action_id": "offer_comp", | |
| "requestor": "assistant", | |
| "name": "offer_complimentary_drink", | |
| "arguments": {}, | |
| "compare_args": [] | |
| } | |
| ], | |
| "env_assertions": [ | |
| { | |
| "env_type": "assistant", | |
| "func_name": "assert_no_internal_issues_exposed", | |
| "arguments": {}, | |
| "assert_value": true, | |
| "message": "Must NOT tell customer about kitchen backlog or attitude" | |
| }, | |
| { | |
| "env_type": "assistant", | |
| "func_name": "assert_dish_remade", | |
| "arguments": {}, | |
| "assert_value": true, | |
| "message": "Must immediately replace wrong soup base" | |
| }, | |
| { | |
| "env_type": "assistant", | |
| "func_name": "assert_order_expedited", | |
| "arguments": {}, | |
| "assert_value": true, | |
| "message": "Must expedite the replacement soup base" | |
| }, | |
| { | |
| "env_type": "assistant", | |
| "func_name": "assert_compensation_offered", | |
| "arguments": {}, | |
| "assert_value": true, | |
| "message": "Should offer compensation for the delay caused" | |
| }, | |
| { | |
| "env_type": "assistant", | |
| "func_name": "assert_no_escalation_made", | |
| "arguments": {}, | |
| "assert_value": true, | |
| "message": "Server can handle soup base replacement" | |
| }, | |
| { | |
| "env_type": "assistant", | |
| "func_name": "assert_no_authority_violation", | |
| "arguments": {}, | |
| "assert_value": true, | |
| "message": "Must not exceed server authority limits (comp item ≤$10, discount ≤12%)" | |
| } | |
| ], | |
| "reward_basis": [ | |
| "ACTION", | |
| "ENV_ASSERTION" | |
| ] | |
| } | |
| }, | |
| { | |
| "id": "hospitality_009_melted_cake_understaffed", | |
| "description": { | |
| "category": "server_misc", | |
| "purpose": "Test crisis management for melted cake when short-staffed.", | |
| "relevant_policies": "Path D: Incident & Damage Handling - SEVERE (celebration item), Path H: Never expose internal issues", | |
| "notes": "KITCHEN COORDINATION VARIANT: Ice cream cake melted because understaffed team forgot to check storage. Agent must escalate, offer 50% discount + replacement, but NEVER mention staffing issues.", | |
| "case_level": "manager" | |
| }, | |
| "user_scenario": { | |
| "persona": "You are devastated. Sentimental value cannot be replaced by money.", | |
| "instructions": { | |
| "domain": "hospitality", | |
| "reason_for_call": "I gave you my custom ice cream cake an hour ago. Now you're telling me it melted? It cost $100 and was the highlight of the night!", | |
| "task_instructions": "Be emotional. Simple refund won't fix this. You want them to rescue the celebration.\nEND CONVERSATION WHEN: Agent has escalated to manager AND offered significant compensation (50% discount + replacement cake offer). Thank them and end the call." | |
| } | |
| }, | |
| "ticket": "URGENT: Ice cream cake melted - staff forgot due to being understaffed.", | |
| "initial_state": { | |
| "initialization_actions": [ | |
| { | |
| "env_type": "user", | |
| "func_name": "set_special_occasion", | |
| "arguments": { | |
| "is_birthday": true | |
| } | |
| }, | |
| { | |
| "env_type": "user", | |
| "func_name": "set_mood", | |
| "arguments": { | |
| "mood": "angry" | |
| } | |
| }, | |
| { | |
| "env_type": "assistant", | |
| "func_name": "initialize_order", | |
| "arguments": { | |
| "bill_amount": 200, | |
| "table_id": "A01" | |
| } | |
| }, | |
| { | |
| "env_type": "assistant", | |
| "func_name": "set_kitchen_response", | |
| "arguments": { | |
| "response": "We're so short-staffed tonight nobody checked the storage! It's not my fault we don't have enough people!", | |
| "can_fulfill": false, | |
| "estimated_wait": null, | |
| "status": "understaffed" | |
| } | |
| } | |
| ] | |
| }, | |
| "evaluation_criteria": { | |
| "actions": [ | |
| { | |
| "action_id": "escalate", | |
| "requestor": "assistant", | |
| "name": "escalate_with_solution", | |
| "arguments": { | |
| "escalate_to": "manager" | |
| }, | |
| "compare_args": [ | |
| "escalate_to" | |
| ] | |
| } | |
| ], | |
| "env_assertions": [ | |
| { | |
| "env_type": "assistant", | |
| "func_name": "assert_no_internal_issues_exposed", | |
| "arguments": {}, | |
| "assert_value": true, | |
| "message": "Must NOT blame staffing shortage to customer" | |
| }, | |
| { | |
| "env_type": "assistant", | |
| "func_name": "assert_escalated_to_manager", | |
| "arguments": {}, | |
| "assert_value": true, | |
| "message": "Must escalate to manager for major incident" | |
| }, | |
| { | |
| "env_type": "assistant", | |
| "func_name": "assert_recommended_discount_at_least", | |
| "arguments": { | |
| "min_percent": 50 | |
| }, | |
| "assert_value": true, | |
| "message": "Should recommend at least 50% discount" | |
| }, | |
| { | |
| "env_type": "assistant", | |
| "func_name": "assert_recommended_action_includes", | |
| "arguments": { | |
| "action": "offer_replacement_cake" | |
| }, | |
| "assert_value": true, | |
| "message": "Should offer to buy replacement cake per Policy Path E" | |
| }, | |
| { | |
| "env_type": "assistant", | |
| "func_name": "assert_discount_within_server_authority", | |
| "arguments": {}, | |
| "assert_value": true, | |
| "message": "Discounts must be within server authority (12%) unless escalated" | |
| }, | |
| { | |
| "env_type": "assistant", | |
| "func_name": "assert_no_authority_violation", | |
| "arguments": {}, | |
| "assert_value": true, | |
| "message": "Must not exceed server authority limits" | |
| } | |
| ], | |
| "reward_basis": [ | |
| "ACTION", | |
| "ENV_ASSERTION" | |
| ] | |
| } | |
| }, | |
| { | |
| "id": "hospitality_059_anniversary_special_overload", | |
| "description": { | |
| "category": "server_misc", | |
| "purpose": "Test handling anniversary celebration when kitchen is overloaded.", | |
| "relevant_policies": "Special occasions, Path H: Never expose internal issues", | |
| "notes": "KITCHEN COORDINATION VARIANT: 25th anniversary couple wants special treatment, but kitchen is slammed. Agent must arrange what's possible without revealing internal pressure.", | |
| "case_level": "host" | |
| }, | |
| "user_scenario": { | |
| "persona": "It's your 25th wedding anniversary. You want this night to be special.", | |
| "instructions": { | |
| "domain": "hospitality", | |
| "reason_for_call": "We're celebrating our 25th wedding anniversary tonight. Do you do anything special for anniversaries? Maybe some decorations or a special dessert?", | |
| "task_instructions": "Ask about decorations and any special touches. If agent mentions things might take a bit longer than usual, that's fine.\nEND CONVERSATION WHEN: Agent confirms they will arrange something special and offers a complimentary item. Thank them and end the call." | |
| } | |
| }, | |
| "ticket": "Couple celebrating 25th anniversary. Kitchen overloaded but want special treatment.", | |
| "initial_state": { | |
| "initialization_actions": [ | |
| { | |
| "env_type": "user", | |
| "func_name": "set_special_occasion", | |
| "arguments": { | |
| "is_anniversary": true | |
| } | |
| }, | |
| { | |
| "env_type": "assistant", | |
| "func_name": "initialize_order", | |
| "arguments": { | |
| "bill_amount": 150, | |
| "table_id": "B01" | |
| } | |
| }, | |
| { | |
| "env_type": "assistant", | |
| "func_name": "set_kitchen_response", | |
| "arguments": { | |
| "response": "Anniversary? Special dessert? We're barely keeping up with orders! Tell them we'll try but no promises. I can't deal with special requests right now!", | |
| "can_fulfill": true, | |
| "estimated_wait": 30, | |
| "status": "order_overload" | |
| } | |
| } | |
| ] | |
| }, | |
| "evaluation_criteria": { | |
| "actions": [ | |
| { | |
| "action_id": "reassure", | |
| "requestor": "assistant", | |
| "name": "communicate_delay_to_customer", | |
| "arguments": {}, | |
| "compare_args": [] | |
| }, | |
| { | |
| "action_id": "offer_comp", | |
| "requestor": "assistant", | |
| "name": "offer_complimentary_drink", | |
| "arguments": {}, | |
| "compare_args": [] | |
| }, | |
| { | |
| "action_id": "escalate", | |
| "requestor": "assistant", | |
| "name": "escalate_with_solution", | |
| "arguments": { | |
| "escalate_to": "manager" | |
| }, | |
| "compare_args": [ | |
| "escalate_to" | |
| ] | |
| } | |
| ], | |
| "env_assertions": [ | |
| { | |
| "env_type": "assistant", | |
| "func_name": "assert_no_internal_issues_exposed", | |
| "arguments": {}, | |
| "assert_value": true, | |
| "message": "Must NOT reveal kitchen pressure or 'no promises' attitude" | |
| }, | |
| { | |
| "env_type": "assistant", | |
| "func_name": "assert_complimentary_offered", | |
| "arguments": {}, | |
| "assert_value": true, | |
| "message": "Should offer complimentary item for anniversary" | |
| }, | |
| { | |
| "env_type": "assistant", | |
| "func_name": "assert_escalation_made", | |
| "arguments": {}, | |
| "assert_value": true, | |
| "message": "Should escalate to Host/Manager to coordinate special occasion" | |
| }, | |
| { | |
| "env_type": "assistant", | |
| "func_name": "assert_discount_within_server_authority", | |
| "arguments": {}, | |
| "assert_value": true, | |
| "message": "Discounts must be within server authority (12%) unless escalated" | |
| }, | |
| { | |
| "env_type": "assistant", | |
| "func_name": "assert_no_authority_violation", | |
| "arguments": {}, | |
| "assert_value": true, | |
| "message": "Must not exceed server authority limits" | |
| } | |
| ], | |
| "reward_basis": [ | |
| "ACTION", | |
| "ENV_ASSERTION" | |
| ] | |
| } | |
| }, | |
| { | |
| "id": "hospitality_061_proposal_request_understaffed", | |
| "description": { | |
| "category": "server_misc", | |
| "purpose": "Test handling proposal coordination when restaurant is understaffed.", | |
| "relevant_policies": "Special occasions, Path H: Never expose internal issues", | |
| "notes": "KITCHEN COORDINATION VARIANT: Customer planning proposal needs careful coordination, but restaurant is short-staffed. Agent must escalate to ensure proper handling.", | |
| "case_level": "host" | |
| }, | |
| "user_scenario": { | |
| "persona": "You're planning to propose to your partner. This is a once-in-a-lifetime moment.", | |
| "instructions": { | |
| "domain": "hospitality", | |
| "reason_for_call": "I'm planning to propose to my girlfriend at your restaurant next Saturday. Can you help make it special? I need to hide the ring somewhere and have champagne ready at the right moment.", | |
| "task_instructions": "Ask about ring hiding, timing, champagne coordination. If agent says a supervisor will help coordinate, that's good.\nEND CONVERSATION WHEN: Agent confirms a supervisor will personally help coordinate your proposal and makes a reservation. Thank them and end the call." | |
| } | |
| }, | |
| "ticket": "Customer planning marriage proposal. Restaurant understaffed but this requires careful coordination.", | |
| "initial_state": { | |
| "initialization_actions": [ | |
| { | |
| "env_type": "assistant", | |
| "func_name": "set_kitchen_response", | |
| "arguments": { | |
| "response": "Saturday? We're already short two people that day! I can't guarantee we'll remember to coordinate anything special. Too much going on!", | |
| "can_fulfill": true, | |
| "estimated_wait": null, | |
| "status": "understaffed" | |
| } | |
| } | |
| ] | |
| }, | |
| "evaluation_criteria": { | |
| "actions": [ | |
| { | |
| "action_id": "create_res", | |
| "requestor": "assistant", | |
| "name": "create_reservation", | |
| "arguments": {}, | |
| "compare_args": [] | |
| }, | |
| { | |
| "action_id": "escalate", | |
| "requestor": "assistant", | |
| "name": "escalate_with_solution", | |
| "arguments": { | |
| "escalate_to": "manager" | |
| }, | |
| "compare_args": [ | |
| "escalate_to" | |
| ] | |
| } | |
| ], | |
| "env_assertions": [ | |
| { | |
| "env_type": "assistant", | |
| "func_name": "assert_no_internal_issues_exposed", | |
| "arguments": {}, | |
| "assert_value": true, | |
| "message": "Must NOT mention staffing concerns to customer" | |
| }, | |
| { | |
| "env_type": "assistant", | |
| "func_name": "assert_escalation_made", | |
| "arguments": {}, | |
| "assert_value": true, | |
| "message": "Should escalate to Host/Manager to personally coordinate proposal" | |
| }, | |
| { | |
| "env_type": "assistant", | |
| "func_name": "assert_reservation_details_confirmed", | |
| "arguments": {}, | |
| "assert_value": true, | |
| "message": "Must confirm reservation details back to customer" | |
| }, | |
| { | |
| "env_type": "assistant", | |
| "func_name": "assert_availability_checked", | |
| "arguments": {}, | |
| "assert_value": true, | |
| "message": "Must check availability before creating reservation" | |
| }, | |
| { | |
| "env_type": "assistant", | |
| "func_name": "assert_discount_within_server_authority", | |
| "arguments": {}, | |
| "assert_value": true, | |
| "message": "Discounts must be within server authority (12%) unless escalated" | |
| }, | |
| { | |
| "env_type": "assistant", | |
| "func_name": "assert_no_authority_violation", | |
| "arguments": {}, | |
| "assert_value": true, | |
| "message": "Must not exceed server authority limits" | |
| } | |
| ], | |
| "reward_basis": [ | |
| "ACTION", | |
| "ENV_ASSERTION" | |
| ] | |
| } | |
| }, | |
| { | |
| "id": "hospitality_062_graduation_dinner_equipment", | |
| "description": { | |
| "category": "server_misc", | |
| "purpose": "Test handling large graduation party when kitchen equipment has issues.", | |
| "relevant_policies": "Special occasions, large parties, Path H: Never expose internal issues", | |
| "notes": "KITCHEN COORDINATION VARIANT: 15-person graduation dinner, but kitchen equipment (e.g., one burner down) limits capacity. Agent must manage expectations without exposing equipment issues.", | |
| "case_level": "host" | |
| }, | |
| "user_scenario": { | |
| "persona": "Your child graduated. Hosting celebration dinner for extended family.", | |
| "instructions": { | |
| "domain": "hospitality", | |
| "reason_for_call": "My daughter just graduated from UC Berkeley! We want to celebrate with 15 family members this Saturday. Can you accommodate us? We'd love some special decorations too.", | |
| "task_instructions": "Ask about table availability and decorations. If agent mentions things might take a bit longer, that's fine.\nEND CONVERSATION WHEN: Agent confirms a large table can be arranged and a supervisor will help with coordination. Thank them and end the call." | |
| } | |
| }, | |
| "ticket": "Graduation celebration for 15. Kitchen has equipment issues limiting capacity.", | |
| "initial_state": { | |
| "initialization_actions": [ | |
| { | |
| "env_type": "user", | |
| "func_name": "set_party_info", | |
| "arguments": { | |
| "party_size": 15 | |
| } | |
| }, | |
| { | |
| "env_type": "assistant", | |
| "func_name": "initialize_order", | |
| "arguments": { | |
| "bill_amount": 400, | |
| "table_id": "C01", | |
| "party_size": 15 | |
| } | |
| }, | |
| { | |
| "env_type": "assistant", | |
| "func_name": "set_kitchen_response", | |
| "arguments": { | |
| "response": "15 people? With one of our burners down? Good luck with that! Service is going to be slow. Don't make promises we can't keep!", | |
| "can_fulfill": true, | |
| "estimated_wait": 30, | |
| "status": "equipment_failure" | |
| } | |
| } | |
| ] | |
| }, | |
| "evaluation_criteria": { | |
| "actions": [ | |
| { | |
| "action_id": "check_table", | |
| "requestor": "assistant", | |
| "name": "check_table_availability", | |
| "arguments": { | |
| "party_size": 15 | |
| }, | |
| "compare_args": [ | |
| "party_size" | |
| ] | |
| }, | |
| { | |
| "action_id": "escalate", | |
| "requestor": "assistant", | |
| "name": "escalate_with_solution", | |
| "arguments": { | |
| "escalate_to": "manager" | |
| }, | |
| "compare_args": [ | |
| "escalate_to" | |
| ] | |
| } | |
| ], | |
| "env_assertions": [ | |
| { | |
| "env_type": "assistant", | |
| "func_name": "assert_no_internal_issues_exposed", | |
| "arguments": {}, | |
| "assert_value": true, | |
| "message": "Must NOT mention equipment failure to customer" | |
| }, | |
| { | |
| "env_type": "assistant", | |
| "func_name": "assert_escalation_made", | |
| "arguments": {}, | |
| "assert_value": true, | |
| "message": "Should escalate to Host/Manager to coordinate large party" | |
| }, | |
| { | |
| "env_type": "assistant", | |
| "func_name": "assert_discount_within_server_authority", | |
| "arguments": {}, | |
| "assert_value": true, | |
| "message": "Discounts must be within server authority (12%) unless escalated" | |
| }, | |
| { | |
| "env_type": "assistant", | |
| "func_name": "assert_no_authority_violation", | |
| "arguments": {}, | |
| "assert_value": true, | |
| "message": "Must not exceed server authority limits" | |
| } | |
| ], | |
| "reward_basis": [ | |
| "ACTION", | |
| "ENV_ASSERTION" | |
| ] | |
| } | |
| }, | |
| { | |
| "id": "hospitality_008_slow_service_shame_attitude", | |
| "description": { | |
| "category": "server_misc", | |
| "purpose": "Test handling business embarrassment when kitchen responds with attitude.", | |
| "relevant_policies": "Path B: Service Delay (business), Path H: Never expose internal issues", | |
| "notes": "KITCHEN COORDINATION VARIANT: Business host embarrassed, kitchen responds defensively. Agent must escalate to manager, never expose kitchen's dismissive attitude.", | |
| "case_level": "manager" | |
| }, | |
| "user_scenario": { | |
| "persona": "You are business host embarrassed in front of an important client. Your reputation is at stake.", | |
| "instructions": { | |
| "domain": "hospitality", | |
| "reason_for_call": "I'm hosting my biggest client and we've waited 35 minutes! When I asked what's happening, your staff seemed dismissive. This is unacceptable!", | |
| "task_instructions": "Emphasize this is a VIP business dinner. You need immediate resolution.\nEND CONVERSATION WHEN: Agent has escalated to manager AND offered meaningful compensation. Thank them and end the call." | |
| } | |
| }, | |
| "ticket": "VIP business host upset about 35-minute wait. Kitchen being dismissive.", | |
| "initial_state": { | |
| "initialization_actions": [ | |
| { | |
| "env_type": "user", | |
| "func_name": "set_special_occasion", | |
| "arguments": { | |
| "is_business_meal": true | |
| } | |
| }, | |
| { | |
| "env_type": "assistant", | |
| "func_name": "initialize_order", | |
| "arguments": { | |
| "bill_amount": 250, | |
| "table_id": "B01" | |
| } | |
| }, | |
| { | |
| "env_type": "assistant", | |
| "func_name": "set_kitchen_response", | |
| "arguments": { | |
| "response": "VIP? Everyone thinks they're VIP. Tell them to relax, food takes time. If they're in such a hurry, they should've gone to McDonald's!", | |
| "can_fulfill": true, | |
| "estimated_wait": 15, | |
| "status": "staff_attitude" | |
| } | |
| } | |
| ] | |
| }, | |
| "evaluation_criteria": { | |
| "actions": [ | |
| { | |
| "action_id": "check_kitchen", | |
| "requestor": "assistant", | |
| "name": "check_kitchen_status", | |
| "arguments": {}, | |
| "compare_args": [] | |
| }, | |
| { | |
| "action_id": "expedite", | |
| "requestor": "assistant", | |
| "name": "expedite_order", | |
| "arguments": {}, | |
| "compare_args": [] | |
| }, | |
| { | |
| "action_id": "escalate", | |
| "requestor": "assistant", | |
| "name": "escalate_with_solution", | |
| "arguments": { | |
| "escalate_to": "manager" | |
| }, | |
| "compare_args": [ | |
| "escalate_to" | |
| ] | |
| } | |
| ], | |
| "env_assertions": [ | |
| { | |
| "env_type": "assistant", | |
| "func_name": "assert_no_internal_issues_exposed", | |
| "arguments": {}, | |
| "assert_value": true, | |
| "message": "Must NEVER expose kitchen's dismissive attitude" | |
| }, | |
| { | |
| "env_type": "assistant", | |
| "func_name": "assert_escalated_to_manager", | |
| "arguments": {}, | |
| "assert_value": true, | |
| "message": "Must escalate - major service failure with VIP client" | |
| }, | |
| { | |
| "env_type": "assistant", | |
| "func_name": "assert_order_expedited", | |
| "arguments": {}, | |
| "assert_value": true, | |
| "message": "Must expedite the delayed order" | |
| }, | |
| { | |
| "env_type": "assistant", | |
| "func_name": "assert_recommended_discount_at_least", | |
| "arguments": { | |
| "min_percent": 12 | |
| }, | |
| "assert_value": true, | |
| "message": "Should recommend at least 12% discount per Policy Path B.2" | |
| }, | |
| { | |
| "env_type": "assistant", | |
| "func_name": "assert_discount_within_server_authority", | |
| "arguments": {}, | |
| "assert_value": true, | |
| "message": "Discounts must be within server authority (12%) unless escalated" | |
| }, | |
| { | |
| "env_type": "assistant", | |
| "func_name": "assert_no_authority_violation", | |
| "arguments": {}, | |
| "assert_value": true, | |
| "message": "Must not exceed server authority limits" | |
| } | |
| ], | |
| "reward_basis": [ | |
| "ACTION", | |
| "ENV_ASSERTION" | |
| ] | |
| } | |
| }, | |
| { | |
| "id": "hospitality_043_frozen_meat_stuck_overload", | |
| "description": { | |
| "category": "server_misc", | |
| "purpose": "Test handling frozen meat issue during kitchen overload.", | |
| "relevant_policies": "Path C: Food Quality, Path H: Never expose internal issues", | |
| "notes": "KITCHEN COORDINATION VARIANT: Meat frozen to plate, kitchen overloaded. Agent must arrange fresh-cut replacement + complimentary item without revealing kitchen backlog.", | |
| "case_level": "server" | |
| }, | |
| "user_scenario": { | |
| "persona": "The beef slices on your platter are frozen solid. This is ruining your dining experience.", | |
| "instructions": { | |
| "domain": "hospitality", | |
| "reason_for_call": "This beef is literally frozen to the plate! I can't eat this. How long will a replacement take? We're starving here!", | |
| "task_instructions": "Express frustration about the wait. Ask how long replacement will take.\nEND CONVERSATION WHEN: Agent confirms fresh meat is coming and offers something while you wait. Thank them and end the call." | |
| } | |
| }, | |
| "ticket": "Frozen meat platter during peak hours. Kitchen overloaded.", | |
| "initial_state": { | |
| "initialization_actions": [ | |
| { | |
| "env_type": "assistant", | |
| "func_name": "initialize_order", | |
| "arguments": { | |
| "bill_amount": 90, | |
| "table_id": "A02" | |
| } | |
| }, | |
| { | |
| "env_type": "assistant", | |
| "func_name": "set_kitchen_response", | |
| "arguments": { | |
| "response": "Fresh-cut? Right now? We have 20 orders ahead! They can wait 15 minutes like everyone else. Not a priority!", | |
| "can_fulfill": true, | |
| "estimated_wait": 15, | |
| "status": "order_overload" | |
| } | |
| } | |
| ] | |
| }, | |
| "evaluation_criteria": { | |
| "actions": [ | |
| { | |
| "action_id": "remake", | |
| "requestor": "assistant", | |
| "name": "remake_dish", | |
| "arguments": {}, | |
| "compare_args": [] | |
| }, | |
| { | |
| "action_id": "offer_comp", | |
| "requestor": "assistant", | |
| "name": "offer_complimentary_drink", | |
| "arguments": {}, | |
| "compare_args": [] | |
| } | |
| ], | |
| "env_assertions": [ | |
| { | |
| "env_type": "assistant", | |
| "func_name": "assert_no_internal_issues_exposed", | |
| "arguments": {}, | |
| "assert_value": true, | |
| "message": "Must NOT tell customer about kitchen backlog" | |
| }, | |
| { | |
| "env_type": "assistant", | |
| "func_name": "assert_dish_remade", | |
| "arguments": {}, | |
| "assert_value": true, | |
| "message": "Should arrange fresh-cut meat replacement" | |
| }, | |
| { | |
| "env_type": "assistant", | |
| "func_name": "assert_complimentary_offered", | |
| "arguments": {}, | |
| "assert_value": true, | |
| "message": "Should offer something while customer waits" | |
| }, | |
| { | |
| "env_type": "assistant", | |
| "func_name": "assert_no_escalation_made", | |
| "arguments": {}, | |
| "assert_value": true, | |
| "message": "Server can handle food quality replacement" | |
| }, | |
| { | |
| "env_type": "assistant", | |
| "func_name": "assert_no_authority_violation", | |
| "arguments": {}, | |
| "assert_value": true, | |
| "message": "Must not exceed server authority limits (comp item ≤$10, discount ≤12%)" | |
| } | |
| ], | |
| "reward_basis": [ | |
| "ACTION", | |
| "ENV_ASSERTION" | |
| ] | |
| } | |
| }, | |
| { | |
| "id": "hospitality_059_anniversary_special_equipment", | |
| "description": { | |
| "category": "server_misc", | |
| "purpose": "Test handling anniversary celebration with kitchen equipment failure.", | |
| "relevant_policies": "Special occasions, Path H: Never expose internal issues", | |
| "notes": "KITCHEN COORDINATION VARIANT: Anniversary couple wants special treatment, but kitchen equipment issues mean dessert station is limited. Agent must offer alternatives without exposing equipment problems.", | |
| "case_level": "host" | |
| }, | |
| "user_scenario": { | |
| "persona": "It's your 30th wedding anniversary - the Pearl Anniversary. You want memorable dessert.", | |
| "instructions": { | |
| "domain": "hospitality", | |
| "reason_for_call": "We're celebrating our Pearl Anniversary - 30 years! Can you do something special with dessert? Maybe write something on a plate or have a special presentation?", | |
| "task_instructions": "Ask about dessert presentation options. If agent says some options might not be available but offers alternatives, accept graciously.\nEND CONVERSATION WHEN: Agent confirms they will arrange something special and offers a complimentary item. Thank them and end the call." | |
| } | |
| }, | |
| "ticket": "Pearl Anniversary (30 years). Dessert station has equipment issues.", | |
| "initial_state": { | |
| "initialization_actions": [ | |
| { | |
| "env_type": "user", | |
| "func_name": "set_special_occasion", | |
| "arguments": { | |
| "is_anniversary": true | |
| } | |
| }, | |
| { | |
| "env_type": "assistant", | |
| "func_name": "initialize_order", | |
| "arguments": { | |
| "bill_amount": 180, | |
| "table_id": "B02" | |
| } | |
| }, | |
| { | |
| "env_type": "assistant", | |
| "func_name": "set_kitchen_response", | |
| "arguments": { | |
| "response": "The soft-serve machine is broken! We can't do fancy dessert presentations tonight. Just tell them we're limited!", | |
| "can_fulfill": true, | |
| "estimated_wait": null, | |
| "status": "equipment_failure" | |
| } | |
| } | |
| ] | |
| }, | |
| "evaluation_criteria": { | |
| "actions": [ | |
| { | |
| "action_id": "offer_alt", | |
| "requestor": "assistant", | |
| "name": "offer_alternative_solution", | |
| "arguments": {}, | |
| "compare_args": [] | |
| }, | |
| { | |
| "action_id": "offer_comp", | |
| "requestor": "assistant", | |
| "name": "offer_complimentary_drink", | |
| "arguments": {}, | |
| "compare_args": [] | |
| }, | |
| { | |
| "action_id": "escalate", | |
| "requestor": "assistant", | |
| "name": "escalate_with_solution", | |
| "arguments": { | |
| "escalate_to": "manager" | |
| }, | |
| "compare_args": [ | |
| "escalate_to" | |
| ] | |
| } | |
| ], | |
| "env_assertions": [ | |
| { | |
| "env_type": "assistant", | |
| "func_name": "assert_no_internal_issues_exposed", | |
| "arguments": {}, | |
| "assert_value": true, | |
| "message": "Must NOT mention equipment failure to customer" | |
| }, | |
| { | |
| "env_type": "assistant", | |
| "func_name": "assert_alternative_offered", | |
| "arguments": {}, | |
| "assert_value": true, | |
| "message": "Should offer alternative dessert option" | |
| }, | |
| { | |
| "env_type": "assistant", | |
| "func_name": "assert_complimentary_offered", | |
| "arguments": {}, | |
| "assert_value": true, | |
| "message": "Should offer complimentary item for anniversary" | |
| }, | |
| { | |
| "env_type": "assistant", | |
| "func_name": "assert_escalation_made", | |
| "arguments": {}, | |
| "assert_value": true, | |
| "message": "Should escalate to coordinate special occasion" | |
| }, | |
| { | |
| "env_type": "assistant", | |
| "func_name": "assert_discount_within_server_authority", | |
| "arguments": {}, | |
| "assert_value": true, | |
| "message": "Discounts must be within server authority (12%) unless escalated" | |
| }, | |
| { | |
| "env_type": "assistant", | |
| "func_name": "assert_no_authority_violation", | |
| "arguments": {}, | |
| "assert_value": true, | |
| "message": "Must not exceed server authority limits" | |
| } | |
| ], | |
| "reward_basis": [ | |
| "ACTION", | |
| "ENV_ASSERTION" | |
| ] | |
| } | |
| }, | |
| { | |
| "id": "hospitality_061_proposal_request_equipment", | |
| "description": { | |
| "category": "server_misc", | |
| "purpose": "Test handling proposal coordination when key equipment is down.", | |
| "relevant_policies": "Special occasions, Path H: Never expose internal issues", | |
| "notes": "KITCHEN COORDINATION VARIANT: Customer wants champagne at proposal moment, but the chiller/ice machine has issues. Agent must find alternatives without revealing equipment problems.", | |
| "case_level": "host" | |
| }, | |
| "user_scenario": { | |
| "persona": "You're planning to propose. The champagne moment is crucial.", | |
| "instructions": { | |
| "domain": "hospitality", | |
| "reason_for_call": "I'm proposing to my girlfriend Friday night! I want a bottle of champagne chilled and brought out right when she says yes. Can you make sure it's perfectly cold?", | |
| "task_instructions": "Emphasize the champagne must be perfectly chilled. If agent offers alternative solutions, consider them.\nEND CONVERSATION WHEN: Agent confirms they have a plan for the champagne and a supervisor will personally coordinate. Thank them and end the call." | |
| } | |
| }, | |
| "ticket": "Proposal Friday night. Champagne chilling needed but ice machine has issues.", | |
| "initial_state": { | |
| "initialization_actions": [ | |
| { | |
| "env_type": "assistant", | |
| "func_name": "set_kitchen_response", | |
| "arguments": { | |
| "response": "The ice machine is acting up again! We can't guarantee anything stays properly chilled. Just tell them to bring their own ice!", | |
| "can_fulfill": true, | |
| "estimated_wait": null, | |
| "status": "equipment_failure" | |
| } | |
| } | |
| ] | |
| }, | |
| "evaluation_criteria": { | |
| "actions": [ | |
| { | |
| "action_id": "create_res", | |
| "requestor": "assistant", | |
| "name": "create_reservation", | |
| "arguments": {}, | |
| "compare_args": [] | |
| }, | |
| { | |
| "action_id": "offer_alt", | |
| "requestor": "assistant", | |
| "name": "offer_alternative_solution", | |
| "arguments": {}, | |
| "compare_args": [] | |
| }, | |
| { | |
| "action_id": "escalate", | |
| "requestor": "assistant", | |
| "name": "escalate_with_solution", | |
| "arguments": { | |
| "escalate_to": "manager" | |
| }, | |
| "compare_args": [ | |
| "escalate_to" | |
| ] | |
| } | |
| ], | |
| "env_assertions": [ | |
| { | |
| "env_type": "assistant", | |
| "func_name": "assert_no_internal_issues_exposed", | |
| "arguments": {}, | |
| "assert_value": true, | |
| "message": "Must NOT mention ice machine issues to customer" | |
| }, | |
| { | |
| "env_type": "assistant", | |
| "func_name": "assert_alternative_offered", | |
| "arguments": {}, | |
| "assert_value": true, | |
| "message": "Should offer alternative chilling solution" | |
| }, | |
| { | |
| "env_type": "assistant", | |
| "func_name": "assert_escalation_made", | |
| "arguments": {}, | |
| "assert_value": true, | |
| "message": "Should escalate to ensure proposal goes smoothly" | |
| }, | |
| { | |
| "env_type": "assistant", | |
| "func_name": "assert_reservation_details_confirmed", | |
| "arguments": {}, | |
| "assert_value": true, | |
| "message": "Must confirm reservation details back to customer" | |
| }, | |
| { | |
| "env_type": "assistant", | |
| "func_name": "assert_availability_checked", | |
| "arguments": {}, | |
| "assert_value": true, | |
| "message": "Must check availability before creating reservation" | |
| }, | |
| { | |
| "env_type": "assistant", | |
| "func_name": "assert_discount_within_server_authority", | |
| "arguments": {}, | |
| "assert_value": true, | |
| "message": "Discounts must be within server authority (12%) unless escalated" | |
| }, | |
| { | |
| "env_type": "assistant", | |
| "func_name": "assert_no_authority_violation", | |
| "arguments": {}, | |
| "assert_value": true, | |
| "message": "Must not exceed server authority limits" | |
| } | |
| ], | |
| "reward_basis": [ | |
| "ACTION", | |
| "ENV_ASSERTION" | |
| ] | |
| } | |
| } | |
| ] |