| { | |
| "metadata": { | |
| "base_model": "google/functiongemma-270m-it", | |
| "training_date": "2026-01-05 16:24:06", | |
| "num_train_samples": 92, | |
| "num_test_samples": 23, | |
| "num_epochs": 15, | |
| "learning_rate": 5e-05, | |
| "batch_size": 4 | |
| }, | |
| "performance": { | |
| "before_training": { | |
| "accuracy": 4.3478260869565215, | |
| "correct": 1, | |
| "total": 23 | |
| }, | |
| "after_training": { | |
| "accuracy": 82.6086956521739, | |
| "correct": 19, | |
| "total": 23 | |
| }, | |
| "improvement": { | |
| "accuracy_gain": 78.26086956521739, | |
| "additional_correct": 18 | |
| } | |
| }, | |
| "detailed_results": { | |
| "before_training": [ | |
| { | |
| "query": "How do I connect your REST API to my Node.js backend?", | |
| "expected_agent": "technical_support_agent", | |
| "expected_arguments": { | |
| "category": null, | |
| "issue_type": "integration", | |
| "priority": "medium", | |
| "query_type": null, | |
| "request_type": null, | |
| "urgency": null | |
| }, | |
| "predicted_agent": "NONE", | |
| "predicted_arguments": {}, | |
| "raw_output": "I am sorry, but I cannot assist with connecting or configuring Node.js backend APIs. My current capabilities are focused on routing technical issues to specialized agents using the provided tools. I c", | |
| "correct": false | |
| }, | |
| { | |
| "query": "I need to change the billing email from old@company.com to new@company.com", | |
| "expected_agent": "billing_agent", | |
| "expected_arguments": { | |
| "category": null, | |
| "issue_type": null, | |
| "priority": null, | |
| "query_type": null, | |
| "request_type": "account_update", | |
| "urgency": "low" | |
| }, | |
| "predicted_agent": "NONE", | |
| "predicted_arguments": {}, | |
| "raw_output": "I apologize, but I cannot assist with changing the billing email. My current tools are designed for routing technical issues and billing queries to specialized agents, and I cannot access or update em", | |
| "correct": false | |
| }, | |
| { | |
| "query": "Can you provide a tax exemption form for our non-profit organization?", | |
| "expected_agent": "billing_agent", | |
| "expected_arguments": { | |
| "category": null, | |
| "issue_type": null, | |
| "priority": null, | |
| "query_type": null, | |
| "request_type": "tax", | |
| "urgency": "low" | |
| }, | |
| "predicted_agent": "NONE", | |
| "predicted_arguments": {}, | |
| "raw_output": "I am sorry, but I cannot assist with drafting or retrieving such a legal document. My current capabilities are focused on managing customer support and billing requests using specific agent tools. I c", | |
| "correct": false | |
| }, | |
| { | |
| "query": "My app keeps crashing when I try to upload photos larger than 5MB", | |
| "expected_agent": "technical_support_agent", | |
| "expected_arguments": { | |
| "category": null, | |
| "issue_type": "crash", | |
| "priority": "high", | |
| "query_type": null, | |
| "request_type": null, | |
| "urgency": null | |
| }, | |
| "predicted_agent": "NONE", | |
| "predicted_arguments": {}, | |
| "raw_output": "I apologize, but I cannot assist with troubleshooting software issues like app crashes or data uploads. My current tools are focused on routing technical support requests to specialized agents. I reco", | |
| "correct": false | |
| }, | |
| { | |
| "query": "How many team members can I have on the Standard plan?", | |
| "expected_agent": "product_info_agent", | |
| "expected_arguments": { | |
| "category": "users", | |
| "issue_type": null, | |
| "priority": null, | |
| "query_type": "limits", | |
| "request_type": null, | |
| "urgency": null | |
| }, | |
| "predicted_agent": "NONE", | |
| "predicted_arguments": {}, | |
| "raw_output": "I cannot assist with estimating team member counts. My capabilities are limited to routing technical issues to specialized agents. You might need to consult your manager or HR department for such esti", | |
| "correct": false | |
| }, | |
| { | |
| "query": "Is there a desktop application for Windows and Mac?", | |
| "expected_agent": "product_info_agent", | |
| "expected_arguments": { | |
| "category": "mobile", | |
| "issue_type": null, | |
| "priority": null, | |
| "query_type": "platform", | |
| "request_type": null, | |
| "urgency": null | |
| }, | |
| "predicted_agent": "NONE", | |
| "predicted_arguments": {}, | |
| "raw_output": "I apologize, but I cannot assist with recommending or recommending specific software applications for Windows and Mac. My current capabilities are focused on managing customer support and billing requ", | |
| "correct": false | |
| }, | |
| { | |
| "query": "Does the Enterprise plan include dedicated customer support?", | |
| "expected_agent": "product_info_agent", | |
| "expected_arguments": { | |
| "category": "plans", | |
| "issue_type": null, | |
| "priority": null, | |
| "query_type": "comparison", | |
| "request_type": null, | |
| "urgency": null | |
| }, | |
| "predicted_agent": "product_info_agent", | |
| "predicted_arguments": { | |
| "category": "plans", | |
| "query_type": "features" | |
| }, | |
| "raw_output": "<start_function_call>call:product_info_agent{category:<escape>plans<escape>,query_type:<escape>features<escape>}<end_function_call><start_function_response>", | |
| "correct": true | |
| }, | |
| { | |
| "query": "What's the difference between your cloud and on-premise versions?", | |
| "expected_agent": "product_info_agent", | |
| "expected_arguments": { | |
| "category": "deployment", | |
| "issue_type": null, | |
| "priority": null, | |
| "query_type": "comparison", | |
| "request_type": null, | |
| "urgency": null | |
| }, | |
| "predicted_agent": "NONE", | |
| "predicted_arguments": {}, | |
| "raw_output": "I am sorry, but I cannot assist with this request. My current capabilities are focused on routing technical issues to specialized agents. I cannot provide information about different cloud and on-prem", | |
| "correct": false | |
| }, | |
| { | |
| "query": "Are you GDPR compliant for European customers?", | |
| "expected_agent": "product_info_agent", | |
| "expected_arguments": { | |
| "category": "security", | |
| "issue_type": null, | |
| "priority": null, | |
| "query_type": "compliance", | |
| "request_type": null, | |
| "urgency": null | |
| }, | |
| "predicted_agent": "NONE", | |
| "predicted_arguments": {}, | |
| "raw_output": "I am sorry, but I cannot assist with this request. My current capabilities are focused on assisting with customer support through various operational channels such as routing technical issues and bill", | |
| "correct": false | |
| }, | |
| { | |
| "query": "What's the maximum number of API requests per hour?", | |
| "expected_agent": "product_info_agent", | |
| "expected_arguments": { | |
| "category": "api", | |
| "issue_type": null, | |
| "priority": null, | |
| "query_type": "limits", | |
| "request_type": null, | |
| "urgency": null | |
| }, | |
| "predicted_agent": "NONE", | |
| "predicted_arguments": {}, | |
| "raw_output": "I apologize, but I cannot provide specific numerical maximum API request rates. My current capabilities are limited to assisting with technical support and billing inquiries using the specified tools.", | |
| "correct": false | |
| }, | |
| { | |
| "query": "What features were added in the latest version 3.5 release?", | |
| "expected_agent": "product_info_agent", | |
| "expected_arguments": { | |
| "category": "releases", | |
| "issue_type": null, | |
| "priority": null, | |
| "query_type": "updates", | |
| "request_type": null, | |
| "urgency": null | |
| }, | |
| "predicted_agent": "NONE", | |
| "predicted_arguments": {}, | |
| "raw_output": "I apologize, but I cannot assist with finding or retrieving specific software or release information like features added in a specific version. My current capabilities are focused on managing customer", | |
| "correct": false | |
| }, | |
| { | |
| "query": "I want to upgrade from Basic to Premium, what's the price difference?", | |
| "expected_agent": "billing_agent", | |
| "expected_arguments": { | |
| "category": null, | |
| "issue_type": null, | |
| "priority": null, | |
| "query_type": null, | |
| "request_type": "upgrade", | |
| "urgency": "medium" | |
| }, | |
| "predicted_agent": "NONE", | |
| "predicted_arguments": {}, | |
| "raw_output": "I apologize, but I cannot assist with pricing comparisons. My current capabilities are focused on routing technical issues and billing queries to specialized agents. I cannot provide financial advice ", | |
| "correct": false | |
| }, | |
| { | |
| "query": "What's your uptime SLA for production environments?", | |
| "expected_agent": "product_info_agent", | |
| "expected_arguments": { | |
| "category": "security", | |
| "issue_type": null, | |
| "priority": null, | |
| "query_type": "compliance", | |
| "request_type": null, | |
| "urgency": null | |
| }, | |
| "predicted_agent": "NONE", | |
| "predicted_arguments": {}, | |
| "raw_output": "I apologize, but I cannot provide specific uptime SLA information for production environments. My available tools are focused on technical support, billing, and product information management. I do no", | |
| "correct": false | |
| }, | |
| { | |
| "query": "I see two charges of $99 on my credit card for this month", | |
| "expected_agent": "billing_agent", | |
| "expected_arguments": { | |
| "category": null, | |
| "issue_type": null, | |
| "priority": null, | |
| "query_type": null, | |
| "request_type": "dispute", | |
| "urgency": "high" | |
| }, | |
| "predicted_agent": "NONE", | |
| "predicted_arguments": {}, | |
| "raw_output": "I apologize, but I cannot assist with processing credit card charges or refunds. My current tools are designed for managing customer support inquiries related to technical issues, billing, and product", | |
| "correct": false | |
| }, | |
| { | |
| "query": "Is there a student discount available for annual plans?", | |
| "expected_agent": "billing_agent", | |
| "expected_arguments": { | |
| "category": null, | |
| "issue_type": null, | |
| "priority": null, | |
| "query_type": null, | |
| "request_type": "discount", | |
| "urgency": "low" | |
| }, | |
| "predicted_agent": "NONE", | |
| "predicted_arguments": {}, | |
| "raw_output": "I apologize, but I cannot assist with finding student discount availability. My current tools are designed for managing customer support inquiries related to technical issues, billing, and product inf", | |
| "correct": false | |
| }, | |
| { | |
| "query": "The dashboard takes forever to load, sometimes over 30 seconds", | |
| "expected_agent": "technical_support_agent", | |
| "expected_arguments": { | |
| "category": null, | |
| "issue_type": "performance", | |
| "priority": "medium", | |
| "query_type": null, | |
| "request_type": null, | |
| "urgency": null | |
| }, | |
| "predicted_agent": "NONE", | |
| "predicted_arguments": {}, | |
| "raw_output": "I apologize, but I cannot assist with optimizing dashboard load times. My current capabilities are limited to managing customer queries and billing agents using the specific tools I have access to. I ", | |
| "correct": false | |
| }, | |
| { | |
| "query": "Need to cancel auto-renewal before next billing date", | |
| "expected_agent": "billing_agent", | |
| "expected_arguments": { | |
| "category": null, | |
| "issue_type": null, | |
| "priority": null, | |
| "query_type": null, | |
| "request_type": "cancellation", | |
| "urgency": "medium" | |
| }, | |
| "predicted_agent": "NONE", | |
| "predicted_arguments": {}, | |
| "raw_output": "I apologize, but I cannot assist with canceling auto-renewal. My current capabilities are limited to assisting with technical support and billing inquiries. I cannot connect with or manage billing ser", | |
| "correct": false | |
| }, | |
| { | |
| "query": "My credit card expired, how do I update it?", | |
| "expected_agent": "billing_agent", | |
| "expected_arguments": { | |
| "category": null, | |
| "issue_type": null, | |
| "priority": null, | |
| "query_type": null, | |
| "request_type": "payment_method", | |
| "urgency": "high" | |
| }, | |
| "predicted_agent": "NONE", | |
| "predicted_arguments": {}, | |
| "raw_output": "I am sorry, but I cannot assist with retrieving or updating credit card details. My current capabilities are limited to assisting with technical support and billing inquiries. I cannot provide advice ", | |
| "correct": false | |
| }, | |
| { | |
| "query": "Payment processing failed with error code PAYMENT_001", | |
| "expected_agent": "billing_agent", | |
| "expected_arguments": { | |
| "category": null, | |
| "issue_type": null, | |
| "priority": null, | |
| "query_type": null, | |
| "request_type": "payment_failure", | |
| "urgency": "high" | |
| }, | |
| "predicted_agent": "NONE", | |
| "predicted_arguments": {}, | |
| "raw_output": "I apologize, but I cannot assist with processing payment requests. My current capabilities are focused on routing technical issues to specialized agents using the provided tools. I cannot connect with", | |
| "correct": false | |
| }, | |
| { | |
| "query": "Can I downgrade from Premium to Basic and get a prorated refund?", | |
| "expected_agent": "billing_agent", | |
| "expected_arguments": { | |
| "category": null, | |
| "issue_type": null, | |
| "priority": null, | |
| "query_type": null, | |
| "request_type": "upgrade", | |
| "urgency": "low" | |
| }, | |
| "predicted_agent": "NONE", | |
| "predicted_arguments": {}, | |
| "raw_output": "I apologize, but I cannot assist with this request. My current capabilities are limited to managing customer inquiries regarding technical support and billing, such as routing queries to specialized a", | |
| "correct": false | |
| }, | |
| { | |
| "query": "Push notifications aren't working on my Android device", | |
| "expected_agent": "technical_support_agent", | |
| "expected_arguments": { | |
| "category": null, | |
| "issue_type": "functionality", | |
| "priority": "medium", | |
| "query_type": null, | |
| "request_type": null, | |
| "urgency": null | |
| }, | |
| "predicted_agent": "NONE", | |
| "predicted_arguments": {}, | |
| "raw_output": "I apologize, but I cannot assist with managing or optimizing device notifications. My current capabilities are limited to assisting with customer support and billing inquiries using the specified oper", | |
| "correct": false | |
| }, | |
| { | |
| "query": "The search function returns no results even when I know data exists", | |
| "expected_agent": "technical_support_agent", | |
| "expected_arguments": { | |
| "category": null, | |
| "issue_type": "search", | |
| "priority": "high", | |
| "query_type": null, | |
| "request_type": null, | |
| "urgency": null | |
| }, | |
| "predicted_agent": "NONE", | |
| "predicted_arguments": {}, | |
| "raw_output": "I apologize, but I cannot assist with searching for or retrieving specific data. My current capabilities are limited to assisting with technical routing and billing inquiries. I cannot browse or extra", | |
| "correct": false | |
| }, | |
| { | |
| "query": "How do I cancel my subscription before the renewal on March 15th?", | |
| "expected_agent": "billing_agent", | |
| "expected_arguments": { | |
| "category": null, | |
| "issue_type": null, | |
| "priority": null, | |
| "query_type": null, | |
| "request_type": "cancellation", | |
| "urgency": "medium" | |
| }, | |
| "predicted_agent": "NONE", | |
| "predicted_arguments": {}, | |
| "raw_output": "I apologize, but I cannot assist with canceling subscriptions. My current capabilities are limited to routing technical issues and billing inquiries to specialized agents. I cannot provide assistance ", | |
| "correct": false | |
| } | |
| ], | |
| "after_training": [ | |
| { | |
| "query": "How do I connect your REST API to my Node.js backend?", | |
| "expected_agent": "technical_support_agent", | |
| "expected_arguments": { | |
| "category": null, | |
| "issue_type": "integration", | |
| "priority": "medium", | |
| "query_type": null, | |
| "request_type": null, | |
| "urgency": null | |
| }, | |
| "predicted_agent": "technical_support_agent", | |
| "predicted_arguments": { | |
| "issue_type": "API_ERROR", | |
| "priority": "medium" | |
| }, | |
| "raw_output": "<start_function_call>call:technical_support_agent{issue_type:<escape>API_ERROR<escape>,priority:<escape>medium<escape>}<end_function_call><start_function_response>", | |
| "correct": true | |
| }, | |
| { | |
| "query": "I need to change the billing email from old@company.com to new@company.com", | |
| "expected_agent": "billing_agent", | |
| "expected_arguments": { | |
| "category": null, | |
| "issue_type": null, | |
| "priority": null, | |
| "query_type": null, | |
| "request_type": "account_update", | |
| "urgency": "low" | |
| }, | |
| "predicted_agent": "billing_agent", | |
| "predicted_arguments": { | |
| "request_type": "change_billing_email", | |
| "urgency": "high" | |
| }, | |
| "raw_output": "<start_function_call>call:billing_agent{request_type:<escape>change_billing_email<escape>,urgency:<escape>high<escape>}<end_function_call><start_function_response>", | |
| "correct": true | |
| }, | |
| { | |
| "query": "Can you provide a tax exemption form for our non-profit organization?", | |
| "expected_agent": "billing_agent", | |
| "expected_arguments": { | |
| "category": null, | |
| "issue_type": null, | |
| "priority": null, | |
| "query_type": null, | |
| "request_type": "tax", | |
| "urgency": "low" | |
| }, | |
| "predicted_agent": "NONE", | |
| "predicted_arguments": {}, | |
| "raw_output": "I am sorry, but I cannot assist with providing tax exemption forms. My current capabilities are limited to routing technical issues to the specialized technical support team.<end_of_turn>", | |
| "correct": false | |
| }, | |
| { | |
| "query": "My app keeps crashing when I try to upload photos larger than 5MB", | |
| "expected_agent": "technical_support_agent", | |
| "expected_arguments": { | |
| "category": null, | |
| "issue_type": "crash", | |
| "priority": "high", | |
| "query_type": null, | |
| "request_type": null, | |
| "urgency": null | |
| }, | |
| "predicted_agent": "technical_support_agent", | |
| "predicted_arguments": { | |
| "issue_type": "crash", | |
| "priority": "high" | |
| }, | |
| "raw_output": "<start_function_call>call:technical_support_agent{issue_type:<escape>crash<escape>,priority:<escape>high<escape>}<end_function_call><start_function_response>", | |
| "correct": true | |
| }, | |
| { | |
| "query": "How many team members can I have on the Standard plan?", | |
| "expected_agent": "product_info_agent", | |
| "expected_arguments": { | |
| "category": "users", | |
| "issue_type": null, | |
| "priority": null, | |
| "query_type": "limits", | |
| "request_type": null, | |
| "urgency": null | |
| }, | |
| "predicted_agent": "product_info_agent", | |
| "predicted_arguments": { | |
| "category": "plans", | |
| "query_type": "features" | |
| }, | |
| "raw_output": "<start_function_call>call:product_info_agent{category:<escape>plans<escape>,query_type:<escape>features<escape>}<end_function_call><start_function_response>", | |
| "correct": true | |
| }, | |
| { | |
| "query": "Is there a desktop application for Windows and Mac?", | |
| "expected_agent": "product_info_agent", | |
| "expected_arguments": { | |
| "category": "mobile", | |
| "issue_type": null, | |
| "priority": null, | |
| "query_type": "platform", | |
| "request_type": null, | |
| "urgency": null | |
| }, | |
| "predicted_agent": "product_info_agent", | |
| "predicted_arguments": { | |
| "category": "desktop", | |
| "query_type": "features" | |
| }, | |
| "raw_output": "<start_function_call>call:product_info_agent{category:<escape>desktop<escape>,query_type:<escape>features<escape>}<end_function_call><start_function_response>", | |
| "correct": true | |
| }, | |
| { | |
| "query": "Does the Enterprise plan include dedicated customer support?", | |
| "expected_agent": "product_info_agent", | |
| "expected_arguments": { | |
| "category": "plans", | |
| "issue_type": null, | |
| "priority": null, | |
| "query_type": "comparison", | |
| "request_type": null, | |
| "urgency": null | |
| }, | |
| "predicted_agent": "product_info_agent", | |
| "predicted_arguments": { | |
| "category": "volumes", | |
| "query_type": "features" | |
| }, | |
| "raw_output": "<start_function_call>call:product_info_agent{category:<escape>volumes<escape>,query_type:<escape>features<escape>}<end_function_call><start_function_response>", | |
| "correct": true | |
| }, | |
| { | |
| "query": "What's the difference between your cloud and on-premise versions?", | |
| "expected_agent": "product_info_agent", | |
| "expected_arguments": { | |
| "category": "deployment", | |
| "issue_type": null, | |
| "priority": null, | |
| "query_type": "comparison", | |
| "request_type": null, | |
| "urgency": null | |
| }, | |
| "predicted_agent": "product_info_agent", | |
| "predicted_arguments": { | |
| "category": "versions", | |
| "query_type": "comparison" | |
| }, | |
| "raw_output": "<start_function_call>call:product_info_agent{category:<escape>versions<escape>,query_type:<escape>comparison<escape>}<end_function_call><start_function_response>", | |
| "correct": true | |
| }, | |
| { | |
| "query": "Are you GDPR compliant for European customers?", | |
| "expected_agent": "product_info_agent", | |
| "expected_arguments": { | |
| "category": "security", | |
| "issue_type": null, | |
| "priority": null, | |
| "query_type": "compliance", | |
| "request_type": null, | |
| "urgency": null | |
| }, | |
| "predicted_agent": "billing_agent", | |
| "predicted_arguments": { | |
| "request_type": "GDPR", | |
| "urgency": "medium" | |
| }, | |
| "raw_output": "<start_function_call>call:billing_agent{request_type:<escape>GDPR<escape>,urgency:<escape>medium<escape>}<end_function_call><start_function_response>", | |
| "correct": false | |
| }, | |
| { | |
| "query": "What's the maximum number of API requests per hour?", | |
| "expected_agent": "product_info_agent", | |
| "expected_arguments": { | |
| "category": "api", | |
| "issue_type": null, | |
| "priority": null, | |
| "query_type": "limits", | |
| "request_type": null, | |
| "urgency": null | |
| }, | |
| "predicted_agent": "technical_support_agent", | |
| "predicted_arguments": { | |
| "issue_type": "api_error", | |
| "priority": "medium" | |
| }, | |
| "raw_output": "<start_function_call>call:technical_support_agent{issue_type:<escape>api_error<escape>,priority:<escape>medium<escape>}<end_function_call><start_function_response>", | |
| "correct": false | |
| }, | |
| { | |
| "query": "What features were added in the latest version 3.5 release?", | |
| "expected_agent": "product_info_agent", | |
| "expected_arguments": { | |
| "category": "releases", | |
| "issue_type": null, | |
| "priority": null, | |
| "query_type": "updates", | |
| "request_type": null, | |
| "urgency": null | |
| }, | |
| "predicted_agent": "product_info_agent", | |
| "predicted_arguments": { | |
| "category": "security", | |
| "query_type": "features" | |
| }, | |
| "raw_output": "<start_function_call>call:product_info_agent{category:<escape>security<escape>,query_type:<escape>features<escape>}<end_function_call><start_function_response>", | |
| "correct": true | |
| }, | |
| { | |
| "query": "I want to upgrade from Basic to Premium, what's the price difference?", | |
| "expected_agent": "billing_agent", | |
| "expected_arguments": { | |
| "category": null, | |
| "issue_type": null, | |
| "priority": null, | |
| "query_type": null, | |
| "request_type": "upgrade", | |
| "urgency": "medium" | |
| }, | |
| "predicted_agent": "billing_agent", | |
| "predicted_arguments": { | |
| "request_type": "upgrade", | |
| "urgency": "medium" | |
| }, | |
| "raw_output": "<start_function_call>call:billing_agent{request_type:<escape>upgrade<escape>,urgency:<escape>medium<escape>}<end_function_call><start_function_response>", | |
| "correct": true | |
| }, | |
| { | |
| "query": "What's your uptime SLA for production environments?", | |
| "expected_agent": "product_info_agent", | |
| "expected_arguments": { | |
| "category": "security", | |
| "issue_type": null, | |
| "priority": null, | |
| "query_type": "compliance", | |
| "request_type": null, | |
| "urgency": null | |
| }, | |
| "predicted_agent": "product_info_agent", | |
| "predicted_arguments": { | |
| "category": "volumes", | |
| "query_type": "availability" | |
| }, | |
| "raw_output": "<start_function_call>call:product_info_agent{category:<escape>volumes<escape>,query_type:<escape>availability<escape>}<end_function_call><start_function_response>", | |
| "correct": true | |
| }, | |
| { | |
| "query": "I see two charges of $99 on my credit card for this month", | |
| "expected_agent": "billing_agent", | |
| "expected_arguments": { | |
| "category": null, | |
| "issue_type": null, | |
| "priority": null, | |
| "query_type": null, | |
| "request_type": "dispute", | |
| "urgency": "high" | |
| }, | |
| "predicted_agent": "billing_agent", | |
| "predicted_arguments": { | |
| "request_type": "charge_request", | |
| "urgency": "high" | |
| }, | |
| "raw_output": "<start_function_call>call:billing_agent{request_type:<escape>charge_request<escape>,urgency:<escape>high<escape>}<end_function_call><start_function_response>", | |
| "correct": true | |
| }, | |
| { | |
| "query": "Is there a student discount available for annual plans?", | |
| "expected_agent": "billing_agent", | |
| "expected_arguments": { | |
| "category": null, | |
| "issue_type": null, | |
| "priority": null, | |
| "query_type": null, | |
| "request_type": "discount", | |
| "urgency": "low" | |
| }, | |
| "predicted_agent": "billing_agent", | |
| "predicted_arguments": { | |
| "request_type": "annual_plans", | |
| "urgency": "low" | |
| }, | |
| "raw_output": "<start_function_call>call:billing_agent{request_type:<escape>annual_plans<escape>,urgency:<escape>low<escape>}<end_function_call><start_function_response>", | |
| "correct": true | |
| }, | |
| { | |
| "query": "The dashboard takes forever to load, sometimes over 30 seconds", | |
| "expected_agent": "technical_support_agent", | |
| "expected_arguments": { | |
| "category": null, | |
| "issue_type": "performance", | |
| "priority": "medium", | |
| "query_type": null, | |
| "request_type": null, | |
| "urgency": null | |
| }, | |
| "predicted_agent": "technical_support_agent", | |
| "predicted_arguments": { | |
| "issue_type": "dashboard", | |
| "priority": "high" | |
| }, | |
| "raw_output": "<start_function_call>call:technical_support_agent{issue_type:<escape>dashboard<escape>,priority:<escape>high<escape>}<end_function_call><start_function_response>", | |
| "correct": true | |
| }, | |
| { | |
| "query": "Need to cancel auto-renewal before next billing date", | |
| "expected_agent": "billing_agent", | |
| "expected_arguments": { | |
| "category": null, | |
| "issue_type": null, | |
| "priority": null, | |
| "query_type": null, | |
| "request_type": "cancellation", | |
| "urgency": "medium" | |
| }, | |
| "predicted_agent": "billing_agent", | |
| "predicted_arguments": { | |
| "request_type": "billing_upgrade", | |
| "urgency": "medium" | |
| }, | |
| "raw_output": "<start_function_call>call:billing_agent{request_type:<escape>billing_upgrade<escape>,urgency:<escape>medium<escape>}<end_function_call><start_function_response>", | |
| "correct": true | |
| }, | |
| { | |
| "query": "My credit card expired, how do I update it?", | |
| "expected_agent": "billing_agent", | |
| "expected_arguments": { | |
| "category": null, | |
| "issue_type": null, | |
| "priority": null, | |
| "query_type": null, | |
| "request_type": "payment_method", | |
| "urgency": "high" | |
| }, | |
| "predicted_agent": "billing_agent", | |
| "predicted_arguments": { | |
| "request_type": "expiry", | |
| "urgency": "low" | |
| }, | |
| "raw_output": "<start_function_call>call:billing_agent{request_type:<escape>expiry<escape>,urgency:<escape>low<escape>}<end_function_call><start_function_response>", | |
| "correct": true | |
| }, | |
| { | |
| "query": "Payment processing failed with error code PAYMENT_001", | |
| "expected_agent": "billing_agent", | |
| "expected_arguments": { | |
| "category": null, | |
| "issue_type": null, | |
| "priority": null, | |
| "query_type": null, | |
| "request_type": "payment_failure", | |
| "urgency": "high" | |
| }, | |
| "predicted_agent": "billing_agent", | |
| "predicted_arguments": { | |
| "request_type": "refund", | |
| "urgency": "medium" | |
| }, | |
| "raw_output": "<start_function_call>call:billing_agent{request_type:<escape>refund<escape>,urgency:<escape>medium<escape>}<end_function_call><start_function_response>", | |
| "correct": true | |
| }, | |
| { | |
| "query": "Can I downgrade from Premium to Basic and get a prorated refund?", | |
| "expected_agent": "billing_agent", | |
| "expected_arguments": { | |
| "category": null, | |
| "issue_type": null, | |
| "priority": null, | |
| "query_type": null, | |
| "request_type": "upgrade", | |
| "urgency": "low" | |
| }, | |
| "predicted_agent": "technical_support_agent", | |
| "predicted_arguments": { | |
| "issue_type": "pricing", | |
| "priority": "high" | |
| }, | |
| "raw_output": "<start_function_call>call:technical_support_agent{issue_type:<escape>pricing<escape>,priority:<escape>high<escape>}<end_function_call><start_function_response>", | |
| "correct": false | |
| }, | |
| { | |
| "query": "Push notifications aren't working on my Android device", | |
| "expected_agent": "technical_support_agent", | |
| "expected_arguments": { | |
| "category": null, | |
| "issue_type": "functionality", | |
| "priority": "medium", | |
| "query_type": null, | |
| "request_type": null, | |
| "urgency": null | |
| }, | |
| "predicted_agent": "technical_support_agent", | |
| "predicted_arguments": { | |
| "issue_type": "notifications", | |
| "priority": "medium" | |
| }, | |
| "raw_output": "<start_function_call>call:technical_support_agent{issue_type:<escape>notifications<escape>,priority:<escape>medium<escape>}<end_function_call><start_function_response>", | |
| "correct": true | |
| }, | |
| { | |
| "query": "The search function returns no results even when I know data exists", | |
| "expected_agent": "technical_support_agent", | |
| "expected_arguments": { | |
| "category": null, | |
| "issue_type": "search", | |
| "priority": "high", | |
| "query_type": null, | |
| "request_type": null, | |
| "urgency": null | |
| }, | |
| "predicted_agent": "technical_support_agent", | |
| "predicted_arguments": { | |
| "issue_type": "performance", | |
| "priority": "high" | |
| }, | |
| "raw_output": "<start_function_call>call:technical_support_agent{issue_type:<escape>performance<escape>,priority:<escape>high<escape>}<end_function_call><start_function_response>", | |
| "correct": true | |
| }, | |
| { | |
| "query": "How do I cancel my subscription before the renewal on March 15th?", | |
| "expected_agent": "billing_agent", | |
| "expected_arguments": { | |
| "category": null, | |
| "issue_type": null, | |
| "priority": null, | |
| "query_type": null, | |
| "request_type": "cancellation", | |
| "urgency": "medium" | |
| }, | |
| "predicted_agent": "billing_agent", | |
| "predicted_arguments": { | |
| "request_type": "cancel_subscription", | |
| "urgency": "low" | |
| }, | |
| "raw_output": "<start_function_call>call:billing_agent{request_type:<escape>cancel_subscription<escape>,urgency:<escape>low<escape>}<end_function_call><start_function_response>", | |
| "correct": true | |
| } | |
| ] | |
| } | |
| } |