| { |
| "repository": "giux78/nev", |
| "revision": "c5c84121263b423f8846811a24790ab96e66cee3", |
| "scope": "16 curated demonstration requests (8 EN/IT pairs), selected AFTER inspecting 32 fixed candidate outputs. Every selected pair succeeded in both languages on both v2 and v3. This selected subset is not an accuracy benchmark.", |
| "cases_sha256": "a5b690415136601fc5815dbbc546c56c773424c93a9d9cf4ae7a1c1e8dc5c62b", |
| "runner_sha256": "9f2ed73f8fbd471a033f32e7a48529ec96a12ce8df9d8ffe8bdd9d2d31661ce9", |
| "python": "3.12.13", |
| "torch": "2.10.0+cu126", |
| "gpu": "NVIDIA A100 80GB PCIe", |
| "precision": "FP32 native cached inference", |
| "timing_scope": "Synchronized wall time after one warmup; shared GPU, illustrative only", |
| "selection_rule": "A pair qualifies for a variant only if both English and Italian choices match the expected labels. No prompt revisions based on predictions.", |
| "models": { |
| "v2": { |
| "cases": [ |
| { |
| "id": "routing/invoice/en", |
| "pair": "routing/invoice", |
| "domain": "routing", |
| "language": "en", |
| "expected": "billing", |
| "predicted": "billing", |
| "correct": true, |
| "elapsed_ms": 64.01175912469625, |
| "response": { |
| "answers": { |
| "decision": { |
| "type": "choice", |
| "choice": "billing", |
| "top_probability": 0.9991890788078308, |
| "probabilities": { |
| "shipping": 0.0005261492915451527, |
| "billing": 0.9991890788078308, |
| "account": 3.5730514355236664e-05, |
| "returns": 0.00016648616292513907, |
| "technical": 8.2566482888069e-05 |
| } |
| } |
| }, |
| "usage": { |
| "state_tokens": 26, |
| "branch_tokens": 61, |
| "questions": 1 |
| }, |
| "latency_ms": 63.913991674780846, |
| "form": "cached" |
| } |
| }, |
| { |
| "id": "routing/invoice/it", |
| "pair": "routing/invoice", |
| "domain": "routing", |
| "language": "it", |
| "expected": "billing", |
| "predicted": "billing", |
| "correct": true, |
| "elapsed_ms": 58.88347327709198, |
| "response": { |
| "answers": { |
| "decision": { |
| "type": "choice", |
| "choice": "billing", |
| "top_probability": 0.9995648264884949, |
| "probabilities": { |
| "shipping": 0.00029537794762291014, |
| "billing": 0.9995648264884949, |
| "account": 9.76422888925299e-05, |
| "returns": 3.3540887670824304e-05, |
| "technical": 8.686401997692883e-06 |
| } |
| } |
| }, |
| "usage": { |
| "state_tokens": 43, |
| "branch_tokens": 91, |
| "questions": 1 |
| }, |
| "latency_ms": 58.789825066924095, |
| "form": "cached" |
| } |
| }, |
| { |
| "id": "routing/exchange/en", |
| "pair": "routing/exchange", |
| "domain": "routing", |
| "language": "en", |
| "expected": "returns", |
| "predicted": "returns", |
| "correct": true, |
| "elapsed_ms": 55.94767443835735, |
| "response": { |
| "answers": { |
| "decision": { |
| "type": "choice", |
| "choice": "returns", |
| "top_probability": 0.9671907424926758, |
| "probabilities": { |
| "shipping": 0.02828586846590042, |
| "billing": 0.0012693244498223066, |
| "account": 0.0002531065256334841, |
| "returns": 0.9671907424926758, |
| "technical": 0.0030009758193045855 |
| } |
| } |
| }, |
| "usage": { |
| "state_tokens": 30, |
| "branch_tokens": 61, |
| "questions": 1 |
| }, |
| "latency_ms": 55.85542321205139, |
| "form": "cached" |
| } |
| }, |
| { |
| "id": "routing/exchange/it", |
| "pair": "routing/exchange", |
| "domain": "routing", |
| "language": "it", |
| "expected": "returns", |
| "predicted": "returns", |
| "correct": true, |
| "elapsed_ms": 57.992350310087204, |
| "response": { |
| "answers": { |
| "decision": { |
| "type": "choice", |
| "choice": "returns", |
| "top_probability": 0.7482154965400696, |
| "probabilities": { |
| "shipping": 0.21125353872776031, |
| "billing": 0.008127794601023197, |
| "account": 0.0016545143444091082, |
| "returns": 0.7482154965400696, |
| "technical": 0.03074869140982628 |
| } |
| } |
| }, |
| "usage": { |
| "state_tokens": 35, |
| "branch_tokens": 91, |
| "questions": 1 |
| }, |
| "latency_ms": 57.917254976928234, |
| "form": "cached" |
| } |
| }, |
| { |
| "id": "function_selection/document/en", |
| "pair": "function_selection/document", |
| "domain": "function_selection", |
| "language": "en", |
| "expected": "get_invoice", |
| "predicted": "get_invoice", |
| "correct": true, |
| "elapsed_ms": 56.741369888186455, |
| "response": { |
| "answers": { |
| "decision": { |
| "type": "choice", |
| "choice": "get_invoice", |
| "top_probability": 0.9994400143623352, |
| "probabilities": { |
| "get_order_status": 0.0004359894955996424, |
| "get_invoice": 0.9994400143623352, |
| "search_help_center": 4.692937363870442e-05, |
| "list_transactions": 3.328855655126972e-06, |
| "no_tool": 7.371994433924556e-05 |
| } |
| } |
| }, |
| "usage": { |
| "state_tokens": 26, |
| "branch_tokens": 84, |
| "questions": 1 |
| }, |
| "latency_ms": 56.673137471079826, |
| "form": "cached" |
| }, |
| "proposed_call": { |
| "name": "get_invoice", |
| "arguments": { |
| "order_id": "ORD-2048" |
| }, |
| "executed": false, |
| "argument_source": "synthetic application context; not model output" |
| } |
| }, |
| { |
| "id": "function_selection/document/it", |
| "pair": "function_selection/document", |
| "domain": "function_selection", |
| "language": "it", |
| "expected": "get_invoice", |
| "predicted": "get_invoice", |
| "correct": true, |
| "elapsed_ms": 61.38090416789055, |
| "response": { |
| "answers": { |
| "decision": { |
| "type": "choice", |
| "choice": "get_invoice", |
| "top_probability": 0.9809324741363525, |
| "probabilities": { |
| "get_order_status": 0.008516545407474041, |
| "get_invoice": 0.9809324741363525, |
| "search_help_center": 0.005564273335039616, |
| "list_transactions": 0.0001961796369869262, |
| "no_tool": 0.00479046069085598 |
| } |
| } |
| }, |
| "usage": { |
| "state_tokens": 35, |
| "branch_tokens": 130, |
| "questions": 1 |
| }, |
| "latency_ms": 61.311095021665096, |
| "form": "cached" |
| }, |
| "proposed_call": { |
| "name": "get_invoice", |
| "arguments": { |
| "order_id": "ORD-2048" |
| }, |
| "executed": false, |
| "argument_source": "synthetic application context; not model output" |
| } |
| }, |
| { |
| "id": "function_selection/guide/en", |
| "pair": "function_selection/guide", |
| "domain": "function_selection", |
| "language": "en", |
| "expected": "search_help_center", |
| "predicted": "search_help_center", |
| "correct": true, |
| "elapsed_ms": 56.838685646653175, |
| "response": { |
| "answers": { |
| "decision": { |
| "type": "choice", |
| "choice": "search_help_center", |
| "top_probability": 0.9849840998649597, |
| "probabilities": { |
| "get_order_status": 0.00019662412523757666, |
| "get_invoice": 1.293711920880014e-05, |
| "search_help_center": 0.9849840998649597, |
| "list_transactions": 2.8651791694755957e-07, |
| "no_tool": 0.014806032180786133 |
| } |
| } |
| }, |
| "usage": { |
| "state_tokens": 22, |
| "branch_tokens": 84, |
| "questions": 1 |
| }, |
| "latency_ms": 56.77655432373285, |
| "form": "cached" |
| }, |
| "proposed_call": { |
| "name": "search_help_center", |
| "arguments": { |
| "help_query": "PX-7 Wi-Fi setup" |
| }, |
| "executed": false, |
| "argument_source": "synthetic application context; not model output" |
| } |
| }, |
| { |
| "id": "function_selection/guide/it", |
| "pair": "function_selection/guide", |
| "domain": "function_selection", |
| "language": "it", |
| "expected": "search_help_center", |
| "predicted": "search_help_center", |
| "correct": true, |
| "elapsed_ms": 66.24926347285509, |
| "response": { |
| "answers": { |
| "decision": { |
| "type": "choice", |
| "choice": "search_help_center", |
| "top_probability": 0.9986178874969482, |
| "probabilities": { |
| "get_order_status": 3.9409202145179734e-05, |
| "get_invoice": 5.8520322454569396e-06, |
| "search_help_center": 0.9986178874969482, |
| "list_transactions": 1.5702799771588616e-07, |
| "no_tool": 0.0013367385836318135 |
| } |
| } |
| }, |
| "usage": { |
| "state_tokens": 31, |
| "branch_tokens": 130, |
| "questions": 1 |
| }, |
| "latency_ms": 66.18964951485395, |
| "form": "cached" |
| }, |
| "proposed_call": { |
| "name": "search_help_center", |
| "arguments": { |
| "help_query": "PX-7 Wi-Fi setup" |
| }, |
| "executed": false, |
| "argument_source": "synthetic application context; not model output" |
| } |
| }, |
| { |
| "id": "news_topic/earnings/en", |
| "pair": "news_topic/earnings", |
| "domain": "news_topic", |
| "language": "en", |
| "expected": "business", |
| "predicted": "business", |
| "correct": true, |
| "elapsed_ms": 50.876121036708355, |
| "response": { |
| "answers": { |
| "decision": { |
| "type": "choice", |
| "choice": "business", |
| "top_probability": 0.9987465143203735, |
| "probabilities": { |
| "world": 0.0007115165353752673, |
| "sports": 4.0678613004274666e-05, |
| "business": 0.9987465143203735, |
| "science_technology": 0.000501250964589417 |
| } |
| } |
| }, |
| "usage": { |
| "state_tokens": 23, |
| "branch_tokens": 55, |
| "questions": 1 |
| }, |
| "latency_ms": 50.81331077963114, |
| "form": "cached" |
| } |
| }, |
| { |
| "id": "news_topic/earnings/it", |
| "pair": "news_topic/earnings", |
| "domain": "news_topic", |
| "language": "it", |
| "expected": "business", |
| "predicted": "business", |
| "correct": true, |
| "elapsed_ms": 58.704317547380924, |
| "response": { |
| "answers": { |
| "decision": { |
| "type": "choice", |
| "choice": "business", |
| "top_probability": 0.97694993019104, |
| "probabilities": { |
| "world": 0.02171821892261505, |
| "sports": 0.00034243689151480794, |
| "business": 0.97694993019104, |
| "science_technology": 0.0009894066024571657 |
| } |
| } |
| }, |
| "usage": { |
| "state_tokens": 52, |
| "branch_tokens": 80, |
| "questions": 1 |
| }, |
| "latency_ms": 58.64487774670124, |
| "form": "cached" |
| } |
| }, |
| { |
| "id": "news_topic/research/en", |
| "pair": "news_topic/research", |
| "domain": "news_topic", |
| "language": "en", |
| "expected": "science_technology", |
| "predicted": "science_technology", |
| "correct": true, |
| "elapsed_ms": 55.586288683116436, |
| "response": { |
| "answers": { |
| "decision": { |
| "type": "choice", |
| "choice": "science_technology", |
| "top_probability": 0.9960021376609802, |
| "probabilities": { |
| "world": 0.0003835761162918061, |
| "sports": 0.00013404859055299312, |
| "business": 0.003480165498331189, |
| "science_technology": 0.9960021376609802 |
| } |
| } |
| }, |
| "usage": { |
| "state_tokens": 25, |
| "branch_tokens": 55, |
| "questions": 1 |
| }, |
| "latency_ms": 55.52917718887329, |
| "form": "cached" |
| } |
| }, |
| { |
| "id": "news_topic/research/it", |
| "pair": "news_topic/research", |
| "domain": "news_topic", |
| "language": "it", |
| "expected": "science_technology", |
| "predicted": "science_technology", |
| "correct": true, |
| "elapsed_ms": 58.31121001392603, |
| "response": { |
| "answers": { |
| "decision": { |
| "type": "choice", |
| "choice": "science_technology", |
| "top_probability": 0.9942295551300049, |
| "probabilities": { |
| "world": 0.00024186023802030832, |
| "sports": 3.7743277061963454e-05, |
| "business": 0.005490866024047136, |
| "science_technology": 0.9942295551300049 |
| } |
| } |
| }, |
| "usage": { |
| "state_tokens": 46, |
| "branch_tokens": 80, |
| "questions": 1 |
| }, |
| "latency_ms": 58.25440306216478, |
| "form": "cached" |
| } |
| }, |
| { |
| "id": "banking_intent/atm/en", |
| "pair": "banking_intent/atm", |
| "domain": "banking_intent", |
| "language": "en", |
| "expected": "cash_withdrawal", |
| "predicted": "cash_withdrawal", |
| "correct": true, |
| "elapsed_ms": 56.72840401530266, |
| "response": { |
| "answers": { |
| "decision": { |
| "type": "choice", |
| "choice": "cash_withdrawal", |
| "top_probability": 0.9999985694885254, |
| "probabilities": { |
| "cash_withdrawal": 0.9999985694885254, |
| "card_payment": 6.230372378013271e-07, |
| "bank_transfer": 8.770895760790154e-07, |
| "card_delivery": 2.870218729356111e-08, |
| "card_pin": 7.743351737588e-09, |
| "exchange_rate": 1.1956832746129464e-10 |
| } |
| } |
| }, |
| "usage": { |
| "state_tokens": 26, |
| "branch_tokens": 89, |
| "questions": 1 |
| }, |
| "latency_ms": 56.67061824351549, |
| "form": "cached" |
| } |
| }, |
| { |
| "id": "banking_intent/atm/it", |
| "pair": "banking_intent/atm", |
| "domain": "banking_intent", |
| "language": "it", |
| "expected": "cash_withdrawal", |
| "predicted": "cash_withdrawal", |
| "correct": true, |
| "elapsed_ms": 55.428968742489815, |
| "response": { |
| "answers": { |
| "decision": { |
| "type": "choice", |
| "choice": "cash_withdrawal", |
| "top_probability": 0.9999879598617554, |
| "probabilities": { |
| "cash_withdrawal": 0.9999879598617554, |
| "card_payment": 6.86843532093917e-06, |
| "bank_transfer": 5.0999055929423776e-06, |
| "card_delivery": 8.232988513157125e-09, |
| "card_pin": 2.9984728033838337e-09, |
| "exchange_rate": 7.62557031408484e-11 |
| } |
| } |
| }, |
| "usage": { |
| "state_tokens": 39, |
| "branch_tokens": 119, |
| "questions": 1 |
| }, |
| "latency_ms": 55.37245608866215, |
| "form": "cached" |
| } |
| }, |
| { |
| "id": "banking_intent/transfer/en", |
| "pair": "banking_intent/transfer", |
| "domain": "banking_intent", |
| "language": "en", |
| "expected": "bank_transfer", |
| "predicted": "bank_transfer", |
| "correct": true, |
| "elapsed_ms": 54.77554630488157, |
| "response": { |
| "answers": { |
| "decision": { |
| "type": "choice", |
| "choice": "bank_transfer", |
| "top_probability": 0.9998921155929565, |
| "probabilities": { |
| "cash_withdrawal": 0.00010535794717725366, |
| "card_payment": 6.971608854655642e-07, |
| "bank_transfer": 0.9998921155929565, |
| "card_delivery": 1.7839795418694848e-06, |
| "card_pin": 1.6865783436514903e-08, |
| "exchange_rate": 8.67874327958873e-10 |
| } |
| } |
| }, |
| "usage": { |
| "state_tokens": 27, |
| "branch_tokens": 89, |
| "questions": 1 |
| }, |
| "latency_ms": 54.718634113669395, |
| "form": "cached" |
| } |
| }, |
| { |
| "id": "banking_intent/transfer/it", |
| "pair": "banking_intent/transfer", |
| "domain": "banking_intent", |
| "language": "it", |
| "expected": "bank_transfer", |
| "predicted": "bank_transfer", |
| "correct": true, |
| "elapsed_ms": 58.42987261712551, |
| "response": { |
| "answers": { |
| "decision": { |
| "type": "choice", |
| "choice": "bank_transfer", |
| "top_probability": 0.9999046325683594, |
| "probabilities": { |
| "cash_withdrawal": 7.792055839672685e-05, |
| "card_payment": 9.46886939345859e-06, |
| "bank_transfer": 0.9999046325683594, |
| "card_delivery": 7.872226888139267e-06, |
| "card_pin": 2.724057956982051e-08, |
| "exchange_rate": 4.5937365023007715e-09 |
| } |
| } |
| }, |
| "usage": { |
| "state_tokens": 40, |
| "branch_tokens": 119, |
| "questions": 1 |
| }, |
| "latency_ms": 58.37359745055437, |
| "form": "cached" |
| } |
| } |
| ], |
| "summary": { |
| "banking_intent": { |
| "en": { |
| "correct": 2, |
| "n": 2 |
| }, |
| "it": { |
| "correct": 2, |
| "n": 2 |
| }, |
| "successful_bilingual_pairs": [ |
| "banking_intent/atm", |
| "banking_intent/transfer" |
| ] |
| }, |
| "function_selection": { |
| "en": { |
| "correct": 2, |
| "n": 2 |
| }, |
| "it": { |
| "correct": 2, |
| "n": 2 |
| }, |
| "successful_bilingual_pairs": [ |
| "function_selection/document", |
| "function_selection/guide" |
| ] |
| }, |
| "news_topic": { |
| "en": { |
| "correct": 2, |
| "n": 2 |
| }, |
| "it": { |
| "correct": 2, |
| "n": 2 |
| }, |
| "successful_bilingual_pairs": [ |
| "news_topic/earnings", |
| "news_topic/research" |
| ] |
| }, |
| "routing": { |
| "en": { |
| "correct": 2, |
| "n": 2 |
| }, |
| "it": { |
| "correct": 2, |
| "n": 2 |
| }, |
| "successful_bilingual_pairs": [ |
| "routing/exchange", |
| "routing/invoice" |
| ] |
| } |
| } |
| }, |
| "v3": { |
| "cases": [ |
| { |
| "id": "routing/invoice/en", |
| "pair": "routing/invoice", |
| "domain": "routing", |
| "language": "en", |
| "expected": "billing", |
| "predicted": "billing", |
| "correct": true, |
| "elapsed_ms": 56.05360306799412, |
| "response": { |
| "answers": { |
| "decision": { |
| "type": "choice", |
| "choice": "billing", |
| "top_probability": 0.9985558390617371, |
| "probabilities": { |
| "shipping": 0.0008598362328484654, |
| "billing": 0.9985558390617371, |
| "account": 2.9924891350674443e-05, |
| "returns": 9.053786925505847e-05, |
| "technical": 0.00046377076068893075 |
| } |
| } |
| }, |
| "usage": { |
| "state_tokens": 26, |
| "branch_tokens": 61, |
| "questions": 1 |
| }, |
| "latency_ms": 55.988483130931854, |
| "form": "cached" |
| } |
| }, |
| { |
| "id": "routing/invoice/it", |
| "pair": "routing/invoice", |
| "domain": "routing", |
| "language": "it", |
| "expected": "billing", |
| "predicted": "billing", |
| "correct": true, |
| "elapsed_ms": 58.90964437276125, |
| "response": { |
| "answers": { |
| "decision": { |
| "type": "choice", |
| "choice": "billing", |
| "top_probability": 0.9996041655540466, |
| "probabilities": { |
| "shipping": 0.00035056372871622443, |
| "billing": 0.9996041655540466, |
| "account": 2.1824813302373514e-05, |
| "returns": 1.1230722520849667e-05, |
| "technical": 1.2304177289479412e-05 |
| } |
| } |
| }, |
| "usage": { |
| "state_tokens": 43, |
| "branch_tokens": 91, |
| "questions": 1 |
| }, |
| "latency_ms": 58.847760781645775, |
| "form": "cached" |
| } |
| }, |
| { |
| "id": "routing/exchange/en", |
| "pair": "routing/exchange", |
| "domain": "routing", |
| "language": "en", |
| "expected": "returns", |
| "predicted": "returns", |
| "correct": true, |
| "elapsed_ms": 57.27338790893555, |
| "response": { |
| "answers": { |
| "decision": { |
| "type": "choice", |
| "choice": "returns", |
| "top_probability": 0.9819230437278748, |
| "probabilities": { |
| "shipping": 0.011836539953947067, |
| "billing": 0.0002842044341377914, |
| "account": 0.00011940801778109744, |
| "returns": 0.9819230437278748, |
| "technical": 0.005836740601807833 |
| } |
| } |
| }, |
| "usage": { |
| "state_tokens": 30, |
| "branch_tokens": 61, |
| "questions": 1 |
| }, |
| "latency_ms": 57.21500888466835, |
| "form": "cached" |
| } |
| }, |
| { |
| "id": "routing/exchange/it", |
| "pair": "routing/exchange", |
| "domain": "routing", |
| "language": "it", |
| "expected": "returns", |
| "predicted": "returns", |
| "correct": true, |
| "elapsed_ms": 58.14587324857712, |
| "response": { |
| "answers": { |
| "decision": { |
| "type": "choice", |
| "choice": "returns", |
| "top_probability": 0.7278805375099182, |
| "probabilities": { |
| "shipping": 0.19424377381801605, |
| "billing": 0.003338765585795045, |
| "account": 0.0008692944538779557, |
| "returns": 0.7278805375099182, |
| "technical": 0.07366769015789032 |
| } |
| } |
| }, |
| "usage": { |
| "state_tokens": 35, |
| "branch_tokens": 91, |
| "questions": 1 |
| }, |
| "latency_ms": 58.08808282017708, |
| "form": "cached" |
| } |
| }, |
| { |
| "id": "function_selection/document/en", |
| "pair": "function_selection/document", |
| "domain": "function_selection", |
| "language": "en", |
| "expected": "get_invoice", |
| "predicted": "get_invoice", |
| "correct": true, |
| "elapsed_ms": 56.49128742516041, |
| "response": { |
| "answers": { |
| "decision": { |
| "type": "choice", |
| "choice": "get_invoice", |
| "top_probability": 0.9980605244636536, |
| "probabilities": { |
| "get_order_status": 0.0014994319062680006, |
| "get_invoice": 0.9980605244636536, |
| "search_help_center": 0.00027331800083629787, |
| "list_transactions": 1.5344045095844194e-05, |
| "no_tool": 0.00015137618174776435 |
| } |
| } |
| }, |
| "usage": { |
| "state_tokens": 26, |
| "branch_tokens": 84, |
| "questions": 1 |
| }, |
| "latency_ms": 56.43379129469395, |
| "form": "cached" |
| }, |
| "proposed_call": { |
| "name": "get_invoice", |
| "arguments": { |
| "order_id": "ORD-2048" |
| }, |
| "executed": false, |
| "argument_source": "synthetic application context; not model output" |
| } |
| }, |
| { |
| "id": "function_selection/document/it", |
| "pair": "function_selection/document", |
| "domain": "function_selection", |
| "language": "it", |
| "expected": "get_invoice", |
| "predicted": "get_invoice", |
| "correct": true, |
| "elapsed_ms": 61.52856070548296, |
| "response": { |
| "answers": { |
| "decision": { |
| "type": "choice", |
| "choice": "get_invoice", |
| "top_probability": 0.9552146196365356, |
| "probabilities": { |
| "get_order_status": 0.019960561767220497, |
| "get_invoice": 0.9552146196365356, |
| "search_help_center": 0.01574959233403206, |
| "list_transactions": 0.000660978548694402, |
| "no_tool": 0.008414213545620441 |
| } |
| } |
| }, |
| "usage": { |
| "state_tokens": 35, |
| "branch_tokens": 130, |
| "questions": 1 |
| }, |
| "latency_ms": 61.4694906398654, |
| "form": "cached" |
| }, |
| "proposed_call": { |
| "name": "get_invoice", |
| "arguments": { |
| "order_id": "ORD-2048" |
| }, |
| "executed": false, |
| "argument_source": "synthetic application context; not model output" |
| } |
| }, |
| { |
| "id": "function_selection/guide/en", |
| "pair": "function_selection/guide", |
| "domain": "function_selection", |
| "language": "en", |
| "expected": "search_help_center", |
| "predicted": "search_help_center", |
| "correct": true, |
| "elapsed_ms": 57.06825014203787, |
| "response": { |
| "answers": { |
| "decision": { |
| "type": "choice", |
| "choice": "search_help_center", |
| "top_probability": 0.975777268409729, |
| "probabilities": { |
| "get_order_status": 0.00015870117931626737, |
| "get_invoice": 1.5348514352808706e-05, |
| "search_help_center": 0.975777268409729, |
| "list_transactions": 5.755619554292934e-07, |
| "no_tool": 0.0240480899810791 |
| } |
| } |
| }, |
| "usage": { |
| "state_tokens": 22, |
| "branch_tokens": 84, |
| "questions": 1 |
| }, |
| "latency_ms": 57.01077729463577, |
| "form": "cached" |
| }, |
| "proposed_call": { |
| "name": "search_help_center", |
| "arguments": { |
| "help_query": "PX-7 Wi-Fi setup" |
| }, |
| "executed": false, |
| "argument_source": "synthetic application context; not model output" |
| } |
| }, |
| { |
| "id": "function_selection/guide/it", |
| "pair": "function_selection/guide", |
| "domain": "function_selection", |
| "language": "it", |
| "expected": "search_help_center", |
| "predicted": "search_help_center", |
| "correct": true, |
| "elapsed_ms": 60.271876864135265, |
| "response": { |
| "answers": { |
| "decision": { |
| "type": "choice", |
| "choice": "search_help_center", |
| "top_probability": 0.9975995421409607, |
| "probabilities": { |
| "get_order_status": 1.6757219782448374e-05, |
| "get_invoice": 3.7196009543549735e-06, |
| "search_help_center": 0.9975995421409607, |
| "list_transactions": 1.5623901106209814e-07, |
| "no_tool": 0.00237982627004385 |
| } |
| } |
| }, |
| "usage": { |
| "state_tokens": 31, |
| "branch_tokens": 130, |
| "questions": 1 |
| }, |
| "latency_ms": 60.214655473828316, |
| "form": "cached" |
| }, |
| "proposed_call": { |
| "name": "search_help_center", |
| "arguments": { |
| "help_query": "PX-7 Wi-Fi setup" |
| }, |
| "executed": false, |
| "argument_source": "synthetic application context; not model output" |
| } |
| }, |
| { |
| "id": "news_topic/earnings/en", |
| "pair": "news_topic/earnings", |
| "domain": "news_topic", |
| "language": "en", |
| "expected": "business", |
| "predicted": "business", |
| "correct": true, |
| "elapsed_ms": 55.95214292407036, |
| "response": { |
| "answers": { |
| "decision": { |
| "type": "choice", |
| "choice": "business", |
| "top_probability": 0.9986342787742615, |
| "probabilities": { |
| "world": 0.0006480682059191167, |
| "sports": 2.9242844902910292e-05, |
| "business": 0.9986342787742615, |
| "science_technology": 0.0006883948226459324 |
| } |
| } |
| }, |
| "usage": { |
| "state_tokens": 23, |
| "branch_tokens": 55, |
| "questions": 1 |
| }, |
| "latency_ms": 55.894918739795685, |
| "form": "cached" |
| } |
| }, |
| { |
| "id": "news_topic/earnings/it", |
| "pair": "news_topic/earnings", |
| "domain": "news_topic", |
| "language": "it", |
| "expected": "business", |
| "predicted": "business", |
| "correct": true, |
| "elapsed_ms": 59.04611013829708, |
| "response": { |
| "answers": { |
| "decision": { |
| "type": "choice", |
| "choice": "business", |
| "top_probability": 0.9882785081863403, |
| "probabilities": { |
| "world": 0.010226842947304249, |
| "sports": 0.00016904894437175244, |
| "business": 0.9882785081863403, |
| "science_technology": 0.0013256433885544538 |
| } |
| } |
| }, |
| "usage": { |
| "state_tokens": 52, |
| "branch_tokens": 80, |
| "questions": 1 |
| }, |
| "latency_ms": 58.98899771273136, |
| "form": "cached" |
| } |
| }, |
| { |
| "id": "news_topic/research/en", |
| "pair": "news_topic/research", |
| "domain": "news_topic", |
| "language": "en", |
| "expected": "science_technology", |
| "predicted": "science_technology", |
| "correct": true, |
| "elapsed_ms": 61.07739731669426, |
| "response": { |
| "answers": { |
| "decision": { |
| "type": "choice", |
| "choice": "science_technology", |
| "top_probability": 0.9968498349189758, |
| "probabilities": { |
| "world": 0.00022324999736156315, |
| "sports": 0.00011827343405457214, |
| "business": 0.002808679360896349, |
| "science_technology": 0.9968498349189758 |
| } |
| } |
| }, |
| "usage": { |
| "state_tokens": 25, |
| "branch_tokens": 55, |
| "questions": 1 |
| }, |
| "latency_ms": 61.00280303508043, |
| "form": "cached" |
| } |
| }, |
| { |
| "id": "news_topic/research/it", |
| "pair": "news_topic/research", |
| "domain": "news_topic", |
| "language": "it", |
| "expected": "science_technology", |
| "predicted": "science_technology", |
| "correct": true, |
| "elapsed_ms": 53.32445818930864, |
| "response": { |
| "answers": { |
| "decision": { |
| "type": "choice", |
| "choice": "science_technology", |
| "top_probability": 0.99574214220047, |
| "probabilities": { |
| "world": 0.00017849514551926404, |
| "sports": 2.3704824343440123e-05, |
| "business": 0.004055578261613846, |
| "science_technology": 0.99574214220047 |
| } |
| } |
| }, |
| "usage": { |
| "state_tokens": 46, |
| "branch_tokens": 80, |
| "questions": 1 |
| }, |
| "latency_ms": 53.264837712049484, |
| "form": "cached" |
| } |
| }, |
| { |
| "id": "banking_intent/atm/en", |
| "pair": "banking_intent/atm", |
| "domain": "banking_intent", |
| "language": "en", |
| "expected": "cash_withdrawal", |
| "predicted": "cash_withdrawal", |
| "correct": true, |
| "elapsed_ms": 56.83506093919277, |
| "response": { |
| "answers": { |
| "decision": { |
| "type": "choice", |
| "choice": "cash_withdrawal", |
| "top_probability": 0.9999995231628418, |
| "probabilities": { |
| "cash_withdrawal": 0.9999995231628418, |
| "card_payment": 3.635780103650177e-07, |
| "bank_transfer": 1.2333165955169534e-07, |
| "card_delivery": 2.776579854923966e-09, |
| "card_pin": 2.3044925967496965e-09, |
| "exchange_rate": 1.999729375146142e-11 |
| } |
| } |
| }, |
| "usage": { |
| "state_tokens": 26, |
| "branch_tokens": 89, |
| "questions": 1 |
| }, |
| "latency_ms": 56.77527189254761, |
| "form": "cached" |
| } |
| }, |
| { |
| "id": "banking_intent/atm/it", |
| "pair": "banking_intent/atm", |
| "domain": "banking_intent", |
| "language": "it", |
| "expected": "cash_withdrawal", |
| "predicted": "cash_withdrawal", |
| "correct": true, |
| "elapsed_ms": 61.042129062116146, |
| "response": { |
| "answers": { |
| "decision": { |
| "type": "choice", |
| "choice": "cash_withdrawal", |
| "top_probability": 0.9999978542327881, |
| "probabilities": { |
| "cash_withdrawal": 0.9999978542327881, |
| "card_payment": 9.771449640538776e-07, |
| "bank_transfer": 1.2317531172811869e-06, |
| "card_delivery": 8.043636312571323e-10, |
| "card_pin": 7.989157668752966e-10, |
| "exchange_rate": 1.0329305986933601e-11 |
| } |
| } |
| }, |
| "usage": { |
| "state_tokens": 39, |
| "branch_tokens": 119, |
| "questions": 1 |
| }, |
| "latency_ms": 60.9825374558568, |
| "form": "cached" |
| } |
| }, |
| { |
| "id": "banking_intent/transfer/en", |
| "pair": "banking_intent/transfer", |
| "domain": "banking_intent", |
| "language": "en", |
| "expected": "bank_transfer", |
| "predicted": "bank_transfer", |
| "correct": true, |
| "elapsed_ms": 65.01384731382132, |
| "response": { |
| "answers": { |
| "decision": { |
| "type": "choice", |
| "choice": "bank_transfer", |
| "top_probability": 0.9999748468399048, |
| "probabilities": { |
| "cash_withdrawal": 2.3217038688017055e-05, |
| "card_payment": 1.4152047924653743e-06, |
| "bank_transfer": 0.9999748468399048, |
| "card_delivery": 5.215157443672069e-07, |
| "card_pin": 1.2648628811007256e-08, |
| "exchange_rate": 2.70978267602473e-10 |
| } |
| } |
| }, |
| "usage": { |
| "state_tokens": 27, |
| "branch_tokens": 89, |
| "questions": 1 |
| }, |
| "latency_ms": 64.92940057069063, |
| "form": "cached" |
| } |
| }, |
| { |
| "id": "banking_intent/transfer/it", |
| "pair": "banking_intent/transfer", |
| "domain": "banking_intent", |
| "language": "it", |
| "expected": "bank_transfer", |
| "predicted": "bank_transfer", |
| "correct": true, |
| "elapsed_ms": 60.956964269280434, |
| "response": { |
| "answers": { |
| "decision": { |
| "type": "choice", |
| "choice": "bank_transfer", |
| "top_probability": 0.9999954700469971, |
| "probabilities": { |
| "cash_withdrawal": 2.807738155752304e-06, |
| "card_payment": 1.276975581276929e-06, |
| "bank_transfer": 0.9999954700469971, |
| "card_delivery": 3.7989903489688004e-07, |
| "card_pin": 2.268327081722532e-09, |
| "exchange_rate": 1.7042040001413739e-10 |
| } |
| } |
| }, |
| "usage": { |
| "state_tokens": 40, |
| "branch_tokens": 119, |
| "questions": 1 |
| }, |
| "latency_ms": 60.88253017514944, |
| "form": "cached" |
| } |
| } |
| ], |
| "summary": { |
| "banking_intent": { |
| "en": { |
| "correct": 2, |
| "n": 2 |
| }, |
| "it": { |
| "correct": 2, |
| "n": 2 |
| }, |
| "successful_bilingual_pairs": [ |
| "banking_intent/atm", |
| "banking_intent/transfer" |
| ] |
| }, |
| "function_selection": { |
| "en": { |
| "correct": 2, |
| "n": 2 |
| }, |
| "it": { |
| "correct": 2, |
| "n": 2 |
| }, |
| "successful_bilingual_pairs": [ |
| "function_selection/document", |
| "function_selection/guide" |
| ] |
| }, |
| "news_topic": { |
| "en": { |
| "correct": 2, |
| "n": 2 |
| }, |
| "it": { |
| "correct": 2, |
| "n": 2 |
| }, |
| "successful_bilingual_pairs": [ |
| "news_topic/earnings", |
| "news_topic/research" |
| ] |
| }, |
| "routing": { |
| "en": { |
| "correct": 2, |
| "n": 2 |
| }, |
| "it": { |
| "correct": 2, |
| "n": 2 |
| }, |
| "successful_bilingual_pairs": [ |
| "routing/exchange", |
| "routing/invoice" |
| ] |
| } |
| } |
| } |
| }, |
| "status": "complete" |
| } |
|
|
