Spaces:
Running
Running
Download index.html from c41/laya-evaluation: direct link, hf CLI and curl.
- Browser
- Download file 69.6 kB
-
https://huggingface.co/spaces/c41/laya-evaluation/resolve/main/index.html
- Command line
-
hf download hf://spaces/c41/laya-evaluation/index.html
-
curl -L -o index.html https://huggingface.co/spaces/c41/laya-evaluation/resolve/main/index.html
69.6 kB
| <html lang="en"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width,initial-scale=1"><title>deepopen | A Laya-based Intent Classification Framework</title><style> | |
| :root{--ink:#142c37;--muted:#647981;--line:#dce5e5;--teal:#007b76;--bg:#f3f6f5}*{box-sizing:border-box}body{margin:0;background:var(--bg);font:15px/1.6 Inter,"Segoe UI","Microsoft YaHei",sans-serif;color:var(--ink)}header{padding:22px 5%;border-bottom:1px solid var(--line);display:flex;align-items:center;justify-content:space-between;background:white}.brand{font-weight:800;font-size:25px;letter-spacing:-1px}.brand span{color:var(--teal)}nav a{margin-left:24px;color:var(--muted);text-decoration:none;font-size:13px}main{max-width:1280px;margin:auto;padding:48px 30px}small,.eyebrow{letter-spacing:2px;font-size:11px;font-weight:700;color:var(--teal)}h1{font-size:46px;line-height:1.2;margin:14px 0;letter-spacing:-2px}h2{font-size:24px;margin:0}h3{font-size:16px;margin:0}.sub{color:var(--muted);max-width:750px}.status{display:inline-block;background:#e2efeb;color:#32665e;padding:5px 11px;border-radius:20px;font-size:12px}.hero{display:flex;justify-content:space-between;align-items:center;margin-bottom:30px}.hero-right{text-align:right;font-size:12px;color:var(--muted)}.cards{display:grid;grid-template-columns:repeat(2,1fr);gap:16px}.card{background:white;border:1px solid var(--line);border-radius:14px;padding:23px}.card.featured{background:#103e43;color:white;border-color:#103e43}.value{font-size:38px;font-weight:750;letter-spacing:-1.7px;margin:14px 0 4px}.value em{font-size:18px;font-style:normal;letter-spacing:0}.caption{font-size:12px;color:var(--muted)}.featured .caption{color:#b9d5d1}.rule{margin:16px 0 36px;font-size:12px;color:var(--muted)}section{margin-top:38px}.heading{display:flex;justify-content:space-between;gap:20px;align-items:center;margin-bottom:18px}.panel{background:white;border:1px solid var(--line);border-radius:14px;padding:25px}.tabs{display:flex;gap:7px;flex-wrap:wrap}button,select,input{font:inherit;border:1px solid var(--line);border-radius:8px;padding:9px 13px;background:white;color:var(--ink)}button{cursor:pointer}button.active{background:var(--ink);color:white;border-color:var(--ink)}.split{display:grid;grid-template-columns:1.4fr 1fr;gap:24px}.barrow{display:grid;grid-template-columns:190px 1fr 84px;align-items:center;gap:12px;margin:17px 0;font-size:13px}.track{height:10px;border-radius:5px;background:#edf1f2;overflow:hidden}.fill{height:100%;background:#abbcc4;border-radius:5px}.ours .fill{background:var(--teal)}.ours{font-weight:700;color:var(--teal)}.barrow strong{text-align:right}.note{font-size:12px;color:var(--muted);margin-top:15px}.callout{background:#f3f7f6;border-radius:10px;padding:20px}.callout p{font-size:13px;color:var(--muted)}a{color:var(--teal);text-underline-offset:3px}.toolbar{display:flex;gap:10px;flex-wrap:wrap;margin:16px 0}.toolbar input{flex:1;min-width:160px}.tablewrap{max-height:390px;overflow:auto}table{width:100%;border-collapse:collapse;font-size:13px}th{text-align:left;font-size:11px;color:var(--muted);position:sticky;top:0;background:#f5f8f7;padding:12px}td{border-bottom:1px solid #edf1f1;padding:11px 12px}td.num{text-align:right;font-variant-numeric:tabular-nums}.mini{height:5px;background:#e7efed;width:80px;display:inline-block;margin-right:10px;vertical-align:middle}.mini i{display:block;height:5px;background:var(--teal)}.downloads{display:flex;gap:12px;flex-wrap:wrap;margin-top:20px}.downloads a{font-size:13px;border:1px solid var(--line);padding:8px 13px;border-radius:7px;text-decoration:none}.footer{font-size:12px;color:var(--muted);border-top:1px solid var(--line);padding-top:24px;margin-top:40px}.footgrid{display:grid;grid-template-columns:1fr 1fr;gap:28px}.pill{font-size:11px;background:#edf3f1;border-radius:5px;padding:3px 7px}details{font-size:13px;margin-top:15px}summary{cursor:pointer} .bigstat{font-size:27px;font-weight:700;color:var(--teal);margin:10px 0} | |
| @media(max-width:850px){.cards{grid-template-columns:1fr 1fr}.split,.footgrid{grid-template-columns:1fr}.hero{display:block}.hero-right{text-align:left;margin-top:18px}.heading{align-items:flex-start;flex-direction:column}h1{font-size:34px}main{padding:25px 18px}.barrow{grid-template-columns:145px 1fr 70px}nav{display:none}}@media print{header,button,.toolbar{display:none}main{padding:0}.tablewrap{max-height:none}.panel,.card{break-inside:avoid}body{background:white}} | |
| .flow{display:flex;align-items:center;gap:12px;padding:22px 0;flex-wrap:wrap}.flow>div{background:#edf5f3;border:1px solid #d0e2dd;padding:15px;border-radius:8px;text-align:center;font-size:13px;flex:1;min-width:130px}.flow>b{color:#007b76}.flow .branches{min-width:245px}.branches>div+div{border-top:1px solid #bcd4ce;margin-top:12px;padding-top:12px}@media(max-width:650px){.flow{flex-direction:column}.flow>div{width:100%}.flow>b{transform:rotate(90deg)}}</style></head><body> | |
| <header><div class="brand">deepopen<span> / </span>Benchmarks</div><nav><a href="#comparison">Results</a><a href="#architecture">Method</a><a href="#results">Predictions</a><a href="#evidence">Reproducibility</a><a href="#model-releases">Models</a></nav><span class="status">Author-reported results</span></header> | |
| <main><div class="hero"><div><div class="eyebrow">INTENT CLASSIFICATION</div><h1>deepopen<br>A Laya-based Intent Classification Framework</h1><p class="sub">Benchmark results, dataset-specific configurations, and reproducible prediction evidence for the deepopen framework.</p></div><div class="hero-right"><span class="status">SOTA2 submission pending</span><p>Reference snapshot: 7 October 2026<br>Maintained by the method authors</p></div></div> | |
| <div class="cards"><div class="card"><small>BANKING77</small><div class="value">94.1234<em>%</em></div><h3>Single classifier / SAM seed 13</h3><p class="caption">2,899 / 3,080 correct · Best observed of 3 SAM seeds</p></div><div class="card featured"><small style="color:#89d6bd">CLINC150 / ENSEMBLE</small><div class="value">98.0222<em>%</em></div><h3>Four-encoder ensemble</h3><p class="caption">4,411 / 4,500 correct · 150 classes</p></div></div> | |
| <p class="rule">Metric: test accuracy. CLINC150 covers in-scope intents only.</p> | |
| <section id="comparison"><div class="heading"><h2>Benchmark comparison</h2><div class="tabs"><button class="active" data-task="bank">BANKING77</button><button data-task="clinc">CLINC150</button></div></div><div class="panel"><div id="charttitle" class="caption"></div><div class="tablewrap"><table><thead><tr><th>Method</th><th>Test accuracy</th><th>Result source</th></tr></thead><tbody id="bars"></tbody></table></div><div id="compareNote" class="note"></div></div></section> | |
| <section><div class="heading"><h2>Evaluation protocol</h2></div><div class="panel"><table><thead><tr><th>Dataset</th><th>Evaluation scope</th><th>Test examples</th><th>Aggregation</th></tr></thead><tbody><tr><td>BANKING77</td><td>77-way intent classification</td><td>3,080</td><td>SAM seed 13; best observed of three</td></tr><tr><td>CLINC150</td><td>150 in-scope intents; no OOS rejection</td><td>4,500</td><td>Four-encoder ensemble</td></tr></tbody></table><p class="note">Historical BANKING77 training used the full training set, including a previously used development subset. That subset is not an independent validation set for that model. The existing test results have been observed; the new continuation results are retrospective test comparisons, not fresh independent confirmation. No further tuning follows these test results.</p></div></section> | |
| <section id="banking-update"><div class="heading"><h2>BANKING77 continuation results</h2><a href="banking_continuation_summary.json" download>Full evaluation summary</a></div><div class="panel"><p>The headline is the best observed single-model result across three fixed SAM seeds, selected for display after test evaluation. All candidates started from the same historical full-train checkpoint.</p><table><thead><tr><th>Candidate</th><th>Correct / 3,080</th><th>Test accuracy</th></tr></thead><tbody><tr><td>SAM seed 13</td><td>2,899</td><td>94.1234%</td></tr><tr><td>SAM seed 42</td><td>2,896</td><td>94.0260%</td></tr><tr><td>SAM seed 87</td><td>2,894</td><td>93.9610%</td></tr><tr><td>Three-seed mean</td><td>--</td><td>94.0368%</td></tr></tbody></table><p class="note">Uniform evaluation: eager FP32, SDPA, batch 32, maximum 128 tokens. The historical checkpoint scores 93.9935% under this path. SAM mean gain: +0.0433 percentage points (shared-row 95% CI: -0.2273 to +0.3247); no significant improvement is established. The best-seed score is subject to test-selection bias.</p><p class="note">Archived historical accuracy: 94.0909%. Current replay of the original BF16 script: 94.0584%. Exact historical prediction reproduction remains unresolved despite matching data and weight hashes. Robust improvement across execution settings is not established.</p><p><a href="banking_execution_audit.json" download>Execution audit</a> / <a href="banking_continuation_protocol.json" download>Training and evaluation configuration</a></p></div></section><section id="architecture"><div class="heading"><h2>deepopen framework</h2><span class="pill">One framework · dataset-specific configurations</span></div><div class="panel"><h3>A Laya-based Framework for Intent Classification</h3><p>deepopen organizes intent classification around a common processing pipeline. Input construction, prediction heads, training objectives, and optional fusion are configured for each dataset.</p><div class="flow" role="img" aria-label="The deepopen framework: input text, task-specific input construction, encoding and prediction, optional probability fusion, and final intent output."><div>Input text</div><b>→</b><div>Task-specific<br>input construction</div><b>→</b><div>Encoding<br>& prediction</div><b>→</b><div>Probability fusion<br><strong>optional</strong></div><b>→</b><div>Intent output</div></div><div class="tablewrap"><table><thead><tr><th>Dataset</th><th>Input & prediction configuration</th><th>Training / fusion configuration</th></tr></thead><tbody><tr><td>BANKING77</td><td>Text tokenization; one Laya-based English encoder and a 77-class classification head.</td><td>Full-train historical classifier + two fixed epochs of SAM (rho 0.05). Seed 13 selected for display after evaluating three continuation seeds; no ensemble.</td></tr><tr><td>CLINC150</td><td>Member-specific tokenization; one Laya-based specialist and three DeBERTa-v3-large classifiers; 150 intents.</td><td>R-Drop-trained specialist (seed 42); probability fusion: 1/2 specialist + 1/6 each DeBERTa member (seeds 13/42/87). Four encoders at inference.</td></tr></tbody></table></div><p class="note">The shared diagram describes the framework workflow. Dataset configurations differ in architecture and training strategy, not only numerical hyperparameters. Reported scores are tied to the configurations above, rather than one universal checkpoint.</p></div></section><section id="configuration"><div class="heading"><h2>Training & execution environment</h2><a href="reproducibility_config.json" download>Download verified configuration (JSON)</a></div><div class="panel"><p class="note">Historical BANKING77 and CLINC150 GPU allocation, CUDA versions, and full training hyperparameters are not established by the release-loading record. They are marked unverified here rather than replaced with current-machine values. No throughput or latency claim is made.</p><h3>Verified historical-package loading environment</h3><div class="tablewrap"><table><thead><tr><th>Component</th><th>Version / setting</th></tr></thead><tbody><tr><td>Python</td><td>3.12.14; isolated environment; no inherited system packages</td></tr><tr><td>PyTorch / Transformers</td><td>2.9.1 / 4.57.6</td></tr><tr><td>NumPy / tokenizers / safetensors</td><td>2.5.3 / 0.23.0rc0 / 0.9.0rc0</td></tr><tr><td>Loading validation</td><td>Dependency check passed; manifest hashes verified; two synthetic inputs matched prior outputs.</td></tr></tbody></table></div><p class="note">This is the clean release-loading environment, not a claim about the original training environment or a reproduction of test accuracy. Additional recorded dependencies are in the downloadable JSON.</p></div></section><section id="results"><div class="heading"><div><small>PREDICTION EVIDENCE</small><h2>Per-class results</h2></div><select id="dataset" aria-label="Select evaluated system"><option value="banking">BANKING77 · single</option><option value="clinc_ensemble">CLINC150 · ensemble</option></select></div><div class="panel"><div id="resultstat"></div><div class="toolbar"><input id="search" aria-label="Search class labels" placeholder="Search labels: card, transfer, alarm..."><select id="sort" aria-label="Sort classes"><option value="accuracy">Lowest recall first</option><option value="label">Class name</option></select></div><div class="tablewrap"><table><thead><tr><th>Gold class</th><th>Correct / support</th><th>Recall</th><th>Precision</th><th>F1</th></tr></thead><tbody id="classes"></tbody></table></div><div class="downloads"><a id="predDownload" download>Per-example predictions (CSV)</a><a id="classDownload" download>Per-class metrics (CSV)</a><a href="evidence_manifest.json" download>Evidence manifest & SHA-256</a></div><p class="note">Historical prediction rows preserve artifact order, not official dataset IDs. Test utterances are not redistributed. Metrics are recounted from saved predictions; BANKING77 now uses the fixed-checkpoint retrospective test predictions of SAM seed 13.</p><details><summary>Most frequent classification errors</summary><div id="confusions"></div></details></div></section> | |
| <section id="evidence"><div class="heading"><h2>Reproducibility & limitations</h2></div><div class="panel footgrid"><div><h3>Artifacts</h3><p>Prediction files and class metrics are available above. BANKING77 and CLINC150 model releases are public and verified at the linked revisions below, including file hashes and complete frozen-test predictions. This delivery statement covers those three model packages only.</p></div><div><h3>External comparison</h3><p>Reference results are attributed to SOTA2 and have not been rerun by us. Training resources, model sizes, and protocols may differ. Selected references are not an exhaustive leaderboard.</p><p>This is an author-maintained submission resource, not an official ranking or a claim of SOTA2 acceptance.</p></div></div></section> | |
| <!-- banking-clinc-verified-releases:start --> | |
| <section id="model-releases"> | |
| <div class="heading"><h2>Download verified models</h2><span class="pill">BANKING77 & CLINC150</span></div> | |
| <div class="panel"><p>Public model weights, inference code, dependency locks, licenses, training-source archives, and per-example evaluation evidence. Each downloaded model was checked against file hashes and its documented prediction reference. CLINC150 single reproduces the historical total score with two disclosed prediction differences; both historical and current-reference predictions are preserved.</p> | |
| <div class="tablewrap"><table><thead><tr><th>Release</th><th>Frozen test accuracy</th><th>Downloads</th><th>Prediction comparison</th></tr></thead><tbody><tr><td>BANKING77 / SAM seed 13</td><td>94.1234% (2899/3080)</td><td><a href="https://huggingface.co/c41/deepopen-banking77-sam13-v1" target="_blank" rel="noopener">Model & inference</a> · <a href="https://huggingface.co/c41/deepopen-banking77-sam13-v1/tree/67b50aea2f7200eb93800f0657037fcd44a68b96" target="_blank" rel="noopener">Frozen revision</a></td><td>0 historical mismatches</td></tr><tr><td>CLINC150 / single encoder</td><td>97.7556% (4399/4500)</td><td><a href="https://huggingface.co/c41/deepopen-clinc150-single-v1" target="_blank" rel="noopener">Model & inference</a> · <a href="https://huggingface.co/c41/deepopen-clinc150-single-v1/tree/801a0c38951db01a64c0834a92918d0fee1a258b" target="_blank" rel="noopener">Frozen revision</a></td><td>2 historical differences; 0 current-reference mismatches</td></tr><tr><td>CLINC150 / four-encoder ensemble</td><td>98.0222% (4411/4500)</td><td><a href="https://huggingface.co/c41/deepopen-clinc150-ensemble-v1" target="_blank" rel="noopener">Model & inference</a> · <a href="https://huggingface.co/c41/deepopen-clinc150-ensemble-v1/tree/e646713055c49c2110e0c831ff057ab1ff59eb2d" target="_blank" rel="noopener">Frozen revision</a></td><td>0 historical mismatches</td></tr></tbody></table></div> | |
| <p class="note">BANKING77 is the best observed SAM seed after test evaluation; it is not a new independent validation-selected improvement. CLINC150 excludes out-of-scope inputs; single and ensemble results remain separate. Reproduction verifies the fixed historical artifacts and does not establish an official leaderboard rank.</p> | |
| <div class="downloads"><a href="banking_clinc_release_verification.json" download>Release revisions & verification</a><a href="banking_clinc_results.csv" download>Results for review</a></div> | |
| <p class="note">For a Hugging Face cache snapshot, run from the downloaded repository directory using <code>python -m infer_banking</code>, <code>python -m infer_clinc</code>, or <code>python -m infer_hybrid</code>, followed by <code>--checkpoint . --text "your text"</code>. Module execution preserves sibling imports when snapshot files are symbolic links.</p> | |
| </div></section> | |
| <!-- banking-clinc-verified-releases:end --> | |
| <div class="footer">deepopen · Author-maintained benchmark submission resource<br>Reference sources: <a href="https://www.sota2.com/research/sota/intent-classification-on-banking77">SOTA2 BANKING77</a> · <a href="https://www.sota2.com/research/sota/intent-classification-on-clinc150">SOTA2 CLINC150</a></div></main> | |
| <script> | |
| const DATA={"banking": {"name": "deepopen BANKING77 SAM seed 13 (best observed)", "n": 3080, "correct": 2899, "accuracy": 94.12337662337663, "classes": [{"label": "Refund_not_showing_up", "n": 40, "correct": 40, "accuracy": 100.0, "precision": 97.5609756097561, "f1": 98.76543209876543}, {"label": "activate_my_card", "n": 40, "correct": 39, "accuracy": 97.5, "precision": 100.0, "f1": 98.73417721518987}, {"label": "age_limit", "n": 40, "correct": 40, "accuracy": 100.0, "precision": 97.5609756097561, "f1": 98.76543209876543}, {"label": "apple_pay_or_google_pay", "n": 40, "correct": 40, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "atm_support", "n": 40, "correct": 40, "accuracy": 100.0, "precision": 97.5609756097561, "f1": 98.76543209876543}, {"label": "automatic_top_up", "n": 40, "correct": 39, "accuracy": 97.5, "precision": 97.5, "f1": 97.5}, {"label": "balance_not_updated_after_bank_transfer", "n": 40, "correct": 33, "accuracy": 82.5, "precision": 76.74418604651163, "f1": 79.51807228915662}, {"label": "balance_not_updated_after_cheque_or_cash_deposit", "n": 40, "correct": 39, "accuracy": 97.5, "precision": 97.5, "f1": 97.5}, {"label": "beneficiary_not_allowed", "n": 40, "correct": 37, "accuracy": 92.5, "precision": 97.36842105263158, "f1": 94.87179487179488}, {"label": "cancel_transfer", "n": 40, "correct": 40, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "card_about_to_expire", "n": 40, "correct": 39, "accuracy": 97.5, "precision": 100.0, "f1": 98.73417721518987}, {"label": "card_acceptance", "n": 40, "correct": 37, "accuracy": 92.5, "precision": 100.0, "f1": 96.1038961038961}, {"label": "card_arrival", "n": 40, "correct": 35, "accuracy": 87.5, "precision": 92.10526315789474, "f1": 89.74358974358974}, {"label": "card_delivery_estimate", "n": 40, "correct": 37, "accuracy": 92.5, "precision": 88.0952380952381, "f1": 90.2439024390244}, {"label": "card_linking", "n": 40, "correct": 39, "accuracy": 97.5, "precision": 95.1219512195122, "f1": 96.29629629629629}, {"label": "card_not_working", "n": 40, "correct": 39, "accuracy": 97.5, "precision": 86.66666666666667, "f1": 91.76470588235294}, {"label": "card_payment_fee_charged", "n": 40, "correct": 36, "accuracy": 90.0, "precision": 87.8048780487805, "f1": 88.88888888888889}, {"label": "card_payment_not_recognised", "n": 40, "correct": 34, "accuracy": 85.0, "precision": 82.92682926829268, "f1": 83.95061728395062}, {"label": "card_payment_wrong_exchange_rate", "n": 40, "correct": 37, "accuracy": 92.5, "precision": 90.2439024390244, "f1": 91.35802469135803}, {"label": "card_swallowed", "n": 40, "correct": 40, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "cash_withdrawal_charge", "n": 40, "correct": 38, "accuracy": 95.0, "precision": 97.43589743589743, "f1": 96.20253164556962}, {"label": "cash_withdrawal_not_recognised", "n": 40, "correct": 39, "accuracy": 97.5, "precision": 92.85714285714286, "f1": 95.1219512195122}, {"label": "change_pin", "n": 40, "correct": 38, "accuracy": 95.0, "precision": 97.43589743589743, "f1": 96.20253164556962}, {"label": "compromised_card", "n": 40, "correct": 33, "accuracy": 82.5, "precision": 86.84210526315789, "f1": 84.61538461538461}, {"label": "contactless_not_working", "n": 40, "correct": 38, "accuracy": 95.0, "precision": 100.0, "f1": 97.43589743589743}, {"label": "country_support", "n": 40, "correct": 40, "accuracy": 100.0, "precision": 95.23809523809524, "f1": 97.5609756097561}, {"label": "declined_card_payment", "n": 40, "correct": 38, "accuracy": 95.0, "precision": 88.37209302325581, "f1": 91.56626506024097}, {"label": "declined_cash_withdrawal", "n": 40, "correct": 40, "accuracy": 100.0, "precision": 95.23809523809524, "f1": 97.5609756097561}, {"label": "declined_transfer", "n": 40, "correct": 32, "accuracy": 80.0, "precision": 100.0, "f1": 88.88888888888889}, {"label": "direct_debit_payment_not_recognised", "n": 40, "correct": 35, "accuracy": 87.5, "precision": 92.10526315789474, "f1": 89.74358974358974}, {"label": "disposable_card_limits", "n": 40, "correct": 37, "accuracy": 92.5, "precision": 90.2439024390244, "f1": 91.35802469135803}, {"label": "edit_personal_details", "n": 40, "correct": 40, "accuracy": 100.0, "precision": 97.5609756097561, "f1": 98.76543209876543}, {"label": "exchange_charge", "n": 40, "correct": 36, "accuracy": 90.0, "precision": 94.73684210526316, "f1": 92.3076923076923}, {"label": "exchange_rate", "n": 40, "correct": 38, "accuracy": 95.0, "precision": 86.36363636363636, "f1": 90.47619047619048}, {"label": "exchange_via_app", "n": 40, "correct": 38, "accuracy": 95.0, "precision": 95.0, "f1": 95.0}, {"label": "extra_charge_on_statement", "n": 40, "correct": 39, "accuracy": 97.5, "precision": 97.5, "f1": 97.5}, {"label": "failed_transfer", "n": 40, "correct": 37, "accuracy": 92.5, "precision": 90.2439024390244, "f1": 91.35802469135803}, {"label": "fiat_currency_support", "n": 40, "correct": 36, "accuracy": 90.0, "precision": 92.3076923076923, "f1": 91.13924050632912}, {"label": "get_disposable_virtual_card", "n": 40, "correct": 38, "accuracy": 95.0, "precision": 95.0, "f1": 95.0}, {"label": "get_physical_card", "n": 40, "correct": 40, "accuracy": 100.0, "precision": 95.23809523809524, "f1": 97.5609756097561}, {"label": "getting_spare_card", "n": 40, "correct": 39, "accuracy": 97.5, "precision": 97.5, "f1": 97.5}, {"label": "getting_virtual_card", "n": 40, "correct": 39, "accuracy": 97.5, "precision": 90.69767441860465, "f1": 93.97590361445783}, {"label": "lost_or_stolen_card", "n": 40, "correct": 38, "accuracy": 95.0, "precision": 86.36363636363636, "f1": 90.47619047619048}, {"label": "lost_or_stolen_phone", "n": 40, "correct": 39, "accuracy": 97.5, "precision": 100.0, "f1": 98.73417721518987}, {"label": "order_physical_card", "n": 40, "correct": 37, "accuracy": 92.5, "precision": 94.87179487179488, "f1": 93.67088607594937}, {"label": "passcode_forgotten", "n": 40, "correct": 40, "accuracy": 100.0, "precision": 97.5609756097561, "f1": 98.76543209876543}, {"label": "pending_card_payment", "n": 40, "correct": 39, "accuracy": 97.5, "precision": 97.5, "f1": 97.5}, {"label": "pending_cash_withdrawal", "n": 40, "correct": 39, "accuracy": 97.5, "precision": 95.1219512195122, "f1": 96.29629629629629}, {"label": "pending_top_up", "n": 40, "correct": 37, "accuracy": 92.5, "precision": 94.87179487179488, "f1": 93.67088607594937}, {"label": "pending_transfer", "n": 40, "correct": 31, "accuracy": 77.5, "precision": 88.57142857142857, "f1": 82.66666666666667}, {"label": "pin_blocked", "n": 40, "correct": 36, "accuracy": 90.0, "precision": 94.73684210526316, "f1": 92.3076923076923}, {"label": "receiving_money", "n": 40, "correct": 38, "accuracy": 95.0, "precision": 100.0, "f1": 97.43589743589743}, {"label": "request_refund", "n": 40, "correct": 39, "accuracy": 97.5, "precision": 100.0, "f1": 98.73417721518987}, {"label": "reverted_card_payment?", "n": 40, "correct": 39, "accuracy": 97.5, "precision": 92.85714285714286, "f1": 95.1219512195122}, {"label": "supported_cards_and_currencies", "n": 40, "correct": 39, "accuracy": 97.5, "precision": 92.85714285714286, "f1": 95.1219512195122}, {"label": "terminate_account", "n": 40, "correct": 40, "accuracy": 100.0, "precision": 97.5609756097561, "f1": 98.76543209876543}, {"label": "top_up_by_bank_transfer_charge", "n": 40, "correct": 35, "accuracy": 87.5, "precision": 92.10526315789474, "f1": 89.74358974358974}, {"label": "top_up_by_card_charge", "n": 40, "correct": 38, "accuracy": 95.0, "precision": 97.43589743589743, "f1": 96.20253164556962}, {"label": "top_up_by_cash_or_cheque", "n": 40, "correct": 39, "accuracy": 97.5, "precision": 100.0, "f1": 98.73417721518987}, {"label": "top_up_failed", "n": 40, "correct": 36, "accuracy": 90.0, "precision": 85.71428571428571, "f1": 87.8048780487805}, {"label": "top_up_limits", "n": 40, "correct": 40, "accuracy": 100.0, "precision": 97.5609756097561, "f1": 98.76543209876543}, {"label": "top_up_reverted", "n": 40, "correct": 34, "accuracy": 85.0, "precision": 87.17948717948718, "f1": 86.07594936708861}, {"label": "topping_up_by_card", "n": 40, "correct": 33, "accuracy": 82.5, "precision": 89.1891891891892, "f1": 85.71428571428571}, {"label": "transaction_charged_twice", "n": 40, "correct": 40, "accuracy": 100.0, "precision": 93.02325581395348, "f1": 96.3855421686747}, {"label": "transfer_fee_charged", "n": 40, "correct": 37, "accuracy": 92.5, "precision": 90.2439024390244, "f1": 91.35802469135803}, {"label": "transfer_into_account", "n": 40, "correct": 38, "accuracy": 95.0, "precision": 90.47619047619048, "f1": 92.6829268292683}, {"label": "transfer_not_received_by_recipient", "n": 40, "correct": 35, "accuracy": 87.5, "precision": 83.33333333333333, "f1": 85.36585365853658}, {"label": "transfer_timing", "n": 40, "correct": 36, "accuracy": 90.0, "precision": 87.8048780487805, "f1": 88.88888888888889}, {"label": "unable_to_verify_identity", "n": 40, "correct": 37, "accuracy": 92.5, "precision": 97.36842105263158, "f1": 94.87179487179488}, {"label": "verify_my_identity", "n": 40, "correct": 37, "accuracy": 92.5, "precision": 94.87179487179488, "f1": 93.67088607594937}, {"label": "verify_source_of_funds", "n": 40, "correct": 40, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "verify_top_up", "n": 40, "correct": 40, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "virtual_card_not_working", "n": 40, "correct": 37, "accuracy": 92.5, "precision": 100.0, "f1": 96.1038961038961}, {"label": "visa_or_mastercard", "n": 40, "correct": 38, "accuracy": 95.0, "precision": 100.0, "f1": 97.43589743589743}, {"label": "why_verify_identity", "n": 40, "correct": 39, "accuracy": 97.5, "precision": 95.1219512195122, "f1": 96.29629629629629}, {"label": "wrong_amount_of_cash_received", "n": 40, "correct": 36, "accuracy": 90.0, "precision": 100.0, "f1": 94.73684210526316}, {"label": "wrong_exchange_rate_for_cash_withdrawal", "n": 40, "correct": 35, "accuracy": 87.5, "precision": 89.74358974358974, "f1": 88.60759493670886}], "confusions": [{"gold": "pending_transfer", "predicted": "balance_not_updated_after_bank_transfer", "count": 5}, {"gold": "card_arrival", "predicted": "card_delivery_estimate", "count": 4}, {"gold": "card_payment_not_recognised", "predicted": "compromised_card", "count": 4}, {"gold": "balance_not_updated_after_bank_transfer", "predicted": "transfer_timing", "count": 4}, {"gold": "card_delivery_estimate", "predicted": "card_arrival", "count": 3}, {"gold": "top_up_by_bank_transfer_charge", "predicted": "transfer_fee_charged", "count": 3}, {"gold": "top_up_reverted", "predicted": "top_up_failed", "count": 3}, {"gold": "compromised_card", "predicted": "lost_or_stolen_card", "count": 3}, {"gold": "wrong_exchange_rate_for_cash_withdrawal", "predicted": "card_payment_wrong_exchange_rate", "count": 3}, {"gold": "exchange_charge", "predicted": "exchange_rate", "count": 3}, {"gold": "fiat_currency_support", "predicted": "exchange_via_app", "count": 2}, {"gold": "exchange_via_app", "predicted": "fiat_currency_support", "count": 2}, {"gold": "contactless_not_working", "predicted": "card_not_working", "count": 2}, {"gold": "pending_top_up", "predicted": "top_up_reverted", "count": 2}, {"gold": "wrong_amount_of_cash_received", "predicted": "pending_cash_withdrawal", "count": 2}]}, "clinc_single": {"name": "deepopen \u00b7 CLINC150 single", "n": 4500, "correct": 4399, "accuracy": 97.75555555555556, "classes": [{"label": "accept_reservations", "n": 30, "correct": 29, "accuracy": 96.66666666666667, "precision": 93.54838709677419, "f1": 95.08196721311475}, {"label": "account_blocked", "n": 30, "correct": 29, "accuracy": 96.66666666666667, "precision": 96.66666666666667, "f1": 96.66666666666667}, {"label": "alarm", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 96.7741935483871, "f1": 98.36065573770492}, {"label": "application_status", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "apr", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 96.7741935483871, "f1": 98.36065573770492}, {"label": "are_you_a_bot", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "balance", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 96.7741935483871, "f1": 98.36065573770492}, {"label": "bill_balance", "n": 30, "correct": 28, "accuracy": 93.33333333333333, "precision": 96.55172413793103, "f1": 94.91525423728814}, {"label": "bill_due", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 90.9090909090909, "f1": 95.23809523809524}, {"label": "book_flight", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "book_hotel", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "calculator", "n": 30, "correct": 28, "accuracy": 93.33333333333333, "precision": 96.55172413793103, "f1": 94.91525423728814}, {"label": "calendar", "n": 30, "correct": 27, "accuracy": 90.0, "precision": 93.10344827586206, "f1": 91.52542372881356}, {"label": "calendar_update", "n": 30, "correct": 29, "accuracy": 96.66666666666667, "precision": 96.66666666666667, "f1": 96.66666666666667}, {"label": "calories", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "cancel", "n": 30, "correct": 28, "accuracy": 93.33333333333333, "precision": 93.33333333333333, "f1": 93.33333333333333}, {"label": "cancel_reservation", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 90.9090909090909, "f1": 95.23809523809524}, {"label": "car_rental", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "card_declined", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "carry_on", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "change_accent", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 96.7741935483871, "f1": 98.36065573770492}, {"label": "change_ai_name", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 85.71428571428571, "f1": 92.3076923076923}, {"label": "change_language", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 96.7741935483871, "f1": 98.36065573770492}, {"label": "change_speed", "n": 30, "correct": 29, "accuracy": 96.66666666666667, "precision": 100.0, "f1": 98.30508474576271}, {"label": "change_user_name", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 96.7741935483871, "f1": 98.36065573770492}, {"label": "change_volume", "n": 30, "correct": 24, "accuracy": 80.0, "precision": 100.0, "f1": 88.88888888888889}, {"label": "confirm_reservation", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "cook_time", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "credit_limit", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 96.7741935483871, "f1": 98.36065573770492}, {"label": "credit_limit_change", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "credit_score", "n": 30, "correct": 29, "accuracy": 96.66666666666667, "precision": 100.0, "f1": 98.30508474576271}, {"label": "current_location", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 93.75, "f1": 96.7741935483871}, {"label": "damaged_card", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "date", "n": 30, "correct": 29, "accuracy": 96.66666666666667, "precision": 100.0, "f1": 98.30508474576271}, {"label": "definition", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "direct_deposit", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "directions", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 96.7741935483871, "f1": 98.36065573770492}, {"label": "distance", "n": 30, "correct": 28, "accuracy": 93.33333333333333, "precision": 96.55172413793103, "f1": 94.91525423728814}, {"label": "do_you_have_pets", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "exchange_rate", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "expiration_date", "n": 30, "correct": 29, "accuracy": 96.66666666666667, "precision": 100.0, "f1": 98.30508474576271}, {"label": "find_phone", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "flight_status", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 96.7741935483871, "f1": 98.36065573770492}, {"label": "flip_coin", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "food_last", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "freeze_account", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "fun_fact", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "gas", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "gas_type", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "goodbye", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 96.7741935483871, "f1": 98.36065573770492}, {"label": "greeting", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 96.7741935483871, "f1": 98.36065573770492}, {"label": "how_busy", "n": 30, "correct": 28, "accuracy": 93.33333333333333, "precision": 96.55172413793103, "f1": 94.91525423728814}, {"label": "how_old_are_you", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "improve_credit_score", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 96.7741935483871, "f1": 98.36065573770492}, {"label": "income", "n": 30, "correct": 29, "accuracy": 96.66666666666667, "precision": 100.0, "f1": 98.30508474576271}, {"label": "ingredient_substitution", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "ingredients_list", "n": 30, "correct": 27, "accuracy": 90.0, "precision": 96.42857142857143, "f1": 93.10344827586206}, {"label": "insurance", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "insurance_change", "n": 30, "correct": 29, "accuracy": 96.66666666666667, "precision": 100.0, "f1": 98.30508474576271}, {"label": "interest_rate", "n": 30, "correct": 29, "accuracy": 96.66666666666667, "precision": 100.0, "f1": 98.30508474576271}, {"label": "international_fees", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "international_visa", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "jump_start", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 96.7741935483871, "f1": 98.36065573770492}, {"label": "last_maintenance", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "lost_luggage", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "make_call", "n": 30, "correct": 29, "accuracy": 96.66666666666667, "precision": 96.66666666666667, "f1": 96.66666666666667}, {"label": "maybe", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "meal_suggestion", "n": 30, "correct": 29, "accuracy": 96.66666666666667, "precision": 87.87878787878788, "f1": 92.06349206349206}, {"label": "meaning_of_life", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "measurement_conversion", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "meeting_schedule", "n": 30, "correct": 29, "accuracy": 96.66666666666667, "precision": 93.54838709677419, "f1": 95.08196721311475}, {"label": "min_payment", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "mpg", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "new_card", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 93.75, "f1": 96.7741935483871}, {"label": "next_holiday", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 96.7741935483871, "f1": 98.36065573770492}, {"label": "next_song", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "no", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "nutrition_info", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "oil_change_how", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 93.75, "f1": 96.7741935483871}, {"label": "oil_change_when", "n": 30, "correct": 28, "accuracy": 93.33333333333333, "precision": 90.3225806451613, "f1": 91.80327868852459}, {"label": "order", "n": 30, "correct": 29, "accuracy": 96.66666666666667, "precision": 100.0, "f1": 98.30508474576271}, {"label": "order_checks", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "order_status", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "pay_bill", "n": 30, "correct": 29, "accuracy": 96.66666666666667, "precision": 96.66666666666667, "f1": 96.66666666666667}, {"label": "payday", "n": 30, "correct": 29, "accuracy": 96.66666666666667, "precision": 100.0, "f1": 98.30508474576271}, {"label": "pin_change", "n": 30, "correct": 25, "accuracy": 83.33333333333333, "precision": 100.0, "f1": 90.9090909090909}, {"label": "play_music", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "plug_type", "n": 30, "correct": 29, "accuracy": 96.66666666666667, "precision": 100.0, "f1": 98.30508474576271}, {"label": "pto_balance", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 93.75, "f1": 96.7741935483871}, {"label": "pto_request", "n": 30, "correct": 29, "accuracy": 96.66666666666667, "precision": 100.0, "f1": 98.30508474576271}, {"label": "pto_request_status", "n": 30, "correct": 29, "accuracy": 96.66666666666667, "precision": 100.0, "f1": 98.30508474576271}, {"label": "pto_used", "n": 30, "correct": 28, "accuracy": 93.33333333333333, "precision": 100.0, "f1": 96.55172413793103}, {"label": "recipe", "n": 30, "correct": 28, "accuracy": 93.33333333333333, "precision": 90.3225806451613, "f1": 91.80327868852459}, {"label": "redeem_rewards", "n": 30, "correct": 26, "accuracy": 86.66666666666667, "precision": 100.0, "f1": 92.85714285714286}, {"label": "reminder", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "reminder_update", "n": 30, "correct": 29, "accuracy": 96.66666666666667, "precision": 100.0, "f1": 98.30508474576271}, {"label": "repeat", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "replacement_card_duration", "n": 30, "correct": 27, "accuracy": 90.0, "precision": 96.42857142857143, "f1": 93.10344827586206}, {"label": "report_fraud", "n": 30, "correct": 29, "accuracy": 96.66666666666667, "precision": 87.87878787878788, "f1": 92.06349206349206}, {"label": "report_lost_card", "n": 30, "correct": 28, "accuracy": 93.33333333333333, "precision": 96.55172413793103, "f1": 94.91525423728814}, {"label": "reset_settings", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "restaurant_reservation", "n": 30, "correct": 27, "accuracy": 90.0, "precision": 93.10344827586206, "f1": 91.52542372881356}, {"label": "restaurant_reviews", "n": 30, "correct": 27, "accuracy": 90.0, "precision": 100.0, "f1": 94.73684210526316}, {"label": "restaurant_suggestion", "n": 30, "correct": 29, "accuracy": 96.66666666666667, "precision": 100.0, "f1": 98.30508474576271}, {"label": "rewards_balance", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 88.23529411764706, "f1": 93.75}, {"label": "roll_dice", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "rollover_401k", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "routing", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 96.7741935483871, "f1": 98.36065573770492}, {"label": "schedule_maintenance", "n": 30, "correct": 27, "accuracy": 90.0, "precision": 100.0, "f1": 94.73684210526316}, {"label": "schedule_meeting", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 96.7741935483871, "f1": 98.36065573770492}, {"label": "share_location", "n": 30, "correct": 29, "accuracy": 96.66666666666667, "precision": 100.0, "f1": 98.30508474576271}, {"label": "shopping_list", "n": 30, "correct": 28, "accuracy": 93.33333333333333, "precision": 100.0, "f1": 96.55172413793103}, {"label": "shopping_list_update", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 96.7741935483871, "f1": 98.36065573770492}, {"label": "smart_home", "n": 30, "correct": 29, "accuracy": 96.66666666666667, "precision": 96.66666666666667, "f1": 96.66666666666667}, {"label": "spelling", "n": 30, "correct": 29, "accuracy": 96.66666666666667, "precision": 96.66666666666667, "f1": 96.66666666666667}, {"label": "spending_history", "n": 30, "correct": 29, "accuracy": 96.66666666666667, "precision": 85.29411764705883, "f1": 90.625}, {"label": "sync_device", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "taxes", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "tell_joke", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "text", "n": 30, "correct": 28, "accuracy": 93.33333333333333, "precision": 100.0, "f1": 96.55172413793103}, {"label": "thank_you", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 93.75, "f1": 96.7741935483871}, {"label": "time", "n": 30, "correct": 29, "accuracy": 96.66666666666667, "precision": 96.66666666666667, "f1": 96.66666666666667}, {"label": "timer", "n": 30, "correct": 29, "accuracy": 96.66666666666667, "precision": 96.66666666666667, "f1": 96.66666666666667}, {"label": "timezone", "n": 30, "correct": 29, "accuracy": 96.66666666666667, "precision": 100.0, "f1": 98.30508474576271}, {"label": "tire_change", "n": 30, "correct": 29, "accuracy": 96.66666666666667, "precision": 100.0, "f1": 98.30508474576271}, {"label": "tire_pressure", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 96.7741935483871, "f1": 98.36065573770492}, {"label": "todo_list", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 96.7741935483871, "f1": 98.36065573770492}, {"label": "todo_list_update", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "traffic", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "transactions", "n": 30, "correct": 26, "accuracy": 86.66666666666667, "precision": 96.29629629629629, "f1": 91.2280701754386}, {"label": "transfer", "n": 30, "correct": 29, "accuracy": 96.66666666666667, "precision": 100.0, "f1": 98.30508474576271}, {"label": "translate", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "travel_alert", "n": 30, "correct": 29, "accuracy": 96.66666666666667, "precision": 100.0, "f1": 98.30508474576271}, {"label": "travel_notification", "n": 30, "correct": 29, "accuracy": 96.66666666666667, "precision": 100.0, "f1": 98.30508474576271}, {"label": "travel_suggestion", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 96.7741935483871, "f1": 98.36065573770492}, {"label": "uber", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "update_playlist", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "user_name", "n": 30, "correct": 29, "accuracy": 96.66666666666667, "precision": 100.0, "f1": 98.30508474576271}, {"label": "vaccines", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "w2", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "weather", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 93.75, "f1": 96.7741935483871}, {"label": "what_are_your_hobbies", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 96.7741935483871, "f1": 98.36065573770492}, {"label": "what_can_i_ask_you", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "what_is_your_name", "n": 30, "correct": 28, "accuracy": 93.33333333333333, "precision": 96.55172413793103, "f1": 94.91525423728814}, {"label": "what_song", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "where_are_you_from", "n": 30, "correct": 29, "accuracy": 96.66666666666667, "precision": 100.0, "f1": 98.30508474576271}, {"label": "whisper_mode", "n": 30, "correct": 29, "accuracy": 96.66666666666667, "precision": 87.87878787878788, "f1": 92.06349206349206}, {"label": "who_do_you_work_for", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 93.75, "f1": 96.7741935483871}, {"label": "who_made_you", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 96.7741935483871, "f1": 98.36065573770492}, {"label": "yes", "n": 30, "correct": 26, "accuracy": 86.66666666666667, "precision": 100.0, "f1": 92.85714285714286}], "confusions": [{"gold": "pin_change", "predicted": "change_ai_name", "count": 5}, {"gold": "change_volume", "predicted": "whisper_mode", "count": 4}, {"gold": "redeem_rewards", "predicted": "rewards_balance", "count": 4}, {"gold": "transactions", "predicted": "spending_history", "count": 3}, {"gold": "schedule_maintenance", "predicted": "oil_change_when", "count": 3}, {"gold": "ingredients_list", "predicted": "recipe", "count": 3}, {"gold": "restaurant_reservation", "predicted": "accept_reservations", "count": 2}, {"gold": "oil_change_when", "predicted": "oil_change_how", "count": 2}, {"gold": "yes", "predicted": "thank_you", "count": 2}, {"gold": "calendar", "predicted": "meeting_schedule", "count": 2}, {"gold": "calculator", "predicted": "spending_history", "count": 2}, {"gold": "pto_used", "predicted": "pto_balance", "count": 2}, {"gold": "report_lost_card", "predicted": "report_fraud", "count": 2}, {"gold": "restaurant_reviews", "predicted": "meal_suggestion", "count": 2}, {"gold": "what_is_your_name", "predicted": "who_do_you_work_for", "count": 2}]}, "clinc_ensemble": {"name": "deepopen \u00b7 CLINC150 ensemble", "n": 4500, "correct": 4411, "accuracy": 98.02222222222223, "classes": [{"label": "accept_reservations", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 93.75, "f1": 96.7741935483871}, {"label": "account_blocked", "n": 30, "correct": 29, "accuracy": 96.66666666666667, "precision": 96.66666666666667, "f1": 96.66666666666667}, {"label": "alarm", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 96.7741935483871, "f1": 98.36065573770492}, {"label": "application_status", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "apr", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 96.7741935483871, "f1": 98.36065573770492}, {"label": "are_you_a_bot", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "balance", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "bill_balance", "n": 30, "correct": 28, "accuracy": 93.33333333333333, "precision": 96.55172413793103, "f1": 94.91525423728814}, {"label": "bill_due", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 90.9090909090909, "f1": 95.23809523809524}, {"label": "book_flight", "n": 30, "correct": 29, "accuracy": 96.66666666666667, "precision": 100.0, "f1": 98.30508474576271}, {"label": "book_hotel", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "calculator", "n": 30, "correct": 28, "accuracy": 93.33333333333333, "precision": 96.55172413793103, "f1": 94.91525423728814}, {"label": "calendar", "n": 30, "correct": 27, "accuracy": 90.0, "precision": 96.42857142857143, "f1": 93.10344827586206}, {"label": "calendar_update", "n": 30, "correct": 29, "accuracy": 96.66666666666667, "precision": 96.66666666666667, "f1": 96.66666666666667}, {"label": "calories", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "cancel", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 93.75, "f1": 96.7741935483871}, {"label": "cancel_reservation", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 93.75, "f1": 96.7741935483871}, {"label": "car_rental", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 96.7741935483871, "f1": 98.36065573770492}, {"label": "card_declined", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "carry_on", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "change_accent", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "change_ai_name", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 85.71428571428571, "f1": 92.3076923076923}, {"label": "change_language", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "change_speed", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "change_user_name", "n": 30, "correct": 29, "accuracy": 96.66666666666667, "precision": 100.0, "f1": 98.30508474576271}, {"label": "change_volume", "n": 30, "correct": 25, "accuracy": 83.33333333333333, "precision": 100.0, "f1": 90.9090909090909}, {"label": "confirm_reservation", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "cook_time", "n": 30, "correct": 29, "accuracy": 96.66666666666667, "precision": 100.0, "f1": 98.30508474576271}, {"label": "credit_limit", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 96.7741935483871, "f1": 98.36065573770492}, {"label": "credit_limit_change", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "credit_score", "n": 30, "correct": 29, "accuracy": 96.66666666666667, "precision": 100.0, "f1": 98.30508474576271}, {"label": "current_location", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 96.7741935483871, "f1": 98.36065573770492}, {"label": "damaged_card", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "date", "n": 30, "correct": 29, "accuracy": 96.66666666666667, "precision": 100.0, "f1": 98.30508474576271}, {"label": "definition", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "direct_deposit", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "directions", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "distance", "n": 30, "correct": 28, "accuracy": 93.33333333333333, "precision": 100.0, "f1": 96.55172413793103}, {"label": "do_you_have_pets", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "exchange_rate", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "expiration_date", "n": 30, "correct": 29, "accuracy": 96.66666666666667, "precision": 100.0, "f1": 98.30508474576271}, {"label": "find_phone", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "flight_status", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 93.75, "f1": 96.7741935483871}, {"label": "flip_coin", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "food_last", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "freeze_account", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "fun_fact", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "gas", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "gas_type", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "goodbye", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "greeting", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "how_busy", "n": 30, "correct": 29, "accuracy": 96.66666666666667, "precision": 96.66666666666667, "f1": 96.66666666666667}, {"label": "how_old_are_you", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "improve_credit_score", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 96.7741935483871, "f1": 98.36065573770492}, {"label": "income", "n": 30, "correct": 29, "accuracy": 96.66666666666667, "precision": 100.0, "f1": 98.30508474576271}, {"label": "ingredient_substitution", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "ingredients_list", "n": 30, "correct": 27, "accuracy": 90.0, "precision": 96.42857142857143, "f1": 93.10344827586206}, {"label": "insurance", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "insurance_change", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "interest_rate", "n": 30, "correct": 29, "accuracy": 96.66666666666667, "precision": 100.0, "f1": 98.30508474576271}, {"label": "international_fees", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "international_visa", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "jump_start", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 96.7741935483871, "f1": 98.36065573770492}, {"label": "last_maintenance", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "lost_luggage", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "make_call", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 96.7741935483871, "f1": 98.36065573770492}, {"label": "maybe", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "meal_suggestion", "n": 30, "correct": 29, "accuracy": 96.66666666666667, "precision": 87.87878787878788, "f1": 92.06349206349206}, {"label": "meaning_of_life", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "measurement_conversion", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "meeting_schedule", "n": 30, "correct": 29, "accuracy": 96.66666666666667, "precision": 93.54838709677419, "f1": 95.08196721311475}, {"label": "min_payment", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "mpg", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "new_card", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 96.7741935483871, "f1": 98.36065573770492}, {"label": "next_holiday", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "next_song", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "no", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "nutrition_info", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "oil_change_how", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 93.75, "f1": 96.7741935483871}, {"label": "oil_change_when", "n": 30, "correct": 28, "accuracy": 93.33333333333333, "precision": 90.3225806451613, "f1": 91.80327868852459}, {"label": "order", "n": 30, "correct": 29, "accuracy": 96.66666666666667, "precision": 100.0, "f1": 98.30508474576271}, {"label": "order_checks", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "order_status", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "pay_bill", "n": 30, "correct": 29, "accuracy": 96.66666666666667, "precision": 96.66666666666667, "f1": 96.66666666666667}, {"label": "payday", "n": 30, "correct": 29, "accuracy": 96.66666666666667, "precision": 100.0, "f1": 98.30508474576271}, {"label": "pin_change", "n": 30, "correct": 25, "accuracy": 83.33333333333333, "precision": 100.0, "f1": 90.9090909090909}, {"label": "play_music", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "plug_type", "n": 30, "correct": 29, "accuracy": 96.66666666666667, "precision": 100.0, "f1": 98.30508474576271}, {"label": "pto_balance", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 93.75, "f1": 96.7741935483871}, {"label": "pto_request", "n": 30, "correct": 29, "accuracy": 96.66666666666667, "precision": 96.66666666666667, "f1": 96.66666666666667}, {"label": "pto_request_status", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "pto_used", "n": 30, "correct": 28, "accuracy": 93.33333333333333, "precision": 100.0, "f1": 96.55172413793103}, {"label": "recipe", "n": 30, "correct": 28, "accuracy": 93.33333333333333, "precision": 90.3225806451613, "f1": 91.80327868852459}, {"label": "redeem_rewards", "n": 30, "correct": 26, "accuracy": 86.66666666666667, "precision": 100.0, "f1": 92.85714285714286}, {"label": "reminder", "n": 30, "correct": 29, "accuracy": 96.66666666666667, "precision": 100.0, "f1": 98.30508474576271}, {"label": "reminder_update", "n": 30, "correct": 29, "accuracy": 96.66666666666667, "precision": 100.0, "f1": 98.30508474576271}, {"label": "repeat", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "replacement_card_duration", "n": 30, "correct": 27, "accuracy": 90.0, "precision": 96.42857142857143, "f1": 93.10344827586206}, {"label": "report_fraud", "n": 30, "correct": 29, "accuracy": 96.66666666666667, "precision": 87.87878787878788, "f1": 92.06349206349206}, {"label": "report_lost_card", "n": 30, "correct": 28, "accuracy": 93.33333333333333, "precision": 96.55172413793103, "f1": 94.91525423728814}, {"label": "reset_settings", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "restaurant_reservation", "n": 30, "correct": 28, "accuracy": 93.33333333333333, "precision": 96.55172413793103, "f1": 94.91525423728814}, {"label": "restaurant_reviews", "n": 30, "correct": 27, "accuracy": 90.0, "precision": 100.0, "f1": 94.73684210526316}, {"label": "restaurant_suggestion", "n": 30, "correct": 29, "accuracy": 96.66666666666667, "precision": 100.0, "f1": 98.30508474576271}, {"label": "rewards_balance", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 88.23529411764706, "f1": 93.75}, {"label": "roll_dice", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "rollover_401k", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "routing", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 96.7741935483871, "f1": 98.36065573770492}, {"label": "schedule_maintenance", "n": 30, "correct": 27, "accuracy": 90.0, "precision": 100.0, "f1": 94.73684210526316}, {"label": "schedule_meeting", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 96.7741935483871, "f1": 98.36065573770492}, {"label": "share_location", "n": 30, "correct": 29, "accuracy": 96.66666666666667, "precision": 100.0, "f1": 98.30508474576271}, {"label": "shopping_list", "n": 30, "correct": 29, "accuracy": 96.66666666666667, "precision": 100.0, "f1": 98.30508474576271}, {"label": "shopping_list_update", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 96.7741935483871, "f1": 98.36065573770492}, {"label": "smart_home", "n": 30, "correct": 29, "accuracy": 96.66666666666667, "precision": 100.0, "f1": 98.30508474576271}, {"label": "spelling", "n": 30, "correct": 29, "accuracy": 96.66666666666667, "precision": 96.66666666666667, "f1": 96.66666666666667}, {"label": "spending_history", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 85.71428571428571, "f1": 92.3076923076923}, {"label": "sync_device", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "taxes", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "tell_joke", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "text", "n": 30, "correct": 28, "accuracy": 93.33333333333333, "precision": 100.0, "f1": 96.55172413793103}, {"label": "thank_you", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 96.7741935483871, "f1": 98.36065573770492}, {"label": "time", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 96.7741935483871, "f1": 98.36065573770492}, {"label": "timer", "n": 30, "correct": 29, "accuracy": 96.66666666666667, "precision": 93.54838709677419, "f1": 95.08196721311475}, {"label": "timezone", "n": 30, "correct": 29, "accuracy": 96.66666666666667, "precision": 100.0, "f1": 98.30508474576271}, {"label": "tire_change", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "tire_pressure", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "todo_list", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 96.7741935483871, "f1": 98.36065573770492}, {"label": "todo_list_update", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "traffic", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "transactions", "n": 30, "correct": 26, "accuracy": 86.66666666666667, "precision": 96.29629629629629, "f1": 91.2280701754386}, {"label": "transfer", "n": 30, "correct": 29, "accuracy": 96.66666666666667, "precision": 100.0, "f1": 98.30508474576271}, {"label": "translate", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 96.7741935483871, "f1": 98.36065573770492}, {"label": "travel_alert", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "travel_notification", "n": 30, "correct": 29, "accuracy": 96.66666666666667, "precision": 100.0, "f1": 98.30508474576271}, {"label": "travel_suggestion", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 93.75, "f1": 96.7741935483871}, {"label": "uber", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "update_playlist", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "user_name", "n": 30, "correct": 29, "accuracy": 96.66666666666667, "precision": 96.66666666666667, "f1": 96.66666666666667}, {"label": "vaccines", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "w2", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "weather", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 96.7741935483871, "f1": 98.36065573770492}, {"label": "what_are_your_hobbies", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "what_can_i_ask_you", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "what_is_your_name", "n": 30, "correct": 28, "accuracy": 93.33333333333333, "precision": 96.55172413793103, "f1": 94.91525423728814}, {"label": "what_song", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "where_are_you_from", "n": 30, "correct": 29, "accuracy": 96.66666666666667, "precision": 100.0, "f1": 98.30508474576271}, {"label": "whisper_mode", "n": 30, "correct": 29, "accuracy": 96.66666666666667, "precision": 87.87878787878788, "f1": 92.06349206349206}, {"label": "who_do_you_work_for", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 93.75, "f1": 96.7741935483871}, {"label": "who_made_you", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 96.7741935483871, "f1": 98.36065573770492}, {"label": "yes", "n": 30, "correct": 27, "accuracy": 90.0, "precision": 100.0, "f1": 94.73684210526316}], "confusions": [{"gold": "pin_change", "predicted": "change_ai_name", "count": 5}, {"gold": "change_volume", "predicted": "whisper_mode", "count": 4}, {"gold": "redeem_rewards", "predicted": "rewards_balance", "count": 4}, {"gold": "transactions", "predicted": "spending_history", "count": 3}, {"gold": "schedule_maintenance", "predicted": "oil_change_when", "count": 3}, {"gold": "ingredients_list", "predicted": "recipe", "count": 3}, {"gold": "restaurant_reservation", "predicted": "accept_reservations", "count": 2}, {"gold": "distance", "predicted": "flight_status", "count": 2}, {"gold": "oil_change_when", "predicted": "oil_change_how", "count": 2}, {"gold": "calendar", "predicted": "meeting_schedule", "count": 2}, {"gold": "calculator", "predicted": "spending_history", "count": 2}, {"gold": "pto_used", "predicted": "pto_balance", "count": 2}, {"gold": "report_lost_card", "predicted": "report_fraud", "count": 2}, {"gold": "restaurant_reviews", "predicted": "meal_suggestion", "count": 2}, {"gold": "what_is_your_name", "predicted": "who_do_you_work_for", "count": 2}]}}; | |
| const refs={bank:[['SPACE',94.94],['SPACE 2.0',94.77],['Llama 2 7B (FT)',94.35],['ConvFiT',94.16],['deepopen (SAM, best observed)',94.1234],['PPTOD',93.86]],clinc:[['TOATOD-small',98.45],['deepopen (ensemble)',98.0222],['TOATOD-base',98.01],['SPACE',97.89],['SPACE 2.0',97.8]]}; | |
| const notes={bank:'Selected published references. SOTA2 entries reporting accuracy above 100% are excluded as anomalous. No official rank is assigned.',clinc:'The reported CLINC150 system combines four encoders. Its accuracy is an ensemble result, with higher inference cost than a single model.'}; | |
| function compare(k){document.querySelectorAll('[data-task]').forEach(b=>b.classList.toggle('active',b.dataset.task===k));document.getElementById('charttitle').textContent='Selected reference results and author-reported test results';document.getElementById('bars').innerHTML=refs[k].map(([n,v])=>`<tr class="${n.startsWith('deepopen')?'ours':''}"><td>${n}</td><td>${v.toFixed(n.startsWith('deepopen')?4:2)}%</td><td>${n.startsWith('deepopen')?'This work \u00b7 saved predictions':'SOTA2 reference'}</td></tr>`).join('');document.getElementById('compareNote').textContent=notes[k]} | |
| function render(){const k=document.getElementById('dataset').value,d=DATA[k],q=document.getElementById('search').value.toLowerCase();document.getElementById('resultstat').innerHTML=`<h3>${d.name}</h3><div class="bigstat">${d.correct.toLocaleString()} <span style="font-size:15px;color:var(--muted)">/ ${d.n.toLocaleString()} correct \u00b7 ${d.classes.length} classes \u00b7 ${d.accuracy.toFixed(4)}% accuracy</span></div>`;let rows=d.classes.filter(x=>x.label.toLowerCase().includes(q)).sort(document.getElementById('sort').value==='label'?(a,b)=>a.label.localeCompare(b.label):(a,b)=>a.accuracy-b.accuracy||a.label.localeCompare(b.label));document.getElementById('classes').innerHTML=rows.length?rows.map(x=>`<tr><td>${x.label}</td><td>${x.correct} / ${x.n}</td><td>${x.accuracy.toFixed(2)}%</td><td>${x.precision.toFixed(2)}%</td><td>${x.f1.toFixed(2)}%</td></tr>`).join(''):'<tr><td colspan="5">No matching classes.</td></tr>';document.getElementById('predDownload').href=k+'_predictions.csv';document.getElementById('classDownload').href=k+'_classes.csv';document.getElementById('confusions').innerHTML='<table><tr><th>Gold class</th><th>Predicted class</th><th>Count</th></tr>'+d.confusions.map(x=>`<tr><td>${x.gold}</td><td>${x.predicted}</td><td>${x.count}</td></tr>`).join('')+'</table>'} | |
| document.querySelectorAll('[data-task]').forEach(b=>b.onclick=()=>compare(b.dataset.task));['dataset','search','sort'].forEach(id=>document.getElementById(id).addEventListener('input',render));compare('bank');render(); | |
| </script></body></html> |