c41 commited on
Commit
1df5a8f
·
verified ·
1 Parent(s): a83f3a9

Publish evaluation preview and verified historical classification tables

Browse files
Files changed (2) hide show
  1. README.md +4 -2
  2. index.html +7 -7
README.md CHANGED
@@ -1,5 +1,5 @@
1
  ---
2
- title: Laya Evaluation Explorer
3
  emoji: 📊
4
  colorFrom: blue
5
  colorTo: green
@@ -8,4 +8,6 @@ app_file: index.html
8
  pinned: false
9
  ---
10
 
11
- Laya author-maintained evaluation display. Not an official SOTA2 leaderboard. Sources and scope are linked in the page. Prediction rows are artifact indices, not official IDs. No test utterances or weights are redistributed here.
 
 
 
1
  ---
2
+ title: deepopen Evaluation Explorer
3
  emoji: 📊
4
  colorFrom: blue
5
  colorTo: green
 
8
  pinned: false
9
  ---
10
 
11
+ deepopen author-maintained evaluation display. Not an official SOTA2 leaderboard. Sources and scope are linked in the page. Prediction rows are artifact indices, not official IDs. No test utterances or weights are redistributed here.
12
+
13
+ Method display name: deepopen. Historical artifacts and model lineage retain the name Laya; renaming does not change predictions or scores.
index.html CHANGED
@@ -1,20 +1,20 @@
1
- <!doctype html><html lang="zh-CN"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width,initial-scale=1"><title>Laya &#183; &#24847;&#22270;&#20998;&#31867;&#35780;&#27979;</title>
2
  <style>
3
  :root{--ink:#142c37;--muted:#647981;--line:#dce5e5;--teal:#007b76;--bg:#f3f6f5}*{box-sizing:border-box}body{margin:0;background:var(--bg);font:15px/1.6 Inter,"Segoe UI","Microsoft YaHei",sans-serif;color:var(--ink)}header{padding:22px 5%;border-bottom:1px solid var(--line);display:flex;align-items:center;justify-content:space-between;background:white}.brand{font-weight:800;font-size:25px;letter-spacing:-1px}.brand span{color:var(--teal)}nav a{margin-left:24px;color:var(--muted);text-decoration:none;font-size:13px}main{max-width:1280px;margin:auto;padding:48px 30px}small,.eyebrow{letter-spacing:2px;font-size:11px;font-weight:700;color:var(--teal)}h1{font-size:46px;line-height:1.2;margin:14px 0;letter-spacing:-2px}h2{font-size:24px;margin:0}h3{font-size:16px;margin:0}.sub{color:var(--muted);max-width:750px}.status{display:inline-block;background:#e2efeb;color:#32665e;padding:5px 11px;border-radius:20px;font-size:12px}.hero{display:flex;justify-content:space-between;align-items:center;margin-bottom:30px}.hero-right{text-align:right;font-size:12px;color:var(--muted)}.cards{display:grid;grid-template-columns:repeat(4,1fr);gap:16px}.card{background:white;border:1px solid var(--line);border-radius:14px;padding:23px}.card.featured{background:#103e43;color:white;border-color:#103e43}.value{font-size:38px;font-weight:750;letter-spacing:-1.7px;margin:14px 0 4px}.value em{font-size:18px;font-style:normal;letter-spacing:0}.caption{font-size:12px;color:var(--muted)}.featured .caption{color:#b9d5d1}.rule{margin:16px 0 36px;font-size:12px;color:var(--muted)}section{margin-top:38px}.heading{display:flex;justify-content:space-between;gap:20px;align-items:center;margin-bottom:18px}.panel{background:white;border:1px solid var(--line);border-radius:14px;padding:25px}.tabs{display:flex;gap:7px;flex-wrap:wrap}button,select,input{font:inherit;border:1px solid var(--line);border-radius:8px;padding:9px 13px;background:white;color:var(--ink)}button{cursor:pointer}button.active{background:var(--ink);color:white;border-color:var(--ink)}.split{display:grid;grid-template-columns:1.4fr 1fr;gap:24px}.barrow{display:grid;grid-template-columns:190px 1fr 84px;align-items:center;gap:12px;margin:17px 0;font-size:13px}.track{height:10px;border-radius:5px;background:#edf1f2;overflow:hidden}.fill{height:100%;background:#abbcc4;border-radius:5px}.ours .fill{background:var(--teal)}.ours{font-weight:700;color:var(--teal)}.barrow strong{text-align:right}.note{font-size:12px;color:var(--muted);margin-top:15px}.callout{background:#f3f7f6;border-radius:10px;padding:20px}.callout p{font-size:13px;color:var(--muted)}a{color:var(--teal);text-underline-offset:3px}.toolbar{display:flex;gap:10px;flex-wrap:wrap;margin:16px 0}.toolbar input{flex:1;min-width:160px}.tablewrap{max-height:390px;overflow:auto}table{width:100%;border-collapse:collapse;font-size:13px}th{text-align:left;font-size:11px;color:var(--muted);position:sticky;top:0;background:#f5f8f7;padding:12px}td{border-bottom:1px solid #edf1f1;padding:11px 12px}td.num{text-align:right;font-variant-numeric:tabular-nums}.mini{height:5px;background:#e7efed;width:80px;display:inline-block;margin-right:10px;vertical-align:middle}.mini i{display:block;height:5px;background:var(--teal)}.downloads{display:flex;gap:12px;flex-wrap:wrap;margin-top:20px}.downloads a{font-size:13px;border:1px solid var(--line);padding:8px 13px;border-radius:7px;text-decoration:none}.footer{font-size:12px;color:var(--muted);border-top:1px solid var(--line);padding-top:24px;margin-top:40px}.footgrid{display:grid;grid-template-columns:1fr 1fr;gap:28px}.pill{font-size:11px;background:#edf3f1;border-radius:5px;padding:3px 7px}details{font-size:13px;margin-top:15px}summary{cursor:pointer} .bigstat{font-size:27px;font-weight:700;color:var(--teal);margin:10px 0}
4
  @media(max-width:850px){.cards{grid-template-columns:1fr 1fr}.split,.footgrid{grid-template-columns:1fr}.hero{display:block}.hero-right{text-align:left;margin-top:18px}.heading{align-items:flex-start;flex-direction:column}h1{font-size:34px}main{padding:25px 18px}.barrow{grid-template-columns:145px 1fr 70px}nav{display:none}}@media print{header,button,.toolbar{display:none}main{padding:0}.tablewrap{max-height:none}.panel,.card{break-inside:avoid}body{background:white}}
5
  </style>
6
- </head><body><header><div class="brand">Laya<span> / </span>Research</div><nav><a href="#comparison">&#20844;&#24320;&#32467;&#26524;&#23545;&#29031;</a><a href="#results">&#20998;&#31867;&#32467;&#26524;</a><a href="#evidence">&#35777;&#25454;&#19982;&#33539;&#22260;</a></nav><span class="status">&#20316;&#32773;&#32500;&#25252; &#183; &#38750;&#23448;&#26041;&#27036;&#21333;</span></header>
7
- <main><div class="hero"><div><div class="eyebrow">INTENT CLASSIFICATION / EVALUATION REPORT</div><h1>&#35753;&#27599;&#19968;&#39033;&#25104;&#32489;&#65292;<br>&#37117;&#26377;&#32467;&#26524;&#21487;&#26597;&#12290;</h1><p class="sub">Laya &#22312;&#38134;&#34892;&#26381;&#21153;&#12289;&#36890;&#29992;&#21161;&#25163;&#19982;&#22810;&#35821;&#35328;&#24847;&#22270;&#20998;&#31867;&#19978;&#30340;&#27979;&#35797;&#34920;&#29616;&#12290;<br>&#26597;&#30475;&#20844;&#24320;&#32467;&#26524;&#23545;&#29031;&#65292;&#28145;&#20837;&#27599;&#20010;&#31867;&#21035;&#65292;&#19979;&#36733;&#21487;&#26680;&#39564;&#30340;&#20998;&#31867;&#35760;&#24405;&#12290;</p></div><div class="hero-right"><span class="status">&#23637;&#31034;&#39044;&#35272;&#29256;</span><p>&#21442;&#32771;&#26469;&#28304;&#26680;&#39564;&#65306;2026.10.07<br>SOTA2 &#25910;&#24405;&#65306;&#23578;&#26410;&#25552;&#20132;</p></div></div>
8
  <div class="cards"><div class="card"><small>BANKING77</small><div class="value">94.0909<em>%</em></div><h3>&#38134;&#34892;&#24847;&#22270; &#183; &#21333;&#27169;&#22411;</h3><p class="caption">2,898 / 3,080 &#27491;&#30830; &#183; 77 &#31867;</p></div><div class="card"><small>CLINC150 / SINGLE</small><div class="value">97.7556<em>%</em></div><h3>&#36890;&#29992;&#21161;&#25163; &#183; &#21333;&#27169;&#22411;</h3><p class="caption">4,399 / 4,500 &#27491;&#30830; &#183; 150 &#31867;</p></div><div class="card featured"><small style="color:#89d6bd">CLINC150 / ENSEMBLE</small><div class="value">98.0222<em>%</em></div><h3>&#36890;&#29992;&#21161;&#25163; &#183; &#22235;&#32534;&#30721;&#22120;&#38598;&#25104;</h3><p class="caption">4,411 / 4,500 &#27491;&#30830; &#183; 150 &#31867;</p></div><div class="card"><small>MASSIVE / MULTILINGUAL</small><div class="value">86.0325<em>%</em></div><h3>51 &#35821;&#35328; &#183; &#19977;&#31181;&#23376;&#24179;&#22343;</h3><p class="caption">&#19968;&#33268;&#24615;&#26041;&#27861; &#183; &#38750;&#21333;&#19968;&#27169;&#22411;&#20998;&#25968;</p></div></div>
9
  <p class="rule">BANKING77 / CLINC150 &#20026;&#21382;&#21490; test &#25104;&#32489;&#65292;&#24050;&#20174;&#39044;&#27979;&#25991;&#20214;&#37325;&#26032;&#35745;&#25968;&#26680;&#23545;&#65307;CLINC150 &#20165;&#21547;&#22495;&#20869;&#31867;&#21035;&#65292;&#19981;&#21547; OOS&#12290;MASSIVE &#20026;&#29420;&#31435;&#30340;&#22810;&#35821;&#35328;&#30740;&#31350;&#35774;&#32622;&#12290;</p>
10
  <section id="comparison"><div class="heading"><h2>&#19982;&#20844;&#24320;&#32467;&#26524;&#25918;&#22312;&#19968;&#36215;&#30475;</h2><div class="tabs" id="tabs"><button class="active" data-task="bank">BANKING77</button><button data-task="clinc">CLINC150</button><button data-task="massive">MASSIVE</button></div></div><div class="panel split"><div><div class="caption" id="charttitle"></div><div id="bars"></div><div class="note">&#26465;&#24418;&#20174; 0% &#36215;&#65292;&#28385;&#21051;&#24230; 100%&#12290;&#20165;&#23637;&#31034;&#36873;&#23450;&#21442;&#32771;&#26465;&#30446;&#65292;&#19981;&#34920;&#31034;&#23448;&#26041;&#25490;&#21517;&#12290;</div></div><div class="callout" id="compareNote"></div></div></section>
11
  <section id="results"><div class="heading"><div><small>VERIFIED PREDICTION COUNTS</small><h2>&#28145;&#20837;&#20998;&#31867;&#32467;&#26524;</h2></div><select id="dataset" aria-label="&#36873;&#25321;&#20998;&#31867;&#27169;&#22411;"><option value="banking">BANKING77 &#183; &#21333;&#27169;&#22411;</option><option value="clinc_single">CLINC150 &#183; &#21333;&#27169;&#22411;</option><option value="clinc_ensemble">CLINC150 &#183; &#22235;&#32534;&#30721;&#22120;&#38598;&#25104;</option><option value="massive">MASSIVE &#183; &#19977;&#31181;&#23376;&#21512;&#24182;&#35760;&#24405;</option></select></div><div class="panel"><div id="resultstat"></div><div class="toolbar"><input id="search" aria-label="&#25628;&#32034;&#31867;&#21035;" placeholder="&#25628;&#32034;&#31867;&#21035;&#65292;&#20363;&#22914; card&#12289;transfer&#12289;alarm&#8230;"><select id="sort" aria-label="&#25490;&#24207;"><option value="accuracy">&#20248;&#20808;&#26597;&#30475;&#21484;&#22238;&#29575;&#36739;&#20302;&#30340;&#31867;&#21035;</option><option value="label">&#25353;&#31867;&#21035;&#21517;&#31216;&#25490;&#24207;</option></select></div><div class="tablewrap"><table><thead><tr><th>&#30495;&#23454;&#31867;&#21035;</th><th>&#27491;&#30830; / &#26679;&#26412;&#25968;</th><th>&#21484;&#22238;&#29575;</th><th>&#31934;&#30830;&#29575;</th><th>F1</th></tr></thead><tbody id="classes"></tbody></table></div><div class="downloads"><a id="predDownload" download>&#19979;&#36733;&#36880;&#26679;&#26412;&#39044;&#27979; CSV &#8599;</a><a id="classDownload" download>&#19979;&#36733;&#20840;&#37096;&#31867;&#21035;&#25351;&#26631; CSV &#8599;</a><a href="evidence_manifest.json" download>&#35777;&#25454;&#26469;&#28304;&#19982; SHA256 &#8599;</a></div><p class="note">&#39044;&#27979;&#34920;&#20351;&#29992;&#21407;&#39044;&#27979;&#25991;&#20214;&#30340;&#34892;&#24207;&#21495;&#65292;&#19981;&#20882;&#20805;&#23448;&#26041;&#26679;&#26412; ID&#65307;&#19981;&#20998;&#21457;&#27979;&#35797;&#25991;&#26412;&#12290;&#31867;&#21035;&#25351;&#26631;&#30001;&#22266;&#23450;&#39044;&#27979;&#37325;&#26032;&#32479;&#35745;&#65292;&#19981;&#26159;&#37325;&#26032;&#35757;&#32451;&#25110;&#35843;&#21442;&#12290;</p><details><summary>&#26597;&#30475;&#26368;&#24120;&#35265;&#30340;&#20998;&#31867;&#28151;&#28102;</summary><div id="confusions"></div></details></div></section>
12
- <section id="evidence"><div class="heading"><h2>&#25104;&#32489;&#30340;&#33539;&#22260;&#65292;&#20063;&#19968;&#36215;&#20844;&#24320;</h2><span class="pill">&#36879;&#26126;&#25259;&#38706;</span></div><div class="panel footgrid"><div><h3>&#22797;&#29616;&#19982;&#23457;&#26680;</h3><p class="sub">&#19977;&#20221;&#21382;&#21490;&#39044;&#27979;&#24050;&#26680;&#23545;&#27491;&#30830;&#25968;&#12290;&#27169;&#22411;&#26435;&#37325;&#27491;&#22312;&#29420;&#31435;&#21457;&#24067;&#65292;&#19979;&#36733;&#21152;&#36733;&#22797;&#39564;&#23436;&#25104;&#21069;&#19981;&#26174;&#31034;&#8220;&#24050;&#39564;&#25910;&#8221;&#12290;&#21512;&#25104;&#26679;&#20363;&#21152;&#36733;&#26816;&#26597;&#19981;&#31561;&#20110;&#23436;&#25972;&#27979;&#35797;&#22797;&#27979;&#12290;</p><p class="sub">MASSIVE &#30340;&#19977;&#31181;&#23376;&#22343;&#20540;&#19981;&#33021;&#32465;&#23450;&#21040;&#19968;&#20010;&#21457;&#24067; checkpoint&#12290;&#20998;&#31867;&#34920;&#27719;&#24635;&#19977;&#31181;&#23376;&#20849; 455,022 &#26465;&#39044;&#27979;&#35760;&#24405;&#65292;&#23545;&#24212;&#21516;&#19968;&#25209; 151,674 &#26465;&#27979;&#35797;&#26679;&#26412;&#65292;&#19981;&#33021;&#24403;&#20316;&#29420;&#31435;&#26679;&#26412;&#37327;&#65307;&#21508;&#35821;&#35328;&#35268;&#27169;&#30456;&#21516;&#65292;&#27719;&#24635;&#20934;&#30830;&#29575;&#31561;&#20110;&#19977;&#31181;&#23376;&#24179;&#22343;&#12290;<a href="massive_locales.csv" download>&#19979;&#36733;&#36880;&#35821;&#35328;&#32467;&#26524;</a>&#12290;</p></div><div><h3>&#27604;&#36739;&#19982;&#30740;&#31350;&#36793;&#30028;</h3><p class="sub">&#22806;&#37096;&#25104;&#32489;&#30001; SOTA2 &#24341;&#29992;&#65292;&#26410;&#30001;&#25105;&#20204;&#37325;&#26032;&#36305;&#27979;&#65307;&#35757;&#32451;&#26465;&#20214;&#19982;&#27169;&#22411;&#35268;&#27169;&#21487;&#33021;&#19981;&#21516;&#12290;BANKING77 &#21382;&#21490;&#20840;&#37327;&#35757;&#32451;&#21253;&#21547;&#26087; dev&#65292;&#19981;&#33021;&#23558;&#26087; dev &#24403;&#20316;&#29420;&#31435;&#39564;&#35777;&#12290;&#26082;&#26377; test &#24050;&#35265;&#65292;&#21518;&#32493;&#35797;&#39564;&#20165;&#29992; train/dev&#12290;</p><p class="sub">&#21518;&#32493;&#30740;&#31350;&#23578;&#26410;&#35777;&#26126;&#26174;&#33879;&#20934;&#30830;&#29575;&#25910;&#30410;&#12290;&#26412;&#39029;&#19981;&#20027;&#24352;&#23448;&#26041;&#25490;&#21517;&#12289;SOTA &#35748;&#35777;&#25110;&#24179;&#21488;&#32972;&#20070;&#12290;</p></div></div></section><div class="footer">Laya Research &#183; &#20316;&#32773;&#33258;&#24314;&#35780;&#27979;&#23637;&#31034;&#39029;<br>&#20844;&#24320;&#21442;&#32771;&#26469;&#28304;&#65306;<a href="https://www.sota2.com/research/sota/intent-classification-on-banking77" target="_blank" rel="noopener">SOTA2 BANKING77</a> &#183; <a href="https://www.sota2.com/research/sota/intent-classification-on-clinc150" target="_blank" rel="noopener">SOTA2 CLINC150</a> &#183; <a href="https://www.sota2.com/research/sota/intent-classification-on-massive-test" target="_blank" rel="noopener">SOTA2 MASSIVE</a></div></main>
13
  <script>
14
  const DATA={"banking": {"name": "BANKING77 \u00b7 \u5355\u6a21\u578b", "n": 3080, "correct": 2898, "accuracy": 94.0909090909091, "classes": [{"label": "Refund_not_showing_up", "n": 40, "correct": 38, "accuracy": 95.0, "precision": 100.0, "f1": 97.43589743589743}, {"label": "activate_my_card", "n": 40, "correct": 39, "accuracy": 97.5, "precision": 100.0, "f1": 98.73417721518987}, {"label": "age_limit", "n": 40, "correct": 40, "accuracy": 100.0, "precision": 97.5609756097561, "f1": 98.76543209876543}, {"label": "apple_pay_or_google_pay", "n": 40, "correct": 40, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "atm_support", "n": 40, "correct": 39, "accuracy": 97.5, "precision": 97.5, "f1": 97.5}, {"label": "automatic_top_up", "n": 40, "correct": 40, "accuracy": 100.0, "precision": 97.5609756097561, "f1": 98.76543209876543}, {"label": "balance_not_updated_after_bank_transfer", "n": 40, "correct": 34, "accuracy": 85.0, "precision": 79.06976744186046, "f1": 81.92771084337349}, {"label": "balance_not_updated_after_cheque_or_cash_deposit", "n": 40, "correct": 39, "accuracy": 97.5, "precision": 97.5, "f1": 97.5}, {"label": "beneficiary_not_allowed", "n": 40, "correct": 37, "accuracy": 92.5, "precision": 94.87179487179488, "f1": 93.67088607594937}, {"label": "cancel_transfer", "n": 40, "correct": 40, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "card_about_to_expire", "n": 40, "correct": 39, "accuracy": 97.5, "precision": 100.0, "f1": 98.73417721518987}, {"label": "card_acceptance", "n": 40, "correct": 37, "accuracy": 92.5, "precision": 100.0, "f1": 96.1038961038961}, {"label": "card_arrival", "n": 40, "correct": 36, "accuracy": 90.0, "precision": 92.3076923076923, "f1": 91.13924050632912}, {"label": "card_delivery_estimate", "n": 40, "correct": 37, "accuracy": 92.5, "precision": 92.5, "f1": 92.5}, {"label": "card_linking", "n": 40, "correct": 39, "accuracy": 97.5, "precision": 95.1219512195122, "f1": 96.29629629629629}, {"label": "card_not_working", "n": 40, "correct": 39, "accuracy": 97.5, "precision": 86.66666666666667, "f1": 91.76470588235294}, {"label": "card_payment_fee_charged", "n": 40, "correct": 36, "accuracy": 90.0, "precision": 87.8048780487805, "f1": 88.88888888888889}, {"label": "card_payment_not_recognised", "n": 40, "correct": 34, "accuracy": 85.0, "precision": 85.0, "f1": 85.0}, {"label": "card_payment_wrong_exchange_rate", "n": 40, "correct": 37, "accuracy": 92.5, "precision": 90.2439024390244, "f1": 91.35802469135803}, {"label": "card_swallowed", "n": 40, "correct": 40, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "cash_withdrawal_charge", "n": 40, "correct": 39, "accuracy": 97.5, "precision": 97.5, "f1": 97.5}, {"label": "cash_withdrawal_not_recognised", "n": 40, "correct": 39, "accuracy": 97.5, "precision": 90.69767441860465, "f1": 93.97590361445783}, {"label": "change_pin", "n": 40, "correct": 38, "accuracy": 95.0, "precision": 97.43589743589743, "f1": 96.20253164556962}, {"label": "compromised_card", "n": 40, "correct": 33, "accuracy": 82.5, "precision": 84.61538461538461, "f1": 83.54430379746836}, {"label": "contactless_not_working", "n": 40, "correct": 39, "accuracy": 97.5, "precision": 100.0, "f1": 98.73417721518987}, {"label": "country_support", "n": 40, "correct": 40, "accuracy": 100.0, "precision": 95.23809523809524, "f1": 97.5609756097561}, {"label": "declined_card_payment", "n": 40, "correct": 38, "accuracy": 95.0, "precision": 88.37209302325581, "f1": 91.56626506024097}, {"label": "declined_cash_withdrawal", "n": 40, "correct": 40, "accuracy": 100.0, "precision": 93.02325581395348, "f1": 96.3855421686747}, {"label": "declined_transfer", "n": 40, "correct": 32, "accuracy": 80.0, "precision": 96.96969696969697, "f1": 87.67123287671232}, {"label": "direct_debit_payment_not_recognised", "n": 40, "correct": 36, "accuracy": 90.0, "precision": 92.3076923076923, "f1": 91.13924050632912}, {"label": "disposable_card_limits", "n": 40, "correct": 36, "accuracy": 90.0, "precision": 90.0, "f1": 90.0}, {"label": "edit_personal_details", "n": 40, "correct": 40, "accuracy": 100.0, "precision": 97.5609756097561, "f1": 98.76543209876543}, {"label": "exchange_charge", "n": 40, "correct": 36, "accuracy": 90.0, "precision": 94.73684210526316, "f1": 92.3076923076923}, {"label": "exchange_rate", "n": 40, "correct": 39, "accuracy": 97.5, "precision": 86.66666666666667, "f1": 91.76470588235294}, {"label": "exchange_via_app", "n": 40, "correct": 37, "accuracy": 92.5, "precision": 94.87179487179488, "f1": 93.67088607594937}, {"label": "extra_charge_on_statement", "n": 40, "correct": 39, "accuracy": 97.5, "precision": 97.5, "f1": 97.5}, {"label": "failed_transfer", "n": 40, "correct": 37, "accuracy": 92.5, "precision": 90.2439024390244, "f1": 91.35802469135803}, {"label": "fiat_currency_support", "n": 40, "correct": 36, "accuracy": 90.0, "precision": 87.8048780487805, "f1": 88.88888888888889}, {"label": "get_disposable_virtual_card", "n": 40, "correct": 38, "accuracy": 95.0, "precision": 95.0, "f1": 95.0}, {"label": "get_physical_card", "n": 40, "correct": 40, "accuracy": 100.0, "precision": 95.23809523809524, "f1": 97.5609756097561}, {"label": "getting_spare_card", "n": 40, "correct": 39, "accuracy": 97.5, "precision": 97.5, "f1": 97.5}, {"label": "getting_virtual_card", "n": 40, "correct": 39, "accuracy": 97.5, "precision": 88.63636363636364, "f1": 92.85714285714286}, {"label": "lost_or_stolen_card", "n": 40, "correct": 38, "accuracy": 95.0, "precision": 86.36363636363636, "f1": 90.47619047619048}, {"label": "lost_or_stolen_phone", "n": 40, "correct": 39, "accuracy": 97.5, "precision": 100.0, "f1": 98.73417721518987}, {"label": "order_physical_card", "n": 40, "correct": 37, "accuracy": 92.5, "precision": 92.5, "f1": 92.5}, {"label": "passcode_forgotten", "n": 40, "correct": 40, "accuracy": 100.0, "precision": 97.5609756097561, "f1": 98.76543209876543}, {"label": "pending_card_payment", "n": 40, "correct": 39, "accuracy": 97.5, "precision": 100.0, "f1": 98.73417721518987}, {"label": "pending_cash_withdrawal", "n": 40, "correct": 39, "accuracy": 97.5, "precision": 97.5, "f1": 97.5}, {"label": "pending_top_up", "n": 40, "correct": 36, "accuracy": 90.0, "precision": 94.73684210526316, "f1": 92.3076923076923}, {"label": "pending_transfer", "n": 40, "correct": 34, "accuracy": 85.0, "precision": 82.92682926829268, "f1": 83.95061728395062}, {"label": "pin_blocked", "n": 40, "correct": 36, "accuracy": 90.0, "precision": 94.73684210526316, "f1": 92.3076923076923}, {"label": "receiving_money", "n": 40, "correct": 38, "accuracy": 95.0, "precision": 100.0, "f1": 97.43589743589743}, {"label": "request_refund", "n": 40, "correct": 39, "accuracy": 97.5, "precision": 100.0, "f1": 98.73417721518987}, {"label": "reverted_card_payment?", "n": 40, "correct": 38, "accuracy": 95.0, "precision": 92.6829268292683, "f1": 93.82716049382717}, {"label": "supported_cards_and_currencies", "n": 40, "correct": 37, "accuracy": 92.5, "precision": 92.5, "f1": 92.5}, {"label": "terminate_account", "n": 40, "correct": 40, "accuracy": 100.0, "precision": 97.5609756097561, "f1": 98.76543209876543}, {"label": "top_up_by_bank_transfer_charge", "n": 40, "correct": 35, "accuracy": 87.5, "precision": 92.10526315789474, "f1": 89.74358974358974}, {"label": "top_up_by_card_charge", "n": 40, "correct": 38, "accuracy": 95.0, "precision": 95.0, "f1": 95.0}, {"label": "top_up_by_cash_or_cheque", "n": 40, "correct": 39, "accuracy": 97.5, "precision": 100.0, "f1": 98.73417721518987}, {"label": "top_up_failed", "n": 40, "correct": 37, "accuracy": 92.5, "precision": 90.2439024390244, "f1": 91.35802469135803}, {"label": "top_up_limits", "n": 40, "correct": 40, "accuracy": 100.0, "precision": 97.5609756097561, "f1": 98.76543209876543}, {"label": "top_up_reverted", "n": 40, "correct": 34, "accuracy": 85.0, "precision": 87.17948717948718, "f1": 86.07594936708861}, {"label": "topping_up_by_card", "n": 40, "correct": 33, "accuracy": 82.5, "precision": 86.84210526315789, "f1": 84.61538461538461}, {"label": "transaction_charged_twice", "n": 40, "correct": 40, "accuracy": 100.0, "precision": 93.02325581395348, "f1": 96.3855421686747}, {"label": "transfer_fee_charged", "n": 40, "correct": 37, "accuracy": 92.5, "precision": 88.0952380952381, "f1": 90.2439024390244}, {"label": "transfer_into_account", "n": 40, "correct": 38, "accuracy": 95.0, "precision": 90.47619047619048, "f1": 92.6829268292683}, {"label": "transfer_not_received_by_recipient", "n": 40, "correct": 34, "accuracy": 85.0, "precision": 87.17948717948718, "f1": 86.07594936708861}, {"label": "transfer_timing", "n": 40, "correct": 36, "accuracy": 90.0, "precision": 90.0, "f1": 90.0}, {"label": "unable_to_verify_identity", "n": 40, "correct": 37, "accuracy": 92.5, "precision": 97.36842105263158, "f1": 94.87179487179488}, {"label": "verify_my_identity", "n": 40, "correct": 37, "accuracy": 92.5, "precision": 94.87179487179488, "f1": 93.67088607594937}, {"label": "verify_source_of_funds", "n": 40, "correct": 39, "accuracy": 97.5, "precision": 100.0, "f1": 98.73417721518987}, {"label": "verify_top_up", "n": 40, "correct": 40, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "virtual_card_not_working", "n": 40, "correct": 37, "accuracy": 92.5, "precision": 100.0, "f1": 96.1038961038961}, {"label": "visa_or_mastercard", "n": 40, "correct": 38, "accuracy": 95.0, "precision": 100.0, "f1": 97.43589743589743}, {"label": "why_verify_identity", "n": 40, "correct": 39, "accuracy": 97.5, "precision": 92.85714285714286, "f1": 95.1219512195122}, {"label": "wrong_amount_of_cash_received", "n": 40, "correct": 36, "accuracy": 90.0, "precision": 100.0, "f1": 94.73684210526316}, {"label": "wrong_exchange_rate_for_cash_withdrawal", "n": 40, "correct": 34, "accuracy": 85.0, "precision": 97.14285714285714, "f1": 90.66666666666667}], "confusions": [{"gold": "card_payment_not_recognised", "predicted": "compromised_card", "count": 4}, {"gold": "pending_transfer", "predicted": "balance_not_updated_after_bank_transfer", "count": 4}, {"gold": "card_arrival", "predicted": "card_delivery_estimate", "count": 3}, {"gold": "card_delivery_estimate", "predicted": "card_arrival", "count": 3}, {"gold": "exchange_via_app", "predicted": "fiat_currency_support", "count": 3}, {"gold": "top_up_by_bank_transfer_charge", "predicted": "transfer_fee_charged", "count": 3}, {"gold": "pending_top_up", "predicted": "top_up_reverted", "count": 3}, {"gold": "top_up_reverted", "predicted": "top_up_failed", "count": 3}, {"gold": "compromised_card", "predicted": "lost_or_stolen_card", "count": 3}, {"gold": "wrong_exchange_rate_for_cash_withdrawal", "predicted": "card_payment_wrong_exchange_rate", "count": 3}, {"gold": "balance_not_updated_after_bank_transfer", "predicted": "transfer_timing", "count": 3}, {"gold": "exchange_charge", "predicted": "exchange_rate", "count": 3}, {"gold": "fiat_currency_support", "predicted": "exchange_via_app", "count": 2}, {"gold": "wrong_amount_of_cash_received", "predicted": "declined_cash_withdrawal", "count": 2}, {"gold": "card_payment_fee_charged", "predicted": "direct_debit_payment_not_recognised", "count": 2}]}, "clinc_single": {"name": "CLINC150 \u00b7 \u5355\u6a21\u578b", "n": 4500, "correct": 4399, "accuracy": 97.75555555555556, "classes": [{"label": "accept_reservations", "n": 30, "correct": 29, "accuracy": 96.66666666666667, "precision": 93.54838709677419, "f1": 95.08196721311475}, {"label": "account_blocked", "n": 30, "correct": 29, "accuracy": 96.66666666666667, "precision": 96.66666666666667, "f1": 96.66666666666667}, {"label": "alarm", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 96.7741935483871, "f1": 98.36065573770492}, {"label": "application_status", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "apr", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 96.7741935483871, "f1": 98.36065573770492}, {"label": "are_you_a_bot", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "balance", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 96.7741935483871, "f1": 98.36065573770492}, {"label": "bill_balance", "n": 30, "correct": 28, "accuracy": 93.33333333333333, "precision": 96.55172413793103, "f1": 94.91525423728814}, {"label": "bill_due", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 90.9090909090909, "f1": 95.23809523809524}, {"label": "book_flight", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "book_hotel", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "calculator", "n": 30, "correct": 28, "accuracy": 93.33333333333333, "precision": 96.55172413793103, "f1": 94.91525423728814}, {"label": "calendar", "n": 30, "correct": 27, "accuracy": 90.0, "precision": 93.10344827586206, "f1": 91.52542372881356}, {"label": "calendar_update", "n": 30, "correct": 29, "accuracy": 96.66666666666667, "precision": 96.66666666666667, "f1": 96.66666666666667}, {"label": "calories", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "cancel", "n": 30, "correct": 28, "accuracy": 93.33333333333333, "precision": 93.33333333333333, "f1": 93.33333333333333}, {"label": "cancel_reservation", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 90.9090909090909, "f1": 95.23809523809524}, {"label": "car_rental", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "card_declined", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "carry_on", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "change_accent", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 96.7741935483871, "f1": 98.36065573770492}, {"label": "change_ai_name", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 85.71428571428571, "f1": 92.3076923076923}, {"label": "change_language", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 96.7741935483871, "f1": 98.36065573770492}, {"label": "change_speed", "n": 30, "correct": 29, "accuracy": 96.66666666666667, "precision": 100.0, "f1": 98.30508474576271}, {"label": "change_user_name", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 96.7741935483871, "f1": 98.36065573770492}, {"label": "change_volume", "n": 30, "correct": 24, "accuracy": 80.0, "precision": 100.0, "f1": 88.88888888888889}, {"label": "confirm_reservation", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "cook_time", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "credit_limit", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 96.7741935483871, "f1": 98.36065573770492}, {"label": "credit_limit_change", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "credit_score", "n": 30, "correct": 29, "accuracy": 96.66666666666667, "precision": 100.0, "f1": 98.30508474576271}, {"label": "current_location", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 93.75, "f1": 96.7741935483871}, {"label": "damaged_card", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "date", "n": 30, "correct": 29, "accuracy": 96.66666666666667, "precision": 100.0, "f1": 98.30508474576271}, {"label": "definition", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "direct_deposit", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "directions", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 96.7741935483871, "f1": 98.36065573770492}, {"label": "distance", "n": 30, "correct": 28, "accuracy": 93.33333333333333, "precision": 96.55172413793103, "f1": 94.91525423728814}, {"label": "do_you_have_pets", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "exchange_rate", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "expiration_date", "n": 30, "correct": 29, "accuracy": 96.66666666666667, "precision": 100.0, "f1": 98.30508474576271}, {"label": "find_phone", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "flight_status", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 96.7741935483871, "f1": 98.36065573770492}, {"label": "flip_coin", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "food_last", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "freeze_account", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "fun_fact", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "gas", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "gas_type", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "goodbye", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 96.7741935483871, "f1": 98.36065573770492}, {"label": "greeting", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 96.7741935483871, "f1": 98.36065573770492}, {"label": "how_busy", "n": 30, "correct": 28, "accuracy": 93.33333333333333, "precision": 96.55172413793103, "f1": 94.91525423728814}, {"label": "how_old_are_you", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "improve_credit_score", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 96.7741935483871, "f1": 98.36065573770492}, {"label": "income", "n": 30, "correct": 29, "accuracy": 96.66666666666667, "precision": 100.0, "f1": 98.30508474576271}, {"label": "ingredient_substitution", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "ingredients_list", "n": 30, "correct": 27, "accuracy": 90.0, "precision": 96.42857142857143, "f1": 93.10344827586206}, {"label": "insurance", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "insurance_change", "n": 30, "correct": 29, "accuracy": 96.66666666666667, "precision": 100.0, "f1": 98.30508474576271}, {"label": "interest_rate", "n": 30, "correct": 29, "accuracy": 96.66666666666667, "precision": 100.0, "f1": 98.30508474576271}, {"label": "international_fees", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "international_visa", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "jump_start", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 96.7741935483871, "f1": 98.36065573770492}, {"label": "last_maintenance", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "lost_luggage", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "make_call", "n": 30, "correct": 29, "accuracy": 96.66666666666667, "precision": 96.66666666666667, "f1": 96.66666666666667}, {"label": "maybe", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "meal_suggestion", "n": 30, "correct": 29, "accuracy": 96.66666666666667, "precision": 87.87878787878788, "f1": 92.06349206349206}, {"label": "meaning_of_life", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "measurement_conversion", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "meeting_schedule", "n": 30, "correct": 29, "accuracy": 96.66666666666667, "precision": 93.54838709677419, "f1": 95.08196721311475}, {"label": "min_payment", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "mpg", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "new_card", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 93.75, "f1": 96.7741935483871}, {"label": "next_holiday", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 96.7741935483871, "f1": 98.36065573770492}, {"label": "next_song", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "no", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "nutrition_info", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "oil_change_how", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 93.75, "f1": 96.7741935483871}, {"label": "oil_change_when", "n": 30, "correct": 28, "accuracy": 93.33333333333333, "precision": 90.3225806451613, "f1": 91.80327868852459}, {"label": "order", "n": 30, "correct": 29, "accuracy": 96.66666666666667, "precision": 100.0, "f1": 98.30508474576271}, {"label": "order_checks", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "order_status", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "pay_bill", "n": 30, "correct": 29, "accuracy": 96.66666666666667, "precision": 96.66666666666667, "f1": 96.66666666666667}, {"label": "payday", "n": 30, "correct": 29, "accuracy": 96.66666666666667, "precision": 100.0, "f1": 98.30508474576271}, {"label": "pin_change", "n": 30, "correct": 25, "accuracy": 83.33333333333333, "precision": 100.0, "f1": 90.9090909090909}, {"label": "play_music", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "plug_type", "n": 30, "correct": 29, "accuracy": 96.66666666666667, "precision": 100.0, "f1": 98.30508474576271}, {"label": "pto_balance", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 93.75, "f1": 96.7741935483871}, {"label": "pto_request", "n": 30, "correct": 29, "accuracy": 96.66666666666667, "precision": 100.0, "f1": 98.30508474576271}, {"label": "pto_request_status", "n": 30, "correct": 29, "accuracy": 96.66666666666667, "precision": 100.0, "f1": 98.30508474576271}, {"label": "pto_used", "n": 30, "correct": 28, "accuracy": 93.33333333333333, "precision": 100.0, "f1": 96.55172413793103}, {"label": "recipe", "n": 30, "correct": 28, "accuracy": 93.33333333333333, "precision": 90.3225806451613, "f1": 91.80327868852459}, {"label": "redeem_rewards", "n": 30, "correct": 26, "accuracy": 86.66666666666667, "precision": 100.0, "f1": 92.85714285714286}, {"label": "reminder", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "reminder_update", "n": 30, "correct": 29, "accuracy": 96.66666666666667, "precision": 100.0, "f1": 98.30508474576271}, {"label": "repeat", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "replacement_card_duration", "n": 30, "correct": 27, "accuracy": 90.0, "precision": 96.42857142857143, "f1": 93.10344827586206}, {"label": "report_fraud", "n": 30, "correct": 29, "accuracy": 96.66666666666667, "precision": 87.87878787878788, "f1": 92.06349206349206}, {"label": "report_lost_card", "n": 30, "correct": 28, "accuracy": 93.33333333333333, "precision": 96.55172413793103, "f1": 94.91525423728814}, {"label": "reset_settings", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "restaurant_reservation", "n": 30, "correct": 27, "accuracy": 90.0, "precision": 93.10344827586206, "f1": 91.52542372881356}, {"label": "restaurant_reviews", "n": 30, "correct": 27, "accuracy": 90.0, "precision": 100.0, "f1": 94.73684210526316}, {"label": "restaurant_suggestion", "n": 30, "correct": 29, "accuracy": 96.66666666666667, "precision": 100.0, "f1": 98.30508474576271}, {"label": "rewards_balance", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 88.23529411764706, "f1": 93.75}, {"label": "roll_dice", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "rollover_401k", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "routing", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 96.7741935483871, "f1": 98.36065573770492}, {"label": "schedule_maintenance", "n": 30, "correct": 27, "accuracy": 90.0, "precision": 100.0, "f1": 94.73684210526316}, {"label": "schedule_meeting", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 96.7741935483871, "f1": 98.36065573770492}, {"label": "share_location", "n": 30, "correct": 29, "accuracy": 96.66666666666667, "precision": 100.0, "f1": 98.30508474576271}, {"label": "shopping_list", "n": 30, "correct": 28, "accuracy": 93.33333333333333, "precision": 100.0, "f1": 96.55172413793103}, {"label": "shopping_list_update", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 96.7741935483871, "f1": 98.36065573770492}, {"label": "smart_home", "n": 30, "correct": 29, "accuracy": 96.66666666666667, "precision": 96.66666666666667, "f1": 96.66666666666667}, {"label": "spelling", "n": 30, "correct": 29, "accuracy": 96.66666666666667, "precision": 96.66666666666667, "f1": 96.66666666666667}, {"label": "spending_history", "n": 30, "correct": 29, "accuracy": 96.66666666666667, "precision": 85.29411764705883, "f1": 90.625}, {"label": "sync_device", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "taxes", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "tell_joke", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "text", "n": 30, "correct": 28, "accuracy": 93.33333333333333, "precision": 100.0, "f1": 96.55172413793103}, {"label": "thank_you", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 93.75, "f1": 96.7741935483871}, {"label": "time", "n": 30, "correct": 29, "accuracy": 96.66666666666667, "precision": 96.66666666666667, "f1": 96.66666666666667}, {"label": "timer", "n": 30, "correct": 29, "accuracy": 96.66666666666667, "precision": 96.66666666666667, "f1": 96.66666666666667}, {"label": "timezone", "n": 30, "correct": 29, "accuracy": 96.66666666666667, "precision": 100.0, "f1": 98.30508474576271}, {"label": "tire_change", "n": 30, "correct": 29, "accuracy": 96.66666666666667, "precision": 100.0, "f1": 98.30508474576271}, {"label": "tire_pressure", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 96.7741935483871, "f1": 98.36065573770492}, {"label": "todo_list", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 96.7741935483871, "f1": 98.36065573770492}, {"label": "todo_list_update", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "traffic", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "transactions", "n": 30, "correct": 26, "accuracy": 86.66666666666667, "precision": 96.29629629629629, "f1": 91.2280701754386}, {"label": "transfer", "n": 30, "correct": 29, "accuracy": 96.66666666666667, "precision": 100.0, "f1": 98.30508474576271}, {"label": "translate", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "travel_alert", "n": 30, "correct": 29, "accuracy": 96.66666666666667, "precision": 100.0, "f1": 98.30508474576271}, {"label": "travel_notification", "n": 30, "correct": 29, "accuracy": 96.66666666666667, "precision": 100.0, "f1": 98.30508474576271}, {"label": "travel_suggestion", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 96.7741935483871, "f1": 98.36065573770492}, {"label": "uber", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "update_playlist", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "user_name", "n": 30, "correct": 29, "accuracy": 96.66666666666667, "precision": 100.0, "f1": 98.30508474576271}, {"label": "vaccines", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "w2", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "weather", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 93.75, "f1": 96.7741935483871}, {"label": "what_are_your_hobbies", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 96.7741935483871, "f1": 98.36065573770492}, {"label": "what_can_i_ask_you", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "what_is_your_name", "n": 30, "correct": 28, "accuracy": 93.33333333333333, "precision": 96.55172413793103, "f1": 94.91525423728814}, {"label": "what_song", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "where_are_you_from", "n": 30, "correct": 29, "accuracy": 96.66666666666667, "precision": 100.0, "f1": 98.30508474576271}, {"label": "whisper_mode", "n": 30, "correct": 29, "accuracy": 96.66666666666667, "precision": 87.87878787878788, "f1": 92.06349206349206}, {"label": "who_do_you_work_for", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 93.75, "f1": 96.7741935483871}, {"label": "who_made_you", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 96.7741935483871, "f1": 98.36065573770492}, {"label": "yes", "n": 30, "correct": 26, "accuracy": 86.66666666666667, "precision": 100.0, "f1": 92.85714285714286}], "confusions": [{"gold": "pin_change", "predicted": "change_ai_name", "count": 5}, {"gold": "change_volume", "predicted": "whisper_mode", "count": 4}, {"gold": "redeem_rewards", "predicted": "rewards_balance", "count": 4}, {"gold": "transactions", "predicted": "spending_history", "count": 3}, {"gold": "schedule_maintenance", "predicted": "oil_change_when", "count": 3}, {"gold": "ingredients_list", "predicted": "recipe", "count": 3}, {"gold": "restaurant_reservation", "predicted": "accept_reservations", "count": 2}, {"gold": "oil_change_when", "predicted": "oil_change_how", "count": 2}, {"gold": "yes", "predicted": "thank_you", "count": 2}, {"gold": "calendar", "predicted": "meeting_schedule", "count": 2}, {"gold": "calculator", "predicted": "spending_history", "count": 2}, {"gold": "pto_used", "predicted": "pto_balance", "count": 2}, {"gold": "report_lost_card", "predicted": "report_fraud", "count": 2}, {"gold": "restaurant_reviews", "predicted": "meal_suggestion", "count": 2}, {"gold": "what_is_your_name", "predicted": "who_do_you_work_for", "count": 2}]}, "clinc_ensemble": {"name": "CLINC150 \u00b7 \u56db\u7f16\u7801\u5668\u96c6\u6210", "n": 4500, "correct": 4411, "accuracy": 98.02222222222223, "classes": [{"label": "accept_reservations", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 93.75, "f1": 96.7741935483871}, {"label": "account_blocked", "n": 30, "correct": 29, "accuracy": 96.66666666666667, "precision": 96.66666666666667, "f1": 96.66666666666667}, {"label": "alarm", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 96.7741935483871, "f1": 98.36065573770492}, {"label": "application_status", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "apr", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 96.7741935483871, "f1": 98.36065573770492}, {"label": "are_you_a_bot", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "balance", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "bill_balance", "n": 30, "correct": 28, "accuracy": 93.33333333333333, "precision": 96.55172413793103, "f1": 94.91525423728814}, {"label": "bill_due", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 90.9090909090909, "f1": 95.23809523809524}, {"label": "book_flight", "n": 30, "correct": 29, "accuracy": 96.66666666666667, "precision": 100.0, "f1": 98.30508474576271}, {"label": "book_hotel", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "calculator", "n": 30, "correct": 28, "accuracy": 93.33333333333333, "precision": 96.55172413793103, "f1": 94.91525423728814}, {"label": "calendar", "n": 30, "correct": 27, "accuracy": 90.0, "precision": 96.42857142857143, "f1": 93.10344827586206}, {"label": "calendar_update", "n": 30, "correct": 29, "accuracy": 96.66666666666667, "precision": 96.66666666666667, "f1": 96.66666666666667}, {"label": "calories", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "cancel", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 93.75, "f1": 96.7741935483871}, {"label": "cancel_reservation", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 93.75, "f1": 96.7741935483871}, {"label": "car_rental", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 96.7741935483871, "f1": 98.36065573770492}, {"label": "card_declined", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "carry_on", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "change_accent", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "change_ai_name", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 85.71428571428571, "f1": 92.3076923076923}, {"label": "change_language", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "change_speed", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "change_user_name", "n": 30, "correct": 29, "accuracy": 96.66666666666667, "precision": 100.0, "f1": 98.30508474576271}, {"label": "change_volume", "n": 30, "correct": 25, "accuracy": 83.33333333333333, "precision": 100.0, "f1": 90.9090909090909}, {"label": "confirm_reservation", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "cook_time", "n": 30, "correct": 29, "accuracy": 96.66666666666667, "precision": 100.0, "f1": 98.30508474576271}, {"label": "credit_limit", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 96.7741935483871, "f1": 98.36065573770492}, {"label": "credit_limit_change", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "credit_score", "n": 30, "correct": 29, "accuracy": 96.66666666666667, "precision": 100.0, "f1": 98.30508474576271}, {"label": "current_location", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 96.7741935483871, "f1": 98.36065573770492}, {"label": "damaged_card", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "date", "n": 30, "correct": 29, "accuracy": 96.66666666666667, "precision": 100.0, "f1": 98.30508474576271}, {"label": "definition", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "direct_deposit", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "directions", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "distance", "n": 30, "correct": 28, "accuracy": 93.33333333333333, "precision": 100.0, "f1": 96.55172413793103}, {"label": "do_you_have_pets", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "exchange_rate", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "expiration_date", "n": 30, "correct": 29, "accuracy": 96.66666666666667, "precision": 100.0, "f1": 98.30508474576271}, {"label": "find_phone", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "flight_status", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 93.75, "f1": 96.7741935483871}, {"label": "flip_coin", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "food_last", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "freeze_account", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "fun_fact", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "gas", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "gas_type", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "goodbye", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "greeting", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "how_busy", "n": 30, "correct": 29, "accuracy": 96.66666666666667, "precision": 96.66666666666667, "f1": 96.66666666666667}, {"label": "how_old_are_you", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "improve_credit_score", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 96.7741935483871, "f1": 98.36065573770492}, {"label": "income", "n": 30, "correct": 29, "accuracy": 96.66666666666667, "precision": 100.0, "f1": 98.30508474576271}, {"label": "ingredient_substitution", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "ingredients_list", "n": 30, "correct": 27, "accuracy": 90.0, "precision": 96.42857142857143, "f1": 93.10344827586206}, {"label": "insurance", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "insurance_change", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "interest_rate", "n": 30, "correct": 29, "accuracy": 96.66666666666667, "precision": 100.0, "f1": 98.30508474576271}, {"label": "international_fees", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "international_visa", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "jump_start", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 96.7741935483871, "f1": 98.36065573770492}, {"label": "last_maintenance", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "lost_luggage", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "make_call", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 96.7741935483871, "f1": 98.36065573770492}, {"label": "maybe", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "meal_suggestion", "n": 30, "correct": 29, "accuracy": 96.66666666666667, "precision": 87.87878787878788, "f1": 92.06349206349206}, {"label": "meaning_of_life", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "measurement_conversion", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "meeting_schedule", "n": 30, "correct": 29, "accuracy": 96.66666666666667, "precision": 93.54838709677419, "f1": 95.08196721311475}, {"label": "min_payment", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "mpg", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "new_card", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 96.7741935483871, "f1": 98.36065573770492}, {"label": "next_holiday", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "next_song", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "no", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "nutrition_info", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "oil_change_how", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 93.75, "f1": 96.7741935483871}, {"label": "oil_change_when", "n": 30, "correct": 28, "accuracy": 93.33333333333333, "precision": 90.3225806451613, "f1": 91.80327868852459}, {"label": "order", "n": 30, "correct": 29, "accuracy": 96.66666666666667, "precision": 100.0, "f1": 98.30508474576271}, {"label": "order_checks", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "order_status", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "pay_bill", "n": 30, "correct": 29, "accuracy": 96.66666666666667, "precision": 96.66666666666667, "f1": 96.66666666666667}, {"label": "payday", "n": 30, "correct": 29, "accuracy": 96.66666666666667, "precision": 100.0, "f1": 98.30508474576271}, {"label": "pin_change", "n": 30, "correct": 25, "accuracy": 83.33333333333333, "precision": 100.0, "f1": 90.9090909090909}, {"label": "play_music", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "plug_type", "n": 30, "correct": 29, "accuracy": 96.66666666666667, "precision": 100.0, "f1": 98.30508474576271}, {"label": "pto_balance", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 93.75, "f1": 96.7741935483871}, {"label": "pto_request", "n": 30, "correct": 29, "accuracy": 96.66666666666667, "precision": 96.66666666666667, "f1": 96.66666666666667}, {"label": "pto_request_status", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "pto_used", "n": 30, "correct": 28, "accuracy": 93.33333333333333, "precision": 100.0, "f1": 96.55172413793103}, {"label": "recipe", "n": 30, "correct": 28, "accuracy": 93.33333333333333, "precision": 90.3225806451613, "f1": 91.80327868852459}, {"label": "redeem_rewards", "n": 30, "correct": 26, "accuracy": 86.66666666666667, "precision": 100.0, "f1": 92.85714285714286}, {"label": "reminder", "n": 30, "correct": 29, "accuracy": 96.66666666666667, "precision": 100.0, "f1": 98.30508474576271}, {"label": "reminder_update", "n": 30, "correct": 29, "accuracy": 96.66666666666667, "precision": 100.0, "f1": 98.30508474576271}, {"label": "repeat", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "replacement_card_duration", "n": 30, "correct": 27, "accuracy": 90.0, "precision": 96.42857142857143, "f1": 93.10344827586206}, {"label": "report_fraud", "n": 30, "correct": 29, "accuracy": 96.66666666666667, "precision": 87.87878787878788, "f1": 92.06349206349206}, {"label": "report_lost_card", "n": 30, "correct": 28, "accuracy": 93.33333333333333, "precision": 96.55172413793103, "f1": 94.91525423728814}, {"label": "reset_settings", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "restaurant_reservation", "n": 30, "correct": 28, "accuracy": 93.33333333333333, "precision": 96.55172413793103, "f1": 94.91525423728814}, {"label": "restaurant_reviews", "n": 30, "correct": 27, "accuracy": 90.0, "precision": 100.0, "f1": 94.73684210526316}, {"label": "restaurant_suggestion", "n": 30, "correct": 29, "accuracy": 96.66666666666667, "precision": 100.0, "f1": 98.30508474576271}, {"label": "rewards_balance", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 88.23529411764706, "f1": 93.75}, {"label": "roll_dice", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "rollover_401k", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "routing", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 96.7741935483871, "f1": 98.36065573770492}, {"label": "schedule_maintenance", "n": 30, "correct": 27, "accuracy": 90.0, "precision": 100.0, "f1": 94.73684210526316}, {"label": "schedule_meeting", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 96.7741935483871, "f1": 98.36065573770492}, {"label": "share_location", "n": 30, "correct": 29, "accuracy": 96.66666666666667, "precision": 100.0, "f1": 98.30508474576271}, {"label": "shopping_list", "n": 30, "correct": 29, "accuracy": 96.66666666666667, "precision": 100.0, "f1": 98.30508474576271}, {"label": "shopping_list_update", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 96.7741935483871, "f1": 98.36065573770492}, {"label": "smart_home", "n": 30, "correct": 29, "accuracy": 96.66666666666667, "precision": 100.0, "f1": 98.30508474576271}, {"label": "spelling", "n": 30, "correct": 29, "accuracy": 96.66666666666667, "precision": 96.66666666666667, "f1": 96.66666666666667}, {"label": "spending_history", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 85.71428571428571, "f1": 92.3076923076923}, {"label": "sync_device", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "taxes", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "tell_joke", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "text", "n": 30, "correct": 28, "accuracy": 93.33333333333333, "precision": 100.0, "f1": 96.55172413793103}, {"label": "thank_you", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 96.7741935483871, "f1": 98.36065573770492}, {"label": "time", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 96.7741935483871, "f1": 98.36065573770492}, {"label": "timer", "n": 30, "correct": 29, "accuracy": 96.66666666666667, "precision": 93.54838709677419, "f1": 95.08196721311475}, {"label": "timezone", "n": 30, "correct": 29, "accuracy": 96.66666666666667, "precision": 100.0, "f1": 98.30508474576271}, {"label": "tire_change", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "tire_pressure", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "todo_list", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 96.7741935483871, "f1": 98.36065573770492}, {"label": "todo_list_update", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "traffic", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "transactions", "n": 30, "correct": 26, "accuracy": 86.66666666666667, "precision": 96.29629629629629, "f1": 91.2280701754386}, {"label": "transfer", "n": 30, "correct": 29, "accuracy": 96.66666666666667, "precision": 100.0, "f1": 98.30508474576271}, {"label": "translate", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 96.7741935483871, "f1": 98.36065573770492}, {"label": "travel_alert", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "travel_notification", "n": 30, "correct": 29, "accuracy": 96.66666666666667, "precision": 100.0, "f1": 98.30508474576271}, {"label": "travel_suggestion", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 93.75, "f1": 96.7741935483871}, {"label": "uber", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "update_playlist", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "user_name", "n": 30, "correct": 29, "accuracy": 96.66666666666667, "precision": 96.66666666666667, "f1": 96.66666666666667}, {"label": "vaccines", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "w2", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "weather", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 96.7741935483871, "f1": 98.36065573770492}, {"label": "what_are_your_hobbies", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "what_can_i_ask_you", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "what_is_your_name", "n": 30, "correct": 28, "accuracy": 93.33333333333333, "precision": 96.55172413793103, "f1": 94.91525423728814}, {"label": "what_song", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "where_are_you_from", "n": 30, "correct": 29, "accuracy": 96.66666666666667, "precision": 100.0, "f1": 98.30508474576271}, {"label": "whisper_mode", "n": 30, "correct": 29, "accuracy": 96.66666666666667, "precision": 87.87878787878788, "f1": 92.06349206349206}, {"label": "who_do_you_work_for", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 93.75, "f1": 96.7741935483871}, {"label": "who_made_you", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 96.7741935483871, "f1": 98.36065573770492}, {"label": "yes", "n": 30, "correct": 27, "accuracy": 90.0, "precision": 100.0, "f1": 94.73684210526316}], "confusions": [{"gold": "pin_change", "predicted": "change_ai_name", "count": 5}, {"gold": "change_volume", "predicted": "whisper_mode", "count": 4}, {"gold": "redeem_rewards", "predicted": "rewards_balance", "count": 4}, {"gold": "transactions", "predicted": "spending_history", "count": 3}, {"gold": "schedule_maintenance", "predicted": "oil_change_when", "count": 3}, {"gold": "ingredients_list", "predicted": "recipe", "count": 3}, {"gold": "restaurant_reservation", "predicted": "accept_reservations", "count": 2}, {"gold": "distance", "predicted": "flight_status", "count": 2}, {"gold": "oil_change_when", "predicted": "oil_change_how", "count": 2}, {"gold": "calendar", "predicted": "meeting_schedule", "count": 2}, {"gold": "calculator", "predicted": "spending_history", "count": 2}, {"gold": "pto_used", "predicted": "pto_balance", "count": 2}, {"gold": "report_lost_card", "predicted": "report_fraud", "count": 2}, {"gold": "restaurant_reviews", "predicted": "meal_suggestion", "count": 2}, {"gold": "what_is_your_name", "predicted": "who_do_you_work_for", "count": 2}]}, "massive": {"name": "MASSIVE \u00b7 \u4e00\u81f4\u6027\u65b9\u6cd5 \u00b7 \u4e09\u79cd\u5b50\u5408\u5e76\u8bb0\u5f55", "n": 455022, "correct": 391467, "accuracy": 86.03254348141408, "classes": [{"label": "alarm_query", "n": 5202, "correct": 4809, "accuracy": 92.44521337946944, "precision": 96.56626506024097, "f1": 94.46081319976429}, {"label": "alarm_remove", "n": 3213, "correct": 3081, "accuracy": 95.89169000933707, "precision": 94.65437788018433, "f1": 95.26901669758813}, {"label": "alarm_set", "n": 6273, "correct": 5686, "accuracy": 90.64243583612307, "precision": 90.38308694961056, "f1": 90.51257561286215}, {"label": "audio_volume_down", "n": 1683, "correct": 1523, "accuracy": 90.4931669637552, "precision": 88.44367015098723, "f1": 89.45668135095448}, {"label": "audio_volume_mute", "n": 4896, "correct": 4452, "accuracy": 90.93137254901961, "precision": 83.52720450281426, "f1": 87.07216898102875}, {"label": "audio_volume_other", "n": 918, "correct": 489, "accuracy": 53.26797385620915, "precision": 88.26714801444044, "f1": 66.44021739130434}, {"label": "audio_volume_up", "n": 1989, "correct": 1554, "accuracy": 78.12971342383108, "precision": 71.1864406779661, "f1": 74.49664429530202}, {"label": "calendar_query", "n": 19278, "correct": 15782, "accuracy": 81.86533872808383, "precision": 75.30658014028725, "f1": 78.44911147011308}, {"label": "calendar_remove", "n": 10251, "correct": 9657, "accuracy": 94.20544337137841, "precision": 87.7829288246523, "f1": 90.88085827216263}, {"label": "calendar_set", "n": 31977, "correct": 29012, "accuracy": 90.72771054195202, "precision": 90.89541951250078, "f1": 90.81148759683856}, {"label": "cooking_query", "n": 0, "correct": 0, "accuracy": 0, "precision": 0.0, "f1": 0.0}, {"label": "cooking_recipe", "n": 11016, "correct": 9451, "accuracy": 85.79339143064634, "precision": 91.59720876138786, "f1": 88.60035623886753}, {"label": "datetime_convert", "n": 2295, "correct": 1790, "accuracy": 77.99564270152506, "precision": 91.18695873662762, "f1": 84.0770314701738}, {"label": "datetime_query", "n": 13464, "correct": 12780, "accuracy": 94.91978609625669, "precision": 90.8703071672355, "f1": 92.8509154315606}, {"label": "email_addcontact", "n": 1836, "correct": 1507, "accuracy": 82.0806100217865, "precision": 66.15452151009657, "f1": 73.2620320855615}, {"label": "email_query", "n": 18207, "correct": 16877, "accuracy": 92.69511726259131, "precision": 96.96638896868716, "f1": 94.78265753116926}, {"label": "email_querycontact", "n": 3978, "correct": 3174, "accuracy": 79.78883861236802, "precision": 72.18558107800773, "f1": 75.79701492537313}, {"label": "email_sendemail", "n": 17442, "correct": 16255, "accuracy": 93.19458777663112, "precision": 95.01402852466683, "f1": 94.09551374819102}, {"label": "general_greet", "n": 153, "correct": 48, "accuracy": 31.372549019607842, "precision": 9.6579476861167, "f1": 14.76923076923077}, {"label": "general_joke", "n": 2907, "correct": 2644, "accuracy": 90.95287237702098, "precision": 94.80100394406597, "f1": 92.83707865168539}, {"label": "general_quirky", "n": 25857, "correct": 14801, "accuracy": 57.24175271686584, "precision": 65.66839700075425, "f1": 61.16621208364327}, {"label": "iot_cleaning", "n": 3978, "correct": 3653, "accuracy": 91.83006535947712, "precision": 97.43931715124033, "f1": 94.55157240843795}, {"label": "iot_coffee", "n": 5508, "correct": 5316, "accuracy": 96.51416122004358, "precision": 92.45217391304348, "f1": 94.4395096820039}, {"label": "iot_hue_lightchange", "n": 5508, "correct": 5092, "accuracy": 92.44734931009441, "precision": 84.92328218812541, "f1": 88.52573018080668}, {"label": "iot_hue_lightdim", "n": 3213, "correct": 2670, "accuracy": 83.0999066293184, "precision": 88.79281676089126, "f1": 85.85209003215434}, {"label": "iot_hue_lightoff", "n": 6579, "correct": 6118, "accuracy": 92.99285605715154, "precision": 92.80946601941747, "f1": 92.90107053374838}, {"label": "iot_hue_lighton", "n": 459, "correct": 239, "accuracy": 52.069716775599126, "precision": 45.78544061302682, "f1": 48.72579001019368}, {"label": "iot_hue_lightup", "n": 4131, "correct": 3432, "accuracy": 83.07915758896151, "precision": 91.0828025477707, "f1": 86.89707557918724}, {"label": "iot_wemo_off", "n": 2754, "correct": 2319, "accuracy": 84.20479302832244, "precision": 95.94538684319404, "f1": 89.69251595436086}, {"label": "iot_wemo_on", "n": 1530, "correct": 1191, "accuracy": 77.84313725490196, "precision": 82.19461697722568, "f1": 79.95971802618328}, {"label": "lists_createoradd", "n": 5967, "correct": 5139, "accuracy": 86.1236802413273, "precision": 85.57868442964197, "f1": 85.85031740728367}, {"label": "lists_query", "n": 7803, "correct": 6544, "accuracy": 83.86518005895168, "precision": 87.83892617449665, "f1": 85.80607093686488}, {"label": "lists_remove", "n": 7956, "correct": 6770, "accuracy": 85.0930115635998, "precision": 95.41930937279774, "f1": 89.96079994684739}, {"label": "music_dislikeness", "n": 612, "correct": 427, "accuracy": 69.77124183006536, "precision": 85.4, "f1": 76.79856115107914}, {"label": "music_likeness", "n": 5508, "correct": 4395, "accuracy": 79.7930283224401, "precision": 88.60887096774194, "f1": 83.97019487963317}, {"label": "music_query", "n": 5355, "correct": 4351, "accuracy": 81.25116713352007, "precision": 89.54517390409549, "f1": 85.19678872136284}, {"label": "music_settings", "n": 918, "correct": 356, "accuracy": 38.77995642701525, "precision": 43.520782396088016, "f1": 41.013824884792626}, {"label": "news_query", "n": 18972, "correct": 16133, "accuracy": 85.03584229390681, "precision": 83.34452652787105, "f1": 84.18169010409872}, {"label": "play_audiobook", "n": 6273, "correct": 5064, "accuracy": 80.726924916308, "precision": 79.52261306532664, "f1": 80.12024365160984}, {"label": "play_game", "n": 5355, "correct": 4346, "accuracy": 81.1577964519141, "precision": 89.03913132554804, "f1": 84.91598280578351}, {"label": "play_music", "n": 26928, "correct": 24078, "accuracy": 89.4162210338681, "precision": 86.23925501432664, "f1": 87.79900816802801}, {"label": "play_podcasts", "n": 9639, "correct": 8684, "accuracy": 90.09233322958814, "precision": 92.55035702866887, "f1": 91.3048049626748}, {"label": "play_radio", "n": 11016, "correct": 9817, "accuracy": 89.1158315177923, "precision": 84.1216795201371, "f1": 86.5467689323812}, {"label": "qa_currency", "n": 5967, "correct": 5745, "accuracy": 96.27953745600804, "precision": 92.49718241829014, "f1": 94.35046805715224}, {"label": "qa_definition", "n": 8721, "correct": 7363, "accuracy": 84.42839123953675, "precision": 84.55443270555811, "f1": 84.49136496643526}, {"label": "qa_factoid", "n": 21573, "correct": 17786, "accuracy": 82.44564965466091, "precision": 78.22491973435369, "f1": 80.2798465357707}, {"label": "qa_maths", "n": 3825, "correct": 3408, "accuracy": 89.09803921568627, "precision": 89.09803921568627, "f1": 89.09803921568627}, {"label": "qa_stock", "n": 3978, "correct": 3706, "accuracy": 93.16239316239316, "precision": 82.11832483935298, "f1": 87.29242727593923}, {"label": "recommendation_events", "n": 6579, "correct": 4971, "accuracy": 75.55859553123575, "precision": 76.71296296296296, "f1": 76.13140362968068}, {"label": "recommendation_locations", "n": 4743, "correct": 4290, "accuracy": 90.44908285895004, "precision": 76.171875, "f1": 82.6987951807229}, {"label": "recommendation_movies", "n": 3060, "correct": 2458, "accuracy": 80.32679738562092, "precision": 69.25894618202311, "f1": 74.38341655318506}, {"label": "social_post", "n": 12393, "correct": 11072, "accuracy": 89.34075687888324, "precision": 96.15284411636995, "f1": 92.62171658022419}, {"label": "social_query", "n": 3825, "correct": 3146, "accuracy": 82.2483660130719, "precision": 83.20550119016133, "f1": 82.7241651327899}, {"label": "takeaway_order", "n": 3366, "correct": 2900, "accuracy": 86.15567439096851, "precision": 77.97795106211348, "f1": 81.86309103740297}, {"label": "takeaway_query", "n": 5355, "correct": 4251, "accuracy": 79.38375350140056, "precision": 92.59420605532564, "f1": 85.48160064347476}, {"label": "transport_query", "n": 7803, "correct": 5941, "accuracy": 76.13738305779829, "precision": 83.34736251402919, "f1": 79.57939856674034}, {"label": "transport_taxi", "n": 3519, "correct": 3327, "accuracy": 94.54390451832907, "precision": 90.43218265833107, "f1": 92.44234509585996}, {"label": "transport_ticket", "n": 5355, "correct": 4946, "accuracy": 92.36227824463118, "precision": 94.04829815554288, "f1": 93.1976634633503}, {"label": "transport_traffic", "n": 2295, "correct": 2188, "accuracy": 95.33769063180829, "precision": 71.95001644195987, "f1": 82.00899550224888}, {"label": "weather_query", "n": 23868, "correct": 22463, "accuracy": 94.11345734875147, "precision": 92.39850273538727, "f1": 93.24809564333007}], "confusions": [{"gold": "general_quirky", "predicted": "qa_factoid", "count": 2270}, {"gold": "general_quirky", "predicted": "calendar_query", "count": 1455}, {"gold": "general_quirky", "predicted": "news_query", "count": 1428}, {"gold": "calendar_set", "predicted": "calendar_query", "count": 1372}, {"gold": "qa_factoid", "predicted": "general_quirky", "count": 1292}, {"gold": "calendar_query", "predicted": "calendar_set", "count": 909}, {"gold": "play_radio", "predicted": "play_music", "count": 900}, {"gold": "news_query", "predicted": "general_quirky", "count": 664}, {"gold": "lists_remove", "predicted": "calendar_remove", "count": 619}, {"gold": "email_sendemail", "predicted": "email_addcontact", "count": 606}, {"gold": "calendar_query", "predicted": "recommendation_events", "count": 600}, {"gold": "weather_query", "predicted": "general_quirky", "count": 597}, {"gold": "play_music", "predicted": "play_radio", "count": 597}, {"gold": "general_quirky", "predicted": "weather_query", "count": 591}, {"gold": "recommendation_events", "predicted": "calendar_query", "count": 523}]}};
15
- const refs={bank:[['SPACE',94.94],['SPACE 2.0',94.77],['Llama 2 7B (FT)',94.35],['ConvFiT',94.16],['Laya \u00b7 \u5355\u6a21\u578b',94.0909],['PPTOD',93.86]],clinc:[['TOATOD-small',98.45],['Laya \u00b7 \u56db\u7f16\u7801\u5668\u96c6\u6210',98.0222],['TOATOD-base',98.01],['SPACE',97.89],['SPACE 2.0',97.8],['Laya \u00b7 \u5355\u6a21\u578b',97.7556]],massive:[['Laya \u00b7 \u6570\u636e\u589e\u5f3a',85.9873],['Laya \u00b7 \u4e00\u81f4\u6027\u65b9\u6cd5',86.0325]]};
16
- const notes={bank:'<h3>\u4e0e SPACE \u53c2\u8003\u503c\u76f8\u5dee 0.8491 \u4e2a\u767e\u5206\u70b9</h3><p>Laya\uff1a94.0909%\uff1b\u516c\u5f00\u53c2\u8003 SPACE\uff1a94.94%\u3002\u8fd9\u662f\u9009\u5b9a\u6761\u76ee\u7684\u6570\u503c\u5bf9\u7167\uff0c\u4e0d\u662f\u7edf\u4e00\u91cd\u6d4b\u6216\u5b98\u65b9\u6392\u540d\u3002</p><p>SOTA2 \u539f\u9875\u9762\u5b58\u5728\u5927\u4e8e 100% \u7684\u5f02\u5e38\u51c6\u786e\u7387\u8bb0\u5f55\uff0c\u672c\u9875\u672a\u7eb3\u5165\u3002\u5c55\u793a\u9009\u5b9a\u7684\u5e38\u89c1\u76d1\u7763\u5206\u7c7b\u53c2\u8003\uff0c\u975e\u5b8c\u6574\u699c\u5355\u3002</p>',clinc:'<h3>\u96c6\u6210 98.0222%\uff0c\u5355\u6a21\u578b 97.7556%</h3><p>\u96c6\u6210\u8f83\u672c\u9879\u76ee\u5355\u6a21\u578b\u591a\u6b63\u786e 12 \u6761\uff1b\u4e0e TOATOD-small \u53c2\u8003\u5206\u6570\u76f8\u5dee 0.4278 \u4e2a\u767e\u5206\u70b9\u3002</p><p>\u56db\u7f16\u7801\u5668\u96c6\u6210\u4e0e\u5355\u6a21\u578b\u8ba1\u7b97\u6210\u672c\u4e0d\u540c\uff0c\u5206\u522b\u6807\u6ce8\uff0c\u4e0d\u5408\u5e76\u6210\u540c\u4e00\u6a21\u578b\u7684\u6210\u7ee9\u3002</p>',massive:'<h3>51 \u8bed\u8a00\u7ed3\u679c\uff0c\u72ec\u7acb\u5c55\u793a</h3><p>\u4e24\u65b9\u6cd5\u5747\u4e3a\u4e09\u79cd\u5b50 test \u5e73\u5747\u3002\u4e00\u81f4\u6027\u76f8\u5bf9\u589e\u5f3a\u589e\u52a0\u7ea6 0.0452 \u4e2a\u767e\u5206\u70b9\uff0c95% \u533a\u95f4\u5305\u542b 0\uff0c\u672a\u8bc1\u660e\u663e\u8457\u63d0\u5347\u3002</p><p>SOTA2 \u5bf9\u5e94\u53c2\u8003\u91c7\u7528\u82f1\u8bed\u8bbe\u7f6e\uff0c\u8bad\u7ec3\u6761\u4ef6\u4e0d\u540c\uff0c\u672c\u9875\u4e0d\u5c06 51 \u8bed\u8a00\u5747\u503c\u4e0e\u82f1\u8bed\u5206\u6570\u6392\u5217\u540d\u6b21\u3002</p>'};
17
- function compare(key){document.querySelectorAll('[data-task]').forEach(b=>b.classList.toggle('active',b.dataset.task===key));document.getElementById('charttitle').textContent=key==='massive'?'51 \u8bed\u8a00 \u00b7 \u4e09\u79cd\u5b50\u5e73\u5747\u51c6\u786e\u7387':'\u6d4b\u8bd5\u51c6\u786e\u7387 \u00b7 \u4f5c\u8005\u7ed3\u679c\u4e0e\u516c\u5f00\u53c2\u8003';document.getElementById('bars').innerHTML=refs[key].map(([n,v])=>`<div class="barrow ${n.startsWith('Laya')?'ours':''}"><span>${n}</span><div class="track"><div class="fill" style="width:${v}%"></div></div><strong>${v.toFixed(n.startsWith('Laya')?4:2)}%</strong></div>`).join('');document.getElementById('compareNote').innerHTML=notes[key]}
18
  function render(){const k=document.getElementById('dataset').value,d=DATA[k],q=document.getElementById('search').value.toLowerCase();document.getElementById('resultstat').innerHTML=`<h3>${d.name}</h3><div class="bigstat">${d.correct.toLocaleString()} <span style="font-size:15px;color:var(--muted)">/ ${d.n.toLocaleString()} \u6761\u9884\u6d4b\u6b63\u786e \u00b7 ${d.classes.length} \u7c7b \u00b7 \u51c6\u786e\u7387 ${d.accuracy.toFixed(4)}%</span></div>`;let rows=d.classes.filter(x=>x.label.toLowerCase().includes(q)).sort(document.getElementById('sort').value==='label'?(a,b)=>a.label.localeCompare(b.label):(a,b)=>a.accuracy-b.accuracy||a.label.localeCompare(b.label));document.getElementById('classes').innerHTML=rows.length?rows.map(x=>`<tr><td>${x.label}</td><td>${x.correct} / ${x.n}</td><td><span class="mini"><i style="width:${x.accuracy}%"></i></span>${x.accuracy.toFixed(2)}%</td><td>${x.precision.toFixed(2)}%</td><td>${x.f1.toFixed(2)}%</td></tr>`).join(''):'<tr><td colspan="5">\u6ca1\u6709\u5339\u914d\u7684\u7c7b\u522b\uff0c\u8bf7\u66f4\u6362\u641c\u7d22\u8bcd\u3002</td></tr>';document.getElementById('predDownload').href=k+'_predictions.csv'+(k==='massive'?'.gz':'');document.getElementById('classDownload').href=k+'_classes.csv';document.getElementById('confusions').innerHTML='<table><tr><th>\u771f\u5b9e\u7c7b\u522b</th><th>\u9884\u6d4b\u4e3a</th><th>\u6761\u6570</th></tr>'+d.confusions.map(x=>`<tr><td>${x.gold}</td><td>${x.predicted}</td><td>${x.count}</td></tr>`).join('')+'</table>'}
19
  document.querySelectorAll('[data-task]').forEach(b=>b.onclick=()=>compare(b.dataset.task));['dataset','search','sort'].forEach(id=>document.getElementById(id).addEventListener('input',render));compare('bank');render();
20
  </script></body></html>
 
1
+ <!doctype html><html lang="zh-CN"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width,initial-scale=1"><title>deepopen &#183; &#24847;&#22270;&#20998;&#31867;&#35780;&#27979;</title>
2
  <style>
3
  :root{--ink:#142c37;--muted:#647981;--line:#dce5e5;--teal:#007b76;--bg:#f3f6f5}*{box-sizing:border-box}body{margin:0;background:var(--bg);font:15px/1.6 Inter,"Segoe UI","Microsoft YaHei",sans-serif;color:var(--ink)}header{padding:22px 5%;border-bottom:1px solid var(--line);display:flex;align-items:center;justify-content:space-between;background:white}.brand{font-weight:800;font-size:25px;letter-spacing:-1px}.brand span{color:var(--teal)}nav a{margin-left:24px;color:var(--muted);text-decoration:none;font-size:13px}main{max-width:1280px;margin:auto;padding:48px 30px}small,.eyebrow{letter-spacing:2px;font-size:11px;font-weight:700;color:var(--teal)}h1{font-size:46px;line-height:1.2;margin:14px 0;letter-spacing:-2px}h2{font-size:24px;margin:0}h3{font-size:16px;margin:0}.sub{color:var(--muted);max-width:750px}.status{display:inline-block;background:#e2efeb;color:#32665e;padding:5px 11px;border-radius:20px;font-size:12px}.hero{display:flex;justify-content:space-between;align-items:center;margin-bottom:30px}.hero-right{text-align:right;font-size:12px;color:var(--muted)}.cards{display:grid;grid-template-columns:repeat(4,1fr);gap:16px}.card{background:white;border:1px solid var(--line);border-radius:14px;padding:23px}.card.featured{background:#103e43;color:white;border-color:#103e43}.value{font-size:38px;font-weight:750;letter-spacing:-1.7px;margin:14px 0 4px}.value em{font-size:18px;font-style:normal;letter-spacing:0}.caption{font-size:12px;color:var(--muted)}.featured .caption{color:#b9d5d1}.rule{margin:16px 0 36px;font-size:12px;color:var(--muted)}section{margin-top:38px}.heading{display:flex;justify-content:space-between;gap:20px;align-items:center;margin-bottom:18px}.panel{background:white;border:1px solid var(--line);border-radius:14px;padding:25px}.tabs{display:flex;gap:7px;flex-wrap:wrap}button,select,input{font:inherit;border:1px solid var(--line);border-radius:8px;padding:9px 13px;background:white;color:var(--ink)}button{cursor:pointer}button.active{background:var(--ink);color:white;border-color:var(--ink)}.split{display:grid;grid-template-columns:1.4fr 1fr;gap:24px}.barrow{display:grid;grid-template-columns:190px 1fr 84px;align-items:center;gap:12px;margin:17px 0;font-size:13px}.track{height:10px;border-radius:5px;background:#edf1f2;overflow:hidden}.fill{height:100%;background:#abbcc4;border-radius:5px}.ours .fill{background:var(--teal)}.ours{font-weight:700;color:var(--teal)}.barrow strong{text-align:right}.note{font-size:12px;color:var(--muted);margin-top:15px}.callout{background:#f3f7f6;border-radius:10px;padding:20px}.callout p{font-size:13px;color:var(--muted)}a{color:var(--teal);text-underline-offset:3px}.toolbar{display:flex;gap:10px;flex-wrap:wrap;margin:16px 0}.toolbar input{flex:1;min-width:160px}.tablewrap{max-height:390px;overflow:auto}table{width:100%;border-collapse:collapse;font-size:13px}th{text-align:left;font-size:11px;color:var(--muted);position:sticky;top:0;background:#f5f8f7;padding:12px}td{border-bottom:1px solid #edf1f1;padding:11px 12px}td.num{text-align:right;font-variant-numeric:tabular-nums}.mini{height:5px;background:#e7efed;width:80px;display:inline-block;margin-right:10px;vertical-align:middle}.mini i{display:block;height:5px;background:var(--teal)}.downloads{display:flex;gap:12px;flex-wrap:wrap;margin-top:20px}.downloads a{font-size:13px;border:1px solid var(--line);padding:8px 13px;border-radius:7px;text-decoration:none}.footer{font-size:12px;color:var(--muted);border-top:1px solid var(--line);padding-top:24px;margin-top:40px}.footgrid{display:grid;grid-template-columns:1fr 1fr;gap:28px}.pill{font-size:11px;background:#edf3f1;border-radius:5px;padding:3px 7px}details{font-size:13px;margin-top:15px}summary{cursor:pointer} .bigstat{font-size:27px;font-weight:700;color:var(--teal);margin:10px 0}
4
  @media(max-width:850px){.cards{grid-template-columns:1fr 1fr}.split,.footgrid{grid-template-columns:1fr}.hero{display:block}.hero-right{text-align:left;margin-top:18px}.heading{align-items:flex-start;flex-direction:column}h1{font-size:34px}main{padding:25px 18px}.barrow{grid-template-columns:145px 1fr 70px}nav{display:none}}@media print{header,button,.toolbar{display:none}main{padding:0}.tablewrap{max-height:none}.panel,.card{break-inside:avoid}body{background:white}}
5
  </style>
6
+ </head><body><header><div class="brand">deepopen<span> / </span>Research</div><nav><a href="#comparison">&#20844;&#24320;&#32467;&#26524;&#23545;&#29031;</a><a href="#results">&#20998;&#31867;&#32467;&#26524;</a><a href="#evidence">&#35777;&#25454;&#19982;&#33539;&#22260;</a></nav><span class="status">&#20316;&#32773;&#32500;&#25252; &#183; &#38750;&#23448;&#26041;&#27036;&#21333;</span></header>
7
+ <main><div class="hero"><div><div class="eyebrow">INTENT CLASSIFICATION / EVALUATION REPORT</div><h1>&#35753;&#27599;&#19968;&#39033;&#25104;&#32489;&#65292;<br>&#37117;&#26377;&#32467;&#26524;&#21487;&#26597;&#12290;</h1><p class="sub">deepopen &#22312;&#38134;&#34892;&#26381;&#21153;&#12289;&#36890;&#29992;&#21161;&#25163;&#19982;&#22810;&#35821;&#35328;&#24847;&#22270;&#20998;&#31867;&#19978;&#30340;&#27979;&#35797;&#34920;&#29616;&#12290;<br>&#26597;&#30475;&#20844;&#24320;&#32467;&#26524;&#23545;&#29031;&#65292;&#28145;&#20837;&#27599;&#20010;&#31867;&#21035;&#65292;&#19979;&#36733;&#21487;&#26680;&#39564;&#30340;&#20998;&#31867;&#35760;&#24405;&#12290;</p></div><div class="hero-right"><span class="status">&#23637;&#31034;&#39044;&#35272;&#29256;</span><p>&#21442;&#32771;&#26469;&#28304;&#26680;&#39564;&#65306;2026.10.07<br>SOTA2 &#25910;&#24405;&#65306;&#23578;&#26410;&#25552;&#20132;</p></div></div>
8
  <div class="cards"><div class="card"><small>BANKING77</small><div class="value">94.0909<em>%</em></div><h3>&#38134;&#34892;&#24847;&#22270; &#183; &#21333;&#27169;&#22411;</h3><p class="caption">2,898 / 3,080 &#27491;&#30830; &#183; 77 &#31867;</p></div><div class="card"><small>CLINC150 / SINGLE</small><div class="value">97.7556<em>%</em></div><h3>&#36890;&#29992;&#21161;&#25163; &#183; &#21333;&#27169;&#22411;</h3><p class="caption">4,399 / 4,500 &#27491;&#30830; &#183; 150 &#31867;</p></div><div class="card featured"><small style="color:#89d6bd">CLINC150 / ENSEMBLE</small><div class="value">98.0222<em>%</em></div><h3>&#36890;&#29992;&#21161;&#25163; &#183; &#22235;&#32534;&#30721;&#22120;&#38598;&#25104;</h3><p class="caption">4,411 / 4,500 &#27491;&#30830; &#183; 150 &#31867;</p></div><div class="card"><small>MASSIVE / MULTILINGUAL</small><div class="value">86.0325<em>%</em></div><h3>51 &#35821;&#35328; &#183; &#19977;&#31181;&#23376;&#24179;&#22343;</h3><p class="caption">&#19968;&#33268;&#24615;&#26041;&#27861; &#183; &#38750;&#21333;&#19968;&#27169;&#22411;&#20998;&#25968;</p></div></div>
9
  <p class="rule">BANKING77 / CLINC150 &#20026;&#21382;&#21490; test &#25104;&#32489;&#65292;&#24050;&#20174;&#39044;&#27979;&#25991;&#20214;&#37325;&#26032;&#35745;&#25968;&#26680;&#23545;&#65307;CLINC150 &#20165;&#21547;&#22495;&#20869;&#31867;&#21035;&#65292;&#19981;&#21547; OOS&#12290;MASSIVE &#20026;&#29420;&#31435;&#30340;&#22810;&#35821;&#35328;&#30740;&#31350;&#35774;&#32622;&#12290;</p>
10
  <section id="comparison"><div class="heading"><h2>&#19982;&#20844;&#24320;&#32467;&#26524;&#25918;&#22312;&#19968;&#36215;&#30475;</h2><div class="tabs" id="tabs"><button class="active" data-task="bank">BANKING77</button><button data-task="clinc">CLINC150</button><button data-task="massive">MASSIVE</button></div></div><div class="panel split"><div><div class="caption" id="charttitle"></div><div id="bars"></div><div class="note">&#26465;&#24418;&#20174; 0% &#36215;&#65292;&#28385;&#21051;&#24230; 100%&#12290;&#20165;&#23637;&#31034;&#36873;&#23450;&#21442;&#32771;&#26465;&#30446;&#65292;&#19981;&#34920;&#31034;&#23448;&#26041;&#25490;&#21517;&#12290;</div></div><div class="callout" id="compareNote"></div></div></section>
11
  <section id="results"><div class="heading"><div><small>VERIFIED PREDICTION COUNTS</small><h2>&#28145;&#20837;&#20998;&#31867;&#32467;&#26524;</h2></div><select id="dataset" aria-label="&#36873;&#25321;&#20998;&#31867;&#27169;&#22411;"><option value="banking">BANKING77 &#183; &#21333;&#27169;&#22411;</option><option value="clinc_single">CLINC150 &#183; &#21333;&#27169;&#22411;</option><option value="clinc_ensemble">CLINC150 &#183; &#22235;&#32534;&#30721;&#22120;&#38598;&#25104;</option><option value="massive">MASSIVE &#183; &#19977;&#31181;&#23376;&#21512;&#24182;&#35760;&#24405;</option></select></div><div class="panel"><div id="resultstat"></div><div class="toolbar"><input id="search" aria-label="&#25628;&#32034;&#31867;&#21035;" placeholder="&#25628;&#32034;&#31867;&#21035;&#65292;&#20363;&#22914; card&#12289;transfer&#12289;alarm&#8230;"><select id="sort" aria-label="&#25490;&#24207;"><option value="accuracy">&#20248;&#20808;&#26597;&#30475;&#21484;&#22238;&#29575;&#36739;&#20302;&#30340;&#31867;&#21035;</option><option value="label">&#25353;&#31867;&#21035;&#21517;&#31216;&#25490;&#24207;</option></select></div><div class="tablewrap"><table><thead><tr><th>&#30495;&#23454;&#31867;&#21035;</th><th>&#27491;&#30830; / &#26679;&#26412;&#25968;</th><th>&#21484;&#22238;&#29575;</th><th>&#31934;&#30830;&#29575;</th><th>F1</th></tr></thead><tbody id="classes"></tbody></table></div><div class="downloads"><a id="predDownload" download>&#19979;&#36733;&#36880;&#26679;&#26412;&#39044;&#27979; CSV &#8599;</a><a id="classDownload" download>&#19979;&#36733;&#20840;&#37096;&#31867;&#21035;&#25351;&#26631; CSV &#8599;</a><a href="evidence_manifest.json" download>&#35777;&#25454;&#26469;&#28304;&#19982; SHA256 &#8599;</a></div><p class="note">&#39044;&#27979;&#34920;&#20351;&#29992;&#21407;&#39044;&#27979;&#25991;&#20214;&#30340;&#34892;&#24207;&#21495;&#65292;&#19981;&#20882;&#20805;&#23448;&#26041;&#26679;&#26412; ID&#65307;&#19981;&#20998;&#21457;&#27979;&#35797;&#25991;&#26412;&#12290;&#31867;&#21035;&#25351;&#26631;&#30001;&#22266;&#23450;&#39044;&#27979;&#37325;&#26032;&#32479;&#35745;&#65292;&#19981;&#26159;&#37325;&#26032;&#35757;&#32451;&#25110;&#35843;&#21442;&#12290;</p><details><summary>&#26597;&#30475;&#26368;&#24120;&#35265;&#30340;&#20998;&#31867;&#28151;&#28102;</summary><div id="confusions"></div></details></div></section>
12
+ <section id="evidence"><div class="heading"><h2>&#25104;&#32489;&#30340;&#33539;&#22260;&#65292;&#20063;&#19968;&#36215;&#20844;&#24320;</h2><span class="pill">&#36879;&#26126;&#25259;&#38706;</span></div><div class="panel footgrid"><div><h3>&#22797;&#29616;&#19982;&#23457;&#26680;</h3><p class="sub">&#19977;&#20221;&#21382;&#21490;&#39044;&#27979;&#24050;&#26680;&#23545;&#27491;&#30830;&#25968;&#12290;&#27169;&#22411;&#26435;&#37325;&#27491;&#22312;&#29420;&#31435;&#21457;&#24067;&#65292;&#19979;&#36733;&#21152;&#36733;&#22797;&#39564;&#23436;&#25104;&#21069;&#19981;&#26174;&#31034;&#8220;&#24050;&#39564;&#25910;&#8221;&#12290;&#21512;&#25104;&#26679;&#20363;&#21152;&#36733;&#26816;&#26597;&#19981;&#31561;&#20110;&#23436;&#25972;&#27979;&#35797;&#22797;&#27979;&#12290;</p><p class="sub">MASSIVE &#30340;&#19977;&#31181;&#23376;&#22343;&#20540;&#19981;&#33021;&#32465;&#23450;&#21040;&#19968;&#20010;&#21457;&#24067; checkpoint&#12290;&#20998;&#31867;&#34920;&#27719;&#24635;&#19977;&#31181;&#23376;&#20849; 455,022 &#26465;&#39044;&#27979;&#35760;&#24405;&#65292;&#23545;&#24212;&#21516;&#19968;&#25209; 151,674 &#26465;&#27979;&#35797;&#26679;&#26412;&#65292;&#19981;&#33021;&#24403;&#20316;&#29420;&#31435;&#26679;&#26412;&#37327;&#65307;&#21508;&#35821;&#35328;&#35268;&#27169;&#30456;&#21516;&#65292;&#27719;&#24635;&#20934;&#30830;&#29575;&#31561;&#20110;&#19977;&#31181;&#23376;&#24179;&#22343;&#12290;<a href="massive_locales.csv" download>&#19979;&#36733;&#36880;&#35821;&#35328;&#32467;&#26524;</a>&#12290;</p></div><div><h3>&#27604;&#36739;&#19982;&#30740;&#31350;&#36793;&#30028;</h3><p class="sub">&#22806;&#37096;&#25104;&#32489;&#30001; SOTA2 &#24341;&#29992;&#65292;&#26410;&#30001;&#25105;&#20204;&#37325;&#26032;&#36305;&#27979;&#65307;&#35757;&#32451;&#26465;&#20214;&#19982;&#27169;&#22411;&#35268;&#27169;&#21487;&#33021;&#19981;&#21516;&#12290;BANKING77 &#21382;&#21490;&#20840;&#37327;&#35757;&#32451;&#21253;&#21547;&#26087; dev&#65292;&#19981;&#33021;&#23558;&#26087; dev &#24403;&#20316;&#29420;&#31435;&#39564;&#35777;&#12290;&#26082;&#26377; test &#24050;&#35265;&#65292;&#21518;&#32493;&#35797;&#39564;&#20165;&#29992; train/dev&#12290;</p><p class="sub">&#21518;&#32493;&#30740;&#31350;&#23578;&#26410;&#35777;&#26126;&#26174;&#33879;&#20934;&#30830;&#29575;&#25910;&#30410;&#12290;&#26412;&#39029;&#19981;&#20027;&#24352;&#23448;&#26041;&#25490;&#21517;&#12289;SOTA &#35748;&#35777;&#25110;&#24179;&#21488;&#32972;&#20070;&#12290;</p></div></div></section><div class="footer">deepopen Research &#183; &#20316;&#32773;&#33258;&#24314;&#35780;&#27979;&#23637;&#31034;&#39029;<br>&#20844;&#24320;&#21442;&#32771;&#26469;&#28304;&#65306;<a href="https://www.sota2.com/research/sota/intent-classification-on-banking77" target="_blank" rel="noopener">SOTA2 BANKING77</a> &#183; <a href="https://www.sota2.com/research/sota/intent-classification-on-clinc150" target="_blank" rel="noopener">SOTA2 CLINC150</a> &#183; <a href="https://www.sota2.com/research/sota/intent-classification-on-massive-test" target="_blank" rel="noopener">SOTA2 MASSIVE</a></div></main>
13
  <script>
14
  const DATA={"banking": {"name": "BANKING77 \u00b7 \u5355\u6a21\u578b", "n": 3080, "correct": 2898, "accuracy": 94.0909090909091, "classes": [{"label": "Refund_not_showing_up", "n": 40, "correct": 38, "accuracy": 95.0, "precision": 100.0, "f1": 97.43589743589743}, {"label": "activate_my_card", "n": 40, "correct": 39, "accuracy": 97.5, "precision": 100.0, "f1": 98.73417721518987}, {"label": "age_limit", "n": 40, "correct": 40, "accuracy": 100.0, "precision": 97.5609756097561, "f1": 98.76543209876543}, {"label": "apple_pay_or_google_pay", "n": 40, "correct": 40, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "atm_support", "n": 40, "correct": 39, "accuracy": 97.5, "precision": 97.5, "f1": 97.5}, {"label": "automatic_top_up", "n": 40, "correct": 40, "accuracy": 100.0, "precision": 97.5609756097561, "f1": 98.76543209876543}, {"label": "balance_not_updated_after_bank_transfer", "n": 40, "correct": 34, "accuracy": 85.0, "precision": 79.06976744186046, "f1": 81.92771084337349}, {"label": "balance_not_updated_after_cheque_or_cash_deposit", "n": 40, "correct": 39, "accuracy": 97.5, "precision": 97.5, "f1": 97.5}, {"label": "beneficiary_not_allowed", "n": 40, "correct": 37, "accuracy": 92.5, "precision": 94.87179487179488, "f1": 93.67088607594937}, {"label": "cancel_transfer", "n": 40, "correct": 40, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "card_about_to_expire", "n": 40, "correct": 39, "accuracy": 97.5, "precision": 100.0, "f1": 98.73417721518987}, {"label": "card_acceptance", "n": 40, "correct": 37, "accuracy": 92.5, "precision": 100.0, "f1": 96.1038961038961}, {"label": "card_arrival", "n": 40, "correct": 36, "accuracy": 90.0, "precision": 92.3076923076923, "f1": 91.13924050632912}, {"label": "card_delivery_estimate", "n": 40, "correct": 37, "accuracy": 92.5, "precision": 92.5, "f1": 92.5}, {"label": "card_linking", "n": 40, "correct": 39, "accuracy": 97.5, "precision": 95.1219512195122, "f1": 96.29629629629629}, {"label": "card_not_working", "n": 40, "correct": 39, "accuracy": 97.5, "precision": 86.66666666666667, "f1": 91.76470588235294}, {"label": "card_payment_fee_charged", "n": 40, "correct": 36, "accuracy": 90.0, "precision": 87.8048780487805, "f1": 88.88888888888889}, {"label": "card_payment_not_recognised", "n": 40, "correct": 34, "accuracy": 85.0, "precision": 85.0, "f1": 85.0}, {"label": "card_payment_wrong_exchange_rate", "n": 40, "correct": 37, "accuracy": 92.5, "precision": 90.2439024390244, "f1": 91.35802469135803}, {"label": "card_swallowed", "n": 40, "correct": 40, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "cash_withdrawal_charge", "n": 40, "correct": 39, "accuracy": 97.5, "precision": 97.5, "f1": 97.5}, {"label": "cash_withdrawal_not_recognised", "n": 40, "correct": 39, "accuracy": 97.5, "precision": 90.69767441860465, "f1": 93.97590361445783}, {"label": "change_pin", "n": 40, "correct": 38, "accuracy": 95.0, "precision": 97.43589743589743, "f1": 96.20253164556962}, {"label": "compromised_card", "n": 40, "correct": 33, "accuracy": 82.5, "precision": 84.61538461538461, "f1": 83.54430379746836}, {"label": "contactless_not_working", "n": 40, "correct": 39, "accuracy": 97.5, "precision": 100.0, "f1": 98.73417721518987}, {"label": "country_support", "n": 40, "correct": 40, "accuracy": 100.0, "precision": 95.23809523809524, "f1": 97.5609756097561}, {"label": "declined_card_payment", "n": 40, "correct": 38, "accuracy": 95.0, "precision": 88.37209302325581, "f1": 91.56626506024097}, {"label": "declined_cash_withdrawal", "n": 40, "correct": 40, "accuracy": 100.0, "precision": 93.02325581395348, "f1": 96.3855421686747}, {"label": "declined_transfer", "n": 40, "correct": 32, "accuracy": 80.0, "precision": 96.96969696969697, "f1": 87.67123287671232}, {"label": "direct_debit_payment_not_recognised", "n": 40, "correct": 36, "accuracy": 90.0, "precision": 92.3076923076923, "f1": 91.13924050632912}, {"label": "disposable_card_limits", "n": 40, "correct": 36, "accuracy": 90.0, "precision": 90.0, "f1": 90.0}, {"label": "edit_personal_details", "n": 40, "correct": 40, "accuracy": 100.0, "precision": 97.5609756097561, "f1": 98.76543209876543}, {"label": "exchange_charge", "n": 40, "correct": 36, "accuracy": 90.0, "precision": 94.73684210526316, "f1": 92.3076923076923}, {"label": "exchange_rate", "n": 40, "correct": 39, "accuracy": 97.5, "precision": 86.66666666666667, "f1": 91.76470588235294}, {"label": "exchange_via_app", "n": 40, "correct": 37, "accuracy": 92.5, "precision": 94.87179487179488, "f1": 93.67088607594937}, {"label": "extra_charge_on_statement", "n": 40, "correct": 39, "accuracy": 97.5, "precision": 97.5, "f1": 97.5}, {"label": "failed_transfer", "n": 40, "correct": 37, "accuracy": 92.5, "precision": 90.2439024390244, "f1": 91.35802469135803}, {"label": "fiat_currency_support", "n": 40, "correct": 36, "accuracy": 90.0, "precision": 87.8048780487805, "f1": 88.88888888888889}, {"label": "get_disposable_virtual_card", "n": 40, "correct": 38, "accuracy": 95.0, "precision": 95.0, "f1": 95.0}, {"label": "get_physical_card", "n": 40, "correct": 40, "accuracy": 100.0, "precision": 95.23809523809524, "f1": 97.5609756097561}, {"label": "getting_spare_card", "n": 40, "correct": 39, "accuracy": 97.5, "precision": 97.5, "f1": 97.5}, {"label": "getting_virtual_card", "n": 40, "correct": 39, "accuracy": 97.5, "precision": 88.63636363636364, "f1": 92.85714285714286}, {"label": "lost_or_stolen_card", "n": 40, "correct": 38, "accuracy": 95.0, "precision": 86.36363636363636, "f1": 90.47619047619048}, {"label": "lost_or_stolen_phone", "n": 40, "correct": 39, "accuracy": 97.5, "precision": 100.0, "f1": 98.73417721518987}, {"label": "order_physical_card", "n": 40, "correct": 37, "accuracy": 92.5, "precision": 92.5, "f1": 92.5}, {"label": "passcode_forgotten", "n": 40, "correct": 40, "accuracy": 100.0, "precision": 97.5609756097561, "f1": 98.76543209876543}, {"label": "pending_card_payment", "n": 40, "correct": 39, "accuracy": 97.5, "precision": 100.0, "f1": 98.73417721518987}, {"label": "pending_cash_withdrawal", "n": 40, "correct": 39, "accuracy": 97.5, "precision": 97.5, "f1": 97.5}, {"label": "pending_top_up", "n": 40, "correct": 36, "accuracy": 90.0, "precision": 94.73684210526316, "f1": 92.3076923076923}, {"label": "pending_transfer", "n": 40, "correct": 34, "accuracy": 85.0, "precision": 82.92682926829268, "f1": 83.95061728395062}, {"label": "pin_blocked", "n": 40, "correct": 36, "accuracy": 90.0, "precision": 94.73684210526316, "f1": 92.3076923076923}, {"label": "receiving_money", "n": 40, "correct": 38, "accuracy": 95.0, "precision": 100.0, "f1": 97.43589743589743}, {"label": "request_refund", "n": 40, "correct": 39, "accuracy": 97.5, "precision": 100.0, "f1": 98.73417721518987}, {"label": "reverted_card_payment?", "n": 40, "correct": 38, "accuracy": 95.0, "precision": 92.6829268292683, "f1": 93.82716049382717}, {"label": "supported_cards_and_currencies", "n": 40, "correct": 37, "accuracy": 92.5, "precision": 92.5, "f1": 92.5}, {"label": "terminate_account", "n": 40, "correct": 40, "accuracy": 100.0, "precision": 97.5609756097561, "f1": 98.76543209876543}, {"label": "top_up_by_bank_transfer_charge", "n": 40, "correct": 35, "accuracy": 87.5, "precision": 92.10526315789474, "f1": 89.74358974358974}, {"label": "top_up_by_card_charge", "n": 40, "correct": 38, "accuracy": 95.0, "precision": 95.0, "f1": 95.0}, {"label": "top_up_by_cash_or_cheque", "n": 40, "correct": 39, "accuracy": 97.5, "precision": 100.0, "f1": 98.73417721518987}, {"label": "top_up_failed", "n": 40, "correct": 37, "accuracy": 92.5, "precision": 90.2439024390244, "f1": 91.35802469135803}, {"label": "top_up_limits", "n": 40, "correct": 40, "accuracy": 100.0, "precision": 97.5609756097561, "f1": 98.76543209876543}, {"label": "top_up_reverted", "n": 40, "correct": 34, "accuracy": 85.0, "precision": 87.17948717948718, "f1": 86.07594936708861}, {"label": "topping_up_by_card", "n": 40, "correct": 33, "accuracy": 82.5, "precision": 86.84210526315789, "f1": 84.61538461538461}, {"label": "transaction_charged_twice", "n": 40, "correct": 40, "accuracy": 100.0, "precision": 93.02325581395348, "f1": 96.3855421686747}, {"label": "transfer_fee_charged", "n": 40, "correct": 37, "accuracy": 92.5, "precision": 88.0952380952381, "f1": 90.2439024390244}, {"label": "transfer_into_account", "n": 40, "correct": 38, "accuracy": 95.0, "precision": 90.47619047619048, "f1": 92.6829268292683}, {"label": "transfer_not_received_by_recipient", "n": 40, "correct": 34, "accuracy": 85.0, "precision": 87.17948717948718, "f1": 86.07594936708861}, {"label": "transfer_timing", "n": 40, "correct": 36, "accuracy": 90.0, "precision": 90.0, "f1": 90.0}, {"label": "unable_to_verify_identity", "n": 40, "correct": 37, "accuracy": 92.5, "precision": 97.36842105263158, "f1": 94.87179487179488}, {"label": "verify_my_identity", "n": 40, "correct": 37, "accuracy": 92.5, "precision": 94.87179487179488, "f1": 93.67088607594937}, {"label": "verify_source_of_funds", "n": 40, "correct": 39, "accuracy": 97.5, "precision": 100.0, "f1": 98.73417721518987}, {"label": "verify_top_up", "n": 40, "correct": 40, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "virtual_card_not_working", "n": 40, "correct": 37, "accuracy": 92.5, "precision": 100.0, "f1": 96.1038961038961}, {"label": "visa_or_mastercard", "n": 40, "correct": 38, "accuracy": 95.0, "precision": 100.0, "f1": 97.43589743589743}, {"label": "why_verify_identity", "n": 40, "correct": 39, "accuracy": 97.5, "precision": 92.85714285714286, "f1": 95.1219512195122}, {"label": "wrong_amount_of_cash_received", "n": 40, "correct": 36, "accuracy": 90.0, "precision": 100.0, "f1": 94.73684210526316}, {"label": "wrong_exchange_rate_for_cash_withdrawal", "n": 40, "correct": 34, "accuracy": 85.0, "precision": 97.14285714285714, "f1": 90.66666666666667}], "confusions": [{"gold": "card_payment_not_recognised", "predicted": "compromised_card", "count": 4}, {"gold": "pending_transfer", "predicted": "balance_not_updated_after_bank_transfer", "count": 4}, {"gold": "card_arrival", "predicted": "card_delivery_estimate", "count": 3}, {"gold": "card_delivery_estimate", "predicted": "card_arrival", "count": 3}, {"gold": "exchange_via_app", "predicted": "fiat_currency_support", "count": 3}, {"gold": "top_up_by_bank_transfer_charge", "predicted": "transfer_fee_charged", "count": 3}, {"gold": "pending_top_up", "predicted": "top_up_reverted", "count": 3}, {"gold": "top_up_reverted", "predicted": "top_up_failed", "count": 3}, {"gold": "compromised_card", "predicted": "lost_or_stolen_card", "count": 3}, {"gold": "wrong_exchange_rate_for_cash_withdrawal", "predicted": "card_payment_wrong_exchange_rate", "count": 3}, {"gold": "balance_not_updated_after_bank_transfer", "predicted": "transfer_timing", "count": 3}, {"gold": "exchange_charge", "predicted": "exchange_rate", "count": 3}, {"gold": "fiat_currency_support", "predicted": "exchange_via_app", "count": 2}, {"gold": "wrong_amount_of_cash_received", "predicted": "declined_cash_withdrawal", "count": 2}, {"gold": "card_payment_fee_charged", "predicted": "direct_debit_payment_not_recognised", "count": 2}]}, "clinc_single": {"name": "CLINC150 \u00b7 \u5355\u6a21\u578b", "n": 4500, "correct": 4399, "accuracy": 97.75555555555556, "classes": [{"label": "accept_reservations", "n": 30, "correct": 29, "accuracy": 96.66666666666667, "precision": 93.54838709677419, "f1": 95.08196721311475}, {"label": "account_blocked", "n": 30, "correct": 29, "accuracy": 96.66666666666667, "precision": 96.66666666666667, "f1": 96.66666666666667}, {"label": "alarm", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 96.7741935483871, "f1": 98.36065573770492}, {"label": "application_status", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "apr", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 96.7741935483871, "f1": 98.36065573770492}, {"label": "are_you_a_bot", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "balance", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 96.7741935483871, "f1": 98.36065573770492}, {"label": "bill_balance", "n": 30, "correct": 28, "accuracy": 93.33333333333333, "precision": 96.55172413793103, "f1": 94.91525423728814}, {"label": "bill_due", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 90.9090909090909, "f1": 95.23809523809524}, {"label": "book_flight", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "book_hotel", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "calculator", "n": 30, "correct": 28, "accuracy": 93.33333333333333, "precision": 96.55172413793103, "f1": 94.91525423728814}, {"label": "calendar", "n": 30, "correct": 27, "accuracy": 90.0, "precision": 93.10344827586206, "f1": 91.52542372881356}, {"label": "calendar_update", "n": 30, "correct": 29, "accuracy": 96.66666666666667, "precision": 96.66666666666667, "f1": 96.66666666666667}, {"label": "calories", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "cancel", "n": 30, "correct": 28, "accuracy": 93.33333333333333, "precision": 93.33333333333333, "f1": 93.33333333333333}, {"label": "cancel_reservation", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 90.9090909090909, "f1": 95.23809523809524}, {"label": "car_rental", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "card_declined", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "carry_on", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "change_accent", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 96.7741935483871, "f1": 98.36065573770492}, {"label": "change_ai_name", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 85.71428571428571, "f1": 92.3076923076923}, {"label": "change_language", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 96.7741935483871, "f1": 98.36065573770492}, {"label": "change_speed", "n": 30, "correct": 29, "accuracy": 96.66666666666667, "precision": 100.0, "f1": 98.30508474576271}, {"label": "change_user_name", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 96.7741935483871, "f1": 98.36065573770492}, {"label": "change_volume", "n": 30, "correct": 24, "accuracy": 80.0, "precision": 100.0, "f1": 88.88888888888889}, {"label": "confirm_reservation", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "cook_time", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "credit_limit", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 96.7741935483871, "f1": 98.36065573770492}, {"label": "credit_limit_change", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "credit_score", "n": 30, "correct": 29, "accuracy": 96.66666666666667, "precision": 100.0, "f1": 98.30508474576271}, {"label": "current_location", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 93.75, "f1": 96.7741935483871}, {"label": "damaged_card", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "date", "n": 30, "correct": 29, "accuracy": 96.66666666666667, "precision": 100.0, "f1": 98.30508474576271}, {"label": "definition", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "direct_deposit", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "directions", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 96.7741935483871, "f1": 98.36065573770492}, {"label": "distance", "n": 30, "correct": 28, "accuracy": 93.33333333333333, "precision": 96.55172413793103, "f1": 94.91525423728814}, {"label": "do_you_have_pets", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "exchange_rate", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "expiration_date", "n": 30, "correct": 29, "accuracy": 96.66666666666667, "precision": 100.0, "f1": 98.30508474576271}, {"label": "find_phone", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "flight_status", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 96.7741935483871, "f1": 98.36065573770492}, {"label": "flip_coin", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "food_last", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "freeze_account", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "fun_fact", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "gas", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "gas_type", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "goodbye", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 96.7741935483871, "f1": 98.36065573770492}, {"label": "greeting", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 96.7741935483871, "f1": 98.36065573770492}, {"label": "how_busy", "n": 30, "correct": 28, "accuracy": 93.33333333333333, "precision": 96.55172413793103, "f1": 94.91525423728814}, {"label": "how_old_are_you", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "improve_credit_score", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 96.7741935483871, "f1": 98.36065573770492}, {"label": "income", "n": 30, "correct": 29, "accuracy": 96.66666666666667, "precision": 100.0, "f1": 98.30508474576271}, {"label": "ingredient_substitution", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "ingredients_list", "n": 30, "correct": 27, "accuracy": 90.0, "precision": 96.42857142857143, "f1": 93.10344827586206}, {"label": "insurance", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "insurance_change", "n": 30, "correct": 29, "accuracy": 96.66666666666667, "precision": 100.0, "f1": 98.30508474576271}, {"label": "interest_rate", "n": 30, "correct": 29, "accuracy": 96.66666666666667, "precision": 100.0, "f1": 98.30508474576271}, {"label": "international_fees", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "international_visa", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "jump_start", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 96.7741935483871, "f1": 98.36065573770492}, {"label": "last_maintenance", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "lost_luggage", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "make_call", "n": 30, "correct": 29, "accuracy": 96.66666666666667, "precision": 96.66666666666667, "f1": 96.66666666666667}, {"label": "maybe", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "meal_suggestion", "n": 30, "correct": 29, "accuracy": 96.66666666666667, "precision": 87.87878787878788, "f1": 92.06349206349206}, {"label": "meaning_of_life", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "measurement_conversion", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "meeting_schedule", "n": 30, "correct": 29, "accuracy": 96.66666666666667, "precision": 93.54838709677419, "f1": 95.08196721311475}, {"label": "min_payment", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "mpg", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "new_card", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 93.75, "f1": 96.7741935483871}, {"label": "next_holiday", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 96.7741935483871, "f1": 98.36065573770492}, {"label": "next_song", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "no", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "nutrition_info", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "oil_change_how", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 93.75, "f1": 96.7741935483871}, {"label": "oil_change_when", "n": 30, "correct": 28, "accuracy": 93.33333333333333, "precision": 90.3225806451613, "f1": 91.80327868852459}, {"label": "order", "n": 30, "correct": 29, "accuracy": 96.66666666666667, "precision": 100.0, "f1": 98.30508474576271}, {"label": "order_checks", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "order_status", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "pay_bill", "n": 30, "correct": 29, "accuracy": 96.66666666666667, "precision": 96.66666666666667, "f1": 96.66666666666667}, {"label": "payday", "n": 30, "correct": 29, "accuracy": 96.66666666666667, "precision": 100.0, "f1": 98.30508474576271}, {"label": "pin_change", "n": 30, "correct": 25, "accuracy": 83.33333333333333, "precision": 100.0, "f1": 90.9090909090909}, {"label": "play_music", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "plug_type", "n": 30, "correct": 29, "accuracy": 96.66666666666667, "precision": 100.0, "f1": 98.30508474576271}, {"label": "pto_balance", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 93.75, "f1": 96.7741935483871}, {"label": "pto_request", "n": 30, "correct": 29, "accuracy": 96.66666666666667, "precision": 100.0, "f1": 98.30508474576271}, {"label": "pto_request_status", "n": 30, "correct": 29, "accuracy": 96.66666666666667, "precision": 100.0, "f1": 98.30508474576271}, {"label": "pto_used", "n": 30, "correct": 28, "accuracy": 93.33333333333333, "precision": 100.0, "f1": 96.55172413793103}, {"label": "recipe", "n": 30, "correct": 28, "accuracy": 93.33333333333333, "precision": 90.3225806451613, "f1": 91.80327868852459}, {"label": "redeem_rewards", "n": 30, "correct": 26, "accuracy": 86.66666666666667, "precision": 100.0, "f1": 92.85714285714286}, {"label": "reminder", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "reminder_update", "n": 30, "correct": 29, "accuracy": 96.66666666666667, "precision": 100.0, "f1": 98.30508474576271}, {"label": "repeat", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "replacement_card_duration", "n": 30, "correct": 27, "accuracy": 90.0, "precision": 96.42857142857143, "f1": 93.10344827586206}, {"label": "report_fraud", "n": 30, "correct": 29, "accuracy": 96.66666666666667, "precision": 87.87878787878788, "f1": 92.06349206349206}, {"label": "report_lost_card", "n": 30, "correct": 28, "accuracy": 93.33333333333333, "precision": 96.55172413793103, "f1": 94.91525423728814}, {"label": "reset_settings", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "restaurant_reservation", "n": 30, "correct": 27, "accuracy": 90.0, "precision": 93.10344827586206, "f1": 91.52542372881356}, {"label": "restaurant_reviews", "n": 30, "correct": 27, "accuracy": 90.0, "precision": 100.0, "f1": 94.73684210526316}, {"label": "restaurant_suggestion", "n": 30, "correct": 29, "accuracy": 96.66666666666667, "precision": 100.0, "f1": 98.30508474576271}, {"label": "rewards_balance", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 88.23529411764706, "f1": 93.75}, {"label": "roll_dice", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "rollover_401k", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "routing", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 96.7741935483871, "f1": 98.36065573770492}, {"label": "schedule_maintenance", "n": 30, "correct": 27, "accuracy": 90.0, "precision": 100.0, "f1": 94.73684210526316}, {"label": "schedule_meeting", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 96.7741935483871, "f1": 98.36065573770492}, {"label": "share_location", "n": 30, "correct": 29, "accuracy": 96.66666666666667, "precision": 100.0, "f1": 98.30508474576271}, {"label": "shopping_list", "n": 30, "correct": 28, "accuracy": 93.33333333333333, "precision": 100.0, "f1": 96.55172413793103}, {"label": "shopping_list_update", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 96.7741935483871, "f1": 98.36065573770492}, {"label": "smart_home", "n": 30, "correct": 29, "accuracy": 96.66666666666667, "precision": 96.66666666666667, "f1": 96.66666666666667}, {"label": "spelling", "n": 30, "correct": 29, "accuracy": 96.66666666666667, "precision": 96.66666666666667, "f1": 96.66666666666667}, {"label": "spending_history", "n": 30, "correct": 29, "accuracy": 96.66666666666667, "precision": 85.29411764705883, "f1": 90.625}, {"label": "sync_device", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "taxes", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "tell_joke", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "text", "n": 30, "correct": 28, "accuracy": 93.33333333333333, "precision": 100.0, "f1": 96.55172413793103}, {"label": "thank_you", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 93.75, "f1": 96.7741935483871}, {"label": "time", "n": 30, "correct": 29, "accuracy": 96.66666666666667, "precision": 96.66666666666667, "f1": 96.66666666666667}, {"label": "timer", "n": 30, "correct": 29, "accuracy": 96.66666666666667, "precision": 96.66666666666667, "f1": 96.66666666666667}, {"label": "timezone", "n": 30, "correct": 29, "accuracy": 96.66666666666667, "precision": 100.0, "f1": 98.30508474576271}, {"label": "tire_change", "n": 30, "correct": 29, "accuracy": 96.66666666666667, "precision": 100.0, "f1": 98.30508474576271}, {"label": "tire_pressure", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 96.7741935483871, "f1": 98.36065573770492}, {"label": "todo_list", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 96.7741935483871, "f1": 98.36065573770492}, {"label": "todo_list_update", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "traffic", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "transactions", "n": 30, "correct": 26, "accuracy": 86.66666666666667, "precision": 96.29629629629629, "f1": 91.2280701754386}, {"label": "transfer", "n": 30, "correct": 29, "accuracy": 96.66666666666667, "precision": 100.0, "f1": 98.30508474576271}, {"label": "translate", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "travel_alert", "n": 30, "correct": 29, "accuracy": 96.66666666666667, "precision": 100.0, "f1": 98.30508474576271}, {"label": "travel_notification", "n": 30, "correct": 29, "accuracy": 96.66666666666667, "precision": 100.0, "f1": 98.30508474576271}, {"label": "travel_suggestion", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 96.7741935483871, "f1": 98.36065573770492}, {"label": "uber", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "update_playlist", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "user_name", "n": 30, "correct": 29, "accuracy": 96.66666666666667, "precision": 100.0, "f1": 98.30508474576271}, {"label": "vaccines", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "w2", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "weather", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 93.75, "f1": 96.7741935483871}, {"label": "what_are_your_hobbies", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 96.7741935483871, "f1": 98.36065573770492}, {"label": "what_can_i_ask_you", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "what_is_your_name", "n": 30, "correct": 28, "accuracy": 93.33333333333333, "precision": 96.55172413793103, "f1": 94.91525423728814}, {"label": "what_song", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "where_are_you_from", "n": 30, "correct": 29, "accuracy": 96.66666666666667, "precision": 100.0, "f1": 98.30508474576271}, {"label": "whisper_mode", "n": 30, "correct": 29, "accuracy": 96.66666666666667, "precision": 87.87878787878788, "f1": 92.06349206349206}, {"label": "who_do_you_work_for", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 93.75, "f1": 96.7741935483871}, {"label": "who_made_you", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 96.7741935483871, "f1": 98.36065573770492}, {"label": "yes", "n": 30, "correct": 26, "accuracy": 86.66666666666667, "precision": 100.0, "f1": 92.85714285714286}], "confusions": [{"gold": "pin_change", "predicted": "change_ai_name", "count": 5}, {"gold": "change_volume", "predicted": "whisper_mode", "count": 4}, {"gold": "redeem_rewards", "predicted": "rewards_balance", "count": 4}, {"gold": "transactions", "predicted": "spending_history", "count": 3}, {"gold": "schedule_maintenance", "predicted": "oil_change_when", "count": 3}, {"gold": "ingredients_list", "predicted": "recipe", "count": 3}, {"gold": "restaurant_reservation", "predicted": "accept_reservations", "count": 2}, {"gold": "oil_change_when", "predicted": "oil_change_how", "count": 2}, {"gold": "yes", "predicted": "thank_you", "count": 2}, {"gold": "calendar", "predicted": "meeting_schedule", "count": 2}, {"gold": "calculator", "predicted": "spending_history", "count": 2}, {"gold": "pto_used", "predicted": "pto_balance", "count": 2}, {"gold": "report_lost_card", "predicted": "report_fraud", "count": 2}, {"gold": "restaurant_reviews", "predicted": "meal_suggestion", "count": 2}, {"gold": "what_is_your_name", "predicted": "who_do_you_work_for", "count": 2}]}, "clinc_ensemble": {"name": "CLINC150 \u00b7 \u56db\u7f16\u7801\u5668\u96c6\u6210", "n": 4500, "correct": 4411, "accuracy": 98.02222222222223, "classes": [{"label": "accept_reservations", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 93.75, "f1": 96.7741935483871}, {"label": "account_blocked", "n": 30, "correct": 29, "accuracy": 96.66666666666667, "precision": 96.66666666666667, "f1": 96.66666666666667}, {"label": "alarm", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 96.7741935483871, "f1": 98.36065573770492}, {"label": "application_status", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "apr", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 96.7741935483871, "f1": 98.36065573770492}, {"label": "are_you_a_bot", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "balance", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "bill_balance", "n": 30, "correct": 28, "accuracy": 93.33333333333333, "precision": 96.55172413793103, "f1": 94.91525423728814}, {"label": "bill_due", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 90.9090909090909, "f1": 95.23809523809524}, {"label": "book_flight", "n": 30, "correct": 29, "accuracy": 96.66666666666667, "precision": 100.0, "f1": 98.30508474576271}, {"label": "book_hotel", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "calculator", "n": 30, "correct": 28, "accuracy": 93.33333333333333, "precision": 96.55172413793103, "f1": 94.91525423728814}, {"label": "calendar", "n": 30, "correct": 27, "accuracy": 90.0, "precision": 96.42857142857143, "f1": 93.10344827586206}, {"label": "calendar_update", "n": 30, "correct": 29, "accuracy": 96.66666666666667, "precision": 96.66666666666667, "f1": 96.66666666666667}, {"label": "calories", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "cancel", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 93.75, "f1": 96.7741935483871}, {"label": "cancel_reservation", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 93.75, "f1": 96.7741935483871}, {"label": "car_rental", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 96.7741935483871, "f1": 98.36065573770492}, {"label": "card_declined", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "carry_on", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "change_accent", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "change_ai_name", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 85.71428571428571, "f1": 92.3076923076923}, {"label": "change_language", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "change_speed", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "change_user_name", "n": 30, "correct": 29, "accuracy": 96.66666666666667, "precision": 100.0, "f1": 98.30508474576271}, {"label": "change_volume", "n": 30, "correct": 25, "accuracy": 83.33333333333333, "precision": 100.0, "f1": 90.9090909090909}, {"label": "confirm_reservation", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "cook_time", "n": 30, "correct": 29, "accuracy": 96.66666666666667, "precision": 100.0, "f1": 98.30508474576271}, {"label": "credit_limit", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 96.7741935483871, "f1": 98.36065573770492}, {"label": "credit_limit_change", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "credit_score", "n": 30, "correct": 29, "accuracy": 96.66666666666667, "precision": 100.0, "f1": 98.30508474576271}, {"label": "current_location", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 96.7741935483871, "f1": 98.36065573770492}, {"label": "damaged_card", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "date", "n": 30, "correct": 29, "accuracy": 96.66666666666667, "precision": 100.0, "f1": 98.30508474576271}, {"label": "definition", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "direct_deposit", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "directions", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "distance", "n": 30, "correct": 28, "accuracy": 93.33333333333333, "precision": 100.0, "f1": 96.55172413793103}, {"label": "do_you_have_pets", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "exchange_rate", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "expiration_date", "n": 30, "correct": 29, "accuracy": 96.66666666666667, "precision": 100.0, "f1": 98.30508474576271}, {"label": "find_phone", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "flight_status", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 93.75, "f1": 96.7741935483871}, {"label": "flip_coin", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "food_last", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "freeze_account", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "fun_fact", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "gas", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "gas_type", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "goodbye", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "greeting", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "how_busy", "n": 30, "correct": 29, "accuracy": 96.66666666666667, "precision": 96.66666666666667, "f1": 96.66666666666667}, {"label": "how_old_are_you", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "improve_credit_score", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 96.7741935483871, "f1": 98.36065573770492}, {"label": "income", "n": 30, "correct": 29, "accuracy": 96.66666666666667, "precision": 100.0, "f1": 98.30508474576271}, {"label": "ingredient_substitution", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "ingredients_list", "n": 30, "correct": 27, "accuracy": 90.0, "precision": 96.42857142857143, "f1": 93.10344827586206}, {"label": "insurance", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "insurance_change", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "interest_rate", "n": 30, "correct": 29, "accuracy": 96.66666666666667, "precision": 100.0, "f1": 98.30508474576271}, {"label": "international_fees", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "international_visa", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "jump_start", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 96.7741935483871, "f1": 98.36065573770492}, {"label": "last_maintenance", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "lost_luggage", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "make_call", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 96.7741935483871, "f1": 98.36065573770492}, {"label": "maybe", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "meal_suggestion", "n": 30, "correct": 29, "accuracy": 96.66666666666667, "precision": 87.87878787878788, "f1": 92.06349206349206}, {"label": "meaning_of_life", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "measurement_conversion", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "meeting_schedule", "n": 30, "correct": 29, "accuracy": 96.66666666666667, "precision": 93.54838709677419, "f1": 95.08196721311475}, {"label": "min_payment", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "mpg", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "new_card", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 96.7741935483871, "f1": 98.36065573770492}, {"label": "next_holiday", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "next_song", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "no", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "nutrition_info", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "oil_change_how", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 93.75, "f1": 96.7741935483871}, {"label": "oil_change_when", "n": 30, "correct": 28, "accuracy": 93.33333333333333, "precision": 90.3225806451613, "f1": 91.80327868852459}, {"label": "order", "n": 30, "correct": 29, "accuracy": 96.66666666666667, "precision": 100.0, "f1": 98.30508474576271}, {"label": "order_checks", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "order_status", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "pay_bill", "n": 30, "correct": 29, "accuracy": 96.66666666666667, "precision": 96.66666666666667, "f1": 96.66666666666667}, {"label": "payday", "n": 30, "correct": 29, "accuracy": 96.66666666666667, "precision": 100.0, "f1": 98.30508474576271}, {"label": "pin_change", "n": 30, "correct": 25, "accuracy": 83.33333333333333, "precision": 100.0, "f1": 90.9090909090909}, {"label": "play_music", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "plug_type", "n": 30, "correct": 29, "accuracy": 96.66666666666667, "precision": 100.0, "f1": 98.30508474576271}, {"label": "pto_balance", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 93.75, "f1": 96.7741935483871}, {"label": "pto_request", "n": 30, "correct": 29, "accuracy": 96.66666666666667, "precision": 96.66666666666667, "f1": 96.66666666666667}, {"label": "pto_request_status", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "pto_used", "n": 30, "correct": 28, "accuracy": 93.33333333333333, "precision": 100.0, "f1": 96.55172413793103}, {"label": "recipe", "n": 30, "correct": 28, "accuracy": 93.33333333333333, "precision": 90.3225806451613, "f1": 91.80327868852459}, {"label": "redeem_rewards", "n": 30, "correct": 26, "accuracy": 86.66666666666667, "precision": 100.0, "f1": 92.85714285714286}, {"label": "reminder", "n": 30, "correct": 29, "accuracy": 96.66666666666667, "precision": 100.0, "f1": 98.30508474576271}, {"label": "reminder_update", "n": 30, "correct": 29, "accuracy": 96.66666666666667, "precision": 100.0, "f1": 98.30508474576271}, {"label": "repeat", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "replacement_card_duration", "n": 30, "correct": 27, "accuracy": 90.0, "precision": 96.42857142857143, "f1": 93.10344827586206}, {"label": "report_fraud", "n": 30, "correct": 29, "accuracy": 96.66666666666667, "precision": 87.87878787878788, "f1": 92.06349206349206}, {"label": "report_lost_card", "n": 30, "correct": 28, "accuracy": 93.33333333333333, "precision": 96.55172413793103, "f1": 94.91525423728814}, {"label": "reset_settings", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "restaurant_reservation", "n": 30, "correct": 28, "accuracy": 93.33333333333333, "precision": 96.55172413793103, "f1": 94.91525423728814}, {"label": "restaurant_reviews", "n": 30, "correct": 27, "accuracy": 90.0, "precision": 100.0, "f1": 94.73684210526316}, {"label": "restaurant_suggestion", "n": 30, "correct": 29, "accuracy": 96.66666666666667, "precision": 100.0, "f1": 98.30508474576271}, {"label": "rewards_balance", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 88.23529411764706, "f1": 93.75}, {"label": "roll_dice", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "rollover_401k", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "routing", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 96.7741935483871, "f1": 98.36065573770492}, {"label": "schedule_maintenance", "n": 30, "correct": 27, "accuracy": 90.0, "precision": 100.0, "f1": 94.73684210526316}, {"label": "schedule_meeting", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 96.7741935483871, "f1": 98.36065573770492}, {"label": "share_location", "n": 30, "correct": 29, "accuracy": 96.66666666666667, "precision": 100.0, "f1": 98.30508474576271}, {"label": "shopping_list", "n": 30, "correct": 29, "accuracy": 96.66666666666667, "precision": 100.0, "f1": 98.30508474576271}, {"label": "shopping_list_update", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 96.7741935483871, "f1": 98.36065573770492}, {"label": "smart_home", "n": 30, "correct": 29, "accuracy": 96.66666666666667, "precision": 100.0, "f1": 98.30508474576271}, {"label": "spelling", "n": 30, "correct": 29, "accuracy": 96.66666666666667, "precision": 96.66666666666667, "f1": 96.66666666666667}, {"label": "spending_history", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 85.71428571428571, "f1": 92.3076923076923}, {"label": "sync_device", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "taxes", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "tell_joke", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "text", "n": 30, "correct": 28, "accuracy": 93.33333333333333, "precision": 100.0, "f1": 96.55172413793103}, {"label": "thank_you", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 96.7741935483871, "f1": 98.36065573770492}, {"label": "time", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 96.7741935483871, "f1": 98.36065573770492}, {"label": "timer", "n": 30, "correct": 29, "accuracy": 96.66666666666667, "precision": 93.54838709677419, "f1": 95.08196721311475}, {"label": "timezone", "n": 30, "correct": 29, "accuracy": 96.66666666666667, "precision": 100.0, "f1": 98.30508474576271}, {"label": "tire_change", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "tire_pressure", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "todo_list", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 96.7741935483871, "f1": 98.36065573770492}, {"label": "todo_list_update", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "traffic", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "transactions", "n": 30, "correct": 26, "accuracy": 86.66666666666667, "precision": 96.29629629629629, "f1": 91.2280701754386}, {"label": "transfer", "n": 30, "correct": 29, "accuracy": 96.66666666666667, "precision": 100.0, "f1": 98.30508474576271}, {"label": "translate", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 96.7741935483871, "f1": 98.36065573770492}, {"label": "travel_alert", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "travel_notification", "n": 30, "correct": 29, "accuracy": 96.66666666666667, "precision": 100.0, "f1": 98.30508474576271}, {"label": "travel_suggestion", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 93.75, "f1": 96.7741935483871}, {"label": "uber", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "update_playlist", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "user_name", "n": 30, "correct": 29, "accuracy": 96.66666666666667, "precision": 96.66666666666667, "f1": 96.66666666666667}, {"label": "vaccines", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "w2", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "weather", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 96.7741935483871, "f1": 98.36065573770492}, {"label": "what_are_your_hobbies", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "what_can_i_ask_you", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "what_is_your_name", "n": 30, "correct": 28, "accuracy": 93.33333333333333, "precision": 96.55172413793103, "f1": 94.91525423728814}, {"label": "what_song", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 100.0, "f1": 100.0}, {"label": "where_are_you_from", "n": 30, "correct": 29, "accuracy": 96.66666666666667, "precision": 100.0, "f1": 98.30508474576271}, {"label": "whisper_mode", "n": 30, "correct": 29, "accuracy": 96.66666666666667, "precision": 87.87878787878788, "f1": 92.06349206349206}, {"label": "who_do_you_work_for", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 93.75, "f1": 96.7741935483871}, {"label": "who_made_you", "n": 30, "correct": 30, "accuracy": 100.0, "precision": 96.7741935483871, "f1": 98.36065573770492}, {"label": "yes", "n": 30, "correct": 27, "accuracy": 90.0, "precision": 100.0, "f1": 94.73684210526316}], "confusions": [{"gold": "pin_change", "predicted": "change_ai_name", "count": 5}, {"gold": "change_volume", "predicted": "whisper_mode", "count": 4}, {"gold": "redeem_rewards", "predicted": "rewards_balance", "count": 4}, {"gold": "transactions", "predicted": "spending_history", "count": 3}, {"gold": "schedule_maintenance", "predicted": "oil_change_when", "count": 3}, {"gold": "ingredients_list", "predicted": "recipe", "count": 3}, {"gold": "restaurant_reservation", "predicted": "accept_reservations", "count": 2}, {"gold": "distance", "predicted": "flight_status", "count": 2}, {"gold": "oil_change_when", "predicted": "oil_change_how", "count": 2}, {"gold": "calendar", "predicted": "meeting_schedule", "count": 2}, {"gold": "calculator", "predicted": "spending_history", "count": 2}, {"gold": "pto_used", "predicted": "pto_balance", "count": 2}, {"gold": "report_lost_card", "predicted": "report_fraud", "count": 2}, {"gold": "restaurant_reviews", "predicted": "meal_suggestion", "count": 2}, {"gold": "what_is_your_name", "predicted": "who_do_you_work_for", "count": 2}]}, "massive": {"name": "MASSIVE \u00b7 \u4e00\u81f4\u6027\u65b9\u6cd5 \u00b7 \u4e09\u79cd\u5b50\u5408\u5e76\u8bb0\u5f55", "n": 455022, "correct": 391467, "accuracy": 86.03254348141408, "classes": [{"label": "alarm_query", "n": 5202, "correct": 4809, "accuracy": 92.44521337946944, "precision": 96.56626506024097, "f1": 94.46081319976429}, {"label": "alarm_remove", "n": 3213, "correct": 3081, "accuracy": 95.89169000933707, "precision": 94.65437788018433, "f1": 95.26901669758813}, {"label": "alarm_set", "n": 6273, "correct": 5686, "accuracy": 90.64243583612307, "precision": 90.38308694961056, "f1": 90.51257561286215}, {"label": "audio_volume_down", "n": 1683, "correct": 1523, "accuracy": 90.4931669637552, "precision": 88.44367015098723, "f1": 89.45668135095448}, {"label": "audio_volume_mute", "n": 4896, "correct": 4452, "accuracy": 90.93137254901961, "precision": 83.52720450281426, "f1": 87.07216898102875}, {"label": "audio_volume_other", "n": 918, "correct": 489, "accuracy": 53.26797385620915, "precision": 88.26714801444044, "f1": 66.44021739130434}, {"label": "audio_volume_up", "n": 1989, "correct": 1554, "accuracy": 78.12971342383108, "precision": 71.1864406779661, "f1": 74.49664429530202}, {"label": "calendar_query", "n": 19278, "correct": 15782, "accuracy": 81.86533872808383, "precision": 75.30658014028725, "f1": 78.44911147011308}, {"label": "calendar_remove", "n": 10251, "correct": 9657, "accuracy": 94.20544337137841, "precision": 87.7829288246523, "f1": 90.88085827216263}, {"label": "calendar_set", "n": 31977, "correct": 29012, "accuracy": 90.72771054195202, "precision": 90.89541951250078, "f1": 90.81148759683856}, {"label": "cooking_query", "n": 0, "correct": 0, "accuracy": 0, "precision": 0.0, "f1": 0.0}, {"label": "cooking_recipe", "n": 11016, "correct": 9451, "accuracy": 85.79339143064634, "precision": 91.59720876138786, "f1": 88.60035623886753}, {"label": "datetime_convert", "n": 2295, "correct": 1790, "accuracy": 77.99564270152506, "precision": 91.18695873662762, "f1": 84.0770314701738}, {"label": "datetime_query", "n": 13464, "correct": 12780, "accuracy": 94.91978609625669, "precision": 90.8703071672355, "f1": 92.8509154315606}, {"label": "email_addcontact", "n": 1836, "correct": 1507, "accuracy": 82.0806100217865, "precision": 66.15452151009657, "f1": 73.2620320855615}, {"label": "email_query", "n": 18207, "correct": 16877, "accuracy": 92.69511726259131, "precision": 96.96638896868716, "f1": 94.78265753116926}, {"label": "email_querycontact", "n": 3978, "correct": 3174, "accuracy": 79.78883861236802, "precision": 72.18558107800773, "f1": 75.79701492537313}, {"label": "email_sendemail", "n": 17442, "correct": 16255, "accuracy": 93.19458777663112, "precision": 95.01402852466683, "f1": 94.09551374819102}, {"label": "general_greet", "n": 153, "correct": 48, "accuracy": 31.372549019607842, "precision": 9.6579476861167, "f1": 14.76923076923077}, {"label": "general_joke", "n": 2907, "correct": 2644, "accuracy": 90.95287237702098, "precision": 94.80100394406597, "f1": 92.83707865168539}, {"label": "general_quirky", "n": 25857, "correct": 14801, "accuracy": 57.24175271686584, "precision": 65.66839700075425, "f1": 61.16621208364327}, {"label": "iot_cleaning", "n": 3978, "correct": 3653, "accuracy": 91.83006535947712, "precision": 97.43931715124033, "f1": 94.55157240843795}, {"label": "iot_coffee", "n": 5508, "correct": 5316, "accuracy": 96.51416122004358, "precision": 92.45217391304348, "f1": 94.4395096820039}, {"label": "iot_hue_lightchange", "n": 5508, "correct": 5092, "accuracy": 92.44734931009441, "precision": 84.92328218812541, "f1": 88.52573018080668}, {"label": "iot_hue_lightdim", "n": 3213, "correct": 2670, "accuracy": 83.0999066293184, "precision": 88.79281676089126, "f1": 85.85209003215434}, {"label": "iot_hue_lightoff", "n": 6579, "correct": 6118, "accuracy": 92.99285605715154, "precision": 92.80946601941747, "f1": 92.90107053374838}, {"label": "iot_hue_lighton", "n": 459, "correct": 239, "accuracy": 52.069716775599126, "precision": 45.78544061302682, "f1": 48.72579001019368}, {"label": "iot_hue_lightup", "n": 4131, "correct": 3432, "accuracy": 83.07915758896151, "precision": 91.0828025477707, "f1": 86.89707557918724}, {"label": "iot_wemo_off", "n": 2754, "correct": 2319, "accuracy": 84.20479302832244, "precision": 95.94538684319404, "f1": 89.69251595436086}, {"label": "iot_wemo_on", "n": 1530, "correct": 1191, "accuracy": 77.84313725490196, "precision": 82.19461697722568, "f1": 79.95971802618328}, {"label": "lists_createoradd", "n": 5967, "correct": 5139, "accuracy": 86.1236802413273, "precision": 85.57868442964197, "f1": 85.85031740728367}, {"label": "lists_query", "n": 7803, "correct": 6544, "accuracy": 83.86518005895168, "precision": 87.83892617449665, "f1": 85.80607093686488}, {"label": "lists_remove", "n": 7956, "correct": 6770, "accuracy": 85.0930115635998, "precision": 95.41930937279774, "f1": 89.96079994684739}, {"label": "music_dislikeness", "n": 612, "correct": 427, "accuracy": 69.77124183006536, "precision": 85.4, "f1": 76.79856115107914}, {"label": "music_likeness", "n": 5508, "correct": 4395, "accuracy": 79.7930283224401, "precision": 88.60887096774194, "f1": 83.97019487963317}, {"label": "music_query", "n": 5355, "correct": 4351, "accuracy": 81.25116713352007, "precision": 89.54517390409549, "f1": 85.19678872136284}, {"label": "music_settings", "n": 918, "correct": 356, "accuracy": 38.77995642701525, "precision": 43.520782396088016, "f1": 41.013824884792626}, {"label": "news_query", "n": 18972, "correct": 16133, "accuracy": 85.03584229390681, "precision": 83.34452652787105, "f1": 84.18169010409872}, {"label": "play_audiobook", "n": 6273, "correct": 5064, "accuracy": 80.726924916308, "precision": 79.52261306532664, "f1": 80.12024365160984}, {"label": "play_game", "n": 5355, "correct": 4346, "accuracy": 81.1577964519141, "precision": 89.03913132554804, "f1": 84.91598280578351}, {"label": "play_music", "n": 26928, "correct": 24078, "accuracy": 89.4162210338681, "precision": 86.23925501432664, "f1": 87.79900816802801}, {"label": "play_podcasts", "n": 9639, "correct": 8684, "accuracy": 90.09233322958814, "precision": 92.55035702866887, "f1": 91.3048049626748}, {"label": "play_radio", "n": 11016, "correct": 9817, "accuracy": 89.1158315177923, "precision": 84.1216795201371, "f1": 86.5467689323812}, {"label": "qa_currency", "n": 5967, "correct": 5745, "accuracy": 96.27953745600804, "precision": 92.49718241829014, "f1": 94.35046805715224}, {"label": "qa_definition", "n": 8721, "correct": 7363, "accuracy": 84.42839123953675, "precision": 84.55443270555811, "f1": 84.49136496643526}, {"label": "qa_factoid", "n": 21573, "correct": 17786, "accuracy": 82.44564965466091, "precision": 78.22491973435369, "f1": 80.2798465357707}, {"label": "qa_maths", "n": 3825, "correct": 3408, "accuracy": 89.09803921568627, "precision": 89.09803921568627, "f1": 89.09803921568627}, {"label": "qa_stock", "n": 3978, "correct": 3706, "accuracy": 93.16239316239316, "precision": 82.11832483935298, "f1": 87.29242727593923}, {"label": "recommendation_events", "n": 6579, "correct": 4971, "accuracy": 75.55859553123575, "precision": 76.71296296296296, "f1": 76.13140362968068}, {"label": "recommendation_locations", "n": 4743, "correct": 4290, "accuracy": 90.44908285895004, "precision": 76.171875, "f1": 82.6987951807229}, {"label": "recommendation_movies", "n": 3060, "correct": 2458, "accuracy": 80.32679738562092, "precision": 69.25894618202311, "f1": 74.38341655318506}, {"label": "social_post", "n": 12393, "correct": 11072, "accuracy": 89.34075687888324, "precision": 96.15284411636995, "f1": 92.62171658022419}, {"label": "social_query", "n": 3825, "correct": 3146, "accuracy": 82.2483660130719, "precision": 83.20550119016133, "f1": 82.7241651327899}, {"label": "takeaway_order", "n": 3366, "correct": 2900, "accuracy": 86.15567439096851, "precision": 77.97795106211348, "f1": 81.86309103740297}, {"label": "takeaway_query", "n": 5355, "correct": 4251, "accuracy": 79.38375350140056, "precision": 92.59420605532564, "f1": 85.48160064347476}, {"label": "transport_query", "n": 7803, "correct": 5941, "accuracy": 76.13738305779829, "precision": 83.34736251402919, "f1": 79.57939856674034}, {"label": "transport_taxi", "n": 3519, "correct": 3327, "accuracy": 94.54390451832907, "precision": 90.43218265833107, "f1": 92.44234509585996}, {"label": "transport_ticket", "n": 5355, "correct": 4946, "accuracy": 92.36227824463118, "precision": 94.04829815554288, "f1": 93.1976634633503}, {"label": "transport_traffic", "n": 2295, "correct": 2188, "accuracy": 95.33769063180829, "precision": 71.95001644195987, "f1": 82.00899550224888}, {"label": "weather_query", "n": 23868, "correct": 22463, "accuracy": 94.11345734875147, "precision": 92.39850273538727, "f1": 93.24809564333007}], "confusions": [{"gold": "general_quirky", "predicted": "qa_factoid", "count": 2270}, {"gold": "general_quirky", "predicted": "calendar_query", "count": 1455}, {"gold": "general_quirky", "predicted": "news_query", "count": 1428}, {"gold": "calendar_set", "predicted": "calendar_query", "count": 1372}, {"gold": "qa_factoid", "predicted": "general_quirky", "count": 1292}, {"gold": "calendar_query", "predicted": "calendar_set", "count": 909}, {"gold": "play_radio", "predicted": "play_music", "count": 900}, {"gold": "news_query", "predicted": "general_quirky", "count": 664}, {"gold": "lists_remove", "predicted": "calendar_remove", "count": 619}, {"gold": "email_sendemail", "predicted": "email_addcontact", "count": 606}, {"gold": "calendar_query", "predicted": "recommendation_events", "count": 600}, {"gold": "weather_query", "predicted": "general_quirky", "count": 597}, {"gold": "play_music", "predicted": "play_radio", "count": 597}, {"gold": "general_quirky", "predicted": "weather_query", "count": 591}, {"gold": "recommendation_events", "predicted": "calendar_query", "count": 523}]}};
15
+ const refs={bank:[['SPACE',94.94],['SPACE 2.0',94.77],['Llama 2 7B (FT)',94.35],['ConvFiT',94.16],['deepopen \u00b7 \u5355\u6a21\u578b',94.0909],['PPTOD',93.86]],clinc:[['TOATOD-small',98.45],['deepopen \u00b7 \u56db\u7f16\u7801\u5668\u96c6\u6210',98.0222],['TOATOD-base',98.01],['SPACE',97.89],['SPACE 2.0',97.8],['deepopen \u00b7 \u5355\u6a21\u578b',97.7556]],massive:[['deepopen \u00b7 \u6570\u636e\u589e\u5f3a',85.9873],['deepopen \u00b7 \u4e00\u81f4\u6027\u65b9\u6cd5',86.0325]]};
16
+ const notes={bank:'<h3>\u4e0e SPACE \u53c2\u8003\u503c\u76f8\u5dee 0.8491 \u4e2a\u767e\u5206\u70b9</h3><p>deepopen\uff1a94.0909%\uff1b\u516c\u5f00\u53c2\u8003 SPACE\uff1a94.94%\u3002\u8fd9\u662f\u9009\u5b9a\u6761\u76ee\u7684\u6570\u503c\u5bf9\u7167\uff0c\u4e0d\u662f\u7edf\u4e00\u91cd\u6d4b\u6216\u5b98\u65b9\u6392\u540d\u3002</p><p>SOTA2 \u539f\u9875\u9762\u5b58\u5728\u5927\u4e8e 100% \u7684\u5f02\u5e38\u51c6\u786e\u7387\u8bb0\u5f55\uff0c\u672c\u9875\u672a\u7eb3\u5165\u3002\u5c55\u793a\u9009\u5b9a\u7684\u5e38\u89c1\u76d1\u7763\u5206\u7c7b\u53c2\u8003\uff0c\u975e\u5b8c\u6574\u699c\u5355\u3002</p>',clinc:'<h3>\u96c6\u6210 98.0222%\uff0c\u5355\u6a21\u578b 97.7556%</h3><p>\u96c6\u6210\u8f83\u672c\u9879\u76ee\u5355\u6a21\u578b\u591a\u6b63\u786e 12 \u6761\uff1b\u4e0e TOATOD-small \u53c2\u8003\u5206\u6570\u76f8\u5dee 0.4278 \u4e2a\u767e\u5206\u70b9\u3002</p><p>\u56db\u7f16\u7801\u5668\u96c6\u6210\u4e0e\u5355\u6a21\u578b\u8ba1\u7b97\u6210\u672c\u4e0d\u540c\uff0c\u5206\u522b\u6807\u6ce8\uff0c\u4e0d\u5408\u5e76\u6210\u540c\u4e00\u6a21\u578b\u7684\u6210\u7ee9\u3002</p>',massive:'<h3>51 \u8bed\u8a00\u7ed3\u679c\uff0c\u72ec\u7acb\u5c55\u793a</h3><p>\u4e24\u65b9\u6cd5\u5747\u4e3a\u4e09\u79cd\u5b50 test \u5e73\u5747\u3002\u4e00\u81f4\u6027\u76f8\u5bf9\u589e\u5f3a\u589e\u52a0\u7ea6 0.0452 \u4e2a\u767e\u5206\u70b9\uff0c95% \u533a\u95f4\u5305\u542b 0\uff0c\u672a\u8bc1\u660e\u663e\u8457\u63d0\u5347\u3002</p><p>SOTA2 \u5bf9\u5e94\u53c2\u8003\u91c7\u7528\u82f1\u8bed\u8bbe\u7f6e\uff0c\u8bad\u7ec3\u6761\u4ef6\u4e0d\u540c\uff0c\u672c\u9875\u4e0d\u5c06 51 \u8bed\u8a00\u5747\u503c\u4e0e\u82f1\u8bed\u5206\u6570\u6392\u5217\u540d\u6b21\u3002</p>'};
17
+ function compare(key){document.querySelectorAll('[data-task]').forEach(b=>b.classList.toggle('active',b.dataset.task===key));document.getElementById('charttitle').textContent=key==='massive'?'51 \u8bed\u8a00 \u00b7 \u4e09\u79cd\u5b50\u5e73\u5747\u51c6\u786e\u7387':'\u6d4b\u8bd5\u51c6\u786e\u7387 \u00b7 \u4f5c\u8005\u7ed3\u679c\u4e0e\u516c\u5f00\u53c2\u8003';document.getElementById('bars').innerHTML=refs[key].map(([n,v])=>`<div class="barrow ${n.startsWith('deepopen')?'ours':''}"><span>${n}</span><div class="track"><div class="fill" style="width:${v}%"></div></div><strong>${v.toFixed(n.startsWith('deepopen')?4:2)}%</strong></div>`).join('');document.getElementById('compareNote').innerHTML=notes[key]}
18
  function render(){const k=document.getElementById('dataset').value,d=DATA[k],q=document.getElementById('search').value.toLowerCase();document.getElementById('resultstat').innerHTML=`<h3>${d.name}</h3><div class="bigstat">${d.correct.toLocaleString()} <span style="font-size:15px;color:var(--muted)">/ ${d.n.toLocaleString()} \u6761\u9884\u6d4b\u6b63\u786e \u00b7 ${d.classes.length} \u7c7b \u00b7 \u51c6\u786e\u7387 ${d.accuracy.toFixed(4)}%</span></div>`;let rows=d.classes.filter(x=>x.label.toLowerCase().includes(q)).sort(document.getElementById('sort').value==='label'?(a,b)=>a.label.localeCompare(b.label):(a,b)=>a.accuracy-b.accuracy||a.label.localeCompare(b.label));document.getElementById('classes').innerHTML=rows.length?rows.map(x=>`<tr><td>${x.label}</td><td>${x.correct} / ${x.n}</td><td><span class="mini"><i style="width:${x.accuracy}%"></i></span>${x.accuracy.toFixed(2)}%</td><td>${x.precision.toFixed(2)}%</td><td>${x.f1.toFixed(2)}%</td></tr>`).join(''):'<tr><td colspan="5">\u6ca1\u6709\u5339\u914d\u7684\u7c7b\u522b\uff0c\u8bf7\u66f4\u6362\u641c\u7d22\u8bcd\u3002</td></tr>';document.getElementById('predDownload').href=k+'_predictions.csv'+(k==='massive'?'.gz':'');document.getElementById('classDownload').href=k+'_classes.csv';document.getElementById('confusions').innerHTML='<table><tr><th>\u771f\u5b9e\u7c7b\u522b</th><th>\u9884\u6d4b\u4e3a</th><th>\u6761\u6570</th></tr>'+d.confusions.map(x=>`<tr><td>${x.gold}</td><td>${x.predicted}</td><td>${x.count}</td></tr>`).join('')+'</table>'}
19
  document.querySelectorAll('[data-task]').forEach(b=>b.onclick=()=>compare(b.dataset.task));['dataset','search','sort'].forEach(id=>document.getElementById(id).addEventListener('input',render));compare('bank');render();
20
  </script></body></html>