Spaces:
Sleeping
Sleeping
| { | |
| "generatedAt": "2026-03-13T16:26:56.022330+00:00", | |
| "sourceRoot": "data/results_real", | |
| "tasks": [ | |
| { | |
| "id": "K-disentQA", | |
| "label": "SCA-QA", | |
| "metricLabel": "Speech Context Faithfulness", | |
| "shortMetric": "Faithfulness", | |
| "lowerBetter": false, | |
| "datasets": [ | |
| { | |
| "id": "history_after_chosun", | |
| "label": "History_after_chosun" | |
| }, | |
| { | |
| "id": "history_before_chosun", | |
| "label": "History_before_chosun" | |
| }, | |
| { | |
| "id": "k-sports", | |
| "label": "K-sports" | |
| }, | |
| { | |
| "id": "kpop", | |
| "label": "K-pop" | |
| } | |
| ] | |
| }, | |
| { | |
| "id": "SQA", | |
| "label": "Speech QA", | |
| "metricLabel": "Accuracy (%)", | |
| "shortMetric": "Acc(%)", | |
| "lowerBetter": false, | |
| "datasets": [ | |
| { | |
| "id": "click", | |
| "label": "CLICk" | |
| }, | |
| { | |
| "id": "click_other", | |
| "label": "CLICk Other" | |
| }, | |
| { | |
| "id": "kobest_boolq", | |
| "label": "KoBest BoolQ" | |
| }, | |
| { | |
| "id": "kobest_boolq_other", | |
| "label": "KoBest BoolQ Other" | |
| }, | |
| { | |
| "id": "kcsat", | |
| "label": "KCSAT" | |
| }, | |
| { | |
| "id": "kcsat_other", | |
| "label": "KCSAT Other" | |
| } | |
| ] | |
| }, | |
| { | |
| "id": "Instruct", | |
| "label": "Speech Instruction", | |
| "metricLabel": "Score (GPT-4o as Judge)", | |
| "shortMetric": "Score (GPT-4o as Judge)", | |
| "lowerBetter": false, | |
| "datasets": [ | |
| { | |
| "id": "alpaca", | |
| "label": "Alpaca" | |
| }, | |
| { | |
| "id": "alpaca_other", | |
| "label": "Alpaca Other" | |
| }, | |
| { | |
| "id": "kudge", | |
| "label": "KUDGE" | |
| }, | |
| { | |
| "id": "kudge_other", | |
| "label": "KUDGE Other" | |
| }, | |
| { | |
| "id": "openhermes", | |
| "label": "OpenHermes" | |
| }, | |
| { | |
| "id": "openhermes_other", | |
| "label": "OpenHermes Other" | |
| }, | |
| { | |
| "id": "vicuna", | |
| "label": "Vicuna" | |
| }, | |
| { | |
| "id": "vicuna_other", | |
| "label": "Vicuna Other" | |
| } | |
| ] | |
| }, | |
| { | |
| "id": "ASR", | |
| "label": "ASR", | |
| "metricLabel": "CER (%)", | |
| "shortMetric": "CER", | |
| "lowerBetter": true, | |
| "datasets": [ | |
| { | |
| "id": "common_voice_korea", | |
| "label": "CommonVoice-KO" | |
| }, | |
| { | |
| "id": "ksponspeech_eval_clean", | |
| "label": "KsponSpeech Clean" | |
| }, | |
| { | |
| "id": "ksponspeech_eval_other", | |
| "label": "KsponSpeech Other" | |
| }, | |
| { | |
| "id": "zeroth_korean_test", | |
| "label": "Zeroth-Korean" | |
| }, | |
| { | |
| "id": "zeroth_korean_test_other", | |
| "label": "Zeroth-Korean Other" | |
| } | |
| ] | |
| }, | |
| { | |
| "id": "Translation", | |
| "label": "Translation", | |
| "metricLabel": "BERTScore", | |
| "shortMetric": "BERTScore", | |
| "lowerBetter": false, | |
| "datasets": [ | |
| { | |
| "id": "etri_tst-COMMON", | |
| "label": "ETRI-TST-Common" | |
| }, | |
| { | |
| "id": "etri_tst-HE", | |
| "label": "ETRI-TST-HE" | |
| } | |
| ] | |
| }, | |
| { | |
| "id": "PA-QA", | |
| "label": "Postion Aware Question Answering", | |
| "metricLabel": "Accuracy (%)", | |
| "shortMetric": "Acc(%)", | |
| "lowerBetter": false, | |
| "datasets": [ | |
| { | |
| "id": "mctest", | |
| "label": "MCTest" | |
| }, | |
| { | |
| "id": "mctest_other", | |
| "label": "MCTest Other" | |
| } | |
| ] | |
| } | |
| ], | |
| "entries": [ | |
| { | |
| "id": "gemini_flash", | |
| "rank_name": "gemini_flash", | |
| "model": "gemini-2.5-flash-lite", | |
| "url": "", | |
| "tasks": { | |
| "ASR": { | |
| "common_voice_korea": { | |
| "value": 13.738049713193117, | |
| "display": "13.74" | |
| }, | |
| "ksponspeech_eval_clean": { | |
| "value": 83.18526725056962, | |
| "display": "83.19" | |
| }, | |
| "ksponspeech_eval_other": { | |
| "value": 45.13922315086276, | |
| "display": "45.14" | |
| }, | |
| "zeroth_korean_test": { | |
| "value": 13.599128992119452, | |
| "display": "13.60" | |
| }, | |
| "zeroth_korean_test_other": { | |
| "value": 14.558274574865202, | |
| "display": "14.56" | |
| } | |
| }, | |
| "Instruct": { | |
| "alpaca": { | |
| "value": 85.94202898550726, | |
| "display": "85.94" | |
| }, | |
| "alpaca_other": { | |
| "value": 86.8840579710145, | |
| "display": "86.88" | |
| }, | |
| "kudge": { | |
| "value": 70.28725314183116, | |
| "display": "70.29" | |
| }, | |
| "kudge_other": { | |
| "value": 70.3859964093356, | |
| "display": "70.39" | |
| }, | |
| "openhermes": { | |
| "value": 82.69230769230771, | |
| "display": "82.69" | |
| }, | |
| "openhermes_other": { | |
| "value": 80.70512820512825, | |
| "display": "80.71" | |
| }, | |
| "vicuna": { | |
| "value": 76.42857142857144, | |
| "display": "76.43" | |
| }, | |
| "vicuna_other": { | |
| "value": 73.28571428571429, | |
| "display": "73.29" | |
| } | |
| }, | |
| "K-disentQA": { | |
| "history_after_chosun": { | |
| "value": 59.72, | |
| "display": "59.72" | |
| }, | |
| "history_before_chosun": { | |
| "value": 66.25, | |
| "display": "66.25" | |
| }, | |
| "k-sports": { | |
| "value": 86.96, | |
| "display": "86.96" | |
| }, | |
| "kpop": { | |
| "value": 78.26, | |
| "display": "78.26" | |
| } | |
| }, | |
| "PA-QA": { | |
| "mctest_other": { | |
| "value": 92.00, | |
| "display": "92.00" | |
| }, | |
| "mctest": { | |
| "value": 92.31, | |
| "display": "92.31" | |
| } | |
| }, | |
| "SQA": { | |
| "click": { | |
| "value": 62.27, | |
| "display": "62.27" | |
| }, | |
| "click_other": { | |
| "value": 66.69, | |
| "display": "66.69" | |
| }, | |
| "kobest_boolq": { | |
| "value": 52.86, | |
| "display": "52.86" | |
| }, | |
| "kobest_boolq_other": { | |
| "value": 54.92, | |
| "display": "54.92" | |
| }, | |
| "kcsat": { | |
| "value": 81.18 , | |
| "display": "81.18 " | |
| }, | |
| "kcsat_other": { | |
| "value": 78.82, | |
| "display": "78.82" | |
| } | |
| }, | |
| "Translation": { | |
| "etri_tst-COMMON": { | |
| "value": 91.60, | |
| "display": "91.60" | |
| }, | |
| "etri_tst-HE": { | |
| "value": 92.17, | |
| "display": "92.17" | |
| } | |
| } | |
| } | |
| }, | |
| { | |
| "id": "gemma3n_vllm", | |
| "rank_name": "gemma3n_vllm", | |
| "model": "google/gemma-3n-E4B-it", | |
| "url": "https://huggingface.co/google/gemma-3n-E4B-it", | |
| "tasks": { | |
| "ASR": { | |
| "common_voice_korea": { | |
| "value": 144.5793499043977, | |
| "display": "144.58" | |
| }, | |
| "ksponspeech_eval_clean": { | |
| "value": 142.99420974518696, | |
| "display": "142.99" | |
| }, | |
| "ksponspeech_eval_other": { | |
| "value": 130.76182558088652, | |
| "display": "130.76" | |
| }, | |
| "zeroth_korean_test": { | |
| "value": 107.88054749066778, | |
| "display": "107.88" | |
| }, | |
| "zeroth_korean_test_other": { | |
| "value": 118.62297801742017, | |
| "display": "118.62" | |
| } | |
| }, | |
| "Instruct": { | |
| "alpaca": { | |
| "value": 82.97101449275362, | |
| "display": "82.97" | |
| }, | |
| "alpaca_other": { | |
| "value": 83.36231884057973, | |
| "display": "83.36" | |
| }, | |
| "kudge": { | |
| "value": 71.38240574506275, | |
| "display": "71.38" | |
| }, | |
| "kudge_other": { | |
| "value": 70.69120287253139, | |
| "display": "70.69" | |
| }, | |
| "openhermes": { | |
| "value": 84.61538461538464, | |
| "display": "84.62" | |
| }, | |
| "openhermes_other": { | |
| "value": 85.96153846153848, | |
| "display": "85.96" | |
| }, | |
| "vicuna": { | |
| "value": 80.21428571428574, | |
| "display": "80.21" | |
| }, | |
| "vicuna_other": { | |
| "value": 80.00000000000003, | |
| "display": "80.00" | |
| } | |
| }, | |
| "K-disentQA": { | |
| "history_after_chosun": { | |
| "value": 45.76, | |
| "display": "45.76" | |
| }, | |
| "history_before_chosun": { | |
| "value": 67.35, | |
| "display": "67.35" | |
| }, | |
| "k-sports": { | |
| "value": 76.60, | |
| "display": "76.60" | |
| }, | |
| "kpop": { | |
| "value": 73.24, | |
| "display": "73.24" | |
| } | |
| }, | |
| "PA-QA": { | |
| "mctest": { | |
| "value": 48.92 , | |
| "display": "48.92 " | |
| }, | |
| "mctest_other": { | |
| "value": 48.92, | |
| "display": "48.92" | |
| } | |
| }, | |
| "SQA": { | |
| "click": { | |
| "value": 35.79, | |
| "display": "35.79" | |
| }, | |
| "click_other": { | |
| "value": 35.708367854183926, | |
| "display": "35.71" | |
| }, | |
| "kobest_boolq": { | |
| "value": 50.89 , | |
| "display": "50.89" | |
| }, | |
| "kobest_boolq_other": { | |
| "value": 50.54, | |
| "display": "50.54" | |
| }, | |
| "kcsat": { | |
| "value": 34.12 , | |
| "display": "34.12" | |
| }, | |
| "kcsat_other": { | |
| "value": 40.00, | |
| "display": "40.00" | |
| } | |
| }, | |
| "Translation": { | |
| "etri_tst-COMMON": { | |
| "value": 87.39, | |
| "display": "87.39" | |
| }, | |
| "etri_tst-HE": { | |
| "value": 87.79, | |
| "display": "87.79" | |
| } | |
| } | |
| } | |
| }, | |
| { | |
| "id": "gpt_audio", | |
| "rank_name": "gpt_audio", | |
| "model": "gpt-audio-mini", | |
| "url": "", | |
| "tasks": { | |
| "ASR": { | |
| "common_voice_korea": { | |
| "value": 33.04971319311664, | |
| "display": "33.05" | |
| }, | |
| "ksponspeech_eval_clean": { | |
| "value": 134.18967787788205, | |
| "display": "134.19" | |
| }, | |
| "ksponspeech_eval_other": { | |
| "value": 63.6444522236322, | |
| "display": "63.64" | |
| }, | |
| "zeroth_korean_test": { | |
| "value": 6.87, | |
| "display": "6.87" | |
| }, | |
| "zeroth_korean_test_other": { | |
| "value": 9.00, | |
| "display": "9.00" | |
| } | |
| }, | |
| "Instruct": { | |
| "alpaca": { | |
| "value": 90.57971014492755, | |
| "display": "90.58" | |
| }, | |
| "alpaca_other": { | |
| "value": 90.57971014492755, | |
| "display": "90.58" | |
| }, | |
| "kudge": { | |
| "value": 74.06642728904846, | |
| "display": "74.07" | |
| }, | |
| "kudge_other": { | |
| "value": 73.98563734290842, | |
| "display": "73.99" | |
| }, | |
| "openhermes": { | |
| "value": 89.42307692307693, | |
| "display": "89.42" | |
| }, | |
| "openhermes_other": { | |
| "value": 89.61538461538464, | |
| "display": "89.62" | |
| }, | |
| "vicuna": { | |
| "value": 82.14285714285717, | |
| "display": "82.14" | |
| }, | |
| "vicuna_other": { | |
| "value": 81.78571428571429, | |
| "display": "81.79" | |
| } | |
| }, | |
| "K-disentQA": { | |
| "history_after_chosun": { | |
| "value": 32.30, | |
| "display": "32.30" | |
| }, | |
| "history_before_chosun": { | |
| "value": 61.40, | |
| "display": "61.40" | |
| }, | |
| "k-sports": { | |
| "value": 39.30, | |
| "display": "39.30" | |
| }, | |
| "kpop": { | |
| "value": 37.50, | |
| "display": "37.50" | |
| } | |
| }, | |
| "PA-QA": { | |
| "mctest_other": { | |
| "value": 79.69, | |
| "display": "79.69" | |
| }, | |
| "mctest": { | |
| "value": 77.23 , | |
| "display": "77.23 " | |
| } | |
| }, | |
| "SQA": { | |
| "click": { | |
| "value": 61.64043082021541, | |
| "display": "61.64" | |
| }, | |
| "click_other": { | |
| "value": 60.06628003314002, | |
| "display": "60.07" | |
| }, | |
| "kobest_boolq": { | |
| "value": 51.878354203935594, | |
| "display": "51.88" | |
| }, | |
| "kobest_boolq_other": { | |
| "value": 50.44722719141323, | |
| "display": "50.45" | |
| }, | |
| "kcsat": { | |
| "value": 52.90 , | |
| "display": "52.90 " | |
| }, | |
| "kcsat_other": { | |
| "value": 47.10, | |
| "display": "47.10" | |
| } | |
| }, | |
| "Translation": { | |
| "etri_tst-COMMON": { | |
| "value": 93.10, | |
| "display": "93.10" | |
| }, | |
| "etri_tst-HE": { | |
| "value": 93.69, | |
| "display": "93.69" | |
| } | |
| } | |
| } | |
| }, | |
| { | |
| "id": "qwen3_onmi", | |
| "rank_name": "qwen3_onmi", | |
| "model": "Qwen/Qwen3-Omni-30B-A3B-Instruct", | |
| "url": "https://huggingface.co/Qwen/Qwen3-Omni-30B-A3B-Instruct", | |
| "tasks": { | |
| "ASR": { | |
| "common_voice_korea": { | |
| "value": 4.961759082217973, | |
| "display": "4.96" | |
| }, | |
| "ksponspeech_eval_clean": { | |
| "value": 8.459624992161208, | |
| "display": "8.46" | |
| }, | |
| "ksponspeech_eval_other": { | |
| "value": 7.907058154290465, | |
| "display": "7.91" | |
| }, | |
| "zeroth_korean_test": { | |
| "value": 3.3336789713811696, | |
| "display": "3.33" | |
| }, | |
| "zeroth_korean_test_other": { | |
| "value": 3.9143508917461634, | |
| "display": "3.91" | |
| } | |
| }, | |
| "Instruct": { | |
| "alpaca": { | |
| "value": 84.05797101449278, | |
| "display": "84.06" | |
| }, | |
| "alpaca_other": { | |
| "value": 83.04347826086959, | |
| "display": "83.04" | |
| }, | |
| "kudge": { | |
| "value": 71.86714542190298, | |
| "display": "71.87" | |
| }, | |
| "kudge_other": { | |
| "value": 71.82226211849184, | |
| "display": "71.82" | |
| }, | |
| "openhermes": { | |
| "value": 86.5384615384616, | |
| "display": "86.54" | |
| }, | |
| "openhermes_other": { | |
| "value": 85.19230769230771, | |
| "display": "85.19" | |
| }, | |
| "vicuna": { | |
| "value": 79.64285714285715, | |
| "display": "79.64" | |
| }, | |
| "vicuna_other": { | |
| "value": 78.42857142857143, | |
| "display": "78.43" | |
| } | |
| }, | |
| "K-disentQA": { | |
| "history_after_chosun": { | |
| "value": 92.31, | |
| "display": "92.31" | |
| }, | |
| "history_before_chosun": { | |
| "value": 94.64, | |
| "display": "94.64" | |
| }, | |
| "k-sports": { | |
| "value": 93.88, | |
| "display": "93.88" | |
| }, | |
| "kpop": { | |
| "value": 95.71, | |
| "display": "95.71" | |
| } | |
| }, | |
| "PA-QA": { | |
| "mctest": { | |
| "value": 93.54, | |
| "display": "93.54" | |
| }, | |
| "mctest_other": { | |
| "value": 93.85, | |
| "display": "93.85" | |
| } | |
| }, | |
| "SQA": { | |
| "click": { | |
| "value": 64.04, | |
| "display": "64.04" | |
| }, | |
| "click_other": { | |
| "value": 62.30, | |
| "display": "62.30" | |
| }, | |
| "kobest_boolq": { | |
| "value": 51.341681574239715, | |
| "display": "51.34" | |
| }, | |
| "kobest_boolq_other": { | |
| "value": 51.16, | |
| "display": "51.16" | |
| }, | |
| "kcsat": { | |
| "value": 83.53, | |
| "display": "83.53" | |
| }, | |
| "kcsat_other": { | |
| "value": 84.71, | |
| "display": "84.71" | |
| } | |
| }, | |
| "Translation": { | |
| "etri_tst-COMMON": { | |
| "value": 93.40, | |
| "display": "93.40" | |
| }, | |
| "etri_tst-HE": { | |
| "value": 93.96, | |
| "display": "93.96" | |
| } | |
| } | |
| } | |
| }, | |
| { | |
| "id": "voxtral", | |
| "rank_name": "voxtral", | |
| "model": "mistralai/Voxtral-Mini-3B-2507", | |
| "url": "https://huggingface.co/mistralai/Voxtral-Mini-3B-2507", | |
| "tasks": { | |
| "ASR": { | |
| "common_voice_korea": { | |
| "value": 60.09560229445506, | |
| "display": "60.10" | |
| }, | |
| "ksponspeech_eval_clean": { | |
| "value": 62.62463680260875, | |
| "display": "62.62" | |
| }, | |
| "ksponspeech_eval_other": { | |
| "value": 56.042240989512685, | |
| "display": "56.04" | |
| }, | |
| "zeroth_korean_test": { | |
| "value": 40.92181667357943, | |
| "display": "40.92" | |
| }, | |
| "zeroth_korean_test_other": { | |
| "value": 39.06574035669846, | |
| "display": "39.07" | |
| } | |
| }, | |
| "Instruct": { | |
| "alpaca": { | |
| "value": 72.89855072463769, | |
| "display": "72.90" | |
| }, | |
| "alpaca_other": { | |
| "value": 72.46376811594205, | |
| "display": "72.46" | |
| }, | |
| "kudge": { | |
| "value": 61.9658886894074, | |
| "display": "61.97" | |
| }, | |
| "kudge_other": { | |
| "value": 61.69658886894065, | |
| "display": "61.70" | |
| }, | |
| "openhermes": { | |
| "value": 69.10256410256412, | |
| "display": "69.10" | |
| }, | |
| "openhermes_other": { | |
| "value": 69.61538461538463, | |
| "display": "69.62" | |
| }, | |
| "vicuna": { | |
| "value": 67.78571428571428, | |
| "display": "67.79" | |
| }, | |
| "vicuna_other": { | |
| "value": 69.49999999999997, | |
| "display": "69.50" | |
| } | |
| }, | |
| "K-disentQA": { | |
| "history_after_chosun": { | |
| "value": 85.71, | |
| "display": "85.71" | |
| }, | |
| "history_before_chosun": { | |
| "value": 85.71, | |
| "display": "85.71" | |
| }, | |
| "k-sports": { | |
| "value": 95.83, | |
| "display": "95.83" | |
| }, | |
| "kpop": { | |
| "value": 88.64, | |
| "display": "88.64" | |
| } | |
| }, | |
| "PA-QA": { | |
| "mctest_other": { | |
| "value": 84.92, | |
| "display": "84.92" | |
| }, | |
| "mctest": { | |
| "value": 86.15, | |
| "display": "86.15" | |
| } | |
| }, | |
| "SQA": { | |
| "click": { | |
| "value": 42.58 , | |
| "display": "42.58 " | |
| }, | |
| "click_other": { | |
| "value": 42.92, | |
| "display": " 42.92" | |
| }, | |
| "kobest_boolq": { | |
| "value": 50.54, | |
| "display": "50.54" | |
| }, | |
| "kobest_boolq_other": { | |
| "value": 50.54, | |
| "display": "50.54" | |
| }, | |
| "kcsat": { | |
| "value": 69.41, | |
| "display": "69.41" | |
| }, | |
| "kcsat_other": { | |
| "value": 72.94, | |
| "display": "72.94" | |
| } | |
| }, | |
| "Translation": { | |
| "etri_tst-COMMON": { | |
| "value": 92.73, | |
| "display": "92.73" | |
| }, | |
| "etri_tst-HE": { | |
| "value": 93.09, | |
| "display": "93.09" | |
| } | |
| } | |
| } | |
| } | |
| ] | |
| } |