Upload organize_model_results.json with huggingface_hub

organize_model_results.json

@@ -13,7 +13,8 @@
             "cascade_whisper_large_v3_llama_3_8b_instruct": 85.2928225451343
         },
         "gpt4o_judge": {
-            "MERaLiON-AudioLLM-Whisper-SEA-LION": 88.77146631439894
+            "MERaLiON-AudioLLM-Whisper-SEA-LION": 88.77146631439894,
+            "Qwen2-Audio-7B-Instruct": 79.0
         }
     },
     "mmau_mini": {
@@ -41,7 +42,8 @@
             "cascade_whisper_large_v3_llama_3_8b_instruct": 55.60000000000001
         },
         "gpt4o_judge": {
-            "MERaLiON-AudioLLM-Whisper-SEA-LION": 63.9
+            "MERaLiON-AudioLLM-Whisper-SEA-LION": 63.9,
+            "Qwen2-Audio-7B-Instruct": 53.0
         }
     },
     "mmau_mini_music": {
@@ -69,7 +71,8 @@
             "cascade_whisper_large_v3_llama_3_8b_instruct": 0.5359281437125748
         },
         "gpt4o_judge": {
-            "MERaLiON-AudioLLM-Whisper-SEA-LION": 0.6347305389221557
+            "MERaLiON-AudioLLM-Whisper-SEA-LION": 0.6347305389221557,
+            "Qwen2-Audio-7B-Instruct": 0.5473684210526316
         }
     },
     "mmau_mini_sound": {
@@ -97,7 +100,8 @@
             "cascade_whisper_large_v3_llama_3_8b_instruct": 0.5105105105105106
         },
         "gpt4o_judge": {
-            "MERaLiON-AudioLLM-Whisper-SEA-LION": 0.6996996996996997
+            "MERaLiON-AudioLLM-Whisper-SEA-LION": 0.6996996996996997,
+            "Qwen2-Audio-7B-Instruct": 0.5980392156862745
         }
     },
     "mmau_mini_speech": {
@@ -125,7 +129,8 @@
             "cascade_whisper_large_v3_llama_3_8b_instruct": 0.6216216216216216
         },
         "gpt4o_judge": {
-            "MERaLiON-AudioLLM-Whisper-SEA-LION": 0.5825825825825826
+            "MERaLiON-AudioLLM-Whisper-SEA-LION": 0.5825825825825826,
+            "Qwen2-Audio-7B-Instruct": 0.44660194174757284
         }
     },
     "slue_p2_sqa5_test": {
@@ -143,6 +148,7 @@
         },
         "gpt4o_judge": {
             "MERaLiON-AudioLLM-Whisper-SEA-LION": 88.23529411764707,
+            "Qwen2-Audio-7B-Instruct": 84.86666666666666,
             "cascade_whisper_large_v3_llama_3_8b_instruct": 87.79411764705883
         }
     },
@@ -160,6 +166,7 @@
         },
         "gpt4o_judge": {
             "MERaLiON-AudioLLM-Whisper-SEA-LION": 14.813295034878948,
+            "Qwen2-Audio-7B-Instruct": 22.666666666666664,
             "cascade_whisper_large_v3_llama_3_8b_instruct": 39.462453836684446
         }
     },
@@ -201,6 +208,7 @@
         },
         "gpt4o_judge": {
             "MERaLiON-AudioLLM-Whisper-SEA-LION": 31.641618497109825,
+            "Qwen2-Audio-7B-Instruct": 34.86666666666667,
             "cascade_whisper_large_v3_llama_3_8b_instruct": 4.61271676300578
         }
     },
@@ -232,6 +240,7 @@
         },
         "gpt4o_judge": {
             "MERaLiON-AudioLLM-Whisper-SEA-LION": 71.6,
+            "Qwen2-Audio-7B-Instruct": 56.0,
             "cascade_whisper_large_v3_llama_3_8b_instruct": 67.0
         }
     },
@@ -272,7 +281,8 @@
             "cascade_whisper_large_v3_llama_3_8b_instruct": 56.44481887110362
         },
         "gpt4o_judge": {
-            "MERaLiON-AudioLLM-Whisper-SEA-LION": 57.87700084245998
+            "MERaLiON-AudioLLM-Whisper-SEA-LION": 57.87700084245998,
+            "Qwen2-Audio-7B-Instruct": 64.66666666666666
         }
     },
     "imda_30s_sqa_human_test": {
@@ -383,6 +393,7 @@
         },
         "gpt4o_judge": {
             "MERaLiON-AudioLLM-Whisper-SEA-LION": 74.2,
+            "Qwen2-Audio-7B-Instruct": 57.199999999999996,
             "cascade_whisper_large_v3_llama_3_8b_instruct": 75.0
         }
     },
@@ -398,7 +409,8 @@
             "cascade_whisper_large_v3_llama_3_8b_instruct": 46.713147410358566
         },
         "gpt4o_judge": {
-            "MERaLiON-AudioLLM-Whisper-SEA-LION": 37.45019920318725
+            "MERaLiON-AudioLLM-Whisper-SEA-LION": 37.45019920318725,
+            "Qwen2-Audio-7B-Instruct": 35.333333333333336
         }
     },
     "public_sg_speech_qa_test": {
@@ -415,6 +427,7 @@
         },
         "gpt4o_judge": {
             "MERaLiON-AudioLLM-Whisper-SEA-LION": 64.18604651162791,
+            "Qwen2-Audio-7B-Instruct": 62.733333333333334,
             "cascade_whisper_large_v3_llama_3_8b_instruct": 73.02325581395348
         }
     },
@@ -458,6 +471,7 @@
         },
         "gpt4o_judge": {
             "MERaLiON-AudioLLM-Whisper-SEA-LION": 74.99159035694262,
+            "Qwen2-Audio-7B-Instruct": 65.6,
             "cascade_whisper_large_v3_llama_3_8b_instruct": 90.12521024107643
         }
     },
@@ -489,7 +503,8 @@
             "cascade_whisper_large_v3_llama_3_8b_instruct": 45.593869731800766
         },
         "gpt4o_judge": {
-            "MERaLiON-AudioLLM-Whisper-SEA-LION": 47.356321839080465
+            "MERaLiON-AudioLLM-Whisper-SEA-LION": 47.356321839080465,
+            "Qwen2-Audio-7B-Instruct": 57.666666666666664
         }
     },
     "imda_part4_30s_sqa_human_test": {
@@ -506,6 +521,7 @@
         },
         "gpt4o_judge": {
             "MERaLiON-AudioLLM-Whisper-SEA-LION": 60.0,
+            "Qwen2-Audio-7B-Instruct": 43.4,
             "cascade_whisper_large_v3_llama_3_8b_instruct": 61.4
         }
     },
@@ -521,7 +537,8 @@
             "cascade_whisper_large_v3_llama_3_8b_instruct": 42.921624948707425
         },
         "gpt4o_judge": {
-            "MERaLiON-AudioLLM-Whisper-SEA-LION": 99.46655724251129
+            "MERaLiON-AudioLLM-Whisper-SEA-LION": 99.46655724251129,
+            "Qwen2-Audio-7B-Instruct": 99.66666666666667
         }
     },
     "imda_gr_dialogue": {
@@ -537,7 +554,8 @@
             "cascade_whisper_large_v3_llama_3_8b_instruct": 25.433333333333337
         },
         "gpt4o_judge": {
-            "MERaLiON-AudioLLM-Whisper-SEA-LION": 93.86666666666666
+            "MERaLiON-AudioLLM-Whisper-SEA-LION": 93.86666666666666,
+            "Qwen2-Audio-7B-Instruct": 61.0
         }
     },
     "imda_30s_ds_human_test": {
@@ -693,6 +711,7 @@
         },
         "gpt4o_judge": {
             "MERaLiON-AudioLLM-Whisper-SEA-LION": 59.2,
+            "Qwen2-Audio-7B-Instruct": 43.2,
             "cascade_whisper_large_v3_llama_3_8b_instruct": 47.400000000000006
         }
     },
@@ -709,7 +728,8 @@
             "cascade_whisper_large_v3_llama_3_8b_instruct": 9.666666666666666
         },
         "gpt4o_judge": {
-            "MERaLiON-AudioLLM-Whisper-SEA-LION": 77.13333333333333
+            "MERaLiON-AudioLLM-Whisper-SEA-LION": 77.13333333333333,
+            "Qwen2-Audio-7B-Instruct": 0.33333333333333337
         }
     },
     "iemocap_gender_test": {
@@ -725,7 +745,8 @@
             "cascade_whisper_large_v3_llama_3_8b_instruct": 44.22310756972111
         },
         "gpt4o_judge": {
-            "MERaLiON-AudioLLM-Whisper-SEA-LION": 98.20717131474103
+            "MERaLiON-AudioLLM-Whisper-SEA-LION": 98.20717131474103,
+            "Qwen2-Audio-7B-Instruct": 98.33333333333333
         }
     },
     "ytb_asr_batch2": {
@@ -766,7 +787,8 @@
             "cascade_whisper_large_v3_llama_3_8b_instruct": 86.4610559330894
         },
         "gpt4o_judge": {
-            "MERaLiON-AudioLLM-Whisper-SEA-LION": 84.31782540512285
+            "MERaLiON-AudioLLM-Whisper-SEA-LION": 84.31782540512285,
+            "Qwen2-Audio-7B-Instruct": 68.66666666666667
         }
     },
     "imda_part5_30s_ds_human_test": {
@@ -783,6 +805,7 @@
         },
         "gpt4o_judge": {
             "MERaLiON-AudioLLM-Whisper-SEA-LION": 64.0,
+            "Qwen2-Audio-7B-Instruct": 50.8,
             "cascade_whisper_large_v3_llama_3_8b_instruct": 56.8
         }
     },
@@ -814,6 +837,7 @@
         },
         "gpt4o_judge": {
             "MERaLiON-AudioLLM-Whisper-SEA-LION": 60.599999999999994,
+            "Qwen2-Audio-7B-Instruct": 50.199999999999996,
             "cascade_whisper_large_v3_llama_3_8b_instruct": 52.800000000000004
         }
     },
@@ -831,6 +855,7 @@
         },
         "gpt4o_judge": {
             "MERaLiON-AudioLLM-Whisper-SEA-LION": 55.199999999999996,
+            "Qwen2-Audio-7B-Instruct": 35.8,
             "cascade_whisper_large_v3_llama_3_8b_instruct": 48.2
         }
     },
@@ -871,6 +896,7 @@
         },
         "gpt4o_judge": {
             "MERaLiON-AudioLLM-Whisper-SEA-LION": 39.29545454545455,
+            "Qwen2-Audio-7B-Instruct": 41.53333333333333,
             "cascade_whisper_large_v3_llama_3_8b_instruct": 4.868181818181818
         }
     },
@@ -887,7 +913,8 @@
             "cascade_whisper_large_v3_llama_3_8b_instruct": 12.416666666666666
         },
         "gpt4o_judge": {
-            "MERaLiON-AudioLLM-Whisper-SEA-LION": 10.116666666666667
+            "MERaLiON-AudioLLM-Whisper-SEA-LION": 10.116666666666667,
+            "Qwen2-Audio-7B-Instruct": 4.666666666666667
         }
     },
     "imda_part6_30s_sqa_test": {
@@ -944,6 +971,7 @@
         },
         "gpt4o_judge": {
             "MERaLiON-AudioLLM-Whisper-SEA-LION": 64.09333981526495,
+            "Qwen2-Audio-7B-Instruct": 56.86666666666667,
             "cascade_whisper_large_v3_llama_3_8b_instruct": 28.076410484229232
         }
     },
@@ -988,6 +1016,7 @@
         },
         "gpt4o_judge": {
             "MERaLiON-AudioLLM-Whisper-SEA-LION": 78.60000000000001,
+            "Qwen2-Audio-7B-Instruct": 61.6,
             "cascade_whisper_large_v3_llama_3_8b_instruct": 77.8
         }
     },
@@ -1028,6 +1057,7 @@
         },
         "gpt4o_judge": {
             "MERaLiON-AudioLLM-Whisper-SEA-LION": 50.60702875399361,
+            "Qwen2-Audio-7B-Instruct": 50.599999999999994,
             "cascade_whisper_large_v3_llama_3_8b_instruct": 14.63258785942492
         }
     },
@@ -1086,6 +1116,7 @@
         },
         "gpt4o_judge": {
             "MERaLiON-AudioLLM-Whisper-SEA-LION": 46.31578947368421,
+            "Qwen2-Audio-7B-Instruct": 48.2,
             "cascade_whisper_large_v3_llama_3_8b_instruct": 14.736842105263158
         }
     },
@@ -1140,7 +1171,8 @@
             "cascade_whisper_large_v3_llama_3_8b_instruct": 36.016666666666666
         },
         "gpt4o_judge": {
-            "MERaLiON-AudioLLM-Whisper-SEA-LION": 66.9
+            "MERaLiON-AudioLLM-Whisper-SEA-LION": 66.9,
+            "Qwen2-Audio-7B-Instruct": 66.33333333333333
         }
     },
     "tedlium3_long_form_test": {
@@ -1170,7 +1202,8 @@
             "cascade_whisper_large_v3_llama_3_8b_instruct": 36.81992337164751
         },
         "gpt4o_judge": {
-            "MERaLiON-AudioLLM-Whisper-SEA-LION": 36.206896551724135
+            "MERaLiON-AudioLLM-Whisper-SEA-LION": 36.206896551724135,
+            "Qwen2-Audio-7B-Instruct": 39.0
         }
     },
     "tedlium3_test": {
@@ -1228,6 +1261,7 @@
         },
         "gpt4o_judge": {
             "MERaLiON-AudioLLM-Whisper-SEA-LION": 66.8,
+            "Qwen2-Audio-7B-Instruct": 58.0,
             "cascade_whisper_large_v3_llama_3_8b_instruct": 64.80000000000001
         }
     },
@@ -1245,6 +1279,7 @@
         },
         "gpt4o_judge": {
             "MERaLiON-AudioLLM-Whisper-SEA-LION": 70.0,
+            "Qwen2-Audio-7B-Instruct": 51.4,
             "cascade_whisper_large_v3_llama_3_8b_instruct": 64.4
         }
     },
@@ -1299,12 +1334,14 @@
     },
     "gigaspeech2_viet": {
         "wer": {
-            "MERaLiON-AudioLLM-Whisper-SEA-LION": 0.4960741822016732
+            "MERaLiON-AudioLLM-Whisper-SEA-LION": 0.4960741822016732,
+            "Qwen2-Audio-7B-Instruct": 1.5011671350211242
         }
     },
     "gigaspeech2_thai": {
         "wer": {
             "MERaLiON-AudioLLM-Whisper-SEA-LION": 0.8901628256099774,
+            "Qwen2-Audio-7B-Instruct": 1.2449725324578913,
             "seallms_audio_7b": 0.3332398502070376
         }
     },