{ "emolia-emo": { "task": "emolia_bench_current", "subset": "emolia-emo", "model": "ensD_4way", "benchmark_dir": "voiceclap/emolia-bench-current", "benchmark_commit": "unknown", "rows_total": 7988, "rows_evaluated_ok": 7988, "missing_audio": 0, "threshold": 0.0, "similarity_distribution": { "min": -0.2042248547077179, "median": 0.14339810609817505, "max": 0.6953250169754028, "mean": 0.17190776765346527 }, "overall_binary_at_thr0": { "n": 7988, "accuracy": 0.6165498247371056, "balanced_accuracy": 0.5615750916527579, "precision": 0.6128376415916352, "recall": 0.9140134286333117, "f1": 0.7337216378336087, "tp": 4220, "fp": 2666, "tn": 705, "fn": 397 }, "optimal_threshold": 0.16227242350578308, "overall_binary_at_optimal_thr": { "n": 7988, "accuracy": 0.671256885327992, "balanced_accuracy": 0.6825126235976609, "precision": 0.7731138545953361, "recall": 0.6103530431015811, "f1": 0.682159283466473, "tp": 2818, "fp": 827, "tn": 2544, "fn": 1799 }, "overall_binary_at_per_prompt_thr": { "n": 7988, "accuracy": 0.6985478217325989, "balanced_accuracy": 0.7084827093865312, "precision": 0.7949265687583444, "recall": 0.6447909898202295, "f1": 0.712030614685482, "tp": 2977, "fp": 768, "tn": 2603, "fn": 1640 }, "per_prompt_thresholds": { "Affection": 0.3331466317176819, "Amusement": 0.22572538256645203, "Anger": 0.18943950533866882, "Astonishment_Surprise": 0.4959396719932556, "Awe": 0.23143336176872253, "Bitterness": 0.13148218393325806, "Concentration": 0.062052492052316666, "Confusion": 0.16997498273849487, "Contemplation": 0.06245122104883194, "Contempt": 0.16315330564975739, "Contentment": 0.1117238700389862, "Disappointment": 0.09065057337284088, "Disgust": 0.272680401802063, "Distress": 0.2060888111591339, "Doubt": 0.10864093899726868, "Elation": 0.1622307449579239, "Embarrassment": 0.11364679038524628, "Emotional_Numbness": 0.10127806663513184, "Fatigue_Exhaustion": 0.17510147392749786, "Fear": 0.11485738307237625, "Helplessness": 0.13407103717327118, "Hope_Enthusiasm_Optimism": 0.11019618064165115, "Impatience_and_Irritability": 0.17070239782333374, "Infatuation": 0.1867411732673645, "Interest": 0.018342573195695877, "Intoxication_Altered_States_of_Consciousness": 0.15875428915023804, "Jealousy___Envy": 0.19299444556236267, "Longing": 0.14367274940013885, "Malevolence_Malice": 0.28945234417915344, "Pain": 0.17142678797245026, "Pleasure_Ecstasy": 0.18039974570274353, "Pride": 0.2631188929080963, "Relief": 0.2775209844112396, "Sadness": 0.1803743988275528, "Sexual_Lust": 0.16225460171699524, "Shame": 0.17446206510066986, "Sourness": 0.08683902025222778, "Teasing": 0.20157286524772644, "Thankfulness_Gratitude": 0.37203627824783325, "Triumph": 0.16911129653453827 }, "spearman_avg": 0.37727163486069065, "spearman_per_prompt": { "Affection": { "n": 201, "spearman": 0.5066440849697682 }, "Amusement": { "n": 229, "spearman": 0.5191409994632501 }, "Anger": { "n": 352, "spearman": 0.5896575793397766 }, "Astonishment_Surprise": { "n": 100, "spearman": 0.1702081315176889 }, "Awe": { "n": 131, "spearman": 0.4850594231020607 }, "Bitterness": { "n": 129, "spearman": 0.4339827010541016 }, "Concentration": { "n": 125, "spearman": 0.11251900480357668 }, "Confusion": { "n": 113, "spearman": 0.45335700853921745 }, "Contemplation": { "n": 131, "spearman": 0.4687195718117067 }, "Contempt": { "n": 189, "spearman": 0.4668975508213962 }, "Contentment": { "n": 250, "spearman": 0.403321293380002 }, "Disappointment": { "n": 125, "spearman": 0.371564914465576 }, "Disgust": { "n": 127, "spearman": 0.5041216359144547 }, "Distress": { "n": 200, "spearman": 0.6454614662084704 }, "Doubt": { "n": 100, "spearman": 0.1500698435014196 }, "Elation": { "n": 325, "spearman": 0.3884938208100936 }, "Embarrassment": { "n": 317, "spearman": 0.3969519241374696 }, "Emotional_Numbness": { "n": 207, "spearman": 0.4188790708398977 }, "Fatigue_Exhaustion": { "n": 244, "spearman": 0.474472976796517 }, "Fear": { "n": 311, "spearman": 0.25902658230137837 }, "Helplessness": { "n": 141, "spearman": 0.5316415130367269 }, "Hope_Enthusiasm_Optimism": { "n": 111, "spearman": 0.2462512071676988 }, "Impatience_and_Irritability": { "n": 238, "spearman": 0.6124228451798376 }, "Infatuation": { "n": 423, "spearman": 0.26488605338154636 }, "Interest": { "n": 101, "spearman": 0.06595055319381218 }, "Intoxication_Altered_States_of_Consciousness": { "n": 606, "spearman": 0.28428624917362105 }, "Jealousy___Envy": { "n": 100, "spearman": 0.2519435402531288 }, "Longing": { "n": 250, "spearman": 0.4356346002123791 }, "Malevolence_Malice": { "n": 100, "spearman": 0.4955207816357248 }, "Pain": { "n": 171, "spearman": 0.3786580774530074 }, "Pleasure_Ecstasy": { "n": 172, "spearman": 0.48498427366383057 }, "Pride": { "n": 150, "spearman": 0.2633846021091064 }, "Relief": { "n": 162, "spearman": 0.39458968028630725 }, "Sadness": { "n": 150, "spearman": 0.5400015741852219 }, "Sexual_Lust": { "n": 280, "spearman": 0.18414398889834926 }, "Shame": { "n": 125, "spearman": 0.3463197328064688 }, "Sourness": { "n": 221, "spearman": 0.12783088832942807 }, "Teasing": { "n": 100, "spearman": 0.4112119575005198 }, "Thankfulness_Gratitude": { "n": 103, "spearman": 0.22999834959540896 }, "Triumph": { "n": 378, "spearman": 0.3226553425876793 } }, "by_benchmark_bucket": [ { "benchmark_bucket": "majority_absent", "n": 2565, "accuracy": 0.197270955165692, "balanced_accuracy": NaN, "precision": 0.0, "recall": NaN, "f1": NaN, "tp": 0, "fp": 2059, "tn": 506, "fn": 0 }, { "benchmark_bucket": "majority_present", "n": 2807, "accuracy": 0.8770929818311365, "balanced_accuracy": NaN, "precision": 1.0, "recall": 0.8770929818311365, "f1": 0.9345226798253938, "tp": 2462, "fp": 0, "tn": 0, "fn": 345 }, { "benchmark_bucket": "single_rater_absent", "n": 1, "accuracy": 0.0, "balanced_accuracy": NaN, "precision": 0.0, "recall": NaN, "f1": NaN, "tp": 0, "fp": 1, "tn": 0, "fn": 0 }, { "benchmark_bucket": "single_rater_present", "n": 1, "accuracy": 1.0, "balanced_accuracy": NaN, "precision": 1.0, "recall": 1.0, "f1": 1.0, "tp": 1, "fp": 0, "tn": 0, "fn": 0 }, { "benchmark_bucket": "unanimous_absent", "n": 805, "accuracy": 0.24720496894409938, "balanced_accuracy": NaN, "precision": 0.0, "recall": NaN, "f1": NaN, "tp": 0, "fp": 606, "tn": 199, "fn": 0 }, { "benchmark_bucket": "unanimous_present", "n": 1809, "accuracy": 0.9712548369264787, "balanced_accuracy": NaN, "precision": 1.0, "recall": 0.9712548369264787, "f1": 0.9854178351093662, "tp": 1757, "fp": 0, "tn": 0, "fn": 52 } ], "by_benchmark_bucket_optimal": [ { "benchmark_bucket": "majority_absent", "n": 2565, "accuracy": 0.7415204678362574, "balanced_accuracy": NaN, "precision": 0.0, "recall": NaN, "f1": NaN, "tp": 0, "fp": 663, "tn": 1902, "fn": 0 }, { "benchmark_bucket": "majority_present", "n": 2807, "accuracy": 0.4976843605272533, "balanced_accuracy": NaN, "precision": 1.0, "recall": 0.4976843605272533, "f1": 0.6646051379638439, "tp": 1397, "fp": 0, "tn": 0, "fn": 1410 }, { "benchmark_bucket": "single_rater_absent", "n": 1, "accuracy": 1.0, "balanced_accuracy": NaN, "precision": NaN, "recall": NaN, "f1": NaN, "tp": 0, "fp": 0, "tn": 1, "fn": 0 }, { "benchmark_bucket": "single_rater_present", "n": 1, "accuracy": 1.0, "balanced_accuracy": NaN, "precision": 1.0, "recall": 1.0, "f1": 1.0, "tp": 1, "fp": 0, "tn": 0, "fn": 0 }, { "benchmark_bucket": "unanimous_absent", "n": 805, "accuracy": 0.7962732919254658, "balanced_accuracy": NaN, "precision": 0.0, "recall": NaN, "f1": NaN, "tp": 0, "fp": 164, "tn": 641, "fn": 0 }, { "benchmark_bucket": "unanimous_present", "n": 1809, "accuracy": 0.7849640685461581, "balanced_accuracy": NaN, "precision": 1.0, "recall": 0.7849640685461581, "f1": 0.8795292660266336, "tp": 1420, "fp": 0, "tn": 0, "fn": 389 } ], "by_n_raters": [ { "n_raters": 1, "n": 2, "accuracy": 0.5, "balanced_accuracy": 0.5, "precision": 0.5, "recall": 1.0, "f1": 0.6666666666666666, "tp": 1, "fp": 1, "tn": 0, "fn": 0 }, { "n_raters": 2, "n": 2, "accuracy": 0.5, "balanced_accuracy": 0.5, "precision": 0.5, "recall": 1.0, "f1": 0.6666666666666666, "tp": 1, "fp": 1, "tn": 0, "fn": 0 }, { "n_raters": 3, "n": 7984, "accuracy": 0.6166082164328658, "balanced_accuracy": 0.5616185364808896, "precision": 0.6129032258064516, "recall": 0.91397616468039, "f1": 0.7337566321649126, "tp": 4218, "fp": 2664, "tn": 705, "fn": 397 } ], "by_n_raters_optimal": [ { "n_raters": 1, "n": 2, "accuracy": 1.0, "balanced_accuracy": 1.0, "precision": 1.0, "recall": 1.0, "f1": 1.0, "tp": 1, "fp": 0, "tn": 1, "fn": 0 }, { "n_raters": 2, "n": 2, "accuracy": 1.0, "balanced_accuracy": 1.0, "precision": 1.0, "recall": 1.0, "f1": 1.0, "tp": 1, "fp": 0, "tn": 1, "fn": 0 }, { "n_raters": 3, "n": 7984, "accuracy": 0.6710921843687375, "balanced_accuracy": 0.6823553738808401, "precision": 0.7729892945374691, "recall": 0.6101841820151679, "f1": 0.6820053281666262, "tp": 2816, "fp": 827, "tn": 2542, "fn": 1799 } ], "ensemble_members": [ { "name": "m0", "snapshot": "finetune_LCO_Embedding_Omni_7B_voiceclap10_lora/epoch_1_hf_salvaged", "weight": 0.25 }, { "name": "m1", "snapshot": "finetune_LCO_7B_mix9corpus_k10_r32_lora/epoch_1_hf", "weight": 0.25 }, { "name": "m2", "snapshot": "finetune_LCO_7B_mix9corpus_r32_lora/epoch_1_hf", "weight": 0.25 }, { "name": "m3", "snapshot": "finetune_BidirLM_Omni_2_5B_voiceclap10_lora/epoch_1_hf_salvaged", "weight": 0.25 } ], "audio_count": 3944, "prompt_count": 40, "outputs": { "predictions": "emolia-emo/predictions.csv", "summary": "emolia-emo/summary.json" } }, "emolia-dim": { "task": "emolia_bench_current", "subset": "emolia-dim", "model": "ensD_4way", "benchmark_dir": "voiceclap/emolia-bench-current", "benchmark_commit": "unknown", "rows_total": 13426, "rows_evaluated_ok": 13426, "missing_audio": 0, "threshold": 0.0, "similarity_distribution": { "min": -0.1761925220489502, "median": 0.17580687999725342, "max": 0.716350793838501, "mean": 0.19329595565795898 }, "overall_binary_at_thr0": { "n": 13426, "accuracy": 0.36295248026217786, "balanced_accuracy": 0.5161562122862273, "precision": 0.33913862085196517, "recall": 0.9710242587601078, "f1": 0.5027036455607884, "tp": 4323, "fp": 8424, "tn": 550, "fn": 129 }, "optimal_threshold": 0.22259671986103058, "overall_binary_at_optimal_thr": { "n": 13426, "accuracy": 0.6385371666914941, "balanced_accuracy": 0.606745307548151, "precision": 0.45960104775337496, "recall": 0.5123539982030548, "f1": 0.48454593733404144, "tp": 2281, "fp": 2682, "tn": 6292, "fn": 2171 }, "overall_binary_at_per_prompt_thr": { "n": 13426, "accuracy": 0.6643825413377029, "balanced_accuracy": 0.6787100190207069, "precision": 0.4958307597282273, "recall": 0.7212488769092543, "f1": 0.5876647144948757, "tp": 3211, "fp": 3265, "tn": 5709, "fn": 1241 }, "per_prompt_thresholds": { "AGEV::0": 0.21076074242591858, "AGEV::1": 0.14853519201278687, "AGEV::2": 0.19873367249965668, "AGEV::3": 0.18441258370876312, "AGEV::4": 0.3172627389431, "AGEV::5": 0.2479257881641388, "AGEV::6": 0.08687320351600647, "AROU::0": 0.18928907811641693, "AROU::1": 0.22878707945346832, "AROU::2": 0.2522868514060974, "AROU::3": 0.08575093746185303, "AROU::4": 0.11359941214323044, "AROU::5": 0.19952324032783508, "AROU::6": 0.4820690453052521, "ARSH::0": 0.18611155450344086, "ARSH::1": 0.40737518668174744, "ARSH::2": 0.14572012424468994, "ARSH::3": 0.19963745772838593, "ARSH::4": 0.1691114604473114, "ARSH::5": 0.2703824043273926, "ARSH::6": 0.34123578667640686, "ATCK::0": 0.3397178053855896, "ATCK::1": 0.1813654750585556, "ATCK::2": 0.13390494883060455, "ATCK::3": 0.1542855203151703, "ATCK::4": -0.013196267187595367, "ATCK::5": 0.23256269097328186, "ATCK::6": 0.33590108156204224, "BKGN::0": 0.15816858410835266, "BKGN::1": 0.1427445113658905, "BKGN::2": 0.5980448126792908, "BKGN::3": 0.1960279941558838, "BKGN::4": 0.17372164130210876, "BKGN::5": 0.2098558247089386, "BKGN::6": 0.2905731201171875, "BRGT::0": 0.24051551520824432, "BRGT::1": 0.29041290283203125, "BRGT::2": 0.07333779335021973, "BRGT::3": 0.06650108844041824, "BRGT::4": 0.04407393932342529, "BRGT::5": 0.14901570975780487, "BRGT::6": 0.23052892088890076, "CHNK::0": 0.20313158631324768, "CHNK::1": 0.11253364384174347, "CHNK::2": 0.19276292622089386, "CHNK::3": 0.04697166755795479, "CHNK::4": 0.09797828644514084, "CHNK::5": -1.0, "CHNK::6": 0.2878555357456207, "CLRT::0": 0.16436068713665009, "CLRT::1": 0.22944454848766327, "CLRT::2": -0.027599509805440903, "CLRT::3": 0.22734105587005615, "CLRT::4": 0.213055819272995, "CLRT::5": 0.13347885012626648, "CLRT::6": 0.19158895313739777, "COGL::0": 0.16901694238185883, "COGL::1": 0.10341300815343857, "COGL::2": 0.18577943742275238, "COGL::3": 0.1187010183930397, "COGL::4": 0.05076289176940918, "COGL::5": 0.20166067779064178, "COGL::6": 0.26348891854286194, "DARC::0": 0.048688020557165146, "DARC::1": 0.17046092450618744, "DARC::2": 0.249573215842247, "DARC::3": 0.5340986251831055, "DARC::4": 0.3806798458099365, "DARC::5": 0.23508307337760925, "DARC::6": 0.5146011710166931, "DFLU::0": 0.127367302775383, "DFLU::1": 0.2532338500022888, "DFLU::2": 0.1492464244365692, "DFLU::3": 0.18774379789829254, "DFLU::4": 0.20842543244361877, "DFLU::5": 0.07988566160202026, "DFLU::6": 0.2222968190908432, "EMPH::0": 0.2299402356147766, "EMPH::1": 0.1568109691143036, "EMPH::2": 0.21271811425685883, "EMPH::3": 0.24060094356536865, "EMPH::4": 0.2897516191005707, "EMPH::5": 0.1243622675538063, "EMPH::6": 0.3064168393611908, "ESTH::0": 0.23354566097259521, "ESTH::1": 0.3341001570224762, "ESTH::2": 0.14206868410110474, "ESTH::3": 0.11171317100524902, "ESTH::4": 0.09295012801885605, "ESTH::5": 0.26779186725616455, "ESTH::6": 0.2625650465488434, "EXPL::0": 0.11043974757194519, "EXPL::1": 0.06393797695636749, "EXPL::2": 0.29873907566070557, "FOCS::0": 0.14203643798828125, "FOCS::1": 0.34050697088241577, "FOCS::2": 0.10388636589050293, "FOCS::3": -0.010454242117702961, "FOCS::4": 0.14554688334465027, "FOCS::5": 0.27183976769447327, "FOCS::6": 0.2853876054286957, "FULL::0": 0.19612716138362885, "FULL::1": 0.18211102485656738, "FULL::2": 0.16294853389263153, "FULL::3": 0.15214647352695465, "FULL::4": 0.14108656346797943, "FULL::5": 0.27591657638549805, "FULL::6": 0.17171204090118408, "GEND::0": 0.07804427295923233, "GEND::1": 0.07704247534275055, "GEND::2": 0.24713216722011566, "GEND::3": 0.2642406225204468, "GEND::4": 0.18882369995117188, "GEND::5": 0.2493027150630951, "GEND::6": 0.16253913938999176, "HARM::0": 0.2610486149787903, "HARM::1": 0.3483065664768219, "HARM::2": 0.17134113609790802, "HARM::3": -1.0, "HARM::4": 0.20183533430099487, "HARM::5": 0.2391233891248703, "HARM::6": 0.2514594793319702, "LANG::Americas": 0.11434783041477203, "LANG::Asian_Pacific": 0.2149960696697235, "LANG::European": 0.1547873467206955, "LANG::Middle_Eastern_African": 0.19445286691188812, "METL::0": 0.21262238919734955, "METL::1": -0.02520637772977352, "METL::2": 0.052812062203884125, "METL::3": 0.28739646077156067, "METL::4": 0.16719752550125122, "METL::5": 0.2563958466053009, "METL::6": 0.2820034325122833, "RANG::0": 0.11330464482307434, "RANG::1": 0.16955408453941345, "RANG::2": 0.15499991178512573, "RANG::3": 0.11899220943450928, "RANG::4": 0.1950940042734146, "RANG::5": 0.22594556212425232, "RANG::6": 0.4265310764312744, "RCQL::0": 0.23168404400348663, "RCQL::1": 0.12006297707557678, "RCQL::2": 0.2869220972061157, "RCQL::3": 0.21034656465053558, "RCQL::4": 0.12921541929244995, "RCQL::5": 0.2019687294960022, "RCQL::6": 0.14100760221481323, "REGS::0": 0.234323650598526, "REGS::1": 0.1465354859828949, "REGS::2": 0.14474964141845703, "REGS::3": 0.2117491364479065, "REGS::4": 0.10497325658798218, "REGS::5": -1.0, "REGS::6": 0.25686293840408325, "RESP::0": 0.1224474310874939, "RESP::1": 0.1928730458021164, "RESP::2": 0.5089949369430542, "RESP::3": 0.37835413217544556, "RESP::4": 0.2824569046497345, "RESP::5": 0.43775245547294617, "RESP::6": 0.34229013323783875, "ROUG::0": 0.3874010145664215, "ROUG::1": 0.3440176546573639, "ROUG::2": 0.03234538063406944, "ROUG::3": -1.0, "ROUG::4": 0.11818694323301315, "ROUG::5": 0.2129768431186676, "ROUG::6": 0.18908154964447021, "R_CHST::0": 0.11539004743099213, "R_CHST::1": 0.22022584080696106, "R_CHST::2": 0.15321068465709686, "R_CHST::3": 0.3039624094963074, "R_CHST::4": 0.25051191449165344, "R_CHST::5": 0.18088345229625702, "R_CHST::6": 0.4219481647014618, "R_HEAD::0": 0.2684696912765503, "R_HEAD::1": -1.0, "R_HEAD::2": 0.3228467106819153, "R_HEAD::3": 0.19648587703704834, "R_HEAD::4": 0.24099424481391907, "R_HEAD::5": 0.23933404684066772, "R_HEAD::6": 0.12007055431604385, "R_MASK::0": 0.41654253005981445, "R_MASK::1": 0.13834431767463684, "R_MASK::2": 0.16790048778057098, "R_MASK::3": 0.1770983636379242, "R_MASK::4": 0.07285335659980774, "R_MASK::5": 0.18938064575195312, "R_MASK::6": 0.17441274225711823, "R_MIXD::0": 0.2797958552837372, "R_MIXD::1": 0.1319204866886139, "R_MIXD::2": 0.037671901285648346, "R_MIXD::3": 0.17433273792266846, "R_MIXD::4": 0.17909455299377441, "R_MIXD::5": 0.2818032503128052, "R_MIXD::6": 0.10028364509344101, "R_NASL::0": 0.3552275598049164, "R_NASL::1": 0.1350976675748825, "R_NASL::2": 0.17290383577346802, "R_NASL::3": 0.1438962072134018, "R_NASL::4": 0.33153241872787476, "R_NASL::5": 0.19503282010555267, "R_NASL::6": 0.2401062250137329, "R_ORAL::0": 0.2773113548755646, "R_ORAL::1": 0.07277868688106537, "R_ORAL::2": 0.18693457543849945, "R_ORAL::3": 0.1876228153705597, "R_ORAL::4": 0.13890193402767181, "R_ORAL::5": 0.20933334529399872, "R_ORAL::6": 0.2393949329853058, "R_THRT::0": 0.1915937066078186, "R_THRT::1": 0.22704793512821198, "R_THRT::2": 0.17992547154426575, "R_THRT::3": 0.21584391593933105, "R_THRT::4": 0.20117142796516418, "R_THRT::5": 0.3739945888519287, "R_THRT::6": 0.3610573709011078, "SMTH::0": 0.1565360277891159, "SMTH::1": 0.1768164485692978, "SMTH::2": 0.32344523072242737, "SMTH::3": 0.06182129308581352, "SMTH::4": 0.14484411478042603, "SMTH::5": 0.24485954642295837, "SMTH::6": 0.30262166261672974, "STNC::0": 0.3388286232948303, "STNC::1": 0.14225773513317108, "STNC::2": 0.09709206968545914, "STNC::3": 0.2593861520290375, "STNC::4": 0.1652897447347641, "STNC::5": 0.39727258682250977, "STNC::6": 0.2357381135225296, "STRU::0": 0.1842246949672699, "STRU::1": 0.30346834659576416, "STRU::2": 0.25403687357902527, "STRU::3": 0.022901276126503944, "STRU::4": 0.1451646387577057, "STRU::5": 0.28555646538734436, "STRU::6": 0.18749591708183289, "S_ASMR::0": 0.14395184814929962, "S_ASMR::1": 0.078745037317276, "S_ASMR::2": 0.2573505938053131, "S_ASMR::3": 0.2519141733646393, "S_ASMR::4": 0.3588062524795532, "S_ASMR::5": 0.22259671986103058, "S_ASMR::6": 0.3014710247516632, "S_AUTH::0": 0.19074152410030365, "S_AUTH::1": 0.2855116128921509, "S_AUTH::2": 0.20288366079330444, "S_AUTH::3": 0.2697114646434784, "S_AUTH::4": 0.3930772542953491, "S_AUTH::5": 0.21111541986465454, "S_AUTH::6": 0.43885713815689087, "S_CART::0": 0.18237030506134033, "S_CART::1": 0.15172432363033295, "S_CART::2": 0.22496220469474792, "S_CART::3": 0.19353927671909332, "S_CART::4": 0.20909184217453003, "S_CART::5": 0.350287526845932, "S_CART::6": 0.19513386487960815, "S_CASU::0": 0.24238410592079163, "S_CASU::1": 0.2050425261259079, "S_CASU::2": 0.18062660098075867, "S_CASU::3": 0.07626665383577347, "S_CASU::4": 0.06166916340589523, "S_CASU::5": 0.2523837089538574, "S_CASU::6": 0.4583868086338043, "S_CONV::0": 0.2682895362377167, "S_CONV::1": 0.27295368909835815, "S_CONV::2": 0.13288025557994843, "S_CONV::3": 0.1126880869269371, "S_CONV::4": 0.1609455645084381, "S_CONV::5": 0.2326298952102661, "S_CONV::6": 0.3375600278377533, "S_DRAM::0": 0.11697396636009216, "S_DRAM::1": 0.10558337718248367, "S_DRAM::2": 0.11618008464574814, "S_DRAM::3": 0.28894245624542236, "S_DRAM::4": 0.111568883061409, "S_DRAM::5": 0.1377236396074295, "S_DRAM::6": 0.29466116428375244, "S_FORM::0": 0.306130051612854, "S_FORM::1": 0.09768883138895035, "S_FORM::2": 0.20208819210529327, "S_FORM::3": 0.22036980092525482, "S_FORM::4": 0.34642741084098816, "S_FORM::5": 0.4346373975276947, "S_FORM::6": 0.19929860532283783, "S_MONO::0": 0.052836064249277115, "S_MONO::1": 0.31257280707359314, "S_MONO::2": 0.04041454195976257, "S_MONO::3": 0.20188474655151367, "S_MONO::4": 0.29442837834358215, "S_MONO::5": 0.34052222967147827, "S_MONO::6": 0.294637531042099, "S_NARR::0": 0.06202767789363861, "S_NARR::1": 0.2288742959499359, "S_NARR::2": 0.2210174798965454, "S_NARR::3": 0.2172921597957611, "S_NARR::4": 0.2630058526992798, "S_NARR::5": 0.306505411863327, "S_NARR::6": 0.19479826092720032, "S_NEWS::0": 0.20682945847511292, "S_NEWS::1": 0.14268521964550018, "S_NEWS::2": 0.05767880380153656, "S_NEWS::3": 0.3163949251174927, "S_NEWS::4": 0.3358045518398285, "S_NEWS::5": 0.1830998957157135, "S_NEWS::6": 0.24874597787857056, "S_PLAY::0": 0.4458395838737488, "S_PLAY::1": 0.19631831347942352, "S_PLAY::2": 0.30044835805892944, "S_PLAY::3": 0.17697328329086304, "S_PLAY::4": 0.39526665210723877, "S_PLAY::5": 0.1910097450017929, "S_PLAY::6": 0.380207359790802, "S_RANT::0": 0.326522558927536, "S_RANT::1": 0.10347118973731995, "S_RANT::2": 0.23348499834537506, "S_RANT::3": 0.15614771842956543, "S_RANT::4": 0.30623531341552734, "S_RANT::5": 0.3516838252544403, "S_RANT::6": 0.36251187324523926, "S_STRY::0": 0.22917526960372925, "S_STRY::1": 0.2637595534324646, "S_STRY::2": 0.2478424459695816, "S_STRY::3": 0.2574775516986847, "S_STRY::4": 0.32923781871795654, "S_STRY::5": 0.4221079349517822, "S_STRY::6": 0.3715190291404724, "S_TECH::0": 0.21200165152549744, "S_TECH::1": 0.008630547672510147, "S_TECH::2": 0.28827959299087524, "S_TECH::3": 0.03316371142864227, "S_TECH::4": 0.28116199374198914, "S_TECH::5": 0.24365605413913727, "S_TECH::6": 0.1286850869655609, "S_WHIS::0": 0.15343983471393585, "S_WHIS::1": 0.2373751401901245, "S_WHIS::2": 0.34007737040519714, "S_WHIS::3": 0.18737497925758362, "S_WHIS::4": 0.4558368921279907, "S_WHIS::5": 0.22039619088172913, "S_WHIS::6": 0.35740312933921814, "TEMP::0": 0.2258777618408203, "TEMP::1": 0.2411680966615677, "TEMP::2": 0.05534553900361061, "TEMP::3": 0.10959947854280472, "TEMP::4": 0.24843010306358337, "TEMP::5": 0.23587726056575775, "TEMP::6": 0.3102848529815674, "TENS::0": 0.13718552887439728, "TENS::1": 0.15986399352550507, "TENS::2": 0.10052742809057236, "TENS::3": 0.13592670857906342, "TENS::4": 0.2761598825454712, "TENS::5": 0.25044670701026917, "TENS::6": 0.40890952944755554, "VALN::0": 0.2822961211204529, "VALN::1": 0.19281697273254395, "VALN::2": 0.31530895829200745, "VALN::3": 0.17367054522037506, "VALN::4": 0.24470268189907074, "VALN::5": 0.1453874260187149, "VALN::6": 0.3235452473163605, "VALS::0": 0.32493966817855835, "VALS::1": 0.1630689650774002, "VALS::2": 0.007327250204980373, "VALS::3": 0.13998670876026154, "VALS::4": 0.19395942986011505, "VALS::5": 0.12620800733566284, "VALS::6": 0.22643305361270905, "VFLX::0": 0.14962132275104523, "VFLX::1": 0.2797451913356781, "VFLX::2": 0.14672639966011047, "VFLX::3": 0.13885095715522766, "VFLX::4": 0.08325840532779694, "VFLX::5": 0.09548235684633255, "VFLX::6": 0.11391308158636093, "VOLT::0": 0.27580544352531433, "VOLT::1": 0.20866966247558594, "VOLT::2": 0.18415161967277527, "VOLT::3": 0.34704795479774475, "VOLT::4": 0.18480698764324188, "VOLT::5": 0.21268771588802338, "VOLT::6": 0.25840890407562256, "VULN::0": 0.24166586995124817, "VULN::1": 0.35293322801589966, "VULN::2": -0.012517829425632954, "VULN::3": 0.17634044587612152, "VULN::4": 0.12965546548366547, "VULN::5": 0.20510028302669525, "VULN::6": 0.1382615715265274, "WARM::0": 0.10377862304449081, "WARM::1": -1.0, "WARM::2": 0.28513598442077637, "WARM::3": 0.24219977855682373, "WARM::4": 0.1823740005493164, "WARM::5": 0.33709338307380676, "WARM::6": 0.14903201162815094 }, "spearman_avg": 0.2037161334071489, "spearman_per_prompt": { "AGEV::0": { "n": 30, "spearman": 0.7613853240192985 }, "AGEV::1": { "n": 33, "spearman": 0.3458457469196514 }, "AGEV::2": { "n": 37, "spearman": 0.07506494913535848 }, "AGEV::3": { "n": 33, "spearman": 0.05497632801434845 }, "AGEV::4": { "n": 39, "spearman": 0.004742586610203935 }, "AGEV::5": { "n": 37, "spearman": 0.31648100744109103 }, "AGEV::6": { "n": 30, "spearman": 0.30079407123696994 }, "AROU::0": { "n": 36, "spearman": 0.28669978924528505 }, "AROU::1": { "n": 36, "spearman": 0.09034741402762311 }, "AROU::2": { "n": 41, "spearman": 0.01787648971314992 }, "AROU::3": { "n": 30, "spearman": 0.04517714706154307 }, "AROU::4": { "n": 30, "spearman": 0.41583331693021985 }, "AROU::5": { "n": 30, "spearman": 0.45782210716831395 }, "AROU::6": { "n": 41, "spearman": 0.4652132397367517 }, "ARSH::0": { "n": 39, "spearman": 0.20652738476797897 }, "ARSH::1": { "n": 37, "spearman": 0.3847626783283741 }, "ARSH::2": { "n": 33, "spearman": 0.4564834069715307 }, "ARSH::3": { "n": 39, "spearman": 0.04266302645179038 }, "ARSH::4": { "n": 39, "spearman": 0.26745968571525097 }, "ARSH::5": { "n": 37, "spearman": 0.6235727661454074 }, "ARSH::6": { "n": 43, "spearman": 0.531280008133028 }, "ATCK::0": { "n": 36, "spearman": 0.2660538018069009 }, "ATCK::1": { "n": 37, "spearman": 0.29163051636389814 }, "ATCK::2": { "n": 38, "spearman": -0.012475931995832563 }, "ATCK::3": { "n": 35, "spearman": 0.22000449789487492 }, "ATCK::4": { "n": 32, "spearman": -0.14577556192446833 }, "ATCK::5": { "n": 39, "spearman": 0.49716067936787867 }, "ATCK::6": { "n": 35, "spearman": 0.7658935805671384 }, "BKGN::0": { "n": 31, "spearman": 0.22088950660390277 }, "BKGN::1": { "n": 39, "spearman": 0.22324653066637812 }, "BKGN::2": { "n": 16, "spearman": -0.2520504151250418 }, "BKGN::3": { "n": 23, "spearman": -0.12795202680775042 }, "BKGN::4": { "n": 36, "spearman": 0.5365780671234863 }, "BKGN::5": { "n": 39, "spearman": 0.6534325400078641 }, "BKGN::6": { "n": 32, "spearman": 0.3967814008308554 }, "BRGT::0": { "n": 22, "spearman": 0.327892735165809 }, "BRGT::1": { "n": 25, "spearman": 0.05944454429259128 }, "BRGT::2": { "n": 35, "spearman": 0.09145474612197715 }, "BRGT::3": { "n": 28, "spearman": 0.045874566285664595 }, "BRGT::4": { "n": 39, "spearman": -0.07836868530429256 }, "BRGT::5": { "n": 32, "spearman": 0.25551118689251745 }, "BRGT::6": { "n": 32, "spearman": 0.10578056210173764 }, "CHNK::0": { "n": 35, "spearman": 0.2679210300771807 }, "CHNK::1": { "n": 36, "spearman": 0.023702031053730196 }, "CHNK::2": { "n": 28, "spearman": 0.040404419000846126 }, "CHNK::3": { "n": 39, "spearman": -0.10118343317977842 }, "CHNK::4": { "n": 37, "spearman": -0.15119922973260608 }, "CHNK::5": { "n": 37, "spearman": -0.08718128114678038 }, "CHNK::6": { "n": 23, "spearman": 0.154205066914225 }, "CLRT::0": { "n": 41, "spearman": 0.5631291436451498 }, "CLRT::1": { "n": 36, "spearman": 0.35785051147258445 }, "CLRT::2": { "n": 38, "spearman": -0.2648556697193051 }, "CLRT::3": { "n": 20, "spearman": -0.007704474260119998 }, "CLRT::4": { "n": 39, "spearman": 0.17678674929520077 }, "CLRT::5": { "n": 38, "spearman": 0.10758593136355796 }, "CLRT::6": { "n": 36, "spearman": 0.04898241178201158 }, "COGL::0": { "n": 38, "spearman": 0.19138079681607154 }, "COGL::1": { "n": 38, "spearman": -0.16068536738211456 }, "COGL::2": { "n": 29, "spearman": 0.30125786715206027 }, "COGL::3": { "n": 22, "spearman": 0.3153087720368796 }, "COGL::4": { "n": 40, "spearman": 0.25767568983822137 }, "COGL::5": { "n": 35, "spearman": -0.07922117928043916 }, "COGL::6": { "n": 37, "spearman": 0.506900854963491 }, "DARC::0": { "n": 38, "spearman": -0.26495636494107044 }, "DARC::1": { "n": 40, "spearman": 0.2612059694875159 }, "DARC::2": { "n": 23, "spearman": 0.2510031665423442 }, "DARC::3": { "n": 37, "spearman": 0.5618192594550492 }, "DARC::4": { "n": 34, "spearman": 0.13274592569908475 }, "DARC::5": { "n": 41, "spearman": 0.05523964939717225 }, "DARC::6": { "n": 38, "spearman": 0.5955711844108232 }, "DFLU::0": { "n": 42, "spearman": 0.07159449167867629 }, "DFLU::1": { "n": 41, "spearman": 0.051871364178935495 }, "DFLU::2": { "n": 30, "spearman": -0.1491755135562042 }, "DFLU::3": { "n": 31, "spearman": 0.6664085521832227 }, "DFLU::4": { "n": 37, "spearman": 0.4499040327963501 }, "DFLU::5": { "n": 43, "spearman": 0.42355908292392813 }, "DFLU::6": { "n": 38, "spearman": 0.5390140600185795 }, "EMPH::0": { "n": 37, "spearman": 0.2811277455107681 }, "EMPH::1": { "n": 37, "spearman": 0.19067199206951166 }, "EMPH::2": { "n": 21, "spearman": 0.3251406622009161 }, "EMPH::3": { "n": 27, "spearman": 0.03685266618818547 }, "EMPH::4": { "n": 43, "spearman": 0.2264482806322159 }, "EMPH::5": { "n": 37, "spearman": 0.4297716615577756 }, "EMPH::6": { "n": 37, "spearman": 0.2778648390635565 }, "ESTH::0": { "n": 32, "spearman": 0.6197285108341206 }, "ESTH::1": { "n": 35, "spearman": 0.20026720118795727 }, "ESTH::2": { "n": 34, "spearman": 0.0621783685468903 }, "ESTH::3": { "n": 24, "spearman": 0.2155607520716949 }, "ESTH::4": { "n": 34, "spearman": -0.16826664110764109 }, "ESTH::5": { "n": 35, "spearman": 0.36513621953427794 }, "ESTH::6": { "n": 30, "spearman": 0.39374669556009156 }, "EXPL::0": { "n": 34, "spearman": 0.08275756101848084 }, "EXPL::1": { "n": 38, "spearman": -0.19177183057899352 }, "EXPL::2": { "n": 38, "spearman": 0.3553608874839528 }, "FOCS::0": { "n": 38, "spearman": 0.43457575454445074 }, "FOCS::1": { "n": 37, "spearman": 0.037687312874648084 }, "FOCS::2": { "n": 38, "spearman": -0.14947821618729293 }, "FOCS::3": { "n": 27, "spearman": -0.17225077552896437 }, "FOCS::4": { "n": 41, "spearman": 0.19818786261571367 }, "FOCS::5": { "n": 36, "spearman": 0.1661151541614146 }, "FOCS::6": { "n": 38, "spearman": 0.1828332388581129 }, "FULL::0": { "n": 20, "spearman": -0.044490356250949464 }, "FULL::1": { "n": 39, "spearman": 0.16642768517348605 }, "FULL::2": { "n": 20, "spearman": 0.225784040455512 }, "FULL::3": { "n": 23, "spearman": -0.02941722452194768 }, "FULL::4": { "n": 37, "spearman": 0.04079416826170334 }, "FULL::5": { "n": 35, "spearman": 0.40241946228813885 }, "FULL::6": { "n": 19, "spearman": 0.16529651228038592 }, "GEND::0": { "n": 41, "spearman": -0.04869821022832882 }, "GEND::1": { "n": 40, "spearman": 0.3164938467749473 }, "GEND::2": { "n": 38, "spearman": 0.23414342483736753 }, "GEND::3": { "n": 42, "spearman": 0.19961484276017502 }, "GEND::4": { "n": 34, "spearman": 0.3290996739065824 }, "GEND::5": { "n": 38, "spearman": 0.2938661570360823 }, "GEND::6": { "n": 33, "spearman": 0.19532297147433475 }, "HARM::0": { "n": 32, "spearman": 0.4400471383432286 }, "HARM::1": { "n": 38, "spearman": 0.029037648901234572 }, "HARM::2": { "n": 41, "spearman": 0.1487084674285227 }, "HARM::3": { "n": 24, "spearman": -0.26439647305269254 }, "HARM::4": { "n": 35, "spearman": 0.13877048837323963 }, "HARM::5": { "n": 20, "spearman": 0.20810639268578882 }, "HARM::6": { "n": 39, "spearman": -0.14482356534401275 }, "LANG::Americas": { "n": 19, "spearman": 0.10687274500887023 }, "LANG::Asian_Pacific": { "n": 15, "spearman": -0.37796447300922714 }, "LANG::European": { "n": 18, "spearman": 0.15539822010288762 }, "LANG::Middle_Eastern_African": { "n": 14, "spearman": 0.11920095660279705 }, "METL::0": { "n": 30, "spearman": 0.0438377139507594 }, "METL::1": { "n": 34, "spearman": -0.0514798221859529 }, "METL::2": { "n": 23, "spearman": -0.08353459382493217 }, "METL::3": { "n": 36, "spearman": -0.08621454987076833 }, "METL::4": { "n": 34, "spearman": -0.0825888017969429 }, "METL::5": { "n": 37, "spearman": 0.03849343973714917 }, "METL::6": { "n": 34, "spearman": 0.4652650930647473 }, "RANG::0": { "n": 19, "spearman": 0.1252448582170299 }, "RANG::1": { "n": 37, "spearman": 0.2411685598879479 }, "RANG::2": { "n": 32, "spearman": 0.3711191922327646 }, "RANG::3": { "n": 23, "spearman": 0.26243194054073904 }, "RANG::4": { "n": 37, "spearman": 0.07961793533457975 }, "RANG::5": { "n": 29, "spearman": 0.0 }, "RANG::6": { "n": 38, "spearman": 0.03637979691370255 }, "RCQL::0": { "n": 36, "spearman": 0.26443184444953477 }, "RCQL::1": { "n": 37, "spearman": 0.17236535347405557 }, "RCQL::2": { "n": 19, "spearman": 0.0 }, "RCQL::3": { "n": 19, "spearman": -0.1452966314513558 }, "RCQL::4": { "n": 23, "spearman": 0.030954782936681764 }, "RCQL::5": { "n": 36, "spearman": 0.30000799786154014 }, "RCQL::6": { "n": 39, "spearman": 0.08964324486987894 }, "REGS::0": { "n": 32, "spearman": 0.49525813832762383 }, "REGS::1": { "n": 34, "spearman": 0.2719367202713395 }, "REGS::2": { "n": 38, "spearman": 0.21661242854144164 }, "REGS::3": { "n": 39, "spearman": 0.213859540784115 }, "REGS::4": { "n": 35, "spearman": 0.3316986156708995 }, "REGS::5": { "n": 34, "spearman": -0.2645202642639394 }, "REGS::6": { "n": 37, "spearman": 0.45421997916613505 }, "RESP::0": { "n": 37, "spearman": 0.25891362607218715 }, "RESP::1": { "n": 36, "spearman": 0.13880541222980436 }, "RESP::2": { "n": 32, "spearman": -0.28146915716559895 }, "RESP::3": { "n": 36, "spearman": 0.48858774099793434 }, "RESP::4": { "n": 31, "spearman": 0.6023167338562568 }, "RESP::5": { "n": 38, "spearman": 0.7081810076364797 }, "RESP::6": { "n": 38, "spearman": 0.6319359461190467 }, "ROUG::0": { "n": 23, "spearman": -0.04821614493254929 }, "ROUG::1": { "n": 29, "spearman": -0.0013622875785415764 }, "ROUG::2": { "n": 27, "spearman": -0.3088185823383824 }, "ROUG::3": { "n": 33, "spearman": -0.20297475761326578 }, "ROUG::4": { "n": 37, "spearman": 0.2984045227222312 }, "ROUG::5": { "n": 37, "spearman": 0.12345365411557863 }, "ROUG::6": { "n": 39, "spearman": 0.619271316599399 }, "R_CHST::0": { "n": 36, "spearman": 0.03315278795376565 }, "R_CHST::1": { "n": 39, "spearman": -0.10841770774686009 }, "R_CHST::2": { "n": 20, "spearman": -0.2867696673382022 }, "R_CHST::3": { "n": 40, "spearman": -0.18919066924585554 }, "R_CHST::4": { "n": 38, "spearman": 0.2017049246110727 }, "R_CHST::5": { "n": 38, "spearman": 0.2407135767441979 }, "R_CHST::6": { "n": 28, "spearman": 0.2574022929869164 }, "R_HEAD::0": { "n": 34, "spearman": 0.24905748750661563 }, "R_HEAD::1": { "n": 41, "spearman": -0.317577065472602 }, "R_HEAD::2": { "n": 32, "spearman": 0.1381179915100817 }, "R_HEAD::3": { "n": 38, "spearman": 0.09213746732419596 }, "R_HEAD::4": { "n": 37, "spearman": 0.5754853009063492 }, "R_HEAD::5": { "n": 37, "spearman": -0.04662510845557401 }, "R_HEAD::6": { "n": 28, "spearman": 0.22059317071155285 }, "R_MASK::0": { "n": 40, "spearman": -0.024159845708815166 }, "R_MASK::1": { "n": 32, "spearman": 0.010118620955418666 }, "R_MASK::2": { "n": 33, "spearman": 0.03921573429525611 }, "R_MASK::3": { "n": 41, "spearman": 0.09765084684397995 }, "R_MASK::4": { "n": 38, "spearman": -0.11122512239104981 }, "R_MASK::5": { "n": 40, "spearman": 0.19525900298982707 }, "R_MASK::6": { "n": 25, "spearman": 0.2795689089716172 }, "R_MIXD::0": { "n": 40, "spearman": 0.4354243976316708 }, "R_MIXD::1": { "n": 29, "spearman": 0.3705079956461819 }, "R_MIXD::2": { "n": 40, "spearman": -0.029080482112521684 }, "R_MIXD::3": { "n": 39, "spearman": -0.0320115141131651 }, "R_MIXD::4": { "n": 35, "spearman": 0.1776520816480779 }, "R_MIXD::5": { "n": 33, "spearman": 0.4248308880277656 }, "R_MIXD::6": { "n": 20, "spearman": 0.08671099695241201 }, "R_NASL::0": { "n": 18, "spearman": 0.07381831222698777 }, "R_NASL::1": { "n": 31, "spearman": 0.02923526731023431 }, "R_NASL::2": { "n": 35, "spearman": 0.2781813952021893 }, "R_NASL::3": { "n": 33, "spearman": 0.11295231675813695 }, "R_NASL::4": { "n": 29, "spearman": 0.005353158578563808 }, "R_NASL::5": { "n": 27, "spearman": -0.015101111262979779 }, "R_NASL::6": { "n": 19, "spearman": 0.23717082451262844 }, "R_ORAL::0": { "n": 32, "spearman": 0.16240573729160407 }, "R_ORAL::1": { "n": 36, "spearman": 0.2055398462125584 }, "R_ORAL::2": { "n": 19, "spearman": -0.282982515378645 }, "R_ORAL::3": { "n": 37, "spearman": 0.01811752771263191 }, "R_ORAL::4": { "n": 39, "spearman": -0.025583332256074526 }, "R_ORAL::5": { "n": 39, "spearman": -0.05493675559942976 }, "R_ORAL::6": { "n": 25, "spearman": 0.2146181695019912 }, "R_THRT::0": { "n": 41, "spearman": 0.10473870389885806 }, "R_THRT::1": { "n": 39, "spearman": 0.019906834435643776 }, "R_THRT::2": { "n": 24, "spearman": 0.3614249843655962 }, "R_THRT::3": { "n": 36, "spearman": 0.3463136644624976 }, "R_THRT::4": { "n": 36, "spearman": 0.3512264892080198 }, "R_THRT::5": { "n": 35, "spearman": 0.28785682342091845 }, "R_THRT::6": { "n": 37, "spearman": 0.30070383744331963 }, "SMTH::0": { "n": 37, "spearman": 0.4174628153928545 }, "SMTH::1": { "n": 41, "spearman": 0.07052845399370603 }, "SMTH::2": { "n": 15, "spearman": 0.3898110954898229 }, "SMTH::3": { "n": 16, "spearman": -0.29623690269952596 }, "SMTH::4": { "n": 39, "spearman": 0.25614527468607534 }, "SMTH::5": { "n": 39, "spearman": 0.4959272731862757 }, "SMTH::6": { "n": 34, "spearman": 0.04103480669292775 }, "STNC::0": { "n": 31, "spearman": 0.3766194922749705 }, "STNC::1": { "n": 39, "spearman": 0.3994579825933368 }, "STNC::2": { "n": 32, "spearman": 0.03978111157211651 }, "STNC::3": { "n": 27, "spearman": 0.037348045344172125 }, "STNC::4": { "n": 31, "spearman": 0.4185789451624787 }, "STNC::5": { "n": 33, "spearman": 0.6126716648539869 }, "STNC::6": { "n": 34, "spearman": 0.36877876679908544 }, "STRU::0": { "n": 33, "spearman": 0.6116789130980063 }, "STRU::1": { "n": 35, "spearman": 0.5359927116712808 }, "STRU::2": { "n": 36, "spearman": 0.17639250785847607 }, "STRU::3": { "n": 21, "spearman": -0.22043459027885934 }, "STRU::4": { "n": 34, "spearman": 0.3441170415074005 }, "STRU::5": { "n": 36, "spearman": 0.46661524123712816 }, "STRU::6": { "n": 19, "spearman": 0.04274909800354809 }, "S_ASMR::0": { "n": 20, "spearman": -0.036681339457633126 }, "S_ASMR::1": { "n": 35, "spearman": 0.17014281125552327 }, "S_ASMR::2": { "n": 38, "spearman": 0.41639945612537554 }, "S_ASMR::3": { "n": 33, "spearman": 0.42103951102930864 }, "S_ASMR::4": { "n": 34, "spearman": 0.37938489489445276 }, "S_ASMR::5": { "n": 41, "spearman": 0.23063292370601768 }, "S_ASMR::6": { "n": 35, "spearman": 0.264661509553792 }, "S_AUTH::0": { "n": 35, "spearman": 0.41749383983611815 }, "S_AUTH::1": { "n": 36, "spearman": 0.4289944452277139 }, "S_AUTH::2": { "n": 37, "spearman": 0.36156467583382995 }, "S_AUTH::3": { "n": 35, "spearman": 0.4300935647079598 }, "S_AUTH::4": { "n": 38, "spearman": 0.11474760636724918 }, "S_AUTH::5": { "n": 41, "spearman": 0.5056401366042045 }, "S_AUTH::6": { "n": 25, "spearman": 0.19482162504979914 }, "S_CART::0": { "n": 28, "spearman": -0.05740647229587608 }, "S_CART::1": { "n": 34, "spearman": 0.29798357294123845 }, "S_CART::2": { "n": 33, "spearman": 0.12569832945592876 }, "S_CART::3": { "n": 33, "spearman": 0.41608064400546657 }, "S_CART::4": { "n": 30, "spearman": 0.5338420050484259 }, "S_CART::5": { "n": 36, "spearman": 0.40086702014162384 }, "S_CART::6": { "n": 34, "spearman": 0.4490658934112721 }, "S_CASU::0": { "n": 20, "spearman": 0.27926526882695474 }, "S_CASU::1": { "n": 31, "spearman": -0.10240900614703717 }, "S_CASU::2": { "n": 34, "spearman": 0.3471722831863234 }, "S_CASU::3": { "n": 41, "spearman": 0.11697244112690927 }, "S_CASU::4": { "n": 36, "spearman": 0.12771164980002853 }, "S_CASU::5": { "n": 31, "spearman": 0.36645539228054763 }, "S_CASU::6": { "n": 22, "spearman": 0.39745965557955426 }, "S_CONV::0": { "n": 38, "spearman": 0.4040386871451768 }, "S_CONV::1": { "n": 35, "spearman": 0.42569509962804813 }, "S_CONV::2": { "n": 38, "spearman": -0.3054899380728124 }, "S_CONV::3": { "n": 32, "spearman": 0.2422301887061677 }, "S_CONV::4": { "n": 41, "spearman": 0.09713457591990396 }, "S_CONV::5": { "n": 35, "spearman": 0.1662306869738883 }, "S_CONV::6": { "n": 24, "spearman": 0.16660407256741183 }, "S_DRAM::0": { "n": 34, "spearman": 0.08058553745536493 }, "S_DRAM::1": { "n": 33, "spearman": -0.035575934967687135 }, "S_DRAM::2": { "n": 34, "spearman": -0.05324673441462065 }, "S_DRAM::3": { "n": 39, "spearman": 0.37429630564185873 }, "S_DRAM::4": { "n": 37, "spearman": 0.3810153729112283 }, "S_DRAM::5": { "n": 36, "spearman": 0.34236965123466084 }, "S_DRAM::6": { "n": 35, "spearman": 0.24075722898727578 }, "S_FORM::0": { "n": 34, "spearman": 0.65610635289492 }, "S_FORM::1": { "n": 42, "spearman": 0.3999592651555707 }, "S_FORM::2": { "n": 35, "spearman": 0.38232517427037455 }, "S_FORM::3": { "n": 33, "spearman": 0.22664116862657405 }, "S_FORM::4": { "n": 35, "spearman": 0.446654358819025 }, "S_FORM::5": { "n": 37, "spearman": 0.5412762296249916 }, "S_FORM::6": { "n": 29, "spearman": 0.5370395138915975 }, "S_MONO::0": { "n": 36, "spearman": -0.06278237296741387 }, "S_MONO::1": { "n": 40, "spearman": 0.436430883142735 }, "S_MONO::2": { "n": 35, "spearman": 0.05919053179927255 }, "S_MONO::3": { "n": 36, "spearman": 0.5966351005468624 }, "S_MONO::4": { "n": 38, "spearman": 0.3664705715858195 }, "S_MONO::5": { "n": 29, "spearman": 0.49492530050638045 }, "S_MONO::6": { "n": 33, "spearman": 0.20532565892208304 }, "S_NARR::0": { "n": 34, "spearman": 0.007950142708601025 }, "S_NARR::1": { "n": 42, "spearman": 0.10681384455667649 }, "S_NARR::2": { "n": 34, "spearman": 0.37245519036076674 }, "S_NARR::3": { "n": 35, "spearman": 0.1693670486127257 }, "S_NARR::4": { "n": 33, "spearman": 0.4429894070633073 }, "S_NARR::5": { "n": 38, "spearman": 0.6600943133983451 }, "S_NARR::6": { "n": 23, "spearman": 0.3595159339879562 }, "S_NEWS::0": { "n": 42, "spearman": 0.30555545954012964 }, "S_NEWS::1": { "n": 36, "spearman": 0.3723485484067104 }, "S_NEWS::2": { "n": 35, "spearman": 0.22725937204344632 }, "S_NEWS::3": { "n": 29, "spearman": 0.17979873805701904 }, "S_NEWS::4": { "n": 37, "spearman": 0.17894457431319738 }, "S_NEWS::5": { "n": 31, "spearman": 0.35410625465358275 }, "S_NEWS::6": { "n": 30, "spearman": 0.6297938217249424 }, "S_PLAY::0": { "n": 32, "spearman": 0.2393431333598355 }, "S_PLAY::1": { "n": 32, "spearman": 0.11845528813230911 }, "S_PLAY::2": { "n": 34, "spearman": -0.006998603360284972 }, "S_PLAY::3": { "n": 35, "spearman": 0.22656151545089798 }, "S_PLAY::4": { "n": 39, "spearman": 0.7266410001076925 }, "S_PLAY::5": { "n": 36, "spearman": 0.28440516442737857 }, "S_PLAY::6": { "n": 39, "spearman": 0.5840651544399243 }, "S_RANT::0": { "n": 38, "spearman": -0.1986201809788762 }, "S_RANT::1": { "n": 37, "spearman": 0.31423475352422064 }, "S_RANT::2": { "n": 42, "spearman": 0.3327079697572583 }, "S_RANT::3": { "n": 34, "spearman": -0.025376862903822267 }, "S_RANT::4": { "n": 43, "spearman": 0.7192408658785577 }, "S_RANT::5": { "n": 38, "spearman": 0.30955305817112494 }, "S_RANT::6": { "n": 37, "spearman": 0.6203040167227801 }, "S_STRY::0": { "n": 34, "spearman": 0.33187230489481356 }, "S_STRY::1": { "n": 35, "spearman": 0.5520048765667759 }, "S_STRY::2": { "n": 33, "spearman": 0.40751880665662393 }, "S_STRY::3": { "n": 35, "spearman": 0.4550141892152125 }, "S_STRY::4": { "n": 34, "spearman": 0.3310345998378245 }, "S_STRY::5": { "n": 35, "spearman": 0.32680562444481637 }, "S_STRY::6": { "n": 28, "spearman": 0.23568114863405154 }, "S_TECH::0": { "n": 28, "spearman": -0.00510529778494188 }, "S_TECH::1": { "n": 37, "spearman": 0.1787215788296572 }, "S_TECH::2": { "n": 37, "spearman": -0.007361353671284814 }, "S_TECH::3": { "n": 29, "spearman": -0.04932437875941278 }, "S_TECH::4": { "n": 37, "spearman": 0.3578336597428029 }, "S_TECH::5": { "n": 34, "spearman": 0.501876153346869 }, "S_TECH::6": { "n": 29, "spearman": 0.43535240394842156 }, "S_WHIS::0": { "n": 38, "spearman": 0.12389074763433164 }, "S_WHIS::1": { "n": 43, "spearman": 0.3953667131967298 }, "S_WHIS::2": { "n": 36, "spearman": 0.4863983605593194 }, "S_WHIS::3": { "n": 32, "spearman": 0.21954067990601195 }, "S_WHIS::4": { "n": 35, "spearman": 0.35761421862853066 }, "S_WHIS::5": { "n": 39, "spearman": 0.4151521803840715 }, "S_WHIS::6": { "n": 32, "spearman": 0.5702413360632035 }, "TEMP::0": { "n": 37, "spearman": 0.25384429559876 }, "TEMP::1": { "n": 38, "spearman": -0.20839057332215022 }, "TEMP::2": { "n": 35, "spearman": 0.21663001092748896 }, "TEMP::3": { "n": 25, "spearman": 0.15285739960952044 }, "TEMP::4": { "n": 41, "spearman": -0.1432180842385041 }, "TEMP::5": { "n": 34, "spearman": 0.04460305075318234 }, "TEMP::6": { "n": 30, "spearman": 0.26333567114532913 }, "TENS::0": { "n": 19, "spearman": 0.4445906192369001 }, "TENS::1": { "n": 36, "spearman": 0.4053062774728426 }, "TENS::2": { "n": 32, "spearman": 0.1538321958968878 }, "TENS::3": { "n": 24, "spearman": -0.07906289324264328 }, "TENS::4": { "n": 33, "spearman": 0.04039282625994107 }, "TENS::5": { "n": 38, "spearman": 0.26647693582324683 }, "TENS::6": { "n": 39, "spearman": 0.34963057207625947 }, "VALN::0": { "n": 36, "spearman": 0.48160692680362477 }, "VALN::1": { "n": 41, "spearman": 0.08541617601785008 }, "VALN::2": { "n": 32, "spearman": -0.02690885998313793 }, "VALN::3": { "n": 38, "spearman": 0.3679791951187824 }, "VALN::4": { "n": 31, "spearman": 0.3318804478633979 }, "VALN::5": { "n": 30, "spearman": 0.4253321847841365 }, "VALN::6": { "n": 37, "spearman": 0.3875006266873016 }, "VALS::0": { "n": 33, "spearman": 0.2343248409464982 }, "VALS::1": { "n": 35, "spearman": 0.25244960965536306 }, "VALS::2": { "n": 22, "spearman": -0.27301331537330936 }, "VALS::3": { "n": 34, "spearman": -0.05925785900401602 }, "VALS::4": { "n": 34, "spearman": 0.04116981158382211 }, "VALS::5": { "n": 30, "spearman": 0.04570999632348523 }, "VALS::6": { "n": 35, "spearman": 0.37884446302804003 }, "VFLX::0": { "n": 33, "spearman": 0.1071873295752654 }, "VFLX::1": { "n": 32, "spearman": 0.2115415994936009 }, "VFLX::2": { "n": 32, "spearman": 0.18927487240533164 }, "VFLX::3": { "n": 27, "spearman": 0.07584717670640533 }, "VFLX::4": { "n": 39, "spearman": -0.05461444215542092 }, "VFLX::5": { "n": 38, "spearman": 0.0343470165871537 }, "VFLX::6": { "n": 34, "spearman": 0.04720751870340899 }, "VOLT::0": { "n": 37, "spearman": 0.1735094741630399 }, "VOLT::1": { "n": 42, "spearman": 0.07180460927344394 }, "VOLT::2": { "n": 26, "spearman": 0.21530403296636383 }, "VOLT::3": { "n": 31, "spearman": -0.362548970861043 }, "VOLT::4": { "n": 29, "spearman": 0.44880402491480903 }, "VOLT::5": { "n": 40, "spearman": 0.40797707318938353 }, "VOLT::6": { "n": 31, "spearman": 0.43267770364648833 }, "VULN::0": { "n": 34, "spearman": 0.5218151083592883 }, "VULN::1": { "n": 39, "spearman": 0.2504951994255044 }, "VULN::2": { "n": 30, "spearman": -0.12343028320611978 }, "VULN::3": { "n": 38, "spearman": 0.153212943530562 }, "VULN::4": { "n": 39, "spearman": 0.2065617196971943 }, "VULN::5": { "n": 37, "spearman": 0.4357303220400481 }, "VULN::6": { "n": 42, "spearman": 0.24952782030556706 }, "WARM::0": { "n": 29, "spearman": -0.41848634790844064 }, "WARM::1": { "n": 39, "spearman": -0.2620308874310017 }, "WARM::2": { "n": 36, "spearman": -0.09451795068984792 }, "WARM::3": { "n": 35, "spearman": -0.03358128487685987 }, "WARM::4": { "n": 33, "spearman": 0.11001684015827304 }, "WARM::5": { "n": 30, "spearman": 0.12711002594784235 }, "WARM::6": { "n": 19, "spearman": 0.16704806721284124 } }, "by_benchmark_bucket": [ { "benchmark_bucket": "majority_no", "n": 973, "accuracy": 0.05755395683453238, "balanced_accuracy": NaN, "precision": 0.0, "recall": NaN, "f1": NaN, "tp": 0, "fp": 917, "tn": 56, "fn": 0 }, { "benchmark_bucket": "majority_yes", "n": 18, "accuracy": 0.8888888888888888, "balanced_accuracy": NaN, "precision": 1.0, "recall": 0.8888888888888888, "f1": 0.9411764705882353, "tp": 16, "fp": 0, "tn": 0, "fn": 2 }, { "benchmark_bucket": "single_rater_no", "n": 7455, "accuracy": 0.06277665995975855, "balanced_accuracy": NaN, "precision": 0.0, "recall": NaN, "f1": NaN, "tp": 0, "fp": 6987, "tn": 468, "fn": 0 }, { "benchmark_bucket": "single_rater_yes", "n": 4069, "accuracy": 0.9722290489063652, "balanced_accuracy": NaN, "precision": 1.0, "recall": 0.9722290489063652, "f1": 0.9859190031152648, "tp": 3956, "fp": 0, "tn": 0, "fn": 113 }, { "benchmark_bucket": "unanimous_no", "n": 546, "accuracy": 0.047619047619047616, "balanced_accuracy": NaN, "precision": 0.0, "recall": NaN, "f1": NaN, "tp": 0, "fp": 520, "tn": 26, "fn": 0 }, { "benchmark_bucket": "unanimous_yes", "n": 365, "accuracy": 0.9616438356164384, "balanced_accuracy": NaN, "precision": 1.0, "recall": 0.9616438356164384, "f1": 0.9804469273743017, "tp": 351, "fp": 0, "tn": 0, "fn": 14 } ], "by_benchmark_bucket_optimal": [ { "benchmark_bucket": "majority_no", "n": 973, "accuracy": 0.6341212744090442, "balanced_accuracy": NaN, "precision": 0.0, "recall": NaN, "f1": NaN, "tp": 0, "fp": 356, "tn": 617, "fn": 0 }, { "benchmark_bucket": "majority_yes", "n": 18, "accuracy": 0.4444444444444444, "balanced_accuracy": NaN, "precision": 1.0, "recall": 0.4444444444444444, "f1": 0.6153846153846153, "tp": 8, "fp": 0, "tn": 0, "fn": 10 }, { "benchmark_bucket": "single_rater_no", "n": 7455, "accuracy": 0.7093226022803487, "balanced_accuracy": NaN, "precision": 0.0, "recall": NaN, "f1": NaN, "tp": 0, "fp": 2167, "tn": 5288, "fn": 0 }, { "benchmark_bucket": "single_rater_yes", "n": 4069, "accuracy": 0.5077414598181371, "balanced_accuracy": NaN, "precision": 1.0, "recall": 0.5077414598181371, "f1": 0.6735126324368378, "tp": 2066, "fp": 0, "tn": 0, "fn": 2003 }, { "benchmark_bucket": "unanimous_no", "n": 546, "accuracy": 0.7087912087912088, "balanced_accuracy": NaN, "precision": 0.0, "recall": NaN, "f1": NaN, "tp": 0, "fp": 159, "tn": 387, "fn": 0 }, { "benchmark_bucket": "unanimous_yes", "n": 365, "accuracy": 0.5671232876712329, "balanced_accuracy": NaN, "precision": 1.0, "recall": 0.5671232876712329, "f1": 0.7237762237762237, "tp": 207, "fp": 0, "tn": 0, "fn": 158 } ], "by_n_raters": [ { "n_raters": 1, "n": 11524, "accuracy": 0.3838944810829573, "balanced_accuracy": 0.5175028544330619, "precision": 0.3615096408663072, "recall": 0.9722290489063652, "f1": 0.5270450306421529, "tp": 3956, "fp": 6987, "tn": 468, "fn": 113 }, { "n_raters": 2, "n": 1843, "accuracy": 0.2311448724905046, "balanced_accuracy": 0.5085796433655503, "precision": 0.1981724728726442, "recall": 0.9638888888888889, "f1": 0.3287541449549976, "tp": 347, "fp": 1404, "tn": 79, "fn": 13 }, { "n_raters": 3, "n": 58, "accuracy": 0.39655172413793105, "balanced_accuracy": 0.47763975155279503, "precision": 0.38461538461538464, "recall": 0.8695652173913043, "f1": 0.5333333333333333, "tp": 20, "fp": 32, "tn": 3, "fn": 3 }, { "n_raters": 4, "n": 1, "accuracy": 0.0, "balanced_accuracy": NaN, "precision": 0.0, "recall": NaN, "f1": NaN, "tp": 0, "fp": 1, "tn": 0, "fn": 0 } ], "by_n_raters_optimal": [ { "n_raters": 1, "n": 11524, "accuracy": 0.6381464769177368, "balanced_accuracy": 0.6085320310492429, "precision": 0.48806992676588706, "recall": 0.5077414598181371, "f1": 0.4977113948446158, "tp": 2066, "fp": 2167, "tn": 5288, "fn": 2003 }, { "n_raters": 2, "n": 1843, "accuracy": 0.6408030385241454, "balanced_accuracy": 0.6137849329437326, "precision": 0.28792134831460675, "recall": 0.5694444444444444, "f1": 0.38246268656716415, "tp": 205, "fp": 507, "tn": 976, "fn": 155 }, { "n_raters": 3, "n": 58, "accuracy": 0.6551724137931034, "balanced_accuracy": 0.6173913043478261, "precision": 0.5882352941176471, "recall": 0.43478260869565216, "f1": 0.5, "tp": 10, "fp": 7, "tn": 28, "fn": 13 }, { "n_raters": 4, "n": 1, "accuracy": 0.0, "balanced_accuracy": NaN, "precision": 0.0, "recall": NaN, "f1": NaN, "tp": 0, "fp": 1, "tn": 0, "fn": 0 } ], "ensemble_members": [ { "name": "m0", "snapshot": "finetune_LCO_Embedding_Omni_7B_voiceclap10_lora/epoch_1_hf_salvaged", "weight": 0.25 }, { "name": "m1", "snapshot": "finetune_LCO_7B_mix9corpus_k10_r32_lora/epoch_1_hf", "weight": 0.25 }, { "name": "m2", "snapshot": "finetune_LCO_7B_mix9corpus_r32_lora/epoch_1_hf", "weight": 0.25 }, { "name": "m3", "snapshot": "finetune_BidirLM_Omni_2_5B_voiceclap10_lora/epoch_1_hf_salvaged", "weight": 0.25 } ], "audio_count": 13426, "prompt_count": 399, "outputs": { "predictions": "emolia-dim/predictions.csv", "summary": "emolia-dim/summary.json" } } }