{ "emolia-emo": { "task": "emolia_bench_current", "subset": "emolia-emo", "model": "baseline_mispeech_GLAP", "benchmark_dir": "benchmarks/emolia-bench-current", "benchmark_commit": "unknown", "rows_total": 7988, "rows_evaluated_ok": 7988, "missing_audio": 0, "threshold": 0.0, "similarity_distribution": { "min": -0.21763966977596283, "median": 0.08229957520961761, "max": 0.17021378874778748, "mean": 0.07702276110649109 }, "overall_binary_at_thr0": { "n": 7988, "accuracy": 0.5769904857285929, "balanced_accuracy": 0.5040971717448581, "precision": 0.5800569063631661, "recall": 0.9714100064977258, "f1": 0.7263746052311929, "tp": 4485, "fp": 3247, "tn": 124, "fn": 132 }, "optimal_threshold": 0.0776447132229805, "overall_binary_at_optimal_thr": { "n": 7988, "accuracy": 0.5379318978467702, "balanced_accuracy": 0.5272695666968454, "precision": 0.6012243113248797, "recall": 0.5956248646307126, "f1": 0.5984114895005984, "tp": 2750, "fp": 1824, "tn": 1547, "fn": 1867 }, "overall_binary_at_per_prompt_thr": { "n": 7988, "accuracy": 0.5803705558337506, "balanced_accuracy": 0.5764706124239884, "precision": 0.6474702727908603, "recall": 0.6014728178470868, "f1": 0.6236245227936223, "tp": 2777, "fp": 1512, "tn": 1859, "fn": 1840 }, "per_prompt_thresholds": { "Affection": 0.06762643158435822, "Amusement": 0.0643991157412529, "Anger": 0.09642507880926132, "Astonishment_Surprise": 0.11093523353338242, "Awe": 0.11825449764728546, "Bitterness": 0.09420523047447205, "Concentration": 0.07591012865304947, "Confusion": 0.08997549116611481, "Contemplation": 0.07895062118768692, "Contempt": 0.07728279381990433, "Contentment": -0.05914847552776337, "Disappointment": 0.06397093087434769, "Disgust": 0.08424420654773712, "Distress": 0.09203492105007172, "Doubt": 0.08729798346757889, "Elation": 0.08350242674350739, "Embarrassment": 0.07003244012594223, "Emotional_Numbness": 0.15691092610359192, "Fatigue_Exhaustion": 0.07304168492555618, "Fear": 0.10943540930747986, "Helplessness": 0.10299661755561829, "Hope_Enthusiasm_Optimism": 0.043617188930511475, "Impatience_and_Irritability": 0.08869136869907379, "Infatuation": 0.03685179352760315, "Interest": 0.0659332200884819, "Intoxication_Altered_States_of_Consciousness": 0.09412436932325363, "Jealousy___Envy": 0.1157766804099083, "Longing": 0.04187215864658356, "Malevolence_Malice": 0.09979461133480072, "Pain": 0.08220703899860382, "Pleasure_Ecstasy": 0.09890707582235336, "Pride": 0.08401366323232651, "Relief": 0.04258570820093155, "Sadness": 0.1357593834400177, "Sexual_Lust": 0.05130108445882797, "Shame": 0.0646560937166214, "Sourness": 0.04722196236252785, "Teasing": 0.06352192908525467, "Thankfulness_Gratitude": 0.02888648211956024, "Triumph": 0.10403099656105042 }, "spearman_avg": 0.06647339143100155, "spearman_per_prompt": { "Affection": { "n": 201, "spearman": -0.023803228864177278 }, "Amusement": { "n": 229, "spearman": 0.5302513583860898 }, "Anger": { "n": 352, "spearman": 0.3885765903801727 }, "Astonishment_Surprise": { "n": 100, "spearman": -0.08152809757652454 }, "Awe": { "n": 131, "spearman": -0.15885204017322674 }, "Bitterness": { "n": 129, "spearman": -0.001742890669642159 }, "Concentration": { "n": 125, "spearman": 0.09507176511857857 }, "Confusion": { "n": 113, "spearman": 0.2762538909413133 }, "Contemplation": { "n": 131, "spearman": -0.02219384297017758 }, "Contempt": { "n": 189, "spearman": 0.2541635664124394 }, "Contentment": { "n": 250, "spearman": -0.23897158922098335 }, "Disappointment": { "n": 125, "spearman": 0.05780036113691394 }, "Disgust": { "n": 127, "spearman": 0.07684427965928382 }, "Distress": { "n": 200, "spearman": 0.2264808884010802 }, "Doubt": { "n": 100, "spearman": 0.14132877480267333 }, "Elation": { "n": 325, "spearman": 0.13642912119540476 }, "Embarrassment": { "n": 317, "spearman": 0.18737062108302777 }, "Emotional_Numbness": { "n": 207, "spearman": -0.43022192005806476 }, "Fatigue_Exhaustion": { "n": 244, "spearman": 0.08221737612633925 }, "Fear": { "n": 311, "spearman": -0.07068692750580474 }, "Helplessness": { "n": 141, "spearman": 0.24978979438323698 }, "Hope_Enthusiasm_Optimism": { "n": 111, "spearman": 0.02972271554300902 }, "Impatience_and_Irritability": { "n": 238, "spearman": 0.3752123826841664 }, "Infatuation": { "n": 423, "spearman": -0.047188675121344135 }, "Interest": { "n": 101, "spearman": 0.09409488666672082 }, "Intoxication_Altered_States_of_Consciousness": { "n": 606, "spearman": 0.14447488737276562 }, "Jealousy___Envy": { "n": 100, "spearman": -0.09330868673722854 }, "Longing": { "n": 250, "spearman": -0.02893409561441725 }, "Malevolence_Malice": { "n": 100, "spearman": 0.1791622782194633 }, "Pain": { "n": 171, "spearman": 0.04761965480037672 }, "Pleasure_Ecstasy": { "n": 172, "spearman": 0.06919205645835917 }, "Pride": { "n": 150, "spearman": 0.1080395435383273 }, "Relief": { "n": 162, "spearman": -0.05571381889162248 }, "Sadness": { "n": 150, "spearman": -0.16720385117466424 }, "Sexual_Lust": { "n": 280, "spearman": -0.03723151871679608 }, "Shame": { "n": 125, "spearman": 0.12222212560146888 }, "Sourness": { "n": 221, "spearman": -0.06028070624001805 }, "Teasing": { "n": 100, "spearman": 0.030962414762378602 }, "Thankfulness_Gratitude": { "n": 103, "spearman": 0.1356258283631025 }, "Triumph": { "n": 378, "spearman": 0.1378903847380617 } }, "by_benchmark_bucket": [ { "benchmark_bucket": "majority_absent", "n": 2565, "accuracy": 0.03274853801169591, "balanced_accuracy": NaN, "precision": 0.0, "recall": NaN, "f1": NaN, "tp": 0, "fp": 2481, "tn": 84, "fn": 0 }, { "benchmark_bucket": "majority_present", "n": 2807, "accuracy": 0.9654435340220876, "balanced_accuracy": NaN, "precision": 1.0, "recall": 0.9654435340220876, "f1": 0.9824179807866594, "tp": 2710, "fp": 0, "tn": 0, "fn": 97 }, { "benchmark_bucket": "single_rater_absent", "n": 1, "accuracy": 0.0, "balanced_accuracy": NaN, "precision": 0.0, "recall": NaN, "f1": NaN, "tp": 0, "fp": 1, "tn": 0, "fn": 0 }, { "benchmark_bucket": "single_rater_present", "n": 1, "accuracy": 1.0, "balanced_accuracy": NaN, "precision": 1.0, "recall": 1.0, "f1": 1.0, "tp": 1, "fp": 0, "tn": 0, "fn": 0 }, { "benchmark_bucket": "unanimous_absent", "n": 805, "accuracy": 0.049689440993788817, "balanced_accuracy": NaN, "precision": 0.0, "recall": NaN, "f1": NaN, "tp": 0, "fp": 765, "tn": 40, "fn": 0 }, { "benchmark_bucket": "unanimous_present", "n": 1809, "accuracy": 0.9806522940851299, "balanced_accuracy": NaN, "precision": 1.0, "recall": 0.9806522940851299, "f1": 0.9902316494557634, "tp": 1774, "fp": 0, "tn": 0, "fn": 35 } ], "by_benchmark_bucket_optimal": [ { "benchmark_bucket": "majority_absent", "n": 2565, "accuracy": 0.45497076023391814, "balanced_accuracy": NaN, "precision": 0.0, "recall": NaN, "f1": NaN, "tp": 0, "fp": 1398, "tn": 1167, "fn": 0 }, { "benchmark_bucket": "majority_present", "n": 2807, "accuracy": 0.5610972568578554, "balanced_accuracy": NaN, "precision": 1.0, "recall": 0.5610972568578554, "f1": 0.7188498402555911, "tp": 1575, "fp": 0, "tn": 0, "fn": 1232 }, { "benchmark_bucket": "single_rater_absent", "n": 1, "accuracy": 0.0, "balanced_accuracy": NaN, "precision": 0.0, "recall": NaN, "f1": NaN, "tp": 0, "fp": 1, "tn": 0, "fn": 0 }, { "benchmark_bucket": "single_rater_present", "n": 1, "accuracy": 0.0, "balanced_accuracy": NaN, "precision": NaN, "recall": 0.0, "f1": NaN, "tp": 0, "fp": 0, "tn": 0, "fn": 1 }, { "benchmark_bucket": "unanimous_absent", "n": 805, "accuracy": 0.4720496894409938, "balanced_accuracy": NaN, "precision": 0.0, "recall": NaN, "f1": NaN, "tp": 0, "fp": 425, "tn": 380, "fn": 0 }, { "benchmark_bucket": "unanimous_present", "n": 1809, "accuracy": 0.6495301271420675, "balanced_accuracy": NaN, "precision": 1.0, "recall": 0.6495301271420675, "f1": 0.7875335120643432, "tp": 1175, "fp": 0, "tn": 0, "fn": 634 } ], "by_n_raters": [ { "n_raters": 1, "n": 2, "accuracy": 0.5, "balanced_accuracy": 0.5, "precision": 0.5, "recall": 1.0, "f1": 0.6666666666666666, "tp": 1, "fp": 1, "tn": 0, "fn": 0 }, { "n_raters": 2, "n": 2, "accuracy": 0.5, "balanced_accuracy": 0.5, "precision": 0.5, "recall": 1.0, "f1": 0.6666666666666666, "tp": 1, "fp": 1, "tn": 0, "fn": 0 }, { "n_raters": 3, "n": 7984, "accuracy": 0.5770290581162325, "balanced_accuracy": 0.5041018952034466, "precision": 0.5800983436853002, "recall": 0.971397616468039, "f1": 0.7264036295876206, "tp": 4483, "fp": 3245, "tn": 124, "fn": 132 } ], "by_n_raters_optimal": [ { "n_raters": 1, "n": 2, "accuracy": 0.0, "balanced_accuracy": 0.0, "precision": 0.0, "recall": 0.0, "f1": NaN, "tp": 0, "fp": 1, "tn": 0, "fn": 1 }, { "n_raters": 2, "n": 2, "accuracy": 0.0, "balanced_accuracy": 0.0, "precision": 0.0, "recall": 0.0, "f1": NaN, "tp": 0, "fp": 1, "tn": 0, "fn": 1 }, { "n_raters": 3, "n": 7984, "accuracy": 0.5382014028056112, "balanced_accuracy": 0.527534846267366, "precision": 0.6014873140857393, "recall": 0.5958829902491874, "f1": 0.5986720365734189, "tp": 2750, "fp": 1822, "tn": 1547, "fn": 1865 } ], "ensemble_members": [ { "name": "baseline_mispeech_GLAP", "snapshot": "mispeech/GLAP", "weight": 1.0 } ], "audio_count": 3944, "prompt_count": 40, "outputs": { "predictions": "emolia-emo/predictions.csv", "summary": "emolia-emo/summary.json" } }, "emolia-dim": { "task": "emolia_bench_current", "subset": "emolia-dim", "model": "baseline_mispeech_GLAP", "benchmark_dir": "benchmarks/emolia-bench-current", "benchmark_commit": "unknown", "rows_total": 18632, "rows_evaluated_ok": 18632, "missing_audio": 0, "threshold": 0.0, "similarity_distribution": { "min": -0.27884507179260254, "median": 0.07990642637014389, "max": 0.23912645876407623, "mean": 0.06187843903899193 }, "overall_binary_at_thr0": { "n": 18632, "accuracy": 0.3633533705452984, "balanced_accuracy": 0.5322630832903793, "precision": 0.25458289054197664, "recall": 0.8571109123434705, "f1": 0.3925645227365834, "tp": 3833, "fp": 11223, "tn": 2937, "fn": 639 }, "optimal_threshold": 0.08698458224534988, "overall_binary_at_optimal_thr": { "n": 18632, "accuracy": 0.5560863031343924, "balanced_accuracy": 0.5427134183317668, "precision": 0.2744865249910958, "recall": 0.516994633273703, "f1": 0.3585886002326483, "tp": 2312, "fp": 6111, "tn": 8049, "fn": 2160 }, "overall_binary_at_per_prompt_thr": { "n": 18632, "accuracy": 0.5927436668097896, "balanced_accuracy": 0.6246616738930495, "precision": 0.3316039775183744, "recall": 0.686046511627907, "f1": 0.4470999708539784, "tp": 3068, "fp": 6184, "tn": 7976, "fn": 1404 }, "per_prompt_thresholds": { "AGEV::0": 0.10953538119792938, "AGEV::1": 0.060368675738573074, "AGEV::2": 0.09743304550647736, "AGEV::3": 0.12286212295293808, "AGEV::4": 0.1094416156411171, "AGEV::5": 0.1256999373435974, "AGEV::6": 0.10960464924573898, "AROU::0": -0.02154511772096157, "AROU::1": 0.06697873026132584, "AROU::2": 0.12043756991624832, "AROU::3": 0.033366065472364426, "AROU::4": -0.03602321073412895, "AROU::5": 0.10664612799882889, "AROU::6": 0.10737410932779312, "ARSH::0": -0.029036870226264, "ARSH::1": 0.0959390327334404, "ARSH::2": 0.09066806733608246, "ARSH::3": 0.1307167112827301, "ARSH::4": 0.12275459617376328, "ARSH::5": 0.10427337884902954, "ARSH::6": 0.11365915834903717, "ATCK::0": 0.027622200548648834, "ATCK::1": 0.16413433849811554, "ATCK::2": 0.01702370122075081, "ATCK::3": 0.14055770635604858, "ATCK::4": 0.14745694398880005, "ATCK::5": 0.0008568876655772328, "ATCK::6": 0.11398088932037354, "BKGN::0": 0.08092733472585678, "BKGN::1": 0.16708151996135712, "BKGN::2": -0.05071578547358513, "BKGN::3": -0.12900029122829437, "BKGN::4": -0.006742800120264292, "BKGN::5": 0.07186874002218246, "BKGN::6": 0.06462223082780838, "BRGT::0": -0.016983043402433395, "BRGT::1": 0.04804827272891998, "BRGT::2": 0.11445468664169312, "BRGT::3": 0.04184621572494507, "BRGT::4": 0.004926270339637995, "BRGT::5": -0.019122309982776642, "BRGT::6": -0.10515641421079636, "CHNK::0": 0.13958938419818878, "CHNK::1": 0.10396471619606018, "CHNK::2": -0.003370558377355337, "CHNK::3": 0.06879455596208572, "CHNK::4": 0.09774190187454224, "CHNK::5": 0.01991010643541813, "CHNK::6": -0.008067650720477104, "CLRT::0": 0.03526310250163078, "CLRT::1": 0.09276550263166428, "CLRT::2": 0.10319698601961136, "CLRT::3": -0.03399959206581116, "CLRT::4": 0.12501868605613708, "CLRT::5": 0.11940887570381165, "CLRT::6": 0.11475671827793121, "COGL::0": 0.12078650295734406, "COGL::1": 0.14301717281341553, "COGL::2": 0.0746966227889061, "COGL::3": 0.11853594332933426, "COGL::4": -0.006991500034928322, "COGL::5": 0.10372251272201538, "COGL::6": 0.11607592552900314, "DARC::0": 0.11425253748893738, "DARC::1": -1.0, "DARC::2": 0.15023741126060486, "DARC::3": 0.1512688845396042, "DARC::4": 0.07120663672685623, "DARC::5": -0.02149660885334015, "DARC::6": 0.007476022932678461, "DFLU::0": 0.13491331040859222, "DFLU::1": 0.01796196587383747, "DFLU::2": -0.01978779397904873, "DFLU::3": 0.022656042128801346, "DFLU::4": -0.05255485326051712, "DFLU::5": 0.047459203749895096, "DFLU::6": 0.08111415803432465, "EMPH::0": 0.14022758603096008, "EMPH::1": -0.0007244576700031757, "EMPH::2": 0.16787926852703094, "EMPH::3": 0.10907556116580963, "EMPH::4": 0.0520220547914505, "EMPH::5": 0.08224809914827347, "EMPH::6": 0.10217167437076569, "ESTH::0": 0.10391226410865784, "ESTH::1": 0.05661708489060402, "ESTH::2": 0.1019371971487999, "ESTH::3": 0.04670919105410576, "ESTH::4": 0.046149395406246185, "ESTH::5": 0.07863558083772659, "ESTH::6": 0.07366733998060226, "EXPL::0": 0.03942381590604782, "EXPL::1": 0.08403811603784561, "EXPL::2": 0.02704774960875511, "FOCS::0": 0.10782276093959808, "FOCS::1": 0.09659777581691742, "FOCS::2": 0.10267269611358643, "FOCS::3": 0.09055275470018387, "FOCS::4": 0.10682406276464462, "FOCS::5": 0.09193345904350281, "FOCS::6": 0.01187092438340187, "FULL::0": 0.15608559548854828, "FULL::1": -0.03877096623182297, "FULL::2": 0.1308971792459488, "FULL::3": 0.02719254419207573, "FULL::4": 0.13019061088562012, "FULL::5": 0.0960002988576889, "FULL::6": 0.06561511754989624, "GEND::0": -0.019883792847394943, "GEND::1": 0.016410959884524345, "GEND::2": 0.015732774510979652, "GEND::3": 0.02469397336244583, "GEND::4": 0.054904207587242126, "GEND::5": -0.03063349425792694, "GEND::6": 0.09430697560310364, "HARM::0": 0.14612874388694763, "HARM::1": 0.018101410940289497, "HARM::2": -0.016795581206679344, "HARM::3": -0.04423942044377327, "HARM::4": 0.07642480731010437, "HARM::5": 0.021985922008752823, "HARM::6": 0.15481030941009521, "LANG::Americas": 0.09252016991376877, "LANG::Asian_Pacific": 0.07026357203722, "LANG::European": 0.06003535911440849, "LANG::Middle_Eastern_African": 0.009839215315878391, "METL::0": -0.07752425223588943, "METL::1": 0.1080096960067749, "METL::2": 0.009690160863101482, "METL::3": 0.14854444563388824, "METL::4": 0.1087089478969574, "METL::5": -0.027268406003713608, "METL::6": 0.07768544554710388, "RANG::0": 0.11397133022546768, "RANG::1": 0.1204928532242775, "RANG::2": 0.06440845876932144, "RANG::3": 0.111471027135849, "RANG::4": 0.0963129922747612, "RANG::5": -0.004094843752682209, "RANG::6": 0.0833931490778923, "RCQL::0": 0.0482536256313324, "RCQL::1": 0.040116697549819946, "RCQL::2": -1.0, "RCQL::3": -0.07178004086017609, "RCQL::4": 0.11101635545492172, "RCQL::5": 0.12998606264591217, "RCQL::6": 0.022064946591854095, "REGS::0": 0.0020490544848144054, "REGS::1": -0.030298959463834763, "REGS::2": -0.035874467343091965, "REGS::3": -0.00032848911359906197, "REGS::4": 0.006346837151795626, "REGS::5": 0.08872274309396744, "REGS::6": -0.0864853709936142, "RESP::0": 0.056206297129392624, "RESP::1": 0.0826893299818039, "RESP::2": 0.012541601434350014, "RESP::3": 0.15230968594551086, "RESP::4": 0.10969672352075577, "RESP::5": 0.12359068542718887, "RESP::6": 0.1305801123380661, "ROUG::0": 0.09153644740581512, "ROUG::1": 0.08486772328615189, "ROUG::2": 0.011005347594618797, "ROUG::3": 0.011642660945653915, "ROUG::4": -0.07530762255191803, "ROUG::5": 0.09332241117954254, "ROUG::6": 0.05801820009946823, "R_CHST::0": 0.12991705536842346, "R_CHST::1": 0.15144780278205872, "R_CHST::2": -1.0, "R_CHST::3": 0.12049907445907593, "R_CHST::4": 0.002036622492596507, "R_CHST::5": 0.1103021427989006, "R_CHST::6": 0.08658602088689804, "R_HEAD::0": 0.010318348184227943, "R_HEAD::1": 0.06624966859817505, "R_HEAD::2": 0.06455060839653015, "R_HEAD::3": 0.10718937963247299, "R_HEAD::4": 0.09664991497993469, "R_HEAD::5": 0.09732336550951004, "R_HEAD::6": 0.1151072159409523, "R_MASK::0": 0.056303780525922775, "R_MASK::1": 0.07632646709680557, "R_MASK::2": 0.0077044665813446045, "R_MASK::3": 0.0004580635577440262, "R_MASK::4": 0.13632144033908844, "R_MASK::5": 0.13055835664272308, "R_MASK::6": 0.1162918359041214, "R_MIXD::0": 0.09803488105535507, "R_MIXD::1": 0.10374975204467773, "R_MIXD::2": 0.08369895070791245, "R_MIXD::3": 0.1444644182920456, "R_MIXD::4": 0.05772794783115387, "R_MIXD::5": 0.0889890044927597, "R_MIXD::6": 0.1296941488981247, "R_NASL::0": 0.11856811493635178, "R_NASL::1": 0.13286802172660828, "R_NASL::2": 0.08860521018505096, "R_NASL::3": 0.10717099905014038, "R_NASL::4": 0.08845214545726776, "R_NASL::5": 0.11766722798347473, "R_NASL::6": 0.10303028672933578, "R_ORAL::0": 0.00041122501716017723, "R_ORAL::1": 0.0878063216805458, "R_ORAL::2": -0.03268809989094734, "R_ORAL::3": 0.019060226157307625, "R_ORAL::4": -0.009355218149721622, "R_ORAL::5": 0.06917467713356018, "R_ORAL::6": -0.0006405734457075596, "R_THRT::0": 0.09974333643913269, "R_THRT::1": 0.12547482550144196, "R_THRT::2": 0.06183483824133873, "R_THRT::3": 0.051760971546173096, "R_THRT::4": 0.12300997972488403, "R_THRT::5": 0.11678268760442734, "R_THRT::6": 0.07575459033250809, "SMTH::0": 0.12543991208076477, "SMTH::1": 0.14755970239639282, "SMTH::2": 0.1151822879910469, "SMTH::3": 0.097053661942482, "SMTH::4": 0.05858442932367325, "SMTH::5": 0.0091582415625453, "SMTH::6": 0.13923074305057526, "STNC::0": 0.10302131623029709, "STNC::1": 0.05826390162110329, "STNC::2": -0.04488897696137428, "STNC::3": 0.09512365609407425, "STNC::4": 0.04035552218556404, "STNC::5": 0.0923076719045639, "STNC::6": 0.07735467702150345, "STRU::0": 0.10379181057214737, "STRU::1": 0.06594016402959824, "STRU::2": 0.0502040758728981, "STRU::3": 0.14925047755241394, "STRU::4": 0.16624370217323303, "STRU::5": 0.13210459053516388, "STRU::6": 0.11345529556274414, "S_ASMR::0": 0.1381698101758957, "S_ASMR::1": 0.008305076509714127, "S_ASMR::2": 0.08693630248308182, "S_ASMR::3": 0.1121397465467453, "S_ASMR::4": 0.1061326116323471, "S_ASMR::5": 0.02428370714187622, "S_ASMR::6": 0.11572737246751785, "S_AUTH::0": 0.1162566989660263, "S_AUTH::1": 0.13325588405132294, "S_AUTH::2": 0.09119447320699692, "S_AUTH::3": 0.13420797884464264, "S_AUTH::4": 0.07305395603179932, "S_AUTH::5": 0.10372130572795868, "S_AUTH::6": 0.1253138780593872, "S_CART::0": 0.08749441802501678, "S_CART::1": 0.1197935938835144, "S_CART::2": 0.07876056432723999, "S_CART::3": -0.006270525511354208, "S_CART::4": 0.020797865465283394, "S_CART::5": 0.020686401054263115, "S_CART::6": 0.08543689548969269, "S_CASU::0": 0.05294925719499588, "S_CASU::1": 0.08698458224534988, "S_CASU::2": 0.03300400823354721, "S_CASU::3": 0.036032535135746, "S_CASU::4": -0.04869630187749863, "S_CASU::5": 0.016573823988437653, "S_CASU::6": 0.04959601163864136, "S_CONV::0": 0.16086755692958832, "S_CONV::1": 0.1162075474858284, "S_CONV::2": -0.006271054036915302, "S_CONV::3": 0.08943931758403778, "S_CONV::4": 0.09187586605548859, "S_CONV::5": 0.09126505255699158, "S_CONV::6": 0.12686699628829956, "S_DRAM::0": 0.0883224830031395, "S_DRAM::1": 0.008189912885427475, "S_DRAM::2": 0.14313101768493652, "S_DRAM::3": 0.06306218355894089, "S_DRAM::4": 0.10897761583328247, "S_DRAM::5": 0.04711352661252022, "S_DRAM::6": 0.02829388529062271, "S_FORM::0": 0.04705023020505905, "S_FORM::1": -0.010890165343880653, "S_FORM::2": 0.11291737109422684, "S_FORM::3": 0.05154227465391159, "S_FORM::4": -0.008273252286016941, "S_FORM::5": 0.07909366488456726, "S_FORM::6": 0.11369432508945465, "S_MONO::0": 0.09588516503572464, "S_MONO::1": 0.111195869743824, "S_MONO::2": 0.09584230184555054, "S_MONO::3": 0.04820165038108826, "S_MONO::4": 0.1052752360701561, "S_MONO::5": 0.13655036687850952, "S_MONO::6": 0.1119413897395134, "S_NARR::0": 0.0035127452574670315, "S_NARR::1": -0.05931573361158371, "S_NARR::2": 0.10199317336082458, "S_NARR::3": 0.03914239630103111, "S_NARR::4": 0.04307229071855545, "S_NARR::5": 0.09808585047721863, "S_NARR::6": 0.04285416379570961, "S_NEWS::0": 0.09055308997631073, "S_NEWS::1": 0.07514863461256027, "S_NEWS::2": 0.11927781999111176, "S_NEWS::3": 0.09291519224643707, "S_NEWS::4": 0.1102030873298645, "S_NEWS::5": 0.04030728340148926, "S_NEWS::6": 0.09840711951255798, "S_PLAY::0": -0.026736851781606674, "S_PLAY::1": 0.04929051920771599, "S_PLAY::2": 0.0870111733675003, "S_PLAY::3": 0.017317332327365875, "S_PLAY::4": 0.11607662588357925, "S_PLAY::5": 0.07196246832609177, "S_PLAY::6": 0.09208178520202637, "S_RANT::0": -0.1141623929142952, "S_RANT::1": 0.12357822060585022, "S_RANT::2": 0.10905550420284271, "S_RANT::3": 0.04282216727733612, "S_RANT::4": 0.10798531770706177, "S_RANT::5": 0.1272832751274109, "S_RANT::6": 0.08307768404483795, "S_STRY::0": 0.09068357199430466, "S_STRY::1": 0.09753724187612534, "S_STRY::2": 0.09250228852033615, "S_STRY::3": 0.089381143450737, "S_STRY::4": 0.04484964907169342, "S_STRY::5": 0.06836923956871033, "S_STRY::6": 0.02616347000002861, "S_TECH::0": 0.05992037430405617, "S_TECH::1": 0.08534892648458481, "S_TECH::2": -0.008079067803919315, "S_TECH::3": 0.13271674513816833, "S_TECH::4": 0.07549376785755157, "S_TECH::5": 0.056778471916913986, "S_TECH::6": 0.03000585548579693, "S_WHIS::0": 0.026971735060214996, "S_WHIS::1": 0.03640476241707802, "S_WHIS::2": 0.10823998600244522, "S_WHIS::3": 0.08109065145254135, "S_WHIS::4": 0.09006915241479874, "S_WHIS::5": 0.07076912373304367, "S_WHIS::6": 0.11272116005420685, "TEMP::0": 0.08969692885875702, "TEMP::1": 0.16771188378334045, "TEMP::2": 0.08225850015878677, "TEMP::3": -1.0, "TEMP::4": 0.11433771997690201, "TEMP::5": 0.03400039300322533, "TEMP::6": 0.046364158391952515, "TENS::0": 0.04201292246580124, "TENS::1": 0.12599065899848938, "TENS::2": 0.15856176614761353, "TENS::3": 0.03510541468858719, "TENS::4": -0.006771760061383247, "TENS::5": 0.06328494101762772, "TENS::6": 0.023761581629514694, "VALN::0": 0.1503811925649643, "VALN::1": 0.11239828169345856, "VALN::2": 0.09759782999753952, "VALN::3": 0.0739356130361557, "VALN::4": 0.02008986473083496, "VALN::5": 0.07129784673452377, "VALN::6": 0.06889262050390244, "VALS::0": 0.04565280303359032, "VALS::1": 0.021934833377599716, "VALS::2": 0.03345334902405739, "VALS::3": 0.1130208969116211, "VALS::4": 0.03894731029868126, "VALS::5": 0.12595583498477936, "VALS::6": -1.0, "VFLX::0": 0.04348582401871681, "VFLX::1": 0.1245725005865097, "VFLX::2": -0.013425376266241074, "VFLX::3": 0.15231597423553467, "VFLX::4": -1.0, "VFLX::5": 0.02115434780716896, "VFLX::6": 0.07864074409008026, "VOLT::0": 0.14280620217323303, "VOLT::1": 0.12069455534219742, "VOLT::2": -0.030226442962884903, "VOLT::3": 0.10013772547245026, "VOLT::4": 0.09378558397293091, "VOLT::5": 0.11149084568023682, "VOLT::6": -0.004130214918404818, "VULN::0": 0.0785403624176979, "VULN::1": 0.131057471036911, "VULN::2": 0.12550419569015503, "VULN::3": 0.08390854299068451, "VULN::4": 0.08736353367567062, "VULN::5": 0.11524645984172821, "VULN::6": 0.12741242349147797, "WARM::0": 0.028625018894672394, "WARM::1": 0.11111150681972504, "WARM::2": 0.15703900158405304, "WARM::3": -0.06614095717668533, "WARM::4": 0.09525936096906662, "WARM::5": 0.04715024307370186, "WARM::6": 0.07855141162872314 }, "spearman_avg": 0.09530876868239857, "spearman_per_prompt": { "AGEV::0": { "n": 50, "spearman": 0.39169124798566074 }, "AGEV::1": { "n": 50, "spearman": 0.45902004946706326 }, "AGEV::2": { "n": 50, "spearman": 0.2662371808380498 }, "AGEV::3": { "n": 50, "spearman": -0.03577506913441619 }, "AGEV::4": { "n": 49, "spearman": 0.07547799732923098 }, "AGEV::5": { "n": 50, "spearman": 0.16350080673632314 }, "AGEV::6": { "n": 50, "spearman": 0.08693424492635686 }, "AROU::0": { "n": 50, "spearman": 0.08499128015470626 }, "AROU::1": { "n": 50, "spearman": 0.174208798464483 }, "AROU::2": { "n": 50, "spearman": 0.33208597566339926 }, "AROU::3": { "n": 47, "spearman": 0.042084394936370575 }, "AROU::4": { "n": 46, "spearman": 0.04895921800664758 }, "AROU::5": { "n": 50, "spearman": 0.1532264164951738 }, "AROU::6": { "n": 50, "spearman": 0.3493364396653926 }, "ARSH::0": { "n": 50, "spearman": -0.16647535612684744 }, "ARSH::1": { "n": 50, "spearman": 0.24203050668289824 }, "ARSH::2": { "n": 50, "spearman": 0.09075561176459727 }, "ARSH::3": { "n": 50, "spearman": 0.09575619124538819 }, "ARSH::4": { "n": 50, "spearman": 0.07277715457220746 }, "ARSH::5": { "n": 50, "spearman": 0.12779185370567459 }, "ARSH::6": { "n": 50, "spearman": -0.06587314798632073 }, "ATCK::0": { "n": 50, "spearman": 0.09767439868366594 }, "ATCK::1": { "n": 50, "spearman": 0.07976696892516867 }, "ATCK::2": { "n": 50, "spearman": -0.04513744502892682 }, "ATCK::3": { "n": 50, "spearman": 0.08283095432710455 }, "ATCK::4": { "n": 50, "spearman": 0.27027896517365446 }, "ATCK::5": { "n": 50, "spearman": -0.2077510147106965 }, "ATCK::6": { "n": 50, "spearman": 0.20397248842528493 }, "BKGN::0": { "n": 50, "spearman": 0.19458630926995063 }, "BKGN::1": { "n": 50, "spearman": 0.047783442538948874 }, "BKGN::2": { "n": 27, "spearman": -0.21278577925238631 }, "BKGN::3": { "n": 38, "spearman": -0.08622720317446589 }, "BKGN::4": { "n": 50, "spearman": 0.30846943468479493 }, "BKGN::5": { "n": 50, "spearman": 0.4421914216734776 }, "BKGN::6": { "n": 50, "spearman": 0.12012267329263093 }, "BRGT::0": { "n": 33, "spearman": 0.39940064111810564 }, "BRGT::1": { "n": 37, "spearman": 0.11318849035582404 }, "BRGT::2": { "n": 50, "spearman": 0.0003581349265168106 }, "BRGT::3": { "n": 50, "spearman": -0.06941366950500662 }, "BRGT::4": { "n": 50, "spearman": 0.09301391870617837 }, "BRGT::5": { "n": 50, "spearman": 0.08597655364180073 }, "BRGT::6": { "n": 50, "spearman": 0.20903208375243532 }, "CHNK::0": { "n": 50, "spearman": 0.3442764276083272 }, "CHNK::1": { "n": 50, "spearman": 0.10758288899546531 }, "CHNK::2": { "n": 37, "spearman": 0.15672509542029098 }, "CHNK::3": { "n": 50, "spearman": 0.06686815695928171 }, "CHNK::4": { "n": 50, "spearman": -0.0950207577580909 }, "CHNK::5": { "n": 50, "spearman": 0.13707494328861503 }, "CHNK::6": { "n": 34, "spearman": -0.05839793339555962 }, "CLRT::0": { "n": 50, "spearman": 0.18146062541898159 }, "CLRT::1": { "n": 50, "spearman": 0.43170440744055116 }, "CLRT::2": { "n": 50, "spearman": 0.10609834177517884 }, "CLRT::3": { "n": 32, "spearman": -0.35827653802710696 }, "CLRT::4": { "n": 50, "spearman": 0.03521553829994411 }, "CLRT::5": { "n": 50, "spearman": -0.20850008143000223 }, "CLRT::6": { "n": 50, "spearman": 0.07564017287016589 }, "COGL::0": { "n": 50, "spearman": 0.010643132780370735 }, "COGL::1": { "n": 50, "spearman": 0.047842956838539855 }, "COGL::2": { "n": 39, "spearman": 0.11177914467157152 }, "COGL::3": { "n": 31, "spearman": 0.1184013571840853 }, "COGL::4": { "n": 50, "spearman": 0.005877710581726085 }, "COGL::5": { "n": 50, "spearman": -0.07126094957665864 }, "COGL::6": { "n": 50, "spearman": 0.03207157575931319 }, "DARC::0": { "n": 50, "spearman": -0.0944126874204164 }, "DARC::1": { "n": 50, "spearman": -0.19363406207313222 }, "DARC::2": { "n": 29, "spearman": -0.0325797477592608 }, "DARC::3": { "n": 50, "spearman": 0.007253003744582718 }, "DARC::4": { "n": 50, "spearman": 0.20142521996922005 }, "DARC::5": { "n": 50, "spearman": -0.0745815389882661 }, "DARC::6": { "n": 50, "spearman": 0.17470361014501656 }, "DFLU::0": { "n": 50, "spearman": 0.057648820434212926 }, "DFLU::1": { "n": 50, "spearman": -0.10677889115061578 }, "DFLU::2": { "n": 40, "spearman": 0.0711021991771683 }, "DFLU::3": { "n": 50, "spearman": 0.00035922335091474023 }, "DFLU::4": { "n": 50, "spearman": -0.15435248204085966 }, "DFLU::5": { "n": 50, "spearman": -0.13693669537725106 }, "DFLU::6": { "n": 50, "spearman": 0.10837493494028712 }, "EMPH::0": { "n": 50, "spearman": -0.08376258028189217 }, "EMPH::1": { "n": 50, "spearman": -0.29093625885602514 }, "EMPH::2": { "n": 30, "spearman": 0.016620314155532077 }, "EMPH::3": { "n": 33, "spearman": -0.1828218390531952 }, "EMPH::4": { "n": 50, "spearman": 0.32436057327975004 }, "EMPH::5": { "n": 50, "spearman": 0.33968190689059036 }, "EMPH::6": { "n": 50, "spearman": 0.021509495158367437 }, "ESTH::0": { "n": 50, "spearman": 0.2879930081140926 }, "ESTH::1": { "n": 50, "spearman": 0.2417639870957918 }, "ESTH::2": { "n": 44, "spearman": 0.21982051633875996 }, "ESTH::3": { "n": 30, "spearman": 0.07254025470164953 }, "ESTH::4": { "n": 50, "spearman": 0.08056200695334594 }, "ESTH::5": { "n": 50, "spearman": -0.07308030813914057 }, "ESTH::6": { "n": 43, "spearman": 0.25392058134725465 }, "EXPL::0": { "n": 50, "spearman": -0.3294049403501385 }, "EXPL::1": { "n": 44, "spearman": 0.3283333795987504 }, "EXPL::2": { "n": 50, "spearman": -0.0036990298132535838 }, "FOCS::0": { "n": 50, "spearman": -0.010109874852939703 }, "FOCS::1": { "n": 50, "spearman": -0.15317434937955976 }, "FOCS::2": { "n": 50, "spearman": -0.0069390103607726485 }, "FOCS::3": { "n": 41, "spearman": 0.1838238526176861 }, "FOCS::4": { "n": 50, "spearman": 0.132284349431672 }, "FOCS::5": { "n": 50, "spearman": 0.12031630062733067 }, "FOCS::6": { "n": 50, "spearman": -0.006031380794323329 }, "FULL::0": { "n": 27, "spearman": -0.06769442707381751 }, "FULL::1": { "n": 50, "spearman": -0.17878881272071873 }, "FULL::2": { "n": 30, "spearman": 0.12497977866831195 }, "FULL::3": { "n": 37, "spearman": 0.18779802780580784 }, "FULL::4": { "n": 50, "spearman": 0.14170209136141337 }, "FULL::5": { "n": 50, "spearman": 0.2465451486190338 }, "FULL::6": { "n": 29, "spearman": 0.16024009614014706 }, "GEND::0": { "n": 50, "spearman": 0.1398544219107407 }, "GEND::1": { "n": 50, "spearman": 0.16820724033110315 }, "GEND::2": { "n": 50, "spearman": 0.06862622143845161 }, "GEND::3": { "n": 50, "spearman": -0.1693487550610255 }, "GEND::4": { "n": 50, "spearman": 0.02593243564938909 }, "GEND::5": { "n": 50, "spearman": -0.04524549548789114 }, "GEND::6": { "n": 50, "spearman": 0.08279629453945549 }, "HARM::0": { "n": 50, "spearman": -0.02419461964761923 }, "HARM::1": { "n": 50, "spearman": 0.12901562360266752 }, "HARM::2": { "n": 50, "spearman": 0.06749883191855699 }, "HARM::3": { "n": 33, "spearman": -0.47939770315676256 }, "HARM::4": { "n": 50, "spearman": 0.2150980096403615 }, "HARM::5": { "n": 30, "spearman": 0.06354384155665702 }, "HARM::6": { "n": 50, "spearman": -0.04205948860676404 }, "LANG::Americas": { "n": 25, "spearman": -0.2675836329549901 }, "LANG::Asian_Pacific": { "n": 25, "spearman": -0.23204774044612855 }, "LANG::European": { "n": 25, "spearman": -0.023782574707724706 }, "LANG::Middle_Eastern_African": { "n": 25, "spearman": -0.1350070860127062 }, "METL::0": { "n": 42, "spearman": 0.1227951412349016 }, "METL::1": { "n": 50, "spearman": -0.11392511986153189 }, "METL::2": { "n": 30, "spearman": -0.02640602822251636 }, "METL::3": { "n": 50, "spearman": 0.004701920877062526 }, "METL::4": { "n": 50, "spearman": 0.049781278974268456 }, "METL::5": { "n": 50, "spearman": -0.06655384312785603 }, "METL::6": { "n": 50, "spearman": 0.05227984000031215 }, "RANG::0": { "n": 25, "spearman": 0.30009379396821323 }, "RANG::1": { "n": 50, "spearman": -0.1230672806531439 }, "RANG::2": { "n": 50, "spearman": -0.08059456608490864 }, "RANG::3": { "n": 27, "spearman": 0.19053855339460338 }, "RANG::4": { "n": 50, "spearman": 0.0565429643372528 }, "RANG::5": { "n": 50, "spearman": -0.08222467291810853 }, "RANG::6": { "n": 50, "spearman": 0.19403688816306947 }, "RCQL::0": { "n": 50, "spearman": -0.14555854976311763 }, "RCQL::1": { "n": 50, "spearman": 0.07895088997444219 }, "RCQL::2": { "n": 25, "spearman": -0.07058747364537932 }, "RCQL::3": { "n": 25, "spearman": -0.23437862516308933 }, "RCQL::4": { "n": 36, "spearman": 0.03161917817312551 }, "RCQL::5": { "n": 50, "spearman": 0.020967338880255176 }, "RCQL::6": { "n": 50, "spearman": -0.06588567737211853 }, "REGS::0": { "n": 50, "spearman": 0.3239702139341537 }, "REGS::1": { "n": 50, "spearman": 0.006532343985613863 }, "REGS::2": { "n": 50, "spearman": -0.06345498681294572 }, "REGS::3": { "n": 50, "spearman": 0.3765150248048017 }, "REGS::4": { "n": 50, "spearman": 0.04367800171997523 }, "REGS::5": { "n": 50, "spearman": 0.23723057648298748 }, "REGS::6": { "n": 50, "spearman": 0.050019785922199696 }, "RESP::0": { "n": 50, "spearman": 0.2319507043668216 }, "RESP::1": { "n": 50, "spearman": 0.11089770185903347 }, "RESP::2": { "n": 44, "spearman": -0.05340873857169867 }, "RESP::3": { "n": 50, "spearman": 0.4208917664706726 }, "RESP::4": { "n": 50, "spearman": 0.3831023387057618 }, "RESP::5": { "n": 50, "spearman": 0.6204307779200203 }, "RESP::6": { "n": 50, "spearman": 0.6008665165353128 }, "ROUG::0": { "n": 27, "spearman": 0.37483682020627224 }, "ROUG::1": { "n": 42, "spearman": 0.23755118559031188 }, "ROUG::2": { "n": 34, "spearman": 0.3838306725184532 }, "ROUG::3": { "n": 50, "spearman": 0.02455470420320599 }, "ROUG::4": { "n": 50, "spearman": -0.06738652314673825 }, "ROUG::5": { "n": 50, "spearman": 0.3680543431339921 }, "ROUG::6": { "n": 50, "spearman": 0.0332107382005356 }, "R_CHST::0": { "n": 50, "spearman": 0.007556950141518567 }, "R_CHST::1": { "n": 50, "spearman": 0.12329528764123682 }, "R_CHST::2": { "n": 30, "spearman": -0.1990180599907465 }, "R_CHST::3": { "n": 50, "spearman": -0.009766285751774524 }, "R_CHST::4": { "n": 50, "spearman": 0.1288877427279088 }, "R_CHST::5": { "n": 50, "spearman": -0.08022749546313758 }, "R_CHST::6": { "n": 40, "spearman": 0.05714602234369475 }, "R_HEAD::0": { "n": 50, "spearman": -0.35040889684280274 }, "R_HEAD::1": { "n": 50, "spearman": 0.19027201842175173 }, "R_HEAD::2": { "n": 50, "spearman": 0.1503418125303865 }, "R_HEAD::3": { "n": 50, "spearman": -0.03748236473757336 }, "R_HEAD::4": { "n": 50, "spearman": 0.22277271794694284 }, "R_HEAD::5": { "n": 50, "spearman": 0.08461836502730558 }, "R_HEAD::6": { "n": 40, "spearman": 0.11570792966469377 }, "R_MASK::0": { "n": 50, "spearman": -0.02133578483685158 }, "R_MASK::1": { "n": 50, "spearman": 0.13070654159849962 }, "R_MASK::2": { "n": 50, "spearman": 0.009108702113414375 }, "R_MASK::3": { "n": 50, "spearman": -0.19259957448867268 }, "R_MASK::4": { "n": 50, "spearman": 0.06800053680542313 }, "R_MASK::5": { "n": 50, "spearman": -0.192027018304317 }, "R_MASK::6": { "n": 32, "spearman": 0.2351237631319213 }, "R_MIXD::0": { "n": 50, "spearman": -0.05348292930928569 }, "R_MIXD::1": { "n": 43, "spearman": 0.1716505773472607 }, "R_MIXD::2": { "n": 50, "spearman": 0.050624855007574396 }, "R_MIXD::3": { "n": 50, "spearman": 0.15903922441622548 }, "R_MIXD::4": { "n": 50, "spearman": 0.43735629001177 }, "R_MIXD::5": { "n": 50, "spearman": 0.1886022208954459 }, "R_MIXD::6": { "n": 25, "spearman": -0.31008683647302115 }, "R_NASL::0": { "n": 28, "spearman": 0.12461959606741299 }, "R_NASL::1": { "n": 50, "spearman": 0.12174773360070332 }, "R_NASL::2": { "n": 50, "spearman": -0.0364712551193493 }, "R_NASL::3": { "n": 50, "spearman": -0.0853502745593937 }, "R_NASL::4": { "n": 41, "spearman": 0.07308781217980431 }, "R_NASL::5": { "n": 35, "spearman": 0.1279501875221359 }, "R_NASL::6": { "n": 25, "spearman": 0.24410177162693392 }, "R_ORAL::0": { "n": 43, "spearman": -0.1826596833089379 }, "R_ORAL::1": { "n": 50, "spearman": 0.20497603402993908 }, "R_ORAL::2": { "n": 34, "spearman": -0.14894753798642732 }, "R_ORAL::3": { "n": 50, "spearman": 0.17678316605482 }, "R_ORAL::4": { "n": 50, "spearman": -0.2750185318349498 }, "R_ORAL::5": { "n": 50, "spearman": 0.022729163222482224 }, "R_ORAL::6": { "n": 34, "spearman": 0.07851245865597486 }, "R_THRT::0": { "n": 50, "spearman": 0.1670280078348353 }, "R_THRT::1": { "n": 50, "spearman": -0.2042793197130171 }, "R_THRT::2": { "n": 36, "spearman": 0.09489025320645658 }, "R_THRT::3": { "n": 50, "spearman": 0.2745363457066217 }, "R_THRT::4": { "n": 50, "spearman": 0.25128102723457546 }, "R_THRT::5": { "n": 50, "spearman": -0.03793400328165982 }, "R_THRT::6": { "n": 50, "spearman": 0.09810692199267464 }, "SMTH::0": { "n": 50, "spearman": 0.3936161458306792 }, "SMTH::1": { "n": 50, "spearman": -0.10734732752560439 }, "SMTH::2": { "n": 37, "spearman": 0.16731791597025944 }, "SMTH::3": { "n": 27, "spearman": 0.1579050729595633 }, "SMTH::4": { "n": 50, "spearman": 0.23136710807022345 }, "SMTH::5": { "n": 50, "spearman": -0.060160467753547095 }, "SMTH::6": { "n": 50, "spearman": 0.16172219808556879 }, "STNC::0": { "n": 50, "spearman": 0.2355336251127382 }, "STNC::1": { "n": 50, "spearman": 0.2659567730511899 }, "STNC::2": { "n": 50, "spearman": -0.1440723720993508 }, "STNC::3": { "n": 42, "spearman": 0.035078108593172674 }, "STNC::4": { "n": 50, "spearman": 0.33632824121030597 }, "STNC::5": { "n": 50, "spearman": 0.41190171209216253 }, "STNC::6": { "n": 50, "spearman": 0.3491407654778817 }, "STRU::0": { "n": 50, "spearman": 0.14844268178267614 }, "STRU::1": { "n": 50, "spearman": 0.37168168564560594 }, "STRU::2": { "n": 50, "spearman": 0.17190173212872933 }, "STRU::3": { "n": 28, "spearman": 0.2791192531242765 }, "STRU::4": { "n": 50, "spearman": 0.14857817774976237 }, "STRU::5": { "n": 50, "spearman": 0.3589852046862805 }, "STRU::6": { "n": 27, "spearman": -0.0750776040805608 }, "S_ASMR::0": { "n": 33, "spearman": 0.25079131226294954 }, "S_ASMR::1": { "n": 50, "spearman": 0.2616078021074984 }, "S_ASMR::2": { "n": 50, "spearman": 0.34840761096280537 }, "S_ASMR::3": { "n": 50, "spearman": 0.023941266411268588 }, "S_ASMR::4": { "n": 50, "spearman": 0.40206063140538184 }, "S_ASMR::5": { "n": 50, "spearman": 0.16069786281697343 }, "S_ASMR::6": { "n": 50, "spearman": 0.46230208579892407 }, "S_AUTH::0": { "n": 50, "spearman": 0.13869663764282175 }, "S_AUTH::1": { "n": 50, "spearman": 0.1844666664469114 }, "S_AUTH::2": { "n": 50, "spearman": 0.10350702775459954 }, "S_AUTH::3": { "n": 50, "spearman": -0.04726240135905552 }, "S_AUTH::4": { "n": 50, "spearman": 0.2509449005108719 }, "S_AUTH::5": { "n": 50, "spearman": 0.31473186792509317 }, "S_AUTH::6": { "n": 38, "spearman": 0.3478199073854092 }, "S_CART::0": { "n": 50, "spearman": -0.04492240083940684 }, "S_CART::1": { "n": 50, "spearman": 0.18649851597899844 }, "S_CART::2": { "n": 47, "spearman": -0.04362822122666542 }, "S_CART::3": { "n": 50, "spearman": -0.0181470697921519 }, "S_CART::4": { "n": 50, "spearman": 0.39826321398685677 }, "S_CART::5": { "n": 50, "spearman": 0.10750825650514326 }, "S_CART::6": { "n": 50, "spearman": 0.2703220076299902 }, "S_CASU::0": { "n": 26, "spearman": 0.2517983258730124 }, "S_CASU::1": { "n": 50, "spearman": -0.02828992979933355 }, "S_CASU::2": { "n": 50, "spearman": -0.11092436974789914 }, "S_CASU::3": { "n": 50, "spearman": 0.009712586739268175 }, "S_CASU::4": { "n": 50, "spearman": -0.11965496100124305 }, "S_CASU::5": { "n": 50, "spearman": 0.0074550375818650885 }, "S_CASU::6": { "n": 40, "spearman": 0.21633705139954407 }, "S_CONV::0": { "n": 50, "spearman": 0.05539340844894327 }, "S_CONV::1": { "n": 50, "spearman": -0.053720427914792444 }, "S_CONV::2": { "n": 49, "spearman": -0.05059650903426634 }, "S_CONV::3": { "n": 50, "spearman": 0.18652166607977466 }, "S_CONV::4": { "n": 50, "spearman": 0.09089265334203304 }, "S_CONV::5": { "n": 50, "spearman": 0.14423695957891366 }, "S_CONV::6": { "n": 34, "spearman": 0.10492738840528186 }, "S_DRAM::0": { "n": 50, "spearman": 0.14168217838461877 }, "S_DRAM::1": { "n": 50, "spearman": -0.01951827757960841 }, "S_DRAM::2": { "n": 50, "spearman": 0.13567002341573356 }, "S_DRAM::3": { "n": 50, "spearman": -0.1500216370557162 }, "S_DRAM::4": { "n": 50, "spearman": 0.12235069893254492 }, "S_DRAM::5": { "n": 50, "spearman": 0.27773925735638977 }, "S_DRAM::6": { "n": 50, "spearman": 0.03328611645049158 }, "S_FORM::0": { "n": 50, "spearman": 0.3987098821418067 }, "S_FORM::1": { "n": 50, "spearman": 0.0860117583811565 }, "S_FORM::2": { "n": 50, "spearman": 0.30831627032796566 }, "S_FORM::3": { "n": 50, "spearman": -0.03263815327489614 }, "S_FORM::4": { "n": 50, "spearman": 0.0013221322262878142 }, "S_FORM::5": { "n": 50, "spearman": 0.4508569389601488 }, "S_FORM::6": { "n": 42, "spearman": 0.05415494271158627 }, "S_MONO::0": { "n": 50, "spearman": -0.06478629194196844 }, "S_MONO::1": { "n": 50, "spearman": 0.33910048934032455 }, "S_MONO::2": { "n": 50, "spearman": -0.04081963220940632 }, "S_MONO::3": { "n": 50, "spearman": 0.14903332775173347 }, "S_MONO::4": { "n": 50, "spearman": 0.33771485471228513 }, "S_MONO::5": { "n": 50, "spearman": 0.4205396517646033 }, "S_MONO::6": { "n": 50, "spearman": 0.20009373099956487 }, "S_NARR::0": { "n": 50, "spearman": -0.1703308327241107 }, "S_NARR::1": { "n": 50, "spearman": 0.14803403024966066 }, "S_NARR::2": { "n": 50, "spearman": 0.04550559460450451 }, "S_NARR::3": { "n": 50, "spearman": 0.29603629034818857 }, "S_NARR::4": { "n": 50, "spearman": 0.10682723988861008 }, "S_NARR::5": { "n": 50, "spearman": 0.49834432249697924 }, "S_NARR::6": { "n": 29, "spearman": -0.25952151661874223 }, "S_NEWS::0": { "n": 50, "spearman": 0.2289823320484076 }, "S_NEWS::1": { "n": 50, "spearman": 0.26042293083333073 }, "S_NEWS::2": { "n": 50, "spearman": 0.29827006885940105 }, "S_NEWS::3": { "n": 50, "spearman": 0.34561470412668865 }, "S_NEWS::4": { "n": 50, "spearman": 0.2149872031255371 }, "S_NEWS::5": { "n": 50, "spearman": 0.24766227323967044 }, "S_NEWS::6": { "n": 40, "spearman": 0.0019748826117451696 }, "S_PLAY::0": { "n": 50, "spearman": -0.1729151563588077 }, "S_PLAY::1": { "n": 50, "spearman": 0.009332352378353236 }, "S_PLAY::2": { "n": 50, "spearman": 0.08846301522700405 }, "S_PLAY::3": { "n": 50, "spearman": 0.10565506025763025 }, "S_PLAY::4": { "n": 50, "spearman": 0.40341719835362955 }, "S_PLAY::5": { "n": 50, "spearman": 0.24570725710623026 }, "S_PLAY::6": { "n": 50, "spearman": 0.12789440889576606 }, "S_RANT::0": { "n": 50, "spearman": -0.17147959027468612 }, "S_RANT::1": { "n": 50, "spearman": -0.01926907141568579 }, "S_RANT::2": { "n": 50, "spearman": 0.5679506562236154 }, "S_RANT::3": { "n": 50, "spearman": 0.1260233241689723 }, "S_RANT::4": { "n": 50, "spearman": 0.5409815848565662 }, "S_RANT::5": { "n": 50, "spearman": 0.12912650294305908 }, "S_RANT::6": { "n": 50, "spearman": 0.5067614429030847 }, "S_STRY::0": { "n": 46, "spearman": 0.16181808926196778 }, "S_STRY::1": { "n": 50, "spearman": 0.44020255773523453 }, "S_STRY::2": { "n": 50, "spearman": 0.09505560019509299 }, "S_STRY::3": { "n": 50, "spearman": 0.5144995009710196 }, "S_STRY::4": { "n": 50, "spearman": 0.23363337894532263 }, "S_STRY::5": { "n": 50, "spearman": 0.403136766926551 }, "S_STRY::6": { "n": 39, "spearman": 0.2589768252014762 }, "S_TECH::0": { "n": 40, "spearman": -0.03475141919652333 }, "S_TECH::1": { "n": 50, "spearman": 0.014534887559282856 }, "S_TECH::2": { "n": 50, "spearman": -0.09228072102011722 }, "S_TECH::3": { "n": 50, "spearman": -0.023241108742642007 }, "S_TECH::4": { "n": 50, "spearman": 0.030920138606175426 }, "S_TECH::5": { "n": 50, "spearman": -0.10216973564044556 }, "S_TECH::6": { "n": 32, "spearman": 0.317502432047664 }, "S_WHIS::0": { "n": 50, "spearman": -0.022333964583174753 }, "S_WHIS::1": { "n": 50, "spearman": 0.3245888911834348 }, "S_WHIS::2": { "n": 50, "spearman": 0.3278272114695854 }, "S_WHIS::3": { "n": 50, "spearman": 0.24963960398403584 }, "S_WHIS::4": { "n": 50, "spearman": 0.4272381324220501 }, "S_WHIS::5": { "n": 50, "spearman": 0.19072797031179078 }, "S_WHIS::6": { "n": 50, "spearman": 0.5358135237291689 }, "TEMP::0": { "n": 49, "spearman": 0.026752262216258413 }, "TEMP::1": { "n": 50, "spearman": -0.1185472432105048 }, "TEMP::2": { "n": 46, "spearman": 0.16903106153852096 }, "TEMP::3": { "n": 31, "spearman": -0.37204880277418995 }, "TEMP::4": { "n": 50, "spearman": -0.05820983295994197 }, "TEMP::5": { "n": 50, "spearman": -0.08162974846801165 }, "TEMP::6": { "n": 42, "spearman": 0.04350562648734837 }, "TENS::0": { "n": 32, "spearman": 0.1436444044664221 }, "TENS::1": { "n": 50, "spearman": 0.14374419493507634 }, "TENS::2": { "n": 40, "spearman": 0.26779486053044227 }, "TENS::3": { "n": 30, "spearman": -0.02941099030559035 }, "TENS::4": { "n": 50, "spearman": 0.3117932385153611 }, "TENS::5": { "n": 50, "spearman": 0.49399790268467436 }, "TENS::6": { "n": 50, "spearman": 0.17339903800855905 }, "VALN::0": { "n": 50, "spearman": -0.1496970977964933 }, "VALN::1": { "n": 50, "spearman": -0.06636980319717926 }, "VALN::2": { "n": 43, "spearman": -0.18515657500944827 }, "VALN::3": { "n": 50, "spearman": -0.05260516506535353 }, "VALN::4": { "n": 50, "spearman": 0.0004564460746168153 }, "VALN::5": { "n": 50, "spearman": 0.13079763747809012 }, "VALN::6": { "n": 50, "spearman": 0.23535898582886897 }, "VALS::0": { "n": 50, "spearman": -0.10322999292711182 }, "VALS::1": { "n": 50, "spearman": -0.1936554647418715 }, "VALS::2": { "n": 28, "spearman": -0.177962633638889 }, "VALS::3": { "n": 50, "spearman": 0.10199364732863556 }, "VALS::4": { "n": 50, "spearman": 0.22659063424511058 }, "VALS::5": { "n": 50, "spearman": 0.2630267108799556 }, "VALS::6": { "n": 50, "spearman": -0.03911368175189116 }, "VFLX::0": { "n": 50, "spearman": 0.2172637067685717 }, "VFLX::1": { "n": 50, "spearman": 0.035935255181098295 }, "VFLX::2": { "n": 50, "spearman": -0.11645438856459563 }, "VFLX::3": { "n": 50, "spearman": -0.10818433423795493 }, "VFLX::4": { "n": 50, "spearman": -0.028446106342860735 }, "VFLX::5": { "n": 50, "spearman": 0.0938972916864888 }, "VFLX::6": { "n": 50, "spearman": -0.009521222187495824 }, "VOLT::0": { "n": 50, "spearman": 0.029783186029139923 }, "VOLT::1": { "n": 50, "spearman": 0.04297117264303347 }, "VOLT::2": { "n": 36, "spearman": -0.3503921803407711 }, "VOLT::3": { "n": 41, "spearman": 0.15145742320961417 }, "VOLT::4": { "n": 41, "spearman": -0.05206757197332257 }, "VOLT::5": { "n": 50, "spearman": 0.24862987643400816 }, "VOLT::6": { "n": 50, "spearman": 0.254927082486896 }, "VULN::0": { "n": 50, "spearman": 0.15863081680502136 }, "VULN::1": { "n": 50, "spearman": -0.07304981615428428 }, "VULN::2": { "n": 50, "spearman": 0.004157069337113513 }, "VULN::3": { "n": 50, "spearman": -0.03495096411879114 }, "VULN::4": { "n": 50, "spearman": 0.18493570888332525 }, "VULN::5": { "n": 50, "spearman": 0.35778993129070513 }, "VULN::6": { "n": 50, "spearman": 0.22953723455830116 }, "WARM::0": { "n": 39, "spearman": 0.20253093693739493 }, "WARM::1": { "n": 50, "spearman": 0.0637998736496052 }, "WARM::2": { "n": 50, "spearman": 0.1760793390932691 }, "WARM::3": { "n": 49, "spearman": -0.16688654436803016 }, "WARM::4": { "n": 50, "spearman": 0.17574797465296763 }, "WARM::5": { "n": 50, "spearman": 0.14890314494643453 }, "WARM::6": { "n": 26, "spearman": -0.3370806048037221 } }, "by_benchmark_bucket": [ { "benchmark_bucket": "majority_no", "n": 6924, "accuracy": 0.1913633737723859, "balanced_accuracy": NaN, "precision": 0.0, "recall": NaN, "f1": NaN, "tp": 0, "fp": 5599, "tn": 1325, "fn": 0 }, { "benchmark_bucket": "majority_yes", "n": 454, "accuracy": 0.8193832599118943, "balanced_accuracy": NaN, "precision": 1.0, "recall": 0.8193832599118943, "f1": 0.9007263922518159, "tp": 372, "fp": 0, "tn": 0, "fn": 82 }, { "benchmark_bucket": "single_rater_no", "n": 3070, "accuracy": 0.2276872964169381, "balanced_accuracy": NaN, "precision": 0.0, "recall": NaN, "f1": NaN, "tp": 0, "fp": 2371, "tn": 699, "fn": 0 }, { "benchmark_bucket": "single_rater_yes", "n": 1577, "accuracy": 0.8585922637920101, "balanced_accuracy": NaN, "precision": 1.0, "recall": 0.8585922637920101, "f1": 0.9239167519617877, "tp": 1354, "fp": 0, "tn": 0, "fn": 223 }, { "benchmark_bucket": "unanimous_no", "n": 4166, "accuracy": 0.21915506481036967, "balanced_accuracy": NaN, "precision": 0.0, "recall": NaN, "f1": NaN, "tp": 0, "fp": 3253, "tn": 913, "fn": 0 }, { "benchmark_bucket": "unanimous_yes", "n": 2441, "accuracy": 0.8631708316263826, "balanced_accuracy": NaN, "precision": 1.0, "recall": 0.8631708316263826, "f1": 0.9265611257695691, "tp": 2107, "fp": 0, "tn": 0, "fn": 334 } ], "by_benchmark_bucket_optimal": [ { "benchmark_bucket": "majority_no", "n": 6924, "accuracy": 0.548526863084922, "balanced_accuracy": NaN, "precision": 0.0, "recall": NaN, "f1": NaN, "tp": 0, "fp": 3126, "tn": 3798, "fn": 0 }, { "benchmark_bucket": "majority_yes", "n": 454, "accuracy": 0.43171806167400884, "balanced_accuracy": NaN, "precision": 1.0, "recall": 0.43171806167400884, "f1": 0.6030769230769231, "tp": 196, "fp": 0, "tn": 0, "fn": 258 }, { "benchmark_bucket": "single_rater_no", "n": 3070, "accuracy": 0.5879478827361564, "balanced_accuracy": NaN, "precision": 0.0, "recall": NaN, "f1": NaN, "tp": 0, "fp": 1265, "tn": 1805, "fn": 0 }, { "benchmark_bucket": "single_rater_yes", "n": 1577, "accuracy": 0.5212428662016487, "balanced_accuracy": NaN, "precision": 1.0, "recall": 0.5212428662016487, "f1": 0.6852855356398501, "tp": 822, "fp": 0, "tn": 0, "fn": 755 }, { "benchmark_bucket": "unanimous_no", "n": 4166, "accuracy": 0.5871339414306289, "balanced_accuracy": NaN, "precision": 0.0, "recall": NaN, "f1": NaN, "tp": 0, "fp": 1720, "tn": 2446, "fn": 0 }, { "benchmark_bucket": "unanimous_yes", "n": 2441, "accuracy": 0.5301106104055715, "balanced_accuracy": NaN, "precision": 1.0, "recall": 0.5301106104055715, "f1": 0.692904953145917, "tp": 1294, "fp": 0, "tn": 0, "fn": 1147 } ], "by_n_raters": [ { "n_raters": 1, "n": 4647, "accuracy": 0.4417904024101571, "balanced_accuracy": 0.5431397801044742, "precision": 0.36348993288590603, "recall": 0.8585922637920101, "f1": 0.5107506601282535, "tp": 1354, "fp": 2371, "tn": 699, "fn": 223 }, { "n_raters": 2, "n": 12562, "accuracy": 0.32494825664703075, "balanced_accuracy": 0.5332640608389124, "precision": 0.19772012578616352, "recall": 0.8642611683848798, "f1": 0.3218170185540627, "tp": 2012, "fp": 8164, "tn": 2070, "fn": 316 }, { "n_raters": 3, "n": 1366, "accuracy": 0.4546120058565154, "balanced_accuracy": 0.5143508849193374, "precision": 0.41335740072202165, "recall": 0.8282097649186256, "f1": 0.551475015051174, "tp": 458, "fp": 650, "tn": 163, "fn": 95 }, { "n_raters": 4, "n": 57, "accuracy": 0.24561403508771928, "balanced_accuracy": 0.3795681063122924, "precision": 0.19148936170212766, "recall": 0.6428571428571429, "f1": 0.29508196721311475, "tp": 9, "fp": 38, "tn": 5, "fn": 5 } ], "by_n_raters_optimal": [ { "n_raters": 1, "n": 4647, "accuracy": 0.5653109533032064, "balanced_accuracy": 0.5545953744689025, "precision": 0.39386679444178246, "recall": 0.5212428662016487, "f1": 0.44868995633187775, "tp": 822, "fp": 1265, "tn": 1805, "fn": 755 }, { "n_raters": 2, "n": 12562, "accuracy": 0.5580321604839994, "balanced_accuracy": 0.5473968837115282, "precision": 0.21689497716894976, "recall": 0.5304982817869416, "f1": 0.30790326601844925, "tp": 1235, "fp": 4459, "tn": 5775, "fn": 1093 }, { "n_raters": 3, "n": 1366, "accuracy": 0.5109809663250366, "balanced_accuracy": 0.5018516912113063, "precision": 0.406807131280389, "recall": 0.4538878842676311, "f1": 0.4290598290598291, "tp": 251, "fp": 366, "tn": 447, "fn": 302 }, { "n_raters": 4, "n": 57, "accuracy": 0.45614035087719296, "balanced_accuracy": 0.39867109634551495, "precision": 0.16, "recall": 0.2857142857142857, "f1": 0.20512820512820512, "tp": 4, "fp": 21, "tn": 22, "fn": 10 } ], "ensemble_members": [ { "name": "baseline_mispeech_GLAP", "snapshot": "mispeech/GLAP", "weight": 1.0 } ], "audio_count": 18632, "prompt_count": 399, "outputs": { "predictions": "emolia-dim/predictions.csv", "summary": "emolia-dim/summary.json" } } }