Hugging Face's logo

Buckets:
Mercity
/
Desicion-Model-Custom

Files
xet
Mercity/Desicion-Model-Custom / jevals /jevals.json
Pranav2748's picture
Pranav2748
download
raw
25.4 kB
{
"ours": {
"base": {
"model": "Qwen/Qwen3.5-4B",
"temps": {
"choice": 1.707683209388825,
"noul": 2.4105066990845945,
"score": 1.2847580560947693
},
"raw": {
"pubmedqa": {
"primitive": "noul",
"ds": 48.05492263372704,
"ci": [
37.76904082341696,
57.150183634721955
],
"accuracy": 0.8066666666666666,
"ece_10_bin": 0.06460862585758344,
"T": 1.0,
"n": 300
},
"helpsteer2": {
"primitive": "score",
"ds": 2.049999831140492,
"ci": [
-9.799345465028715,
13.108881184556378
],
"accuracy": 0.4033333333333333,
"ece_10_bin": 0.15123322991920435,
"T": 1.0,
"n": 300
},
"banking77": {
"primitive": "choice",
"ds": 56.173306098208656,
"ci": [
49.27693100059662,
63.141876285290685
],
"accuracy": 0.7186666666666667,
"ece_10_bin": 0.14011876588219976,
"T": 1.0,
"n": 300
}
},
"calibrated": {
"pubmedqa": {
"primitive": "noul",
"ds": 36.371473995104964,
"ci": [
29.35952403100504,
42.77706939107353
],
"accuracy": 0.8066666666666666,
"ece_10_bin": 0.15700328478531872,
"T": 2.4105066990845945,
"n": 300
},
"helpsteer2": {
"primitive": "score",
"ds": 4.103423043229137,
"ci": [
-6.389453722683416,
13.956330811232187
],
"accuracy": 0.4033333333333333,
"ece_10_bin": 0.08902712405177117,
"T": 1.2847580560947693,
"n": 300
},
"banking77": {
"primitive": "choice",
"ds": 59.3453919929658,
"ci": [
53.69828564607293,
64.93884389399817
],
"accuracy": 0.7186666666666667,
"ece_10_bin": 0.04315345334218913,
"T": 1.707683209388825,
"n": 300
}
}
},
"base_2b": {
"model": "openbmb/MiniCPM5-2B",
"temps": {
"choice": 4.78807442670338,
"noul": 19.999999999982688,
"score": 9.956643342471382
},
"raw": {
"pubmedqa": {
"primitive": "noul",
"ds": 0.06537669220909592,
"ci": [
-12.074739103400578,
12.86250789143286
],
"accuracy": 0.6833333333333333,
"ece_10_bin": 0.2256417780118445,
"T": 1.0,
"n": 300
},
"helpsteer2": {
"primitive": "score",
"ds": -26.542430292496523,
"ci": [
-32.37154921739351,
-19.847259796335507
],
"accuracy": 0.3233333333333333,
"ece_10_bin": 0.4115217030994677,
"T": 1.0,
"n": 300
},
"banking77": {
"primitive": "choice",
"ds": 28.57680388837838,
"ci": [
20.866179600278087,
36.316383050601196
],
"accuracy": 0.57,
"ece_10_bin": 0.3036123204111504,
"T": 1.0,
"n": 300
}
},
"calibrated": {
"pubmedqa": {
"primitive": "noul",
"ds": 7.871329959231909,
"ci": [
4.386042772766913,
11.112613249722312
],
"accuracy": 0.6833333333333333,
"ece_10_bin": 0.125778638387935,
"T": 19.999999999982688,
"n": 300
},
"helpsteer2": {
"primitive": "score",
"ds": -6.307747971961097,
"ci": [
-12.458304188976395,
-0.6936091297558415
],
"accuracy": 0.3233333333333333,
"ece_10_bin": 0.010844527401254841,
"T": 9.956643342471382,
"n": 300
},
"banking77": {
"primitive": "choice",
"ds": 31.930029024290597,
"ci": [
28.786810587023467,
35.1970814222839
],
"accuracy": 0.57,
"ece_10_bin": 0.2619731354805633,
"T": 4.78807442670338,
"n": 300
}
}
},
"base_2b_think": {
"model": "openbmb/MiniCPM5-2B",
"temps": {
"choice": 11.20281383618261,
"noul": 14.170122098039382,
"score": 8.440970884106083
},
"raw": {
"pubmedqa": {
"primitive": "noul",
"ds": 33.5031127255792,
"ci": [
14.994088405964881,
50.14401495430281
],
"accuracy": 0.8433333333333334,
"ece_10_bin": 0.15666666652019579,
"T": 1.0,
"n": 300
},
"helpsteer2": {
"primitive": "score",
"ds": -53.5046810509483,
"ci": [
-70.32589555811374,
-37.2137948447786
],
"accuracy": 0.33666666666666667,
"ece_10_bin": 0.6633333118218515,
"T": 1.0,
"n": 300
},
"banking77": {
"primitive": "choice",
"ds": 36.503652709543644,
"ci": [
25.697439535484303,
47.31038520132706
],
"accuracy": 0.6866666666666666,
"ece_10_bin": 0.3133331416314482,
"T": 1.0,
"n": 300
}
},
"calibrated": {
"pubmedqa": {
"primitive": "noul",
"ds": 43.57061048146679,
"ci": [
29.803221302295093,
55.62367362989394
],
"accuracy": 0.8433333333333334,
"ece_10_bin": 0.025859414384324198,
"T": 14.170122098039382,
"n": 300
},
"helpsteer2": {
"primitive": "score",
"ds": -24.399113849176544,
"ci": [
-38.0616774021222,
-11.746581566453877
],
"accuracy": 0.33666666666666667,
"ece_10_bin": 0.4503987571107928,
"T": 8.440970884106083,
"n": 300
},
"banking77": {
"primitive": "choice",
"ds": 28.77450536506706,
"ci": [
25.907327767216973,
31.63217534210117
],
"accuracy": 0.6866666666666666,
"ece_10_bin": 0.4068075231256754,
"T": 11.20281383618261,
"n": 300
}
}
},
"rl40": {
"model": "runs/v2/checkpoints/rl40_merged",
"temps": {
"choice": 1.1597719952546015,
"noul": 0.8079639789416141,
"score": 0.7152072687033781
},
"raw": {
"pubmedqa": {
"primitive": "noul",
"ds": 58.26569187854867,
"ci": [
47.682670427263815,
67.59957426059007
],
"accuracy": 0.86,
"ece_10_bin": 0.02504456097376074,
"T": 1.0,
"n": 300
},
"helpsteer2": {
"primitive": "score",
"ds": 6.046865704552173,
"ci": [
-2.4892746013859646,
14.387750191722974
],
"accuracy": 0.42,
"ece_10_bin": 0.09333381174925866,
"T": 1.0,
"n": 300
},
"banking77": {
"primitive": "choice",
"ds": 60.86873010019582,
"ci": [
54.64881897913666,
67.22493242614614
],
"accuracy": 0.732,
"ece_10_bin": 0.05956597864906059,
"T": 1.0,
"n": 300
}
},
"calibrated": {
"pubmedqa": {
"primitive": "noul",
"ds": 58.07152293620064,
"ci": [
46.80979761941665,
68.33396374013778
],
"accuracy": 0.86,
"ece_10_bin": 0.03793417592656142,
"T": 0.8079639789416141,
"n": 300
},
"helpsteer2": {
"primitive": "score",
"ds": 2.0690909709641936,
"ci": [
-7.552019439614597,
11.588130422238306
],
"accuracy": 0.42,
"ece_10_bin": 0.15102980959657503,
"T": 0.7152072687033781,
"n": 300
},
"banking77": {
"primitive": "choice",
"ds": 61.28325240743173,
"ci": [
55.474958876375645,
67.25123138884102
],
"accuracy": 0.732,
"ece_10_bin": 0.02114475384467073,
"T": 1.1597719952546015,
"n": 300
}
}
},
"rl60": {
"model": "runs/v2/checkpoints/rl60_merged",
"temps": {
"choice": 1.1599743376333564,
"noul": 0.7983784191901301,
"score": 0.7276446912498098
},
"raw": {
"pubmedqa": {
"primitive": "noul",
"ds": 58.72560050503446,
"ci": [
48.25519209655997,
67.9986456840194
],
"accuracy": 0.8633333333333333,
"ece_10_bin": 0.026845709305701414,
"T": 1.0,
"n": 300
},
"helpsteer2": {
"primitive": "score",
"ds": 6.037515788606118,
"ci": [
-2.400834157155406,
14.35084224020631
],
"accuracy": 0.43666666666666665,
"ece_10_bin": 0.095953934239622,
"T": 1.0,
"n": 300
},
"banking77": {
"primitive": "choice",
"ds": 60.955089258193006,
"ci": [
54.66931758063293,
67.32577013719438
],
"accuracy": 0.7313333333333333,
"ece_10_bin": 0.05904332938166834,
"T": 1.0,
"n": 300
}
},
"calibrated": {
"pubmedqa": {
"primitive": "noul",
"ds": 58.614060949612615,
"ci": [
47.33820737643784,
68.84278523729319
],
"accuracy": 0.8633333333333333,
"ece_10_bin": 0.03571861771195042,
"T": 0.7983784191901301,
"n": 300
},
"helpsteer2": {
"primitive": "score",
"ds": 2.368980700043777,
"ci": [
-7.2405524866967275,
11.746115459613257
],
"accuracy": 0.43666666666666665,
"ece_10_bin": 0.1303236520061676,
"T": 0.7276446912498098,
"n": 300
},
"banking77": {
"primitive": "choice",
"ds": 61.33904164009218,
"ci": [
55.52009100804195,
67.34968363963038
],
"accuracy": 0.7313333333333333,
"ece_10_bin": 0.027606077031200056,
"T": 1.1599743376333564,
"n": 300
}
}
},
"rl60_think": {
"model": "runs/v2/checkpoints/rl60_merged",
"temps": {
"choice": 2.145082715273473,
"noul": 1.9352338162597778,
"score": 1.8069801445316314
},
"raw": {
"pubmedqa": {
"primitive": "noul",
"ds": 58.936313009275274,
"ci": [
44.39233287434623,
72.34563008669913
],
"accuracy": 0.9,
"ece_10_bin": 0.09364023899498322,
"T": 1.0,
"n": 300
},
"helpsteer2": {
"primitive": "score",
"ds": -42.99011458744382,
"ci": [
-61.37940857090847,
-24.665619976798016
],
"accuracy": 0.41333333333333333,
"ece_10_bin": 0.519184866453263,
"T": 1.0,
"n": 300
},
"banking77": {
"primitive": "choice",
"ds": 49.48519474863158,
"ci": [
40.343495405677444,
58.75819407518067
],
"accuracy": 0.7333333333333333,
"ece_10_bin": 0.24588454845319418,
"T": 1.0,
"n": 300
}
},
"calibrated": {
"pubmedqa": {
"primitive": "noul",
"ds": 63.46531221234828,
"ci": [
51.2282704267052,
74.80118608216338
],
"accuracy": 0.9,
"ece_10_bin": 0.045119675748778995,
"T": 1.9352338162597778,
"n": 300
},
"helpsteer2": {
"primitive": "score",
"ds": -20.13990146885871,
"ci": [
-35.33533731665648,
-4.939007706671506
],
"accuracy": 0.41333333333333333,
"ece_10_bin": 0.37806488700901164,
"T": 1.8069801445316314,
"n": 300
},
"banking77": {
"primitive": "choice",
"ds": 59.27752355436782,
"ci": [
53.606719794836316,
64.93246211545883
],
"accuracy": 0.7333333333333333,
"ece_10_bin": 0.13413758897367453,
"T": 2.145082715273473,
"n": 300
}
}
},
"v1_joint": {
"model": "checkpoints/joint_merged",
"temps": {
"choice": 1.2843740495491411,
"noul": 1.1776347713530488,
"score": 1.0535480782391846
},
"raw": {
"pubmedqa": {
"primitive": "noul",
"ds": 49.206682779012525,
"ci": [
36.65442701300594,
59.903266381066885
],
"accuracy": 0.8233333333333334,
"ece_10_bin": 0.06969906794374832,
"T": 1.0,
"n": 300
},
"helpsteer2": {
"primitive": "score",
"ds": 6.7772824916263685,
"ci": [
-2.2223595409748595,
16.175846550495578
],
"accuracy": 0.3933333333333333,
"ece_10_bin": 0.17954795337837168,
"T": 1.0,
"n": 300
},
"banking77": {
"primitive": "choice",
"ds": 60.61334399242887,
"ci": [
54.24958965660347,
66.93369679868799
],
"accuracy": 0.7326666666666667,
"ece_10_bin": 0.07895850726459176,
"T": 1.0,
"n": 300
}
},
"calibrated": {
"pubmedqa": {
"primitive": "noul",
"ds": 50.30267194300714,
"ci": [
38.78645963769695,
60.216646199918934
],
"accuracy": 0.8233333333333334,
"ece_10_bin": 0.06609612462236357,
"T": 1.1776347713530488,
"n": 300
},
"helpsteer2": {
"primitive": "score",
"ds": 7.5280862119676435,
"ci": [
-1.2187682680666836,
16.69347049625058
],
"accuracy": 0.3933333333333333,
"ece_10_bin": 0.16808161152147194,
"T": 1.0535480782391846,
"n": 300
},
"banking77": {
"primitive": "choice",
"ds": 61.483220852549046,
"ci": [
55.919259043009745,
67.11049272666077
],
"accuracy": 0.7326666666666667,
"ece_10_bin": 0.05006183740765189,
"T": 1.2843740495491411,
"n": 300
}
}
},
"v2_2b": {
"model": "runs/v2_2b/checkpoints/v2b_joint_merged",
"temps": {
"choice": 1.0924917038331285,
"noul": 0.6133218472538287,
"score": 0.7621787953970005
},
"raw": {
"pubmedqa": {
"primitive": "noul",
"ds": 43.58234449799754,
"ci": [
32.467084766485215,
53.490310674231864
],
"accuracy": 0.79,
"ece_10_bin": 0.06297236285262527,
"T": 1.0,
"n": 300
},
"helpsteer2": {
"primitive": "score",
"ds": 0.6637233783827412,
"ci": [
-6.537478300305655,
8.270746327898612
],
"accuracy": 0.35,
"ece_10_bin": 0.17140852998786757,
"T": 1.0,
"n": 300
},
"banking77": {
"primitive": "choice",
"ds": 53.92308258456515,
"ci": [
47.8488111622375,
60.00593318759219
],
"accuracy": 0.668,
"ece_10_bin": 0.048110853789338,
"T": 1.0,
"n": 300
}
},
"calibrated": {
"pubmedqa": {
"primitive": "noul",
"ds": 41.560019439645814,
"ci": [
27.87758590950197,
53.45290476044082
],
"accuracy": 0.79,
"ece_10_bin": 0.0857771775727929,
"T": 0.6133218472538287,
"n": 300
},
"helpsteer2": {
"primitive": "score",
"ds": -3.9949134233046912,
"ci": [
-11.642008207818442,
4.443413141629088
],
"accuracy": 0.35,
"ece_10_bin": 0.2241508614885324,
"T": 0.7621787953970005,
"n": 300
},
"banking77": {
"primitive": "choice",
"ds": 54.152173734130216,
"ci": [
48.384491001853675,
59.872845624000156
],
"accuracy": 0.668,
"ece_10_bin": 0.03394477729754107,
"T": 1.0924917038331285,
"n": 300
}
}
},
"v2_2b_ro": {
"model": "runs/v2_2b/checkpoints/v2b_ro_merged",
"temps": {
"choice": 1.0772703967295125,
"noul": 0.6149937401183327,
"score": 0.7973321069676444
},
"raw": {
"pubmedqa": {
"primitive": "noul",
"ds": 47.04643260903809,
"ci": [
36.009090370403115,
56.80703000977964
],
"accuracy": 0.82,
"ece_10_bin": 0.015477258953326474,
"T": 1.0,
"n": 300
},
"helpsteer2": {
"primitive": "score",
"ds": 2.676392608952649,
"ci": [
-4.610753531368588,
10.187235755087887
],
"accuracy": 0.4033333333333333,
"ece_10_bin": 0.10693574766283558,
"T": 1.0,
"n": 300
},
"banking77": {
"primitive": "choice",
"ds": 53.88407802182469,
"ci": [
48.02925310834641,
60.06044789809394
],
"accuracy": 0.6793333333333333,
"ece_10_bin": 0.026937718645406474,
"T": 1.0,
"n": 300
}
},
"calibrated": {
"pubmedqa": {
"primitive": "noul",
"ds": 44.575293822302086,
"ci": [
31.521025085187834,
56.53223179946575
],
"accuracy": 0.82,
"ece_10_bin": 0.07335495778853046,
"T": 0.6149937401183327,
"n": 300
},
"helpsteer2": {
"primitive": "score",
"ds": -0.3802936765010312,
"ci": [
-8.192748138473686,
7.9471501430822675
],
"accuracy": 0.4033333333333333,
"ece_10_bin": 0.13772908624584113,
"T": 0.7973321069676444,
"n": 300
},
"banking77": {
"primitive": "choice",
"ds": 53.90635910616057,
"ci": [
48.35518692964494,
59.81217350423332
],
"accuracy": 0.6793333333333333,
"ece_10_bin": 0.045755905574880795,
"T": 1.0772703967295125,
"n": 300
}
}
},
"v2_joint": {
"model": "runs/v2/checkpoints/v2_joint_merged",
"temps": {
"choice": 1.1592816013856364,
"noul": 0.8031824332990728,
"score": 0.716891860990727
},
"raw": {
"pubmedqa": {
"primitive": "noul",
"ds": 57.9754923387608,
"ci": [
47.474841970587384,
67.33647236474492
],
"accuracy": 0.8566666666666667,
"ece_10_bin": 0.024792356261037377,
"T": 1.0,
"n": 300
},
"helpsteer2": {
"primitive": "score",
"ds": 6.072330512589563,
"ci": [
-2.447039948281713,
14.421161550147108
],
"accuracy": 0.43,
"ece_10_bin": 0.0758746614948931,
"T": 1.0,
"n": 300
},
"banking77": {
"primitive": "choice",
"ds": 60.90446514198684,
"ci": [
54.643973597052714,
67.26811195597078
],
"accuracy": 0.734,
"ece_10_bin": 0.05840767711375198,
"T": 1.0,
"n": 300
}
},
"calibrated": {
"pubmedqa": {
"primitive": "noul",
"ds": 57.786338685636565,
"ci": [
46.39802375142668,
68.04438733775511
],
"accuracy": 0.8566666666666667,
"ece_10_bin": 0.033081921946533914,
"T": 0.8031824332990728,
"n": 300
},
"helpsteer2": {
"primitive": "score",
"ds": 2.314214972593698,
"ci": [
-7.267135153938562,
11.84096549506603
],
"accuracy": 0.43,
"ece_10_bin": 0.1364261756101774,
"T": 0.716891860990727,
"n": 300
},
"banking77": {
"primitive": "choice",
"ds": 61.31478198739936,
"ci": [
55.423784855833,
67.28099088659312
],
"accuracy": 0.734,
"ece_10_bin": 0.02052732067327072,
"T": 1.1592816013856364,
"n": 300
}
}
},
"v2_joint_think": {
"model": "runs/v2/checkpoints/v2_joint_merged",
"temps": {
"choice": 2.117644532766411,
"noul": 1.8910262876636832,
"score": 1.6148431811799775
},
"raw": {
"pubmedqa": {
"primitive": "noul",
"ds": 53.8314296697503,
"ci": [
38.461630538462174,
67.87374086095278
],
"accuracy": 0.8833333333333333,
"ece_10_bin": 0.1058673775031327,
"T": 1.0,
"n": 300
},
"helpsteer2": {
"primitive": "score",
"ds": -26.198138966218607,
"ci": [
-41.16874622377119,
-10.352952534125588
],
"accuracy": 0.4066666666666667,
"ece_10_bin": 0.43834127630132536,
"T": 1.0,
"n": 300
},
"banking77": {
"primitive": "choice",
"ds": 51.88605431545722,
"ci": [
43.24874709777629,
61.27065017189058
],
"accuracy": 0.7333333333333333,
"ece_10_bin": 0.22046967225320246,
"T": 1.0,
"n": 300
}
},
"calibrated": {
"pubmedqa": {
"primitive": "noul",
"ds": 60.202782831455316,
"ci": [
47.716787071167964,
71.90589744680015
],
"accuracy": 0.8833333333333333,
"ece_10_bin": 0.05377978866712399,
"T": 1.8910262876636832,
"n": 300
},
"helpsteer2": {
"primitive": "score",
"ds": -13.044833077924034,
"ci": [
-26.61036521386943,
0.5771093535241466
],
"accuracy": 0.4066666666666667,
"ece_10_bin": 0.344897159660971,
"T": 1.6148431811799775,
"n": 300
},
"banking77": {
"primitive": "choice",
"ds": 59.32358836134435,
"ci": [
53.92375680330441,
65.06160011264834
],
"accuracy": 0.7333333333333333,
"ece_10_bin": 0.11909380664834994,
"T": 2.117644532766411,
"n": 300
}
}
}
},
"board": [
{
"board": "banking77",
"name": "Gemini 3.8 Flash",
"ds": 74.107,
"ci": [
68.4397,
79.7893
],
"accuracy": 0.846,
"ece": 0.0319,
"p95_ms": 3938,
"cost_per_1k": 1.367931,
"interface": "adapter"
},
{
"board": "banking77",
"name": "Jev",
"ds": 67.7806,
"ci": [
61.3393,
74.4909
],
"accuracy": 0.7967,
"ece": 0.0981,
"p95_ms": 693,
"cost_per_1k": 0.043276,
"interface": "native"
},
{
"board": "banking77",
"name": "GLM-5.3",
"ds": 66.8264,
"ci": [
61.9567,
71.7196
],
"accuracy": 0.7887,
"ece": 0.0616,
"p95_ms": 3186,
"cost_per_1k": 0.718505,
"interface": "adapter"
},
{
"board": "banking77",
"name": "DeepSeek V4.1 Flash",
"ds": 63.9292,
"ci": [
58.5065,
69.6119
],
"accuracy": 0.7587,
"ece": 0.0276,
"p95_ms": 1309,
"cost_per_1k": 0.138384,
"interface": "adapter"
},
{
"board": "banking77",
"name": "Qwen3.8 Flash",
"ds": 62.4389,
"ci": [
55.9178,
68.8961
],
"accuracy": 0.768,
"ece": 0.0951,
"p95_ms": 3105,
"cost_per_1k": 0.122064,
"interface": "adapter"
},
{
"board": "banking77",
"name": "Mistral Medium 3.5",
"ds": 59.4547,
"ci": [
54.2695,
64.8301
],
"accuracy": 0.7447,
"ece": 0.0807,
"p95_ms": 1444,
"cost_per_1k": 1.515244,
"interface": "adapter"
},
{
"board": "banking77",
"name": "Mercury 2.5",
"ds": 54.0395,
"ci": [
47.8178,
60.6156
],
"accuracy": 0.7033,
"ece": 0.1044,
"p95_ms": 1478,
"cost_per_1k": 0.03606,
"interface": "adapter"
},
{
"board": "banking77",
"name": "Label prior",
"ds": 0,
"ci": [
0,
0
],
"accuracy": 0.0133,
"ece": 0,
"p95_ms": null,
"cost_per_1k": 0,
"interface": "baseline"
},
{
"board": "helpsteer2",
"name": "Jev",
"ds": 9.2034,
"ci": [
-4.365,
20.9655
],
"accuracy": 0.4127,
"ece": 0.1966,
"p95_ms": 670,
"cost_per_1k": 0.036285,
"interface": "native"
},
{
"board": "helpsteer2",
"name": "GLM-5.3",
"ds": 7.7828,
"ci": [
-4.5884,
18.5647
],
"accuracy": 0.43,
"ece": 0.1287,
"p95_ms": 3631,
"cost_per_1k": 0.784848,
"interface": "adapter"
},
{
"board": "helpsteer2",
"name": "Gemini 3.8 Flash",
"ds": 4.5909,
"ci": [
-12.2314,
18.9838
],
"accuracy": 0.424,
"ece": 0.2259,
"p95_ms": 3979,
"cost_per_1k": 1.269273,
"interface": "adapter"
},
{
"board": "helpsteer2",
"name": "Label prior",
"ds": 0,
"ci": [
0,
0
],
"accuracy": 0.4167,
"ece": 0,
"p95_ms": null,
"cost_per_1k": 0,
"interface": "baseline"
},
{
"board": "helpsteer2",
"name": "Qwen3.8 Flash",
"ds": -1.3701,
"ci": [
-15.4058,
11.5108
],
"accuracy": 0.3607,
"ece": 0.2577,
"p95_ms": 2638,
"cost_per_1k": 0.125698,
"interface": "adapter"
},
{
"board": "helpsteer2",
"name": "Mercury 2.5",
"ds": -5.5457,
"ci": [
-16.2223,
3.9934
],
"accuracy": 0.4173,
"ece": 0.3148,
"p95_ms": 1165,
"cost_per_1k": 0.034331,
"interface": "adapter"
},
{
"board": "helpsteer2",
"name": "Mistral Medium 3.5",
"ds": -13.7288,
"ci": [
-28.2641,
-0.1455
],
"accuracy": 0.4387,
"ece": 0.3361,
"p95_ms": 1178,
"cost_per_1k": 1.325036,
"interface": "adapter"
},
{
"board": "helpsteer2",
"name": "DeepSeek V4.1 Flash",
"ds": -18.9946,
"ci": [
-36.0158,
-3.3185
],
"accuracy": 0.3467,
"ece": 0.3348,
"p95_ms": 1203,
"cost_per_1k": 0.125297,
"interface": "adapter"
},
{
"board": "pubmedqa",
"name": "Gemini 3.8 Flash",
"ds": 72.983,
"ci": [
62.0659,
82.2398
],
"accuracy": 0.9253,
"ece": 0.0198,
"p95_ms": 5538,
"cost_per_1k": 0.803439,
"interface": "adapter"
},
{
"board": "pubmedqa",
"name": "Jev",
"ds": 69.0274,
"ci": [
59.7741,
76.5459
],
"accuracy": 0.9127,
"ece": 0.0504,
"p95_ms": 653,
"cost_per_1k": 0.028535,
"interface": "native"
},
{
"board": "pubmedqa",
"name": "Qwen3.8 Flash",
"ds": 62.4422,
"ci": [
50.905,
72.5479
],
"accuracy": 0.8967,
"ece": 0.0255,
"p95_ms": 3515,
"cost_per_1k": 0.08613,
"interface": "adapter"
},
{
"board": "pubmedqa",
"name": "GLM-5.3",
"ds": 60.6328,
"ci": [
49.9596,
69.8189
],
"accuracy": 0.8867,
"ece": 0.0202,
"p95_ms": 2879,
"cost_per_1k": 0.512414,
"interface": "adapter"
},
{
"board": "pubmedqa",
"name": "Mistral Medium 3.5",
"ds": 58.0327,
"ci": [
45.3414,
69.8002
],
"accuracy": 0.888,
"ece": 0.0523,
"p95_ms": 917,
"cost_per_1k": 0.874716,
"interface": "adapter"
},
{
"board": "pubmedqa",
"name": "Mercury 2.5",
"ds": 55.6583,
"ci": [
44.8114,
65.4167
],
"accuracy": 0.8713,
"ece": 0.0468,
"p95_ms": 1036,
"cost_per_1k": 0.022595,
"interface": "adapter"
},
{
"board": "pubmedqa",
"name": "DeepSeek V4.1 Flash",
"ds": 47.5001,
"ci": [
34.7497,
59.0847
],
"accuracy": 0.8367,
"ece": 0.0616,
"p95_ms": 1123,
"cost_per_1k": 0.078411,
"interface": "adapter"
},
{
"board": "pubmedqa",
"name": "Label prior",
"ds": 0,
"ci": [
0,
0
],
"accuracy": 0.62,
"ece": 0,
"p95_ms": null,
"cost_per_1k": 0,
"interface": "baseline"
}
],
"release": "2026-09-18",
"suite": "0.1.0"
}

Xet Storage Details

Size:
25.4 kB
·
Xet hash:
a96f519bf0a13d57db630d3aa29e8eb8fed6c84a1137c68ef25fffc26b0a2252

Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.