Hugging Face's logo

Buckets:
Mercity
/
Desicion-Model-Custom

Files
xet
Mercity/Desicion-Model-Custom / code /results /jevals.json
Pranav2748's picture
Pranav2748
download
raw
14.7 kB
{
"ours": {
"base": {
"model": "Qwen/Qwen3.5-4B",
"temps": {
"choice": 1.707683209388825,
"noul": 2.4105066990845945,
"score": 1.2847580560947693
},
"raw": {
"pubmedqa": {
"primitive": "noul",
"ds": 48.05492263372704,
"ci": [
37.76904082341696,
57.150183634721955
],
"accuracy": 0.8066666666666666,
"ece_10_bin": 0.06460862585758344,
"T": 1.0,
"n": 300
},
"helpsteer2": {
"primitive": "score",
"ds": 2.049999831140492,
"ci": [
-9.799345465028715,
13.108881184556378
],
"accuracy": 0.4033333333333333,
"ece_10_bin": 0.15123322991920435,
"T": 1.0,
"n": 300
},
"banking77": {
"primitive": "choice",
"ds": 56.173306098208656,
"ci": [
49.27693100059662,
63.141876285290685
],
"accuracy": 0.7186666666666667,
"ece_10_bin": 0.14011876588219976,
"T": 1.0,
"n": 300
}
},
"calibrated": {
"pubmedqa": {
"primitive": "noul",
"ds": 36.371473995104964,
"ci": [
29.35952403100504,
42.77706939107353
],
"accuracy": 0.8066666666666666,
"ece_10_bin": 0.15700328478531872,
"T": 2.4105066990845945,
"n": 300
},
"helpsteer2": {
"primitive": "score",
"ds": 4.103423043229137,
"ci": [
-6.389453722683416,
13.956330811232187
],
"accuracy": 0.4033333333333333,
"ece_10_bin": 0.08902712405177117,
"T": 1.2847580560947693,
"n": 300
},
"banking77": {
"primitive": "choice",
"ds": 59.3453919929658,
"ci": [
53.69828564607293,
64.93884389399817
],
"accuracy": 0.7186666666666667,
"ece_10_bin": 0.04315345334218913,
"T": 1.707683209388825,
"n": 300
}
}
},
"rl60": {
"model": "runs/v2/checkpoints/rl60_merged",
"temps": {
"choice": 1.1599743376333564,
"noul": 0.7983784191901301,
"score": 0.7276446912498098
},
"raw": {
"pubmedqa": {
"primitive": "noul",
"ds": 58.72560050503446,
"ci": [
48.25519209655997,
67.9986456840194
],
"accuracy": 0.8633333333333333,
"ece_10_bin": 0.026845709305701414,
"T": 1.0,
"n": 300
},
"helpsteer2": {
"primitive": "score",
"ds": 6.037515788606118,
"ci": [
-2.400834157155406,
14.35084224020631
],
"accuracy": 0.43666666666666665,
"ece_10_bin": 0.095953934239622,
"T": 1.0,
"n": 300
},
"banking77": {
"primitive": "choice",
"ds": 60.955089258193006,
"ci": [
54.66931758063293,
67.32577013719438
],
"accuracy": 0.7313333333333333,
"ece_10_bin": 0.05904332938166834,
"T": 1.0,
"n": 300
}
},
"calibrated": {
"pubmedqa": {
"primitive": "noul",
"ds": 58.614060949612615,
"ci": [
47.33820737643784,
68.84278523729319
],
"accuracy": 0.8633333333333333,
"ece_10_bin": 0.03571861771195042,
"T": 0.7983784191901301,
"n": 300
},
"helpsteer2": {
"primitive": "score",
"ds": 2.368980700043777,
"ci": [
-7.2405524866967275,
11.746115459613257
],
"accuracy": 0.43666666666666665,
"ece_10_bin": 0.1303236520061676,
"T": 0.7276446912498098,
"n": 300
},
"banking77": {
"primitive": "choice",
"ds": 61.33904164009218,
"ci": [
55.52009100804195,
67.34968363963038
],
"accuracy": 0.7313333333333333,
"ece_10_bin": 0.027606077031200056,
"T": 1.1599743376333564,
"n": 300
}
}
},
"rl60_think": {
"model": "runs/v2/checkpoints/rl60_merged",
"temps": {
"choice": 2.145082715273473,
"noul": 1.9352338162597778,
"score": 1.8069801445316314
},
"raw": {
"pubmedqa": {
"primitive": "noul",
"ds": 58.936313009275274,
"ci": [
44.39233287434623,
72.34563008669913
],
"accuracy": 0.9,
"ece_10_bin": 0.09364023899498322,
"T": 1.0,
"n": 300
},
"helpsteer2": {
"primitive": "score",
"ds": -42.99011458744382,
"ci": [
-61.37940857090847,
-24.665619976798016
],
"accuracy": 0.41333333333333333,
"ece_10_bin": 0.519184866453263,
"T": 1.0,
"n": 300
},
"banking77": {
"primitive": "choice",
"ds": 49.48519474863158,
"ci": [
40.343495405677444,
58.75819407518067
],
"accuracy": 0.7333333333333333,
"ece_10_bin": 0.24588454845319418,
"T": 1.0,
"n": 300
}
},
"calibrated": {
"pubmedqa": {
"primitive": "noul",
"ds": 63.46531221234828,
"ci": [
51.2282704267052,
74.80118608216338
],
"accuracy": 0.9,
"ece_10_bin": 0.045119675748778995,
"T": 1.9352338162597778,
"n": 300
},
"helpsteer2": {
"primitive": "score",
"ds": -20.13990146885871,
"ci": [
-35.33533731665648,
-4.939007706671506
],
"accuracy": 0.41333333333333333,
"ece_10_bin": 0.37806488700901164,
"T": 1.8069801445316314,
"n": 300
},
"banking77": {
"primitive": "choice",
"ds": 59.27752355436782,
"ci": [
53.606719794836316,
64.93246211545883
],
"accuracy": 0.7333333333333333,
"ece_10_bin": 0.13413758897367453,
"T": 2.145082715273473,
"n": 300
}
}
},
"v1_joint": {
"model": "checkpoints/joint_merged",
"temps": {
"choice": 1.2843740495491411,
"noul": 1.1776347713530488,
"score": 1.0535480782391846
},
"raw": {
"pubmedqa": {
"primitive": "noul",
"ds": 49.206682779012525,
"ci": [
36.65442701300594,
59.903266381066885
],
"accuracy": 0.8233333333333334,
"ece_10_bin": 0.06969906794374832,
"T": 1.0,
"n": 300
},
"helpsteer2": {
"primitive": "score",
"ds": 6.7772824916263685,
"ci": [
-2.2223595409748595,
16.175846550495578
],
"accuracy": 0.3933333333333333,
"ece_10_bin": 0.17954795337837168,
"T": 1.0,
"n": 300
},
"banking77": {
"primitive": "choice",
"ds": 60.61334399242887,
"ci": [
54.24958965660347,
66.93369679868799
],
"accuracy": 0.7326666666666667,
"ece_10_bin": 0.07895850726459176,
"T": 1.0,
"n": 300
}
},
"calibrated": {
"pubmedqa": {
"primitive": "noul",
"ds": 50.30267194300714,
"ci": [
38.78645963769695,
60.216646199918934
],
"accuracy": 0.8233333333333334,
"ece_10_bin": 0.06609612462236357,
"T": 1.1776347713530488,
"n": 300
},
"helpsteer2": {
"primitive": "score",
"ds": 7.5280862119676435,
"ci": [
-1.2187682680666836,
16.69347049625058
],
"accuracy": 0.3933333333333333,
"ece_10_bin": 0.16808161152147194,
"T": 1.0535480782391846,
"n": 300
},
"banking77": {
"primitive": "choice",
"ds": 61.483220852549046,
"ci": [
55.919259043009745,
67.11049272666077
],
"accuracy": 0.7326666666666667,
"ece_10_bin": 0.05006183740765189,
"T": 1.2843740495491411,
"n": 300
}
}
},
"v2_joint": {
"model": "runs/v2/checkpoints/v2_joint_merged",
"temps": {
"choice": 1.1592816013856364,
"noul": 0.8031824332990728,
"score": 0.716891860990727
},
"raw": {
"pubmedqa": {
"primitive": "noul",
"ds": 57.9754923387608,
"ci": [
47.474841970587384,
67.33647236474492
],
"accuracy": 0.8566666666666667,
"ece_10_bin": 0.024792356261037377,
"T": 1.0,
"n": 300
},
"helpsteer2": {
"primitive": "score",
"ds": 6.072330512589563,
"ci": [
-2.447039948281713,
14.421161550147108
],
"accuracy": 0.43,
"ece_10_bin": 0.0758746614948931,
"T": 1.0,
"n": 300
},
"banking77": {
"primitive": "choice",
"ds": 60.90446514198684,
"ci": [
54.643973597052714,
67.26811195597078
],
"accuracy": 0.734,
"ece_10_bin": 0.05840767711375198,
"T": 1.0,
"n": 300
}
},
"calibrated": {
"pubmedqa": {
"primitive": "noul",
"ds": 57.786338685636565,
"ci": [
46.39802375142668,
68.04438733775511
],
"accuracy": 0.8566666666666667,
"ece_10_bin": 0.033081921946533914,
"T": 0.8031824332990728,
"n": 300
},
"helpsteer2": {
"primitive": "score",
"ds": 2.314214972593698,
"ci": [
-7.267135153938562,
11.84096549506603
],
"accuracy": 0.43,
"ece_10_bin": 0.1364261756101774,
"T": 0.716891860990727,
"n": 300
},
"banking77": {
"primitive": "choice",
"ds": 61.31478198739936,
"ci": [
55.423784855833,
67.28099088659312
],
"accuracy": 0.734,
"ece_10_bin": 0.02052732067327072,
"T": 1.1592816013856364,
"n": 300
}
}
}
},
"board": [
{
"board": "banking77",
"name": "Gemini 3.8 Flash",
"ds": 74.107,
"ci": [
68.4397,
79.7893
],
"accuracy": 0.846,
"ece": 0.0319,
"p95_ms": 3938,
"cost_per_1k": 1.367931,
"interface": "adapter"
},
{
"board": "banking77",
"name": "Jev",
"ds": 67.7806,
"ci": [
61.3393,
74.4909
],
"accuracy": 0.7967,
"ece": 0.0981,
"p95_ms": 693,
"cost_per_1k": 0.043276,
"interface": "native"
},
{
"board": "banking77",
"name": "GLM-5.3",
"ds": 66.8264,
"ci": [
61.9567,
71.7196
],
"accuracy": 0.7887,
"ece": 0.0616,
"p95_ms": 3186,
"cost_per_1k": 0.718505,
"interface": "adapter"
},
{
"board": "banking77",
"name": "DeepSeek V4.1 Flash",
"ds": 63.9292,
"ci": [
58.5065,
69.6119
],
"accuracy": 0.7587,
"ece": 0.0276,
"p95_ms": 1309,
"cost_per_1k": 0.138384,
"interface": "adapter"
},
{
"board": "banking77",
"name": "Qwen3.8 Flash",
"ds": 62.4389,
"ci": [
55.9178,
68.8961
],
"accuracy": 0.768,
"ece": 0.0951,
"p95_ms": 3105,
"cost_per_1k": 0.122064,
"interface": "adapter"
},
{
"board": "banking77",
"name": "Mistral Medium 3.5",
"ds": 59.4547,
"ci": [
54.2695,
64.8301
],
"accuracy": 0.7447,
"ece": 0.0807,
"p95_ms": 1444,
"cost_per_1k": 1.515244,
"interface": "adapter"
},
{
"board": "banking77",
"name": "Mercury 2.5",
"ds": 54.0395,
"ci": [
47.8178,
60.6156
],
"accuracy": 0.7033,
"ece": 0.1044,
"p95_ms": 1478,
"cost_per_1k": 0.03606,
"interface": "adapter"
},
{
"board": "banking77",
"name": "Label prior",
"ds": 0,
"ci": [
0,
0
],
"accuracy": 0.0133,
"ece": 0,
"p95_ms": null,
"cost_per_1k": 0,
"interface": "baseline"
},
{
"board": "helpsteer2",
"name": "Jev",
"ds": 9.2034,
"ci": [
-4.365,
20.9655
],
"accuracy": 0.4127,
"ece": 0.1966,
"p95_ms": 670,
"cost_per_1k": 0.036285,
"interface": "native"
},
{
"board": "helpsteer2",
"name": "GLM-5.3",
"ds": 7.7828,
"ci": [
-4.5884,
18.5647
],
"accuracy": 0.43,
"ece": 0.1287,
"p95_ms": 3631,
"cost_per_1k": 0.784848,
"interface": "adapter"
},
{
"board": "helpsteer2",
"name": "Gemini 3.8 Flash",
"ds": 4.5909,
"ci": [
-12.2314,
18.9838
],
"accuracy": 0.424,
"ece": 0.2259,
"p95_ms": 3979,
"cost_per_1k": 1.269273,
"interface": "adapter"
},
{
"board": "helpsteer2",
"name": "Label prior",
"ds": 0,
"ci": [
0,
0
],
"accuracy": 0.4167,
"ece": 0,
"p95_ms": null,
"cost_per_1k": 0,
"interface": "baseline"
},
{
"board": "helpsteer2",
"name": "Qwen3.8 Flash",
"ds": -1.3701,
"ci": [
-15.4058,
11.5108
],
"accuracy": 0.3607,
"ece": 0.2577,
"p95_ms": 2638,
"cost_per_1k": 0.125698,
"interface": "adapter"
},
{
"board": "helpsteer2",
"name": "Mercury 2.5",
"ds": -5.5457,
"ci": [
-16.2223,
3.9934
],
"accuracy": 0.4173,
"ece": 0.3148,
"p95_ms": 1165,
"cost_per_1k": 0.034331,
"interface": "adapter"
},
{
"board": "helpsteer2",
"name": "Mistral Medium 3.5",
"ds": -13.7288,
"ci": [
-28.2641,
-0.1455
],
"accuracy": 0.4387,
"ece": 0.3361,
"p95_ms": 1178,
"cost_per_1k": 1.325036,
"interface": "adapter"
},
{
"board": "helpsteer2",
"name": "DeepSeek V4.1 Flash",
"ds": -18.9946,
"ci": [
-36.0158,
-3.3185
],
"accuracy": 0.3467,
"ece": 0.3348,
"p95_ms": 1203,
"cost_per_1k": 0.125297,
"interface": "adapter"
},
{
"board": "pubmedqa",
"name": "Gemini 3.8 Flash",
"ds": 72.983,
"ci": [
62.0659,
82.2398
],
"accuracy": 0.9253,
"ece": 0.0198,
"p95_ms": 5538,
"cost_per_1k": 0.803439,
"interface": "adapter"
},
{
"board": "pubmedqa",
"name": "Jev",
"ds": 69.0274,
"ci": [
59.7741,
76.5459
],
"accuracy": 0.9127,
"ece": 0.0504,
"p95_ms": 653,
"cost_per_1k": 0.028535,
"interface": "native"
},
{
"board": "pubmedqa",
"name": "Qwen3.8 Flash",
"ds": 62.4422,
"ci": [
50.905,
72.5479
],
"accuracy": 0.8967,
"ece": 0.0255,
"p95_ms": 3515,
"cost_per_1k": 0.08613,
"interface": "adapter"
},
{
"board": "pubmedqa",
"name": "GLM-5.3",
"ds": 60.6328,
"ci": [
49.9596,
69.8189
],
"accuracy": 0.8867,
"ece": 0.0202,
"p95_ms": 2879,
"cost_per_1k": 0.512414,
"interface": "adapter"
},
{
"board": "pubmedqa",
"name": "Mistral Medium 3.5",
"ds": 58.0327,
"ci": [
45.3414,
69.8002
],
"accuracy": 0.888,
"ece": 0.0523,
"p95_ms": 917,
"cost_per_1k": 0.874716,
"interface": "adapter"
},
{
"board": "pubmedqa",
"name": "Mercury 2.5",
"ds": 55.6583,
"ci": [
44.8114,
65.4167
],
"accuracy": 0.8713,
"ece": 0.0468,
"p95_ms": 1036,
"cost_per_1k": 0.022595,
"interface": "adapter"
},
{
"board": "pubmedqa",
"name": "DeepSeek V4.1 Flash",
"ds": 47.5001,
"ci": [
34.7497,
59.0847
],
"accuracy": 0.8367,
"ece": 0.0616,
"p95_ms": 1123,
"cost_per_1k": 0.078411,
"interface": "adapter"
},
{
"board": "pubmedqa",
"name": "Label prior",
"ds": 0,
"ci": [
0,
0
],
"accuracy": 0.62,
"ece": 0,
"p95_ms": null,
"cost_per_1k": 0,
"interface": "baseline"
}
],
"release": "2026-09-18",
"suite": "0.1.0"
}

Xet Storage Details

Size:
14.7 kB
·
Xet hash:
7e183a0962c9bca1f5223fecde8625dad32cf3cb5c3ce2921463894e697436c0

Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.