| { |
| "overall": { |
| "questions": 908, |
| "answered": 908, |
| "errors": 0, |
| "correct": 783, |
| "accuracyAllQuestions": 0.8623348017621145, |
| "brierAnswered": 0.19616500539754675, |
| "nllAnswered": 0.374025080065555, |
| "ece10Answered": 0.04207503699664424, |
| "confidentWrongAt09": 26, |
| "coverageByFixedThreshold": [ |
| { |
| "threshold": 0.5, |
| "accepted": 889, |
| "coverage": 0.9790748898678414, |
| "errorRate": 0.12373453318335208 |
| }, |
| { |
| "threshold": 0.7, |
| "accepted": 819, |
| "coverage": 0.9019823788546255, |
| "errorRate": 0.08424908424908426 |
| }, |
| { |
| "threshold": 0.8, |
| "accepted": 767, |
| "coverage": 0.8447136563876652, |
| "errorRate": 0.06649282920469361 |
| }, |
| { |
| "threshold": 0.9, |
| "accepted": 676, |
| "coverage": 0.7444933920704846, |
| "errorRate": 0.038461538461538464 |
| }, |
| { |
| "threshold": 0.95, |
| "accepted": 567, |
| "coverage": 0.6244493392070485, |
| "errorRate": 0.012345679012345678 |
| }, |
| { |
| "threshold": 0.99, |
| "accepted": 0, |
| "coverage": 0.0, |
| "errorRate": null |
| } |
| ], |
| "coverageInterpretation": "Observed on this dataset; not a future error guarantee. Thresholds not tuned here." |
| }, |
| "byType": { |
| "choice": { |
| "questions": 492, |
| "answered": 492, |
| "errors": 0, |
| "correct": 400, |
| "accuracyAllQuestions": 0.8130081300813008, |
| "brierAnswered": 0.2615263089500927, |
| "nllAnswered": 0.5247360934156637, |
| "ece10Answered": 0.06533461933177653, |
| "confidentWrongAt09": 20, |
| "coverageByFixedThreshold": [ |
| { |
| "threshold": 0.5, |
| "accepted": 473, |
| "coverage": 0.9613821138211383, |
| "errorRate": 0.16279069767441862 |
| }, |
| { |
| "threshold": 0.7, |
| "accepted": 424, |
| "coverage": 0.8617886178861789, |
| "errorRate": 0.10377358490566038 |
| }, |
| { |
| "threshold": 0.8, |
| "accepted": 386, |
| "coverage": 0.7845528455284553, |
| "errorRate": 0.08031088082901554 |
| }, |
| { |
| "threshold": 0.9, |
| "accepted": 331, |
| "coverage": 0.6727642276422764, |
| "errorRate": 0.06042296072507553 |
| }, |
| { |
| "threshold": 0.95, |
| "accepted": 267, |
| "coverage": 0.5426829268292683, |
| "errorRate": 0.026217228464419477 |
| }, |
| { |
| "threshold": 0.99, |
| "accepted": 0, |
| "coverage": 0.0, |
| "errorRate": null |
| } |
| ], |
| "coverageInterpretation": "Observed on this dataset; not a future error guarantee. Thresholds not tuned here." |
| }, |
| "noul": { |
| "questions": 328, |
| "answered": 328, |
| "errors": 0, |
| "correct": 298, |
| "accuracyAllQuestions": 0.9085365853658537, |
| "brierAnswered": 0.13871514597249474, |
| "nllAnswered": 0.22139737153892355, |
| "ece10Answered": 0.04994271263817423, |
| "confidentWrongAt09": 5, |
| "coverageByFixedThreshold": [ |
| { |
| "threshold": 0.5, |
| "accepted": 328, |
| "coverage": 1.0, |
| "errorRate": 0.09146341463414634 |
| }, |
| { |
| "threshold": 0.7, |
| "accepted": 309, |
| "coverage": 0.9420731707317073, |
| "errorRate": 0.07766990291262135 |
| }, |
| { |
| "threshold": 0.8, |
| "accepted": 296, |
| "coverage": 0.9024390243902439, |
| "errorRate": 0.06418918918918919 |
| }, |
| { |
| "threshold": 0.9, |
| "accepted": 261, |
| "coverage": 0.7957317073170732, |
| "errorRate": 0.019157088122605363 |
| }, |
| { |
| "threshold": 0.95, |
| "accepted": 220, |
| "coverage": 0.6707317073170732, |
| "errorRate": 0.0 |
| }, |
| { |
| "threshold": 0.99, |
| "accepted": 0, |
| "coverage": 0.0, |
| "errorRate": null |
| } |
| ], |
| "coverageInterpretation": "Observed on this dataset; not a future error guarantee. Thresholds not tuned here." |
| }, |
| "score": { |
| "questions": 88, |
| "answered": 88, |
| "errors": 0, |
| "correct": 85, |
| "accuracyAllQuestions": 0.9659090909090909, |
| "brierAnswered": 0.04486719339259727, |
| "nllAnswered": 0.10029860084375529, |
| "ece10Answered": 0.03799778408573792, |
| "confidentWrongAt09": 1, |
| "coverageByFixedThreshold": [ |
| { |
| "threshold": 0.5, |
| "accepted": 88, |
| "coverage": 1.0, |
| "errorRate": 0.03409090909090909 |
| }, |
| { |
| "threshold": 0.7, |
| "accepted": 86, |
| "coverage": 0.9772727272727273, |
| "errorRate": 0.011627906976744186 |
| }, |
| { |
| "threshold": 0.8, |
| "accepted": 85, |
| "coverage": 0.9659090909090909, |
| "errorRate": 0.011764705882352941 |
| }, |
| { |
| "threshold": 0.9, |
| "accepted": 84, |
| "coverage": 0.9545454545454546, |
| "errorRate": 0.011904761904761904 |
| }, |
| { |
| "threshold": 0.95, |
| "accepted": 80, |
| "coverage": 0.9090909090909091, |
| "errorRate": 0.0 |
| }, |
| { |
| "threshold": 0.99, |
| "accepted": 0, |
| "coverage": 0.0, |
| "errorRate": null |
| } |
| ], |
| "coverageInterpretation": "Observed on this dataset; not a future error guarantee. Thresholds not tuned here." |
| } |
| }, |
| "bySource": { |
| "authored-russian-typed": { |
| "questions": 144, |
| "answered": 144, |
| "errors": 0, |
| "correct": 144, |
| "accuracyAllQuestions": 1.0, |
| "brierAnswered": 0.0009998268084272204, |
| "nllAnswered": 0.024466212500916604, |
| "ece10Answered": 0.024139216454452272, |
| "confidentWrongAt09": 0, |
| "coverageByFixedThreshold": [ |
| { |
| "threshold": 0.5, |
| "accepted": 144, |
| "coverage": 1.0, |
| "errorRate": 0.0 |
| }, |
| { |
| "threshold": 0.7, |
| "accepted": 144, |
| "coverage": 1.0, |
| "errorRate": 0.0 |
| }, |
| { |
| "threshold": 0.8, |
| "accepted": 144, |
| "coverage": 1.0, |
| "errorRate": 0.0 |
| }, |
| { |
| "threshold": 0.9, |
| "accepted": 144, |
| "coverage": 1.0, |
| "errorRate": 0.0 |
| }, |
| { |
| "threshold": 0.95, |
| "accepted": 144, |
| "coverage": 1.0, |
| "errorRate": 0.0 |
| }, |
| { |
| "threshold": 0.99, |
| "accepted": 0, |
| "coverage": 0.0, |
| "errorRate": null |
| } |
| ], |
| "coverageInterpretation": "Observed on this dataset; not a future error guarantee. Thresholds not tuned here." |
| }, |
| "composition_holdout": { |
| "questions": 96, |
| "answered": 96, |
| "errors": 0, |
| "correct": 84, |
| "accuracyAllQuestions": 0.875, |
| "brierAnswered": 0.18821988619559113, |
| "nllAnswered": 0.31506340814867745, |
| "ece10Answered": 0.08715274769631735, |
| "confidentWrongAt09": 9, |
| "coverageByFixedThreshold": [ |
| { |
| "threshold": 0.5, |
| "accepted": 96, |
| "coverage": 1.0, |
| "errorRate": 0.125 |
| }, |
| { |
| "threshold": 0.7, |
| "accepted": 92, |
| "coverage": 0.9583333333333334, |
| "errorRate": 0.09782608695652174 |
| }, |
| { |
| "threshold": 0.8, |
| "accepted": 92, |
| "coverage": 0.9583333333333334, |
| "errorRate": 0.09782608695652174 |
| }, |
| { |
| "threshold": 0.9, |
| "accepted": 92, |
| "coverage": 0.9583333333333334, |
| "errorRate": 0.09782608695652174 |
| }, |
| { |
| "threshold": 0.95, |
| "accepted": 86, |
| "coverage": 0.8958333333333334, |
| "errorRate": 0.03488372093023256 |
| }, |
| { |
| "threshold": 0.99, |
| "accepted": 0, |
| "coverage": 0.0, |
| "errorRate": null |
| } |
| ], |
| "coverageInterpretation": "Observed on this dataset; not a future error guarantee. Thresholds not tuned here." |
| }, |
| "emotion": { |
| "questions": 116, |
| "answered": 116, |
| "errors": 0, |
| "correct": 69, |
| "accuracyAllQuestions": 0.5948275862068966, |
| "brierAnswered": 0.5245559470753833, |
| "nllAnswered": 1.0208149336604613, |
| "ece10Answered": 0.12016921458494689, |
| "confidentWrongAt09": 2, |
| "coverageByFixedThreshold": [ |
| { |
| "threshold": 0.5, |
| "accepted": 101, |
| "coverage": 0.8706896551724138, |
| "errorRate": 0.3564356435643564 |
| }, |
| { |
| "threshold": 0.7, |
| "accepted": 70, |
| "coverage": 0.603448275862069, |
| "errorRate": 0.22857142857142856 |
| }, |
| { |
| "threshold": 0.8, |
| "accepted": 44, |
| "coverage": 0.3793103448275862, |
| "errorRate": 0.18181818181818182 |
| }, |
| { |
| "threshold": 0.9, |
| "accepted": 8, |
| "coverage": 0.06896551724137931, |
| "errorRate": 0.25 |
| }, |
| { |
| "threshold": 0.95, |
| "accepted": 0, |
| "coverage": 0.0, |
| "errorRate": null |
| }, |
| { |
| "threshold": 0.99, |
| "accepted": 0, |
| "coverage": 0.0, |
| "errorRate": null |
| } |
| ], |
| "coverageInterpretation": "Observed on this dataset; not a future error guarantee. Thresholds not tuned here." |
| }, |
| "legacy_holdout": { |
| "questions": 80, |
| "answered": 80, |
| "errors": 0, |
| "correct": 77, |
| "accuracyAllQuestions": 0.9625, |
| "brierAnswered": 0.04910351545555701, |
| "nllAnswered": 0.10416551178163372, |
| "ece10Answered": 0.03574970703743059, |
| "confidentWrongAt09": 1, |
| "coverageByFixedThreshold": [ |
| { |
| "threshold": 0.5, |
| "accepted": 80, |
| "coverage": 1.0, |
| "errorRate": 0.0375 |
| }, |
| { |
| "threshold": 0.7, |
| "accepted": 78, |
| "coverage": 0.975, |
| "errorRate": 0.01282051282051282 |
| }, |
| { |
| "threshold": 0.8, |
| "accepted": 77, |
| "coverage": 0.9625, |
| "errorRate": 0.012987012987012988 |
| }, |
| { |
| "threshold": 0.9, |
| "accepted": 76, |
| "coverage": 0.95, |
| "errorRate": 0.013157894736842105 |
| }, |
| { |
| "threshold": 0.95, |
| "accepted": 72, |
| "coverage": 0.9, |
| "errorRate": 0.0 |
| }, |
| { |
| "threshold": 0.99, |
| "accepted": 0, |
| "coverage": 0.0, |
| "errorRate": null |
| } |
| ], |
| "coverageInterpretation": "Observed on this dataset; not a future error guarantee. Thresholds not tuned here." |
| }, |
| "mmlu": { |
| "questions": 116, |
| "answered": 116, |
| "errors": 0, |
| "correct": 86, |
| "accuracyAllQuestions": 0.7413793103448276, |
| "brierAnswered": 0.39416899029285185, |
| "nllAnswered": 0.8394104030580065, |
| "ece10Answered": 0.12528513793721022, |
| "confidentWrongAt09": 9, |
| "coverageByFixedThreshold": [ |
| { |
| "threshold": 0.5, |
| "accepted": 112, |
| "coverage": 0.9655172413793104, |
| "errorRate": 0.23214285714285715 |
| }, |
| { |
| "threshold": 0.7, |
| "accepted": 100, |
| "coverage": 0.8620689655172413, |
| "errorRate": 0.18 |
| }, |
| { |
| "threshold": 0.8, |
| "accepted": 91, |
| "coverage": 0.7844827586206896, |
| "errorRate": 0.14285714285714285 |
| }, |
| { |
| "threshold": 0.9, |
| "accepted": 77, |
| "coverage": 0.6637931034482759, |
| "errorRate": 0.11688311688311688 |
| }, |
| { |
| "threshold": 0.95, |
| "accepted": 42, |
| "coverage": 0.3620689655172414, |
| "errorRate": 0.09523809523809523 |
| }, |
| { |
| "threshold": 0.99, |
| "accepted": 0, |
| "coverage": 0.0, |
| "errorRate": null |
| } |
| ], |
| "coverageInterpretation": "Observed on this dataset; not a future error guarantee. Thresholds not tuned here." |
| }, |
| "paws": { |
| "questions": 80, |
| "answered": 80, |
| "errors": 0, |
| "correct": 68, |
| "accuracyAllQuestions": 0.85, |
| "brierAnswered": 0.20695251414059662, |
| "nllAnswered": 0.3140944708626177, |
| "ece10Answered": 0.08749262200436604, |
| "confidentWrongAt09": 2, |
| "coverageByFixedThreshold": [ |
| { |
| "threshold": 0.5, |
| "accepted": 80, |
| "coverage": 1.0, |
| "errorRate": 0.15 |
| }, |
| { |
| "threshold": 0.7, |
| "accepted": 75, |
| "coverage": 0.9375, |
| "errorRate": 0.12 |
| }, |
| { |
| "threshold": 0.8, |
| "accepted": 73, |
| "coverage": 0.9125, |
| "errorRate": 0.0958904109589041 |
| }, |
| { |
| "threshold": 0.9, |
| "accepted": 65, |
| "coverage": 0.8125, |
| "errorRate": 0.03076923076923077 |
| }, |
| { |
| "threshold": 0.95, |
| "accepted": 58, |
| "coverage": 0.725, |
| "errorRate": 0.0 |
| }, |
| { |
| "threshold": 0.99, |
| "accepted": 0, |
| "coverage": 0.0, |
| "errorRate": null |
| } |
| ], |
| "coverageInterpretation": "Observed on this dataset; not a future error guarantee. Thresholds not tuned here." |
| }, |
| "qnli": { |
| "questions": 80, |
| "answered": 80, |
| "errors": 0, |
| "correct": 74, |
| "accuracyAllQuestions": 0.925, |
| "brierAnswered": 0.1185264487594759, |
| "nllAnswered": 0.19157792439378024, |
| "ece10Answered": 0.07130364771118254, |
| "confidentWrongAt09": 1, |
| "coverageByFixedThreshold": [ |
| { |
| "threshold": 0.5, |
| "accepted": 80, |
| "coverage": 1.0, |
| "errorRate": 0.075 |
| }, |
| { |
| "threshold": 0.7, |
| "accepted": 79, |
| "coverage": 0.9875, |
| "errorRate": 0.0759493670886076 |
| }, |
| { |
| "threshold": 0.8, |
| "accepted": 77, |
| "coverage": 0.9625, |
| "errorRate": 0.06493506493506493 |
| }, |
| { |
| "threshold": 0.9, |
| "accepted": 71, |
| "coverage": 0.8875, |
| "errorRate": 0.014084507042253521 |
| }, |
| { |
| "threshold": 0.95, |
| "accepted": 60, |
| "coverage": 0.75, |
| "errorRate": 0.0 |
| }, |
| { |
| "threshold": 0.99, |
| "accepted": 0, |
| "coverage": 0.0, |
| "errorRate": null |
| } |
| ], |
| "coverageInterpretation": "Observed on this dataset; not a future error guarantee. Thresholds not tuned here." |
| }, |
| "sciq": { |
| "questions": 116, |
| "answered": 116, |
| "errors": 0, |
| "correct": 113, |
| "accuracyAllQuestions": 0.9741379310344828, |
| "brierAnswered": 0.03409531064541125, |
| "nllAnswered": 0.09114362656929587, |
| "ece10Answered": 0.05454464085684041, |
| "confidentWrongAt09": 0, |
| "coverageByFixedThreshold": [ |
| { |
| "threshold": 0.5, |
| "accepted": 116, |
| "coverage": 1.0, |
| "errorRate": 0.02586206896551724 |
| }, |
| { |
| "threshold": 0.7, |
| "accepted": 114, |
| "coverage": 0.9827586206896551, |
| "errorRate": 0.008771929824561403 |
| }, |
| { |
| "threshold": 0.8, |
| "accepted": 111, |
| "coverage": 0.9568965517241379, |
| "errorRate": 0.009009009009009009 |
| }, |
| { |
| "threshold": 0.9, |
| "accepted": 106, |
| "coverage": 0.9137931034482759, |
| "errorRate": 0.0 |
| }, |
| { |
| "threshold": 0.95, |
| "accepted": 91, |
| "coverage": 0.7844827586206896, |
| "errorRate": 0.0 |
| }, |
| { |
| "threshold": 0.99, |
| "accepted": 0, |
| "coverage": 0.0, |
| "errorRate": null |
| } |
| ], |
| "coverageInterpretation": "Observed on this dataset; not a future error guarantee. Thresholds not tuned here." |
| }, |
| "tweet_offensive": { |
| "questions": 80, |
| "answered": 80, |
| "errors": 0, |
| "correct": 68, |
| "accuracyAllQuestions": 0.85, |
| "brierAnswered": 0.24263742159686036, |
| "nllAnswered": 0.38374648265869776, |
| "ece10Answered": 0.05576197181938556, |
| "confidentWrongAt09": 2, |
| "coverageByFixedThreshold": [ |
| { |
| "threshold": 0.5, |
| "accepted": 80, |
| "coverage": 1.0, |
| "errorRate": 0.15 |
| }, |
| { |
| "threshold": 0.7, |
| "accepted": 67, |
| "coverage": 0.8375, |
| "errorRate": 0.13432835820895522 |
| }, |
| { |
| "threshold": 0.8, |
| "accepted": 58, |
| "coverage": 0.725, |
| "errorRate": 0.1206896551724138 |
| }, |
| { |
| "threshold": 0.9, |
| "accepted": 37, |
| "coverage": 0.4625, |
| "errorRate": 0.05405405405405406 |
| }, |
| { |
| "threshold": 0.95, |
| "accepted": 14, |
| "coverage": 0.175, |
| "errorRate": 0.0 |
| }, |
| { |
| "threshold": 0.99, |
| "accepted": 0, |
| "coverage": 0.0, |
| "errorRate": null |
| } |
| ], |
| "coverageInterpretation": "Observed on this dataset; not a future error guarantee. Thresholds not tuned here." |
| } |
| }, |
| "provenance": { |
| "revision": "3ef077cd7cc3c97d696c18150358ab4de3a6884e", |
| "codeZipSha256": "b3b7808d7c2cbe2ce26d4b31ee0dde2b3e894c6776537931b61cd173ef526f8e", |
| "execution": "reference BF16 backbone, FP32 shared adapter unmerged", |
| "dataSha256": "9266d6ea1ce83c9755a60cf48c7469b0fa3264c229e96f4142d683bada8fef91", |
| "gpu": "NVIDIA A10G", |
| "torch": "2.14.0+cu130", |
| "questionIsolation": false |
| }, |
| "servedModels": [ |
| "fev-4b" |
| ], |
| "records": 812, |
| "startedUtc": null, |
| "wallLatencyMs": { |
| "median": 86.00620499998968, |
| "p95": 102.34360900000183 |
| }, |
| "latencyMeaning": "Sequential wall time including client overhead/cold calls; not model-only time or throughput" |
| } |
|
|
