| { |
| "protocol": { |
| "endpoint": "https://openrouter.ai/api/alpha/decisions", |
| "models": [ |
| "typesafe/jev-1.14", |
| "liquid/d1", |
| "perplexity/pplx-decider-v1.2", |
| "jaredpalmer/kev-4b" |
| ], |
| "test_data_sha256": "e2ca7413a306fc2b0e9e044aa685c94d5aa8e0def7cfcec6448941b7369ae625", |
| "questions": 512, |
| "protocol": "One request per question. Identical state, question instructions, answer meanings, and option order; no reference targets sent. Native provider prompts and calibration remain provider-controlled. No prompt optimization or fitting.", |
| "concurrency": "Four models concurrently, one request at a time per model. HTTP latency includes network/provider overhead and is not local GPU compute latency.", |
| "probability_rounding": "Returned distributions within 0.02 of unit mass are renormalized before Brier scoring; raw API responses are retained. Argmax option order breaks probability ties.", |
| "cost_limit_usd": 1.0, |
| "date": "2026-10-21" |
| }, |
| "models": { |
| "liquid/d1": { |
| "questions": 512, |
| "modal_accuracy": 0.662109375, |
| "brier": 0.4156228551936487, |
| "median_http_ms": 153.96152925677598, |
| "cost_usd": 0.0032266, |
| "renormalized_responses": 0, |
| "max_probability_sum_error": 4.440892098500626e-16, |
| "requested_model": "liquid/d1", |
| "served_models": [ |
| "liquid/d1-20260930" |
| ], |
| "by_source": { |
| "helpsteer2": { |
| "questions": 128, |
| "modal_accuracy": 0.5859375, |
| "brier": 0.31772152271267684, |
| "median_http_ms": 156.79931757040322, |
| "cost_usd": 0.00130398 |
| }, |
| "planning": { |
| "questions": 192, |
| "modal_accuracy": 0.7916666666666666, |
| "brier": 0.36154586401436944, |
| "median_http_ms": 160.43087700381875, |
| "cost_usd": 0.00111626 |
| }, |
| "scone": { |
| "questions": 192, |
| "modal_accuracy": 0.5833333333333334, |
| "brier": 0.5349674013602426, |
| "median_http_ms": 146.55435550957918, |
| "cost_usd": 0.00080636 |
| } |
| }, |
| "by_kind": { |
| "choice": { |
| "questions": 241, |
| "modal_accuracy": 0.6970954356846473, |
| "brier": 0.45294190031856313, |
| "median_http_ms": 153.44042284414172, |
| "cost_usd": 0.0013192 |
| }, |
| "noul": { |
| "questions": 94, |
| "modal_accuracy": 0.7127659574468085, |
| "brier": 0.36126946919251096, |
| "median_http_ms": 152.50918315723538, |
| "cost_usd": 0.00040766 |
| }, |
| "score": { |
| "questions": 177, |
| "modal_accuracy": 0.5875706214689266, |
| "brier": 0.39367555806936955, |
| "median_http_ms": 156.5550728701055, |
| "cost_usd": 0.00149974 |
| } |
| } |
| }, |
| "perplexity/pplx-decider-v1.2": { |
| "questions": 512, |
| "modal_accuracy": 0.74609375, |
| "brier": 0.3177875424868089, |
| "median_http_ms": 190.27893454767764, |
| "cost_usd": 0.00346588, |
| "renormalized_responses": 0, |
| "max_probability_sum_error": 4.440892098500626e-16, |
| "requested_model": "perplexity/pplx-decider-v1.2", |
| "served_models": [ |
| "perplexity/pplx-decider-v1.2-20261016" |
| ], |
| "by_source": { |
| "helpsteer2": { |
| "questions": 128, |
| "modal_accuracy": 0.5703125, |
| "brier": 0.2787386993739065, |
| "median_http_ms": 197.84710323438048, |
| "cost_usd": 0.00139706 |
| }, |
| "planning": { |
| "questions": 192, |
| "modal_accuracy": 0.8802083333333334, |
| "brier": 0.26224774345602686, |
| "median_http_ms": 189.46263939142227, |
| "cost_usd": 0.00119936 |
| }, |
| "scone": { |
| "questions": 192, |
| "modal_accuracy": 0.7291666666666666, |
| "brier": 0.3993599035928593, |
| "median_http_ms": 185.11412316001952, |
| "cost_usd": 0.00086946 |
| } |
| }, |
| "by_kind": { |
| "choice": { |
| "questions": 241, |
| "modal_accuracy": 0.8091286307053942, |
| "brier": 0.3446775651495988, |
| "median_http_ms": 189.45799581706524, |
| "cost_usd": 0.0014693 |
| }, |
| "noul": { |
| "questions": 94, |
| "modal_accuracy": 0.8191489361702128, |
| "brier": 0.24647724911756672, |
| "median_http_ms": 183.42466722242534, |
| "cost_usd": 0.00042356 |
| }, |
| "score": { |
| "questions": 177, |
| "modal_accuracy": 0.6214689265536724, |
| "brier": 0.3190455770346982, |
| "median_http_ms": 192.91105587035418, |
| "cost_usd": 0.00157302 |
| } |
| } |
| }, |
| "typesafe/jev-1.14": { |
| "questions": 512, |
| "modal_accuracy": 0.66015625, |
| "brier": 0.38428551299771707, |
| "median_http_ms": 160.70113889873028, |
| "cost_usd": 0.012567408, |
| "renormalized_responses": 6, |
| "max_probability_sum_error": 0.009999999999999898, |
| "requested_model": "typesafe/jev-1.14", |
| "served_models": [ |
| "typesafe/jev-1.14-20261014" |
| ], |
| "by_source": { |
| "helpsteer2": { |
| "questions": 128, |
| "modal_accuracy": 0.5546875, |
| "brier": 0.3501162689075094, |
| "median_http_ms": 161.725880112499, |
| "cost_usd": 0.00407988 |
| }, |
| "planning": { |
| "questions": 192, |
| "modal_accuracy": 0.7447916666666666, |
| "brier": 0.3210494803889059, |
| "median_http_ms": 162.16431255452335, |
| "cost_usd": 0.004622646 |
| }, |
| "scone": { |
| "questions": 192, |
| "modal_accuracy": 0.6458333333333334, |
| "brier": 0.4703010416666667, |
| "median_http_ms": 158.17508311010897, |
| "cost_usd": 0.003864882 |
| } |
| }, |
| "by_kind": { |
| "choice": { |
| "questions": 241, |
| "modal_accuracy": 0.6804979253112033, |
| "brier": 0.4145994016191929, |
| "median_http_ms": 159.9918780848384, |
| "cost_usd": 0.005741064 |
| }, |
| "noul": { |
| "questions": 94, |
| "modal_accuracy": 0.7446808510638298, |
| "brier": 0.2824255319148936, |
| "median_http_ms": 159.92187475785613, |
| "cost_usd": 0.001838382 |
| }, |
| "score": { |
| "questions": 177, |
| "modal_accuracy": 0.5875706214689266, |
| "brier": 0.3971058014949472, |
| "median_http_ms": 162.21308708190918, |
| "cost_usd": 0.004987962 |
| } |
| } |
| }, |
| "jaredpalmer/kev-4b": { |
| "questions": 512, |
| "modal_accuracy": 0.42578125, |
| "brier": 0.5133862273141486, |
| "median_http_ms": 374.62099245749414, |
| "cost_usd": 0.00583884, |
| "renormalized_responses": 166, |
| "max_probability_sum_error": 0.00019999999999997797, |
| "requested_model": "jaredpalmer/kev-4b", |
| "served_models": [ |
| "jaredpalmer/kev-4b-20260924" |
| ], |
| "by_source": { |
| "helpsteer2": { |
| "questions": 128, |
| "modal_accuracy": 0.578125, |
| "brier": 0.2507105103891895, |
| "median_http_ms": 391.88018697313964, |
| "cost_usd": 0.002482536 |
| }, |
| "planning": { |
| "questions": 192, |
| "modal_accuracy": 0.3645833333333333, |
| "brier": 0.5232131482077674, |
| "median_http_ms": 364.3412641249597, |
| "cost_usd": 0.00194796 |
| }, |
| "scone": { |
| "questions": 192, |
| "modal_accuracy": 0.3854166666666667, |
| "brier": 0.6786764510371691, |
| "median_http_ms": 369.4418070372194, |
| "cost_usd": 0.001408344 |
| } |
| }, |
| "by_kind": { |
| "choice": { |
| "questions": 241, |
| "modal_accuracy": 0.26556016597510373, |
| "brier": 0.648331459991458, |
| "median_http_ms": 369.08131977543235, |
| "cost_usd": 0.002369598 |
| }, |
| "noul": { |
| "questions": 94, |
| "modal_accuracy": 0.6170212765957447, |
| "brier": 0.4348702625531915, |
| "median_http_ms": 367.6399344112724, |
| "cost_usd": 0.000701316 |
| }, |
| "score": { |
| "questions": 177, |
| "modal_accuracy": 0.5423728813559322, |
| "brier": 0.37134498218589096, |
| "median_http_ms": 380.40076615288854, |
| "cost_usd": 0.002767926 |
| } |
| } |
| } |
| }, |
| "errors": {}, |
| "total_cost_usd": 0.02509872800000002, |
| "local_results": { |
| "Decisive Dory \u2014 Q8_0 / llama.cpp": { |
| "questions": 512, |
| "modal_accuracy": 0.77734375, |
| "brier": 0.2599162990600157, |
| "median_request_ms": 109.77665497921407, |
| "mean_request_ms": 135.0687600670426, |
| "by_source": { |
| "helpsteer2": { |
| "questions": 128, |
| "modal_accuracy": 0.515625, |
| "brier": 0.23377195573417558, |
| "median_request_ms": 211.45723038353026, |
| "mean_request_ms": 208.28389131929725 |
| }, |
| "planning": { |
| "questions": 192, |
| "modal_accuracy": 0.96875, |
| "brier": 0.14051244957828152, |
| "median_request_ms": 112.26724414154887, |
| "mean_request_ms": 119.26124551003643 |
| }, |
| "scone": { |
| "questions": 192, |
| "modal_accuracy": 0.7604166666666666, |
| "brier": 0.3967497107589765, |
| "median_request_ms": 103.08765154331923, |
| "mean_request_ms": 102.06618712254567 |
| } |
| }, |
| "by_kind": { |
| "choice": { |
| "questions": 241, |
| "modal_accuracy": 0.8257261410788381, |
| "brier": 0.3344495604077365, |
| "median_request_ms": 110.2855778299272, |
| "mean_request_ms": 120.80057657659981 |
| }, |
| "noul": { |
| "questions": 94, |
| "modal_accuracy": 0.9680851063829787, |
| "brier": 0.07220308279158415, |
| "median_request_ms": 101.73197486437857, |
| "mean_request_ms": 97.29551964815944 |
| }, |
| "score": { |
| "questions": 177, |
| "modal_accuracy": 0.6101694915254238, |
| "brier": 0.2581226625878792, |
| "median_request_ms": 134.17465519160032, |
| "mean_request_ms": 174.55642571999024 |
| } |
| }, |
| "model_sha256": "bb36b8ed263ce633649ccfad9c6e1dc065e9c3d333624fc27028f496089a38b8", |
| "loops": 2, |
| "prompt_parity_questions": 512, |
| "command": [ |
| "llama-server", |
| "--model", |
| "dory-decisions-Q8_0.gguf", |
| "--alias", |
| "dory-gguf-eval", |
| "--host", |
| "127.0.0.1", |
| "--port", |
| "18091", |
| "--ctx-size", |
| "4096", |
| "--parallel", |
| "1", |
| "--threads", |
| "4", |
| "--threads-batch", |
| "4", |
| "--batch-size", |
| "512", |
| "--ubatch-size", |
| "512", |
| "--fit", |
| "off", |
| "--gpu-layers", |
| "all", |
| "--no-warmup" |
| ] |
| }, |
| "Decisive Dory \u2014 NVFP4 / Eider": { |
| "questions": 512, |
| "modal_accuracy": 0.771484375, |
| "brier": 0.26463106057883456, |
| "cross_entropy": 1.075109941121327, |
| "mean_request_ms": 80.6613628164996, |
| "median_request_ms": 71.86161540448666, |
| "by_kind": { |
| "choice": { |
| "questions": 241, |
| "modal_accuracy": 0.8215767634854771, |
| "brier": 0.34044989428658823, |
| "cross_entropy": 1.1801933058267577, |
| "mean_request_ms": 75.06443578865827, |
| "median_request_ms": 72.3452023230493 |
| }, |
| "noul": { |
| "questions": 94, |
| "modal_accuracy": 0.9574468085106383, |
| "brier": 0.09152335422286362, |
| "cross_entropy": 0.2881069286288201, |
| "mean_request_ms": 67.15274397581022, |
| "median_request_ms": 67.46977800503373 |
| }, |
| "score": { |
| "questions": 177, |
| "modal_accuracy": 0.6045197740112994, |
| "brier": 0.2533304135386799, |
| "cross_entropy": 1.3499867336653204, |
| "mean_request_ms": 95.45610623307905, |
| "median_request_ms": 77.96838786453009 |
| } |
| }, |
| "by_source": { |
| "helpsteer2": { |
| "questions": 128, |
| "modal_accuracy": 0.5078125, |
| "brier": 0.22775570941129228, |
| "cross_entropy": 1.5323848707632575, |
| "mean_request_ms": 107.58943439577706, |
| "median_request_ms": 97.03774400986731 |
| }, |
| "planning": { |
| "questions": 192, |
| "modal_accuracy": 0.9635416666666666, |
| "brier": 0.155754142490034, |
| "cross_entropy": 0.834348251030471, |
| "mean_request_ms": 74.49704928634067, |
| "median_request_ms": 73.17394809797406 |
| }, |
| "scone": { |
| "questions": 192, |
| "modal_accuracy": 0.7552083333333334, |
| "brier": 0.3980915461126633, |
| "cross_entropy": 1.0110216781175627, |
| "mean_request_ms": 68.87362862714023, |
| "median_request_ms": 68.2264359202236 |
| } |
| }, |
| "model_sha256": "62bc20a6fbafaf157154650dfb4e9ec32a19d67e8e1f60ddcc3318fd37d6ccb4", |
| "loops": 2, |
| "calibrated": false, |
| "binary_sha256": "935335c9df451b4b9108d65fc46882abbfd7a0b6cb20b3ba76f4e8c42fa7ab0c", |
| "quantization_recipe": "NVIDIA Model Optimizer local_hessian", |
| "calibration_examples": 256 |
| } |
| }, |
| "scope": "512 test questions withheld from this fine-tune. Task families are represented in its training. Provider training overlap is unknown. This is not an official source benchmark or a general model ranking.", |
| "scoring": "Accuracy accepts any answer tied for highest target probability. Brier is the sum of squared differences between predicted and target probabilities, averaged over questions.", |
| "cost_scope": "Cost covers scored requests and calls made during the final resumed run. Earlier failed-response and diagnostic probe costs are excluded." |
| } |
|
|
