{
  "protocol": {
    "endpoint": "https://openrouter.ai/api/alpha/decisions",
    "models": [
      "typesafe/jev-1.14",
      "liquid/d1",
      "perplexity/pplx-decider-v1.2",
      "jaredpalmer/kev-4b"
    ],
    "test_data_sha256": "e2ca7413a306fc2b0e9e044aa685c94d5aa8e0def7cfcec6448941b7369ae625",
    "questions": 512,
    "protocol": "One request per question. Identical state, question instructions, answer meanings, and option order; no reference targets sent. Native provider prompts and calibration remain provider-controlled. No prompt optimization or fitting.",
    "concurrency": "Four models concurrently, one request at a time per model. HTTP latency includes network/provider overhead and is not local GPU compute latency.",
    "probability_rounding": "Returned distributions within 0.02 of unit mass are renormalized before Brier scoring; raw API responses are retained. Argmax option order breaks probability ties.",
    "cost_limit_usd": 1.0,
    "date": "2026-10-21"
  },
  "models": {
    "liquid/d1": {
      "questions": 512,
      "modal_accuracy": 0.662109375,
      "brier": 0.4156228551936487,
      "median_http_ms": 153.96152925677598,
      "cost_usd": 0.0032266,
      "renormalized_responses": 0,
      "max_probability_sum_error": 4.440892098500626e-16,
      "requested_model": "liquid/d1",
      "served_models": [
        "liquid/d1-20260930"
      ],
      "by_source": {
        "helpsteer2": {
          "questions": 128,
          "modal_accuracy": 0.5859375,
          "brier": 0.31772152271267684,
          "median_http_ms": 156.79931757040322,
          "cost_usd": 0.00130398
        },
        "planning": {
          "questions": 192,
          "modal_accuracy": 0.7916666666666666,
          "brier": 0.36154586401436944,
          "median_http_ms": 160.43087700381875,
          "cost_usd": 0.00111626
        },
        "scone": {
          "questions": 192,
          "modal_accuracy": 0.5833333333333334,
          "brier": 0.5349674013602426,
          "median_http_ms": 146.55435550957918,
          "cost_usd": 0.00080636
        }
      },
      "by_kind": {
        "choice": {
          "questions": 241,
          "modal_accuracy": 0.6970954356846473,
          "brier": 0.45294190031856313,
          "median_http_ms": 153.44042284414172,
          "cost_usd": 0.0013192
        },
        "noul": {
          "questions": 94,
          "modal_accuracy": 0.7127659574468085,
          "brier": 0.36126946919251096,
          "median_http_ms": 152.50918315723538,
          "cost_usd": 0.00040766
        },
        "score": {
          "questions": 177,
          "modal_accuracy": 0.5875706214689266,
          "brier": 0.39367555806936955,
          "median_http_ms": 156.5550728701055,
          "cost_usd": 0.00149974
        }
      }
    },
    "perplexity/pplx-decider-v1.2": {
      "questions": 512,
      "modal_accuracy": 0.74609375,
      "brier": 0.3177875424868089,
      "median_http_ms": 190.27893454767764,
      "cost_usd": 0.00346588,
      "renormalized_responses": 0,
      "max_probability_sum_error": 4.440892098500626e-16,
      "requested_model": "perplexity/pplx-decider-v1.2",
      "served_models": [
        "perplexity/pplx-decider-v1.2-20261016"
      ],
      "by_source": {
        "helpsteer2": {
          "questions": 128,
          "modal_accuracy": 0.5703125,
          "brier": 0.2787386993739065,
          "median_http_ms": 197.84710323438048,
          "cost_usd": 0.00139706
        },
        "planning": {
          "questions": 192,
          "modal_accuracy": 0.8802083333333334,
          "brier": 0.26224774345602686,
          "median_http_ms": 189.46263939142227,
          "cost_usd": 0.00119936
        },
        "scone": {
          "questions": 192,
          "modal_accuracy": 0.7291666666666666,
          "brier": 0.3993599035928593,
          "median_http_ms": 185.11412316001952,
          "cost_usd": 0.00086946
        }
      },
      "by_kind": {
        "choice": {
          "questions": 241,
          "modal_accuracy": 0.8091286307053942,
          "brier": 0.3446775651495988,
          "median_http_ms": 189.45799581706524,
          "cost_usd": 0.0014693
        },
        "noul": {
          "questions": 94,
          "modal_accuracy": 0.8191489361702128,
          "brier": 0.24647724911756672,
          "median_http_ms": 183.42466722242534,
          "cost_usd": 0.00042356
        },
        "score": {
          "questions": 177,
          "modal_accuracy": 0.6214689265536724,
          "brier": 0.3190455770346982,
          "median_http_ms": 192.91105587035418,
          "cost_usd": 0.00157302
        }
      }
    },
    "typesafe/jev-1.14": {
      "questions": 512,
      "modal_accuracy": 0.66015625,
      "brier": 0.38428551299771707,
      "median_http_ms": 160.70113889873028,
      "cost_usd": 0.012567408,
      "renormalized_responses": 6,
      "max_probability_sum_error": 0.009999999999999898,
      "requested_model": "typesafe/jev-1.14",
      "served_models": [
        "typesafe/jev-1.14-20261014"
      ],
      "by_source": {
        "helpsteer2": {
          "questions": 128,
          "modal_accuracy": 0.5546875,
          "brier": 0.3501162689075094,
          "median_http_ms": 161.725880112499,
          "cost_usd": 0.00407988
        },
        "planning": {
          "questions": 192,
          "modal_accuracy": 0.7447916666666666,
          "brier": 0.3210494803889059,
          "median_http_ms": 162.16431255452335,
          "cost_usd": 0.004622646
        },
        "scone": {
          "questions": 192,
          "modal_accuracy": 0.6458333333333334,
          "brier": 0.4703010416666667,
          "median_http_ms": 158.17508311010897,
          "cost_usd": 0.003864882
        }
      },
      "by_kind": {
        "choice": {
          "questions": 241,
          "modal_accuracy": 0.6804979253112033,
          "brier": 0.4145994016191929,
          "median_http_ms": 159.9918780848384,
          "cost_usd": 0.005741064
        },
        "noul": {
          "questions": 94,
          "modal_accuracy": 0.7446808510638298,
          "brier": 0.2824255319148936,
          "median_http_ms": 159.92187475785613,
          "cost_usd": 0.001838382
        },
        "score": {
          "questions": 177,
          "modal_accuracy": 0.5875706214689266,
          "brier": 0.3971058014949472,
          "median_http_ms": 162.21308708190918,
          "cost_usd": 0.004987962
        }
      }
    },
    "jaredpalmer/kev-4b": {
      "questions": 512,
      "modal_accuracy": 0.42578125,
      "brier": 0.5133862273141486,
      "median_http_ms": 374.62099245749414,
      "cost_usd": 0.00583884,
      "renormalized_responses": 166,
      "max_probability_sum_error": 0.00019999999999997797,
      "requested_model": "jaredpalmer/kev-4b",
      "served_models": [
        "jaredpalmer/kev-4b-20260924"
      ],
      "by_source": {
        "helpsteer2": {
          "questions": 128,
          "modal_accuracy": 0.578125,
          "brier": 0.2507105103891895,
          "median_http_ms": 391.88018697313964,
          "cost_usd": 0.002482536
        },
        "planning": {
          "questions": 192,
          "modal_accuracy": 0.3645833333333333,
          "brier": 0.5232131482077674,
          "median_http_ms": 364.3412641249597,
          "cost_usd": 0.00194796
        },
        "scone": {
          "questions": 192,
          "modal_accuracy": 0.3854166666666667,
          "brier": 0.6786764510371691,
          "median_http_ms": 369.4418070372194,
          "cost_usd": 0.001408344
        }
      },
      "by_kind": {
        "choice": {
          "questions": 241,
          "modal_accuracy": 0.26556016597510373,
          "brier": 0.648331459991458,
          "median_http_ms": 369.08131977543235,
          "cost_usd": 0.002369598
        },
        "noul": {
          "questions": 94,
          "modal_accuracy": 0.6170212765957447,
          "brier": 0.4348702625531915,
          "median_http_ms": 367.6399344112724,
          "cost_usd": 0.000701316
        },
        "score": {
          "questions": 177,
          "modal_accuracy": 0.5423728813559322,
          "brier": 0.37134498218589096,
          "median_http_ms": 380.40076615288854,
          "cost_usd": 0.002767926
        }
      }
    }
  },
  "errors": {},
  "total_cost_usd": 0.02509872800000002,
  "local_results": {
    "Decisive Dory \u2014 Q8_0 / llama.cpp": {
      "questions": 512,
      "modal_accuracy": 0.77734375,
      "brier": 0.2599162990600157,
      "median_request_ms": 109.77665497921407,
      "mean_request_ms": 135.0687600670426,
      "by_source": {
        "helpsteer2": {
          "questions": 128,
          "modal_accuracy": 0.515625,
          "brier": 0.23377195573417558,
          "median_request_ms": 211.45723038353026,
          "mean_request_ms": 208.28389131929725
        },
        "planning": {
          "questions": 192,
          "modal_accuracy": 0.96875,
          "brier": 0.14051244957828152,
          "median_request_ms": 112.26724414154887,
          "mean_request_ms": 119.26124551003643
        },
        "scone": {
          "questions": 192,
          "modal_accuracy": 0.7604166666666666,
          "brier": 0.3967497107589765,
          "median_request_ms": 103.08765154331923,
          "mean_request_ms": 102.06618712254567
        }
      },
      "by_kind": {
        "choice": {
          "questions": 241,
          "modal_accuracy": 0.8257261410788381,
          "brier": 0.3344495604077365,
          "median_request_ms": 110.2855778299272,
          "mean_request_ms": 120.80057657659981
        },
        "noul": {
          "questions": 94,
          "modal_accuracy": 0.9680851063829787,
          "brier": 0.07220308279158415,
          "median_request_ms": 101.73197486437857,
          "mean_request_ms": 97.29551964815944
        },
        "score": {
          "questions": 177,
          "modal_accuracy": 0.6101694915254238,
          "brier": 0.2581226625878792,
          "median_request_ms": 134.17465519160032,
          "mean_request_ms": 174.55642571999024
        }
      },
      "model_sha256": "bb36b8ed263ce633649ccfad9c6e1dc065e9c3d333624fc27028f496089a38b8",
      "loops": 2,
      "prompt_parity_questions": 512,
      "command": [
        "llama-server",
        "--model",
        "dory-decisions-Q8_0.gguf",
        "--alias",
        "dory-gguf-eval",
        "--host",
        "127.0.0.1",
        "--port",
        "18091",
        "--ctx-size",
        "4096",
        "--parallel",
        "1",
        "--threads",
        "4",
        "--threads-batch",
        "4",
        "--batch-size",
        "512",
        "--ubatch-size",
        "512",
        "--fit",
        "off",
        "--gpu-layers",
        "all",
        "--no-warmup"
      ]
    },
    "Decisive Dory \u2014 NVFP4 / Eider": {
      "questions": 512,
      "modal_accuracy": 0.771484375,
      "brier": 0.26463106057883456,
      "cross_entropy": 1.075109941121327,
      "mean_request_ms": 80.6613628164996,
      "median_request_ms": 71.86161540448666,
      "by_kind": {
        "choice": {
          "questions": 241,
          "modal_accuracy": 0.8215767634854771,
          "brier": 0.34044989428658823,
          "cross_entropy": 1.1801933058267577,
          "mean_request_ms": 75.06443578865827,
          "median_request_ms": 72.3452023230493
        },
        "noul": {
          "questions": 94,
          "modal_accuracy": 0.9574468085106383,
          "brier": 0.09152335422286362,
          "cross_entropy": 0.2881069286288201,
          "mean_request_ms": 67.15274397581022,
          "median_request_ms": 67.46977800503373
        },
        "score": {
          "questions": 177,
          "modal_accuracy": 0.6045197740112994,
          "brier": 0.2533304135386799,
          "cross_entropy": 1.3499867336653204,
          "mean_request_ms": 95.45610623307905,
          "median_request_ms": 77.96838786453009
        }
      },
      "by_source": {
        "helpsteer2": {
          "questions": 128,
          "modal_accuracy": 0.5078125,
          "brier": 0.22775570941129228,
          "cross_entropy": 1.5323848707632575,
          "mean_request_ms": 107.58943439577706,
          "median_request_ms": 97.03774400986731
        },
        "planning": {
          "questions": 192,
          "modal_accuracy": 0.9635416666666666,
          "brier": 0.155754142490034,
          "cross_entropy": 0.834348251030471,
          "mean_request_ms": 74.49704928634067,
          "median_request_ms": 73.17394809797406
        },
        "scone": {
          "questions": 192,
          "modal_accuracy": 0.7552083333333334,
          "brier": 0.3980915461126633,
          "cross_entropy": 1.0110216781175627,
          "mean_request_ms": 68.87362862714023,
          "median_request_ms": 68.2264359202236
        }
      },
      "model_sha256": "62bc20a6fbafaf157154650dfb4e9ec32a19d67e8e1f60ddcc3318fd37d6ccb4",
      "loops": 2,
      "calibrated": false,
      "binary_sha256": "935335c9df451b4b9108d65fc46882abbfd7a0b6cb20b3ba76f4e8c42fa7ab0c",
      "quantization_recipe": "NVIDIA Model Optimizer local_hessian",
      "calibration_examples": 256
    }
  },
  "scope": "512 test questions withheld from this fine-tune. Task families are represented in its training. Provider training overlap is unknown. This is not an official source benchmark or a general model ranking.",
  "scoring": "Accuracy accepts any answer tied for highest target probability. Brier is the sum of squared differences between predicted and target probabilities, averaged over questions.",
  "cost_scope": "Cost covers scored requests and calls made during the final resumed run. Earlier failed-response and diagnostic probe costs are excluded."
}