| { |
| "date": "2026-10-08", |
| "weight_version": "v2.2", |
| "runtime": "llama.cpp b11495 / 37ac63456, Linux x86_64, CPU, four threads", |
| "runtime_archive_sha256": "e9272c922c2e098d55ec39c0616b8f545984978a7b61fec8ed1fc312925a27c2", |
| "temperatures": { |
| "choice": 0.85, |
| "noul": 0.85, |
| "score": 0.7 |
| }, |
| "added_metadata_keys": [ |
| "mistral3.decision.type", |
| "tokenizer.chat_template.systemone", |
| "mistral3.decision.temperature.choice", |
| "mistral3.decision.temperature.noul", |
| "mistral3.decision.temperature.score" |
| ], |
| "method": "Compare native /v1/systemone against /completion pre-sampling option logprobs using identical prompts and verified one-token labels; softmax at per-type temperatures. Individual slots cleared before each request; multi-question reference calls execute in the native question order and preserve prefix state after the first question.", |
| "single_call_probability_tolerance": 5e-05, |
| "multi_question_probability_tolerance": 5e-05, |
| "files": { |
| "StandardOne-8B-BF16.gguf": { |
| "passed": true, |
| "cases": [ |
| { |
| "name": "choice", |
| "status": 200, |
| "max_probability_error": 8.67430016704418e-10, |
| "max_expected_score_error": 0.0 |
| }, |
| { |
| "name": "noul", |
| "status": 200, |
| "max_probability_error": 4.154059318750569e-10, |
| "max_expected_score_error": 0.0 |
| }, |
| { |
| "name": "score", |
| "status": 200, |
| "max_probability_error": 1.2847689578876498e-10, |
| "max_expected_score_error": 1.950315464682717e-10 |
| }, |
| { |
| "name": "structured-state", |
| "status": 200, |
| "max_probability_error": 4.71248553374215e-09, |
| "max_expected_score_error": 0.0 |
| }, |
| { |
| "name": "reversed-key-order", |
| "status": 200, |
| "max_probability_error": 1.187641235356196e-09, |
| "max_expected_score_error": 0.0 |
| }, |
| { |
| "name": "multi", |
| "status": 200, |
| "max_probability_error": 1.3165766254985556e-09, |
| "max_expected_score_error": 8.499756454227736e-11 |
| } |
| ], |
| "output_tokens": 0 |
| }, |
| "StandardOne-8B-IQ2_M.gguf": { |
| "passed": true, |
| "cases": [ |
| { |
| "name": "choice", |
| "status": 200, |
| "max_probability_error": 4.4403458687725106e-09, |
| "max_expected_score_error": 0.0 |
| }, |
| { |
| "name": "noul", |
| "status": 200, |
| "max_probability_error": 5.856941598381127e-10, |
| "max_expected_score_error": 0.0 |
| }, |
| { |
| "name": "score", |
| "status": 200, |
| "max_probability_error": 8.091216585626171e-10, |
| "max_expected_score_error": 1.1192298199347306e-09 |
| }, |
| { |
| "name": "structured-state", |
| "status": 200, |
| "max_probability_error": 1.3272716259393746e-12, |
| "max_expected_score_error": 0.0 |
| }, |
| { |
| "name": "reversed-key-order", |
| "status": 200, |
| "max_probability_error": 1.551408697689216e-10, |
| "max_expected_score_error": 0.0 |
| }, |
| { |
| "name": "multi", |
| "status": 200, |
| "max_probability_error": 4.4403458687725106e-09, |
| "max_expected_score_error": 1.346451616868194e-09 |
| } |
| ], |
| "output_tokens": 0 |
| }, |
| "StandardOne-8B-IQ3_M.gguf": { |
| "passed": true, |
| "cases": [ |
| { |
| "name": "choice", |
| "status": 200, |
| "max_probability_error": 2.1058255139649873e-10, |
| "max_expected_score_error": 0.0 |
| }, |
| { |
| "name": "noul", |
| "status": 200, |
| "max_probability_error": 2.212067196083467e-10, |
| "max_expected_score_error": 0.0 |
| }, |
| { |
| "name": "score", |
| "status": 200, |
| "max_probability_error": 2.632024598270277e-10, |
| "max_expected_score_error": 4.130382702527413e-10 |
| }, |
| { |
| "name": "structured-state", |
| "status": 200, |
| "max_probability_error": 1.4432894879234937e-09, |
| "max_expected_score_error": 0.0 |
| }, |
| { |
| "name": "reversed-key-order", |
| "status": 200, |
| "max_probability_error": 1.4257006686335671e-10, |
| "max_expected_score_error": 0.0 |
| }, |
| { |
| "name": "multi", |
| "status": 200, |
| "max_probability_error": 5.107962941508504e-10, |
| "max_expected_score_error": 4.049738322464691e-11 |
| } |
| ], |
| "output_tokens": 0 |
| }, |
| "StandardOne-8B-IQ4_XS.gguf": { |
| "passed": true, |
| "cases": [ |
| { |
| "name": "choice", |
| "status": 200, |
| "max_probability_error": 2.300640788988062e-09, |
| "max_expected_score_error": 0.0 |
| }, |
| { |
| "name": "noul", |
| "status": 200, |
| "max_probability_error": 3.1092288743650442e-09, |
| "max_expected_score_error": 0.0 |
| }, |
| { |
| "name": "score", |
| "status": 200, |
| "max_probability_error": 3.8100922417072525e-10, |
| "max_expected_score_error": 5.71461766796233e-10 |
| }, |
| { |
| "name": "structured-state", |
| "status": 200, |
| "max_probability_error": 1.6641010791573763e-10, |
| "max_expected_score_error": 0.0 |
| }, |
| { |
| "name": "reversed-key-order", |
| "status": 200, |
| "max_probability_error": 1.58454260823504e-09, |
| "max_expected_score_error": 0.0 |
| }, |
| { |
| "name": "multi", |
| "status": 200, |
| "max_probability_error": 2.300640788988062e-09, |
| "max_expected_score_error": 2.9579938498613956e-10 |
| } |
| ], |
| "output_tokens": 0 |
| }, |
| "StandardOne-8B-Q2_K.gguf": { |
| "passed": true, |
| "cases": [ |
| { |
| "name": "choice", |
| "status": 200, |
| "max_probability_error": 2.969889112414137e-09, |
| "max_expected_score_error": 0.0 |
| }, |
| { |
| "name": "noul", |
| "status": 200, |
| "max_probability_error": 2.8542667118713894e-09, |
| "max_expected_score_error": 0.0 |
| }, |
| { |
| "name": "score", |
| "status": 200, |
| "max_probability_error": 5.7223004112927356e-11, |
| "max_expected_score_error": 7.932721146630684e-11 |
| }, |
| { |
| "name": "structured-state", |
| "status": 200, |
| "max_probability_error": 2.2514560216180257e-09, |
| "max_expected_score_error": 0.0 |
| }, |
| { |
| "name": "reversed-key-order", |
| "status": 200, |
| "max_probability_error": 8.267649898030527e-10, |
| "max_expected_score_error": 0.0 |
| }, |
| { |
| "name": "multi", |
| "status": 200, |
| "max_probability_error": 2.969889112414137e-09, |
| "max_expected_score_error": 2.607756233174996e-10 |
| } |
| ], |
| "output_tokens": 0 |
| }, |
| "StandardOne-8B-Q3_K_M.gguf": { |
| "passed": true, |
| "cases": [ |
| { |
| "name": "choice", |
| "status": 200, |
| "max_probability_error": 3.5506686479891414e-09, |
| "max_expected_score_error": 0.0 |
| }, |
| { |
| "name": "noul", |
| "status": 200, |
| "max_probability_error": 7.957721148699193e-10, |
| "max_expected_score_error": 0.0 |
| }, |
| { |
| "name": "score", |
| "status": 200, |
| "max_probability_error": 3.9876990598486373e-10, |
| "max_expected_score_error": 6.00312688447957e-10 |
| }, |
| { |
| "name": "structured-state", |
| "status": 200, |
| "max_probability_error": 1.8685453670452823e-09, |
| "max_expected_score_error": 0.0 |
| }, |
| { |
| "name": "reversed-key-order", |
| "status": 200, |
| "max_probability_error": 5.117114739751349e-10, |
| "max_expected_score_error": 0.0 |
| }, |
| { |
| "name": "multi", |
| "status": 200, |
| "max_probability_error": 3.5506686479891414e-09, |
| "max_expected_score_error": 1.593545295719423e-10 |
| } |
| ], |
| "output_tokens": 0 |
| }, |
| "StandardOne-8B-Q4_K_M.gguf": { |
| "passed": true, |
| "cases": [ |
| { |
| "name": "choice", |
| "status": 200, |
| "max_probability_error": 4.163814071311833e-09, |
| "max_expected_score_error": 0.0 |
| }, |
| { |
| "name": "noul", |
| "status": 200, |
| "max_probability_error": 2.457777537046013e-09, |
| "max_expected_score_error": 0.0 |
| }, |
| { |
| "name": "score", |
| "status": 200, |
| "max_probability_error": 1.0959511076436002e-10, |
| "max_expected_score_error": 1.6773493705102283e-10 |
| }, |
| { |
| "name": "structured-state", |
| "status": 200, |
| "max_probability_error": 5.2668226446783706e-09, |
| "max_expected_score_error": 0.0 |
| }, |
| { |
| "name": "reversed-key-order", |
| "status": 200, |
| "max_probability_error": 5.749417218356423e-11, |
| "max_expected_score_error": 0.0 |
| }, |
| { |
| "name": "multi", |
| "status": 200, |
| "max_probability_error": 4.163814071311833e-09, |
| "max_expected_score_error": 1.3229888295995806e-09 |
| } |
| ], |
| "output_tokens": 0 |
| }, |
| "StandardOne-8B-Q5_K_M.gguf": { |
| "passed": true, |
| "cases": [ |
| { |
| "name": "choice", |
| "status": 200, |
| "max_probability_error": 2.556157429819961e-10, |
| "max_expected_score_error": 0.0 |
| }, |
| { |
| "name": "noul", |
| "status": 200, |
| "max_probability_error": 1.8832481130104384e-09, |
| "max_expected_score_error": 0.0 |
| }, |
| { |
| "name": "score", |
| "status": 200, |
| "max_probability_error": 8.755662861403835e-11, |
| "max_expected_score_error": 1.3074274995972246e-10 |
| }, |
| { |
| "name": "structured-state", |
| "status": 200, |
| "max_probability_error": 2.830105719220377e-09, |
| "max_expected_score_error": 0.0 |
| }, |
| { |
| "name": "reversed-key-order", |
| "status": 200, |
| "max_probability_error": 2.6982107043452275e-10, |
| "max_expected_score_error": 0.0 |
| }, |
| { |
| "name": "multi", |
| "status": 200, |
| "max_probability_error": 2.5231146061344134e-09, |
| "max_expected_score_error": 6.529470297778062e-10 |
| } |
| ], |
| "output_tokens": 0 |
| }, |
| "StandardOne-8B-Q8_0.gguf": { |
| "passed": true, |
| "cases": [ |
| { |
| "name": "choice", |
| "status": 200, |
| "max_probability_error": 3.4088990519265394e-09, |
| "max_expected_score_error": 0.0 |
| }, |
| { |
| "name": "noul", |
| "status": 200, |
| "max_probability_error": 2.811861632423529e-09, |
| "max_expected_score_error": 0.0 |
| }, |
| { |
| "name": "score", |
| "status": 200, |
| "max_probability_error": 3.1254554500037557e-10, |
| "max_expected_score_error": 4.796023578279573e-10 |
| }, |
| { |
| "name": "structured-state", |
| "status": 200, |
| "max_probability_error": 7.075904098763353e-11, |
| "max_expected_score_error": 0.0 |
| }, |
| { |
| "name": "reversed-key-order", |
| "status": 200, |
| "max_probability_error": 1.3301609024310435e-09, |
| "max_expected_score_error": 0.0 |
| }, |
| { |
| "name": "multi", |
| "status": 200, |
| "max_probability_error": 3.4088990519265394e-09, |
| "max_expected_score_error": 4.729217017995779e-10 |
| } |
| ], |
| "output_tokens": 0 |
| } |
| }, |
| "limits": [ |
| "native wording, <=26 options", |
| "text only; no GPU or image validation", |
| "not served preset with temperature 1.65 or extended labels", |
| "state object key order preserved, not sorted as in jev-adapter", |
| "llama.cpp confidence is not adapter entropy confidence", |
| "API/probability parity fixtures, not task accuracy or calibration benchmark" |
| ] |
| } |
|
|
