| { |
| "date": "2026-10-08", |
| "weight_version": "v2.2", |
| "runtime": "llama.cpp b11495 / 37ac63456, Linux x86_64, CPU, four threads", |
| "runtime_archive_sha256": "e9272c922c2e098d55ec39c0616b8f545984978a7b61fec8ed1fc312925a27c2", |
| "temperatures": { |
| "choice": 0.9, |
| "noul": 1.15, |
| "score": 1.2 |
| }, |
| "added_metadata_keys": [ |
| "mistral3.decision.type", |
| "tokenizer.chat_template.systemone", |
| "mistral3.decision.temperature.choice", |
| "mistral3.decision.temperature.noul", |
| "mistral3.decision.temperature.score" |
| ], |
| "method": "Compare native /v1/systemone against /completion pre-sampling option logprobs using identical prompts and verified one-token labels; softmax at per-type temperatures. Individual slots cleared before each request; multi-question reference calls execute in the native question order and preserve prefix state after the first question.", |
| "single_call_probability_tolerance": 5e-05, |
| "multi_question_probability_tolerance": 5e-05, |
| "files": { |
| "StandardOne-3B-BF16.gguf": { |
| "passed": true, |
| "cases": [ |
| { |
| "name": "choice", |
| "status": 200, |
| "max_probability_error": 3.4035333301707382e-09, |
| "max_expected_score_error": 0.0 |
| }, |
| { |
| "name": "noul", |
| "status": 200, |
| "max_probability_error": 1.895569257115426e-09, |
| "max_expected_score_error": 0.0 |
| }, |
| { |
| "name": "score", |
| "status": 200, |
| "max_probability_error": 7.564064152809635e-09, |
| "max_expected_score_error": 9.852553439770873e-09 |
| }, |
| { |
| "name": "structured-state", |
| "status": 200, |
| "max_probability_error": 2.2212667642740325e-10, |
| "max_expected_score_error": 0.0 |
| }, |
| { |
| "name": "reversed-key-order", |
| "status": 200, |
| "max_probability_error": 1.3041013243245136e-09, |
| "max_expected_score_error": 0.0 |
| }, |
| { |
| "name": "multi", |
| "status": 200, |
| "max_probability_error": 6.924828266363647e-09, |
| "max_expected_score_error": 9.041088988581691e-09 |
| } |
| ], |
| "output_tokens": 0 |
| }, |
| "StandardOne-3B-IQ2_M.gguf": { |
| "passed": true, |
| "cases": [ |
| { |
| "name": "choice", |
| "status": 200, |
| "max_probability_error": 3.4227816525511656e-08, |
| "max_expected_score_error": 0.0 |
| }, |
| { |
| "name": "noul", |
| "status": 200, |
| "max_probability_error": 2.8620783520949544e-10, |
| "max_expected_score_error": 0.0 |
| }, |
| { |
| "name": "score", |
| "status": 200, |
| "max_probability_error": 6.059652571388519e-10, |
| "max_expected_score_error": 5.536675562467508e-10 |
| }, |
| { |
| "name": "structured-state", |
| "status": 200, |
| "max_probability_error": 3.0364000203064734e-09, |
| "max_expected_score_error": 0.0 |
| }, |
| { |
| "name": "reversed-key-order", |
| "status": 200, |
| "max_probability_error": 2.0019862278369827e-09, |
| "max_expected_score_error": 0.0 |
| }, |
| { |
| "name": "multi", |
| "status": 200, |
| "max_probability_error": 3.4227816525511656e-08, |
| "max_expected_score_error": 3.2945264294426124e-09 |
| } |
| ], |
| "output_tokens": 0 |
| }, |
| "StandardOne-3B-IQ3_M.gguf": { |
| "passed": true, |
| "cases": [ |
| { |
| "name": "choice", |
| "status": 200, |
| "max_probability_error": 1.2602752313961574e-09, |
| "max_expected_score_error": 0.0 |
| }, |
| { |
| "name": "noul", |
| "status": 200, |
| "max_probability_error": 1.7948651453991715e-09, |
| "max_expected_score_error": 0.0 |
| }, |
| { |
| "name": "score", |
| "status": 200, |
| "max_probability_error": 2.3118781333764105e-09, |
| "max_expected_score_error": 3.1876872252922794e-09 |
| }, |
| { |
| "name": "structured-state", |
| "status": 200, |
| "max_probability_error": 7.9271381125956e-10, |
| "max_expected_score_error": 0.0 |
| }, |
| { |
| "name": "reversed-key-order", |
| "status": 200, |
| "max_probability_error": 4.368192500422974e-10, |
| "max_expected_score_error": 0.0 |
| }, |
| { |
| "name": "multi", |
| "status": 200, |
| "max_probability_error": 5.521665680241483e-09, |
| "max_expected_score_error": 7.26999127564909e-09 |
| } |
| ], |
| "output_tokens": 0 |
| }, |
| "StandardOne-3B-IQ4_XS.gguf": { |
| "passed": true, |
| "cases": [ |
| { |
| "name": "choice", |
| "status": 200, |
| "max_probability_error": 8.738536894092874e-09, |
| "max_expected_score_error": 0.0 |
| }, |
| { |
| "name": "noul", |
| "status": 200, |
| "max_probability_error": 1.1185221637788345e-09, |
| "max_expected_score_error": 0.0 |
| }, |
| { |
| "name": "score", |
| "status": 200, |
| "max_probability_error": 1.5589824986506073e-09, |
| "max_expected_score_error": 2.221572259486493e-09 |
| }, |
| { |
| "name": "structured-state", |
| "status": 200, |
| "max_probability_error": 2.9374810899679415e-09, |
| "max_expected_score_error": 0.0 |
| }, |
| { |
| "name": "reversed-key-order", |
| "status": 200, |
| "max_probability_error": 1.5319889419551647e-09, |
| "max_expected_score_error": 0.0 |
| }, |
| { |
| "name": "multi", |
| "status": 200, |
| "max_probability_error": 8.738536894092874e-09, |
| "max_expected_score_error": 6.359697213298432e-09 |
| } |
| ], |
| "output_tokens": 0 |
| }, |
| "StandardOne-3B-Q2_K.gguf": { |
| "passed": true, |
| "cases": [ |
| { |
| "name": "choice", |
| "status": 200, |
| "max_probability_error": 7.071243257605886e-09, |
| "max_expected_score_error": 0.0 |
| }, |
| { |
| "name": "noul", |
| "status": 200, |
| "max_probability_error": 4.934683772717108e-10, |
| "max_expected_score_error": 0.0 |
| }, |
| { |
| "name": "score", |
| "status": 200, |
| "max_probability_error": 9.269061851391314e-09, |
| "max_expected_score_error": 9.514000698374048e-09 |
| }, |
| { |
| "name": "structured-state", |
| "status": 200, |
| "max_probability_error": 1.3465061704520664e-09, |
| "max_expected_score_error": 0.0 |
| }, |
| { |
| "name": "reversed-key-order", |
| "status": 200, |
| "max_probability_error": 3.3744141925140525e-09, |
| "max_expected_score_error": 0.0 |
| }, |
| { |
| "name": "multi", |
| "status": 200, |
| "max_probability_error": 7.071243257605886e-09, |
| "max_expected_score_error": 5.828444615829653e-09 |
| } |
| ], |
| "output_tokens": 0 |
| }, |
| "StandardOne-3B-Q3_K_M.gguf": { |
| "passed": true, |
| "cases": [ |
| { |
| "name": "choice", |
| "status": 200, |
| "max_probability_error": 6.604746860539024e-09, |
| "max_expected_score_error": 0.0 |
| }, |
| { |
| "name": "noul", |
| "status": 200, |
| "max_probability_error": 1.0567493546886908e-09, |
| "max_expected_score_error": 0.0 |
| }, |
| { |
| "name": "score", |
| "status": 200, |
| "max_probability_error": 8.310530485999834e-09, |
| "max_expected_score_error": 1.1104013930562928e-08 |
| }, |
| { |
| "name": "structured-state", |
| "status": 200, |
| "max_probability_error": 1.1127920807041392e-10, |
| "max_expected_score_error": 0.0 |
| }, |
| { |
| "name": "reversed-key-order", |
| "status": 200, |
| "max_probability_error": 2.25361262984336e-09, |
| "max_expected_score_error": 0.0 |
| }, |
| { |
| "name": "multi", |
| "status": 200, |
| "max_probability_error": 7.031610960162027e-09, |
| "max_expected_score_error": 9.352805641071882e-09 |
| } |
| ], |
| "output_tokens": 0 |
| }, |
| "StandardOne-3B-Q4_K_M.gguf": { |
| "passed": true, |
| "cases": [ |
| { |
| "name": "choice", |
| "status": 200, |
| "max_probability_error": 2.749427241433011e-09, |
| "max_expected_score_error": 0.0 |
| }, |
| { |
| "name": "noul", |
| "status": 200, |
| "max_probability_error": 4.159056432584407e-09, |
| "max_expected_score_error": 0.0 |
| }, |
| { |
| "name": "score", |
| "status": 200, |
| "max_probability_error": 8.781075866437504e-10, |
| "max_expected_score_error": 1.3451164626587797e-09 |
| }, |
| { |
| "name": "structured-state", |
| "status": 200, |
| "max_probability_error": 4.449257628991177e-09, |
| "max_expected_score_error": 0.0 |
| }, |
| { |
| "name": "reversed-key-order", |
| "status": 200, |
| "max_probability_error": 1.0385021878903111e-09, |
| "max_expected_score_error": 0.0 |
| }, |
| { |
| "name": "multi", |
| "status": 200, |
| "max_probability_error": 3.2670162131154257e-09, |
| "max_expected_score_error": 5.518938639426096e-10 |
| } |
| ], |
| "output_tokens": 0 |
| }, |
| "StandardOne-3B-Q5_K_M.gguf": { |
| "passed": true, |
| "cases": [ |
| { |
| "name": "choice", |
| "status": 200, |
| "max_probability_error": 9.42021446959096e-10, |
| "max_expected_score_error": 0.0 |
| }, |
| { |
| "name": "noul", |
| "status": 200, |
| "max_probability_error": 2.287422473656875e-10, |
| "max_expected_score_error": 0.0 |
| }, |
| { |
| "name": "score", |
| "status": 200, |
| "max_probability_error": 3.6667280323143814e-09, |
| "max_expected_score_error": 4.8813184605478455e-09 |
| }, |
| { |
| "name": "structured-state", |
| "status": 200, |
| "max_probability_error": 1.5826660069301335e-09, |
| "max_expected_score_error": 0.0 |
| }, |
| { |
| "name": "reversed-key-order", |
| "status": 200, |
| "max_probability_error": 9.424254493115014e-10, |
| "max_expected_score_error": 0.0 |
| }, |
| { |
| "name": "multi", |
| "status": 200, |
| "max_probability_error": 1.8404174850772392e-09, |
| "max_expected_score_error": 2.23601959170594e-09 |
| } |
| ], |
| "output_tokens": 0 |
| }, |
| "StandardOne-3B-Q8_0.gguf": { |
| "passed": true, |
| "cases": [ |
| { |
| "name": "choice", |
| "status": 200, |
| "max_probability_error": 1.8004621349376748e-08, |
| "max_expected_score_error": 0.0 |
| }, |
| { |
| "name": "noul", |
| "status": 200, |
| "max_probability_error": 2.260874265580526e-09, |
| "max_expected_score_error": 0.0 |
| }, |
| { |
| "name": "score", |
| "status": 200, |
| "max_probability_error": 7.16143355727894e-09, |
| "max_expected_score_error": 9.318077864861607e-09 |
| }, |
| { |
| "name": "structured-state", |
| "status": 200, |
| "max_probability_error": 3.367551348887332e-09, |
| "max_expected_score_error": 0.0 |
| }, |
| { |
| "name": "reversed-key-order", |
| "status": 200, |
| "max_probability_error": 4.823325940039869e-10, |
| "max_expected_score_error": 0.0 |
| }, |
| { |
| "name": "multi", |
| "status": 200, |
| "max_probability_error": 1.8004621349376748e-08, |
| "max_expected_score_error": 7.562626969104258e-09 |
| } |
| ], |
| "output_tokens": 0 |
| } |
| }, |
| "limits": [ |
| "native wording, <=26 options", |
| "text only; no GPU or image validation", |
| "not served preset with temperature 1.65 or extended labels", |
| "state object key order preserved, not sorted as in jev-adapter", |
| "llama.cpp confidence is not adapter entropy confidence", |
| "API/probability parity fixtures, not task accuracy or calibration benchmark" |
| ], |
| "cold_prefill_comparison_note": { |
| "artifact": "StandardOne-3B-Q4_K_M.gguf", |
| "observed_noul_difference": 0.0012071734712206794, |
| "reason": "Native multi-question requests preserve prefix cache between questions; clearing the cache before every reference question changes CPU batching. The validated reference follows the same cache reuse order without modifying the runtime." |
| } |
| } |
|
|
