Add files using upload-large-folder tool

LICENSE

@@ -0,0 +1,201 @@
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright 2023-2024 SGLang Team
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

MERGE_REPORT.json

@@ -0,0 +1,59 @@
+{
+  "snapshot": "mistralai/Ministral-3-3B-Instruct-2512-BF16@b6d637bef2393152b3da2b2fde72eecdee30557e",
+  "adapter": "StandardOne-3B-LoRA (training run final2-3b-long)",
+  "adapter_sha256": "ae5fe8e3be2e9186e6504e48ad2fe7eea5537e7e8382a0cca2ca49d95605f7bf",
+  "changed_params": 182,
+  "unchanged_params": 276,
+  "changed_outside_lm_projections": [],
+  "max_abs_delta": 0.002506256103515625,
+  "changed_sample": [
+    [
+      "model.language_model.layers.0.self_attn.q_proj.weight",
+      0.001220703125
+    ],
+    [
+      "model.language_model.layers.0.self_attn.k_proj.weight",
+      0.0010986328125
+    ],
+    [
+      "model.language_model.layers.0.self_attn.v_proj.weight",
+      0.001373291015625
+    ],
+    [
+      "model.language_model.layers.0.self_attn.o_proj.weight",
+      0.000946044921875
+    ],
+    [
+      "model.language_model.layers.0.mlp.gate_proj.weight",
+      0.00200653076171875
+    ],
+    [
+      "model.language_model.layers.0.mlp.up_proj.weight",
+      0.0017948150634765625
+    ],
+    [
+      "model.language_model.layers.0.mlp.down_proj.weight",
+      0.0006103515625
+    ],
+    [
+      "model.language_model.layers.1.self_attn.q_proj.weight",
+      0.00146484375
+    ]
+  ],
+  "file_sha256": {
+    "SYSTEM_PROMPT.txt": "331b249682cd52226c50e533f59825184997f3b31f9a62b2c3fea940db6999c5",
+    "chat_template.jinja": "0701cfbdc2b7d44fdbad104dff604faee4b0543e8247624568777fe465746f9b",
+    "config.json": "c4446cf970fefd8b65f2e3119c30d406ea7026dca362dfb319c9aa08643131c3",
+    "generation_config.json": "e0923390059f84a9180b00e5501778acc45ea9856cd7f2fd68208b360927c677",
+    "model-00001-of-00002.safetensors": "6f675d7bb2615275072980b36a0d94356c4b866d829b1cfeb472294a0caabc57",
+    "model-00002-of-00002.safetensors": "edcb0fd56e0a5da4e6c057166ac812587636e8cda20fa23776d465d448f3dfdd",
+    "model.safetensors.index.json": "55641440eb385d8bc83a319b0a92f60fe45e6aef4b6992245eee91142237786e",
+    "params.json": "d15587c57574d2a92c66ff2552c6db5173d4c440ad47f7454dc51bad0ecaeb59",
+    "processor_config.json": "84be30b124b50749c56d25fdbec5ccedf564446f6b3b035e88e1e07b986d2491",
+    "special_tokens_map.json": "0a5c981e8c5c6f8886ee007a6d4543a0be6b221cb9ca32a8709384a4c6fc8cbb",
+    "tekken.json": "600bb27946565481ecf51ba8aee252e49b9a68507866080ac9c30185bb312843",
+    "tokenizer.json": "d5f6046775b112f0e2d456ee9dba450684ab964fe5c4e231599bdc6773028135",
+    "tokenizer_config.json": "de368f6e0f14b41c380b29278159f8897f01ec6b3208a91096d2deaf2bcd19f9"
+  },
+  "seconds": 64.6
+}

NOTICE

@@ -0,0 +1,28 @@
+Jev adapter
+
+The initial protocol helpers and their unit tests were extracted from the
+System One extension in the local sglang-jev experiment, commit
+633f05e (Add prefill-only System One decisions and visual game evaluation).
+That repository was based on SGLang upstream commit
+0024efa0de38794ee309ba10ab00ebb891a3d050.
+
+The accompanying Apache License 2.0 is retained from that source tree.
+SGLang project: https://github.com/sgl-project/sglang
+
+This repository is an independent API adapter. It does not include the
+proprietary Jev model and does not bundle the SGLang inference engine.
+
+Benchmark metric definitions and dataset schema mapping follow Jared Palmer's
+Apache-2.0 KEV project, pinned at 4f8110a3f8620cc3a182ae9a708e4398492c4b1a:
+https://github.com/jaredpalmer/kev
+The benchmark runner is an independent implementation. Downloaded benchmark
+data retains upstream provenance and any underlying dataset license terms.
+
+--
+
+This release (StandardOne-3B) merges a LoRA adapter into the following base model's
+weights, licensed under Apache License 2.0:
+
+  mistralai/Ministral-3-3B-Instruct-2512-BF16
+  revision b6d637bef2393152b3da2b2fde72eecdee30557e
+  https://huggingface.co/mistralai/Ministral-3-3B-Instruct-2512-BF16

QUICKSTART.md

@@ -0,0 +1,121 @@
+# Quickstart
+
+## Standard One 8B
+
+1. Clone the repository.
+
+```bash
+git clone https://huggingface.co/StandardThinking/StandardOne-8B
+```
+
+2. Create a venv and install SGLang.
+
+```bash
+uv venv --python 3.12 .venv-sglang
+uv pip install --python .venv-sglang/bin/python 'sglang==0.5.20'
+```
+
+3. Start the engine.
+
+```bash
+CUDA_VISIBLE_DEVICES=0 SGLANG_VLM_CACHE_SIZE_MB=0 .venv-sglang/bin/python -m sglang.launch_server \
+  --model-path ./StandardOne-8B --served-model-name standard-one-8b \
+  --host 127.0.0.1 --port 30000 --tp-size 1 --model-impl sglang --dtype bfloat16 \
+  --context-length 8192 --max-running-requests 32 --mem-fraction-static 0.8 \
+  --chunked-prefill-size -1 --disable-radix-cache --mm-preprocess-cache-size-mb 0 \
+  --model-config-parser hf --load-format safetensors
+```
+
+4. Create a venv and install the adapter.
+
+```bash
+uv venv --python 3.12 .venv-native
+pip install -e ./StandardOne-8B/server[native-tokenizer]
+```
+
+5. Start the adapter.
+
+```bash
+jev-adapter --engine-url http://127.0.0.1:30000 --model standard-one-8b --alias jev-latest \
+  --host 0.0.0.0 --port 30120 --max-concurrency 1 \
+  --tokenizer-model mistralai/Ministral-3-8B-Instruct-2512-BF16 \
+  --tokenizer-revision f6fae9795746f63c9be8344932f01275f3c63734 \
+  --prompt-wording native --native-system-prompt none --default-temperature 1.35
+```
+
+6. Check health and models.
+
+```bash
+curl -s http://127.0.0.1:30120/health
+curl -s http://127.0.0.1:30120/v1/models
+```
+
+7. Send a request.
+
+```bash
+curl -s http://127.0.0.1:30120/v1/systemone -X POST -H 'content-type: application/json' -d '{
+  "model": "jev-latest",
+  "state": "Policy: refunds require a receipt and purchase within 30 days. A customer bought 12 days ago but has no receipt. Issue a refund.",
+  "questions": {
+    "decision": {
+      "type": "noul",
+      "instructions": "Under the stated policy, is the requested action permitted? Treat unproved required conditions as not satisfied.",
+      "criteria": {"true": "Every required condition is established and no prohibition applies.", "false": "A condition is missing or a prohibition applies."}
+    }
+  }
+}'
+```
+
+8. Or run the packaged smoke script.
+
+```bash
+.venv-native/bin/python StandardOne-8B/server/examples/smoke.py --base-url http://127.0.0.1:30120 --request sample-request.json
+```
+
+9. Run JevBench against the endpoint.
+
+```bash
+python -m jevbench.cli run --tasks jevbench-easy,jevbench-original,jevbench-hard \
+  --adapter typesafe --endpoint http://127.0.0.1:30120 --model jev-latest
+```
+
+## Standard One 3B
+
+1. Clone the repository.
+
+```bash
+git clone https://huggingface.co/StandardThinking/StandardOne-3B
+```
+
+2. Start the engine (reuses the `.venv-sglang` venv from the 8B section).
+
+```bash
+.venv-sglang/bin/python -m sglang.launch_server \
+  --model-path ./StandardOne-3B --served-model-name standard-one-3b \
+  --host 127.0.0.1 --port 30000 --tp-size 1 --model-impl sglang --dtype bfloat16 \
+  --context-length 8192 --max-running-requests 32 --mem-fraction-static 0.8 \
+  --chunked-prefill-size -1 --disable-radix-cache --mm-preprocess-cache-size-mb 0 \
+  --model-config-parser hf --load-format safetensors
+```
+
+3. Start the adapter (reuses `.venv-native` and the `StandardOne-8B/server` install from the 8B section).
+
+```bash
+jev-adapter --engine-url http://127.0.0.1:30000 --model standard-one-3b --alias jev-latest \
+  --host 0.0.0.0 --port 30120 --max-concurrency 1 \
+  --tokenizer-model mistralai/Ministral-3-3B-Instruct-2512-BF16 \
+  --tokenizer-revision b6d637bef2393152b3da2b2fde72eecdee30557e \
+  --prompt-wording native --native-system-prompt none --default-temperature 1.45
+```
+
+4. Check health, then send a request or run JevBench exactly as in steps 6-9 above, with
+   `"model": "jev-latest"` unchanged (the adapter's alias, not the served checkpoint name).
+
+## Request format
+
+`POST /v1/systemone` takes a `state` (the scenario, as text, and for supported task families an image
+as a data URL) and a `questions` map. Each question has a `type` of `noul` (yes/no), `choice` (one of
+several labeled options) or `score` (an ordinal scale), plus `instructions` and `criteria` describing
+the labels. An optional `options.temperature` overrides the server's default softmax temperature for
+that request. The response carries one native probability distribution per question; `usage.output_tokens`
+is always `0`, and every probability vector sums to 1 over exactly the caller's label set.

README.md

@@ -0,0 +1,175 @@
+---
+base_model: mistralai/Ministral-3-3B-Instruct-2512-BF16
+library_name: transformers
+license: apache-2.0
+pipeline_tag: text-generation
+language:
+  - en
+  - ko
+tags:
+  - mistral3
+  - decision-model
+  - typed-decisions
+  - jev
+  - jevbench
+  - calibration
+  - decode-free
+---
+
+# Standard One 3B
+
+Decode-free typed decisions on Ministral 3 3B — one forward pass, native probabilities, ~23 ms per
+decision on one H200. The small sibling of `StandardThinking/StandardOne-8B`, same recipe and serving stack,
+roughly a third of the trainable parameters.
+
+![Standard One benchmark card: JevBench public tiers, held-out suites, stated-distribution probability, hard-tier calibration, latency and throughput for Standard One 8B, Standard One 3B and Jev 1.13.](docs/assets/00-benchmark-card.png)
+
+*Same items, same option order, raw probabilities unless noted; measured 24 September 2026.*
+
+## At a glance
+
+- Turns `mistralai/Ministral-3-3B-Instruct-2512` into a typed-decision endpoint: send a piece of state
+  and a bounded rubric (`choice`, `noul`, or `score`), get back a native probability distribution over
+  every option — no decoding, one forward pass, ≈23 ms per decision on one H200.
+- Clear accuracy gains over the untuned base on every suite measured, including the public hard tier
+  (a smaller margin than the other suites — see Limitations).
+- Probabilities are temperature-scaled and calibration-checked (hard-tier ECE, distribution
+  total-variation) — see Benchmarks below.
+- Languages: English and Korean. Image input is supported (the Pixtral vision tower is retained; images
+  are passed as data URLs).
+- Apache-2.0 throughout: base model, adapter and merged weights.
+
+## Quick start
+
+Engine and adapter are the same stack as `StandardThinking/StandardOne-8B` — the server code (`jev-adapter`)
+is not duplicated in this repository; it ships in `StandardThinking/StandardOne-8B/server/`.
+
+Engine (stock SGLang 0.5.20):
+
+```bash
+.venv-sglang/bin/python -m sglang.launch_server \
+  --model-path ./StandardOne-3B --served-model-name standard-one-3b \
+  --host 127.0.0.1 --port 30000 --tp-size 1 --model-impl sglang --dtype bfloat16 \
+  --context-length 8192 --max-running-requests 32 --mem-fraction-static 0.8 \
+  --chunked-prefill-size -1 --disable-radix-cache --mm-preprocess-cache-size-mb 0 \
+  --model-config-parser hf --load-format safetensors
+```
+
+Adapter:
+
+```bash
+jev-adapter --engine-url http://127.0.0.1:30000 --model standard-one-3b --alias jev-latest \
+  --host 0.0.0.0 --port 30120 --max-concurrency 1 \
+  --tokenizer-model mistralai/Ministral-3-3B-Instruct-2512-BF16 \
+  --tokenizer-revision b6d637bef2393152b3da2b2fde72eecdee30557e \
+  --prompt-wording native --native-system-prompt none --default-temperature 1.45
+```
+
+Try it:
+
+```bash
+curl -s http://127.0.0.1:30120/v1/systemone -X POST -H 'content-type: application/json' -d '{
+  "model": "jev-latest",
+  "state": "Policy: refunds require a receipt and purchase within 30 days. A customer bought 12 days ago but has no receipt. Issue a refund.",
+  "questions": {
+    "decision": {
+      "type": "noul",
+      "instructions": "Under the stated policy, is the requested action permitted? Treat unproved required conditions as not satisfied.",
+      "criteria": {"true": "Every required condition is established and no prohibition applies.", "false": "A condition is missing or a prohibition applies."}
+    }
+  }
+}'
+```
+
+Response shape (illustrative values, default temperature applied):
+
+```json
+{
+  "model": "standard-one-3b",
+  "answers": {"decision": {"type": "noul", "noul": 0.09}},
+  "usage": {"input_tokens": 96, "output_tokens": 0},
+  "metadata": {
+    "confidence_method": "1 - normalized_entropy",
+    "temperature": 1.45,
+    "evaluations": 1,
+    "adapter_elapsed_ms": 22.4
+  }
+}
+```
+
+More (client command, 8B variant, request format): see QUICKSTART.md.
+
+## Benchmarks
+
+**(a) Accuracy by suite** — untuned base, offline no-system-prompt runner, native wording, single option order, T=1; Jev 1.13 measured on the same items through its own endpoint, raw probabilities, no temperature applied.
+
+| Suite | Untuned base | **Standard One 3B** | Jev 1.13 |
+|---|---|---|---|
+| JevBench public easy (48) | 97.92 % | **100.00 %** | 100.00 % |
+| JevBench public standard (72) | 70.83 % | **91.67 %** | 98.61 % |
+| JevBench public hard (111) | 47.75 % | **49.55 %** | 72.07 % |
+| judge proxy (600: routing + answer adequacy) | 63.17 % | **85.67 %** | 90.50 % |
+| realistic transfer set (600) | 62.50 % | **88.67 %** | 86.67 % |
+| stated-distribution probability (1,036) | 31.56 % | **79.73 %** | 72.97 % |
+
+Standard One 3B gains over the untuned base on every suite; the margin is smallest on the public hard tier. Hard-tier ECE at served T is 0.158 (T=1.45) against Jev 1.13's 0.099 raw, and mean TV to the stated distributions is 0.135 at served T against Jev's 0.192 raw; full calibration table: [`docs/BENCHMARKS.md`](docs/BENCHMARKS.md#calibration).
+
+**(b) Speed** — serial, one H200, SGLang 0.5.20, 242-decision profile. Qwen checkpoints are untuned and shown for speed only; no accuracy comparison is implied.
+
+| Model | p50 | p95 | Input tokens/decision |
+|---|---|---|---|
+| **Standard One 3B** | **22.6 ms** | **33.2 ms** | ≈280 |
+| Standard One 8B | 25.8 ms | 41.9 ms | ≈280 |
+| Qwen3-8B (untuned) | 28.5 ms | 57.2 ms | 278 |
+| Qwen3.5-4B (untuned) | 48.8 ms | 72.7 ms | 283 |
+
+Throughput has been measured for the 8B only (see `docs/BENCHMARKS.md`); no equivalent sweep has been run for the 3B.
+
+On the public classification and decision suites (400 cases/suite, seed 13, served endpoints): AG News 83.8 %, typed decisions 69.3 %, MASSIVE intent mean 84.0 %, email spam 91.8 %, phishing 89.5 %. Full table, per-language and per-workflow breakdown: [`docs/BENCHMARKS.md`](docs/BENCHMARKS.md#public-classification-and-decision-suites) and [`docs/public-classification-suites.md`](docs/public-classification-suites.md).
+
+A JevBench v1.4.1 run has been requested; the sealed-set result is not yet available. Full report: [`docs/BENCHMARKS.md`](docs/BENCHMARKS.md).
+
+## Model details
+
+- **Base model:** `mistralai/Ministral-3-3B-Instruct-2512-BF16`, revision `b6d637bef2393152b3da2b2fde72eecdee30557e` (Apache-2.0).
+- **Adapter:** LoRA r=16, α=32, dropout 0, on `q_proj k_proj v_proj o_proj gate_proj up_proj down_proj` of the language-model projections only (vision tower and multimodal projector excluded), **24,707,072** trainable parameters, PEFT 0.21.0. Adapter file `adapter_model.safetensors`, 135,113,048 bytes, sha256 `ae5fe8e3be2e9186e6504e48ad2fe7eea5537e7e8382a0cca2ca49d95605f7bf`.
+- **Merged BF16 checkpoint:** merging the adapter into the base changed **182 tensors**, none of them outside the language-model projections, maximum absolute weight change **0.0025**.
+- **Serving details:** native chat-template wording, served without a system prompt, at a fixed temperature (T = 1.45, fitted on held-out calibration data); served model name `standard-one-3b` behind stock SGLang 0.5.20 via `jev-adapter` (`POST /v1/systemone`); single, caller-supplied option order, no rotation ensemble; 8,192-token context.
+
+| Path | Contents |
+|---|---|
+| `*.safetensors` | Merged BF16 checkpoint (base + LoRA) |
+| `config.json`, `tokenizer*`, `chat_template*`, `preprocessor*` | Base model's non-weight files |
+| `docs/`, `docs/public-classification-suites.md` | Benchmark report and figures (shared with `StandardOne-8B`); per-language/per-workflow numbers |
+| `SHA256SUMS`, `release-manifest.json`, `MERGE_REPORT.json`, `evidence/`, `LICENSE`, `README.md` | File hashes, training manifest, merge report, supporting artifacts, licence, this card |
+
+The server code and full quick-start guide live in `StandardThinking/StandardOne-8B`'s `server/` directory and `QUICKSTART.md`, and are reused for this checkpoint — see *Quick start* above for the 3B launch command.
+
+## Training data
+
+Trains on the identical mixture as `StandardThinking/StandardOne-8B` — same cohorts, same row counts, same overlap-audit result, not a reduced subset. Training data spans document and field normalisation, judge/routing and answer-adequacy, stated-distribution probability, verification/scoring, adequacy-rubric style, and game- and Tetris board-state cohorts — every row is synthetic and solver-generated (deterministic generators with visible checkers; a subset was teacher-reviewed), licensed Apache-2.0 (ours); the JevBench public tiers used only for evaluation carry MIT. Full per-cohort breakdown: [`docs/BENCHMARKS.md`](docs/BENCHMARKS.md#training-data-provenance).
+
+An exact-text overlap audit against the public JevBench tiers found 0 exact scenario matches and 181 exact instruction matches — rows in two adequacy-rubric cohorts whose entire instruction field, a generic 58-character adequacy question, is byte-identical to one public hard-tier instruction (0.05 % of the 359,497-row training mixture). These rows are kept and disclosed here rather than regenerated, since the overlap is limited to one rubric question's wording and never touches a scenario or an answer.
+
+## Limitations
+
+- **Public hard tier: 49 %** for Standard One 3B, against **48 %** for the untuned base and **72 %** for Jev
+  1.13 — Standard One's margin over the base here is much smaller than on the other suites.
+- Served probabilities are temperature-scaled by a single global value; if you apply this model to a
+  materially different question distribution, re-fitting that temperature is advisable rather than
+  assuming 1.45 transfers.
+- At most 26 options per question (one uppercase letter per option, `A`–`Z`).
+- The sealed JevBench set has not been measured for this model.
+- Served and offline probabilities can differ on identical prompts (mean total-variation ≈0.07 on the
+  hard tier); served numbers are treated as authoritative.
+- Korean is a small share of multilingual training alongside English.
+- Ten-way support triage (34–40 %) and RAG passage relevance (58–61 %) are weak zero-shot; fine-tune
+  for those.
+
+## Licence
+
+Adapter weights, merged weights and this card: Apache-2.0. Base model `mistralai/Ministral-3-3B-Instruct-2512` (and `-BF16`): Apache-2.0 per its Hugging Face model card, which adds that the model must not be used in a way that infringes, misappropriates, or otherwise violates any third party's rights. `jev-adapter` and SGLang: Apache-2.0. The JevBench harness and public tiers used for evaluation: MIT; other benchmark items keep their own upstream terms.
+
+## Citation
+
+`StandardThinking/StandardOne-3B` (merged weights) · `StandardThinking/StandardOne-3B-LoRA` (adapter + merge recipe).

SHA256SUMS

@@ -0,0 +1,88 @@
+1495e1e757ef4d0925a2350563cf5754bb23c51701a8ec4fb3c5cdcbedae6747  LICENSE
+db003518d8d78b50128bd0166322cd0c230c199b047a01ff646ea6529538f05a  MERGE_REPORT.json
+f10343f951ecb3a72670e12720de6e04d59e9cb872e84b2ceddd7d54853bb7e7  NOTICE
+cccd4aeea4b5ca9c98b6ea5f638e113175847526bfd748c488e417040ecb5930  QUICKSTART.md
+b93c7d30f112209abb6f2188084661dffb73179e1718c4de692610f57ddc7b67  README.md
+331b249682cd52226c50e533f59825184997f3b31f9a62b2c3fea940db6999c5  SYSTEM_PROMPT.txt
+0701cfbdc2b7d44fdbad104dff604faee4b0543e8247624568777fe465746f9b  chat_template.jinja
+c4446cf970fefd8b65f2e3119c30d406ea7026dca362dfb319c9aa08643131c3  config.json
+41f6da260cb61e59780081fa5b5e97c63413721889faf34f7ae4dc65fca7291b  docs/BENCHMARKS.md
+2300b40c53e596eacf93f152375c68bd78a9d77c03fb15ec17e5a76ca9f632bd  docs/assets/00-benchmark-card.png
+4d36cb45e5445a672de2dcbe0cfb273226354af8c842ca8d0925698033efe1a3  docs/assets/00-benchmark-card.svg
+7313f2b0645ee401d082572986dcee0c08d3daed1ae79fa650cfa685570ad7e2  docs/assets/02-latency-vs-qwen.png
+0b48a0dd4b4c9ac7c1842836940348edc2e1ba697184943f0c84c3e3d3c7f2d0  docs/assets/02-latency-vs-qwen.svg
+5f430f8841e393548f080833524840adb93deef1c6f75dca93d0571262a2ac68  docs/assets/04-throughput.png
+dd888be78c0e32894cf04a8e3fbf8757e7218873668fa4443ba47c7c1c48b6dc  docs/assets/04-throughput.svg
+f3211f7d1cbc624b0f558e7f9e7edac3ea51a381287cb7409719a12d3f11edaa  docs/assets/05-gain-over-base.png
+855f5e1d0871d89f5e92dbf8087282d787f9bf7c7c6a2b25b936cffd33f6c219  docs/assets/05-gain-over-base.svg
+4d41f1b782502403109c4074e7705fc42f18dfb9fc865b5aca5771f4c4991264  docs/assets/06-vs-jev.png
+946703a63a228ae328dc713186a6b16b682b193fcfbc6303e17fa143cbd58ddb  docs/assets/06-vs-jev.svg
+c3eacfddd6755a60b741d8929f3a41e84d2a862629dce7458ca96d248b4378fd  docs/public-classification-suites.md
+5da7c598decc1df7d385f835c378e4a1e14aaf8d54c25b84834f0649cd11f713  evidence/calibration-leaderboard-ablate-nosys.json
+b8c5441a673229a58223214f65b5150b67e659947dfd6edc6216df793c79975f  evidence/calibration-leaderboard-merged-native.json
+c887ca6b3d73571207c4f39a619c409685b217a4207b8d0486219650aafb5a71  evidence/served-nosys-easy-report.json
+08ef8e3a16bc49c865d44961d026d282df5647b4e3d3d023baa900ac76272636  evidence/served-nosys-hard-report.json
+7b9b1ee2f93166a817ead61bbc0f4c62984afe7d6c8affefd051007582d0a485  evidence/served-nosys-latency-idle-gpu.json
+22b8e9b0ae331fb195271ae434083059041131ab3b317c6b56f6d7e93593001c  evidence/served-nosys-original-report.json
+e0923390059f84a9180b00e5501778acc45ea9856cd7f2fd68208b360927c677  generation_config.json
+6f675d7bb2615275072980b36a0d94356c4b866d829b1cfeb472294a0caabc57  model-00001-of-00002.safetensors
+edcb0fd56e0a5da4e6c057166ac812587636e8cda20fa23776d465d448f3dfdd  model-00002-of-00002.safetensors
+55641440eb385d8bc83a319b0a92f60fe45e6aef4b6992245eee91142237786e  model.safetensors.index.json
+d15587c57574d2a92c66ff2552c6db5173d4c440ad47f7454dc51bad0ecaeb59  params.json
+ece2373c2ae391bce18785a1810543bc6173a1f28b6767cffab2c35dbea5f002  processor_config.json
+42352f35f7e9424e228f437943b78d67c007cf458613d9076371fb42d9d38a4f  release-manifest.json
+eef2a88a000d2ec294379755770c147327cb79d3d7278c711899bd36797d7a02  server/.gitignore
+1495e1e757ef4d0925a2350563cf5754bb23c51701a8ec4fb3c5cdcbedae6747  server/LICENSE
+0f1f023b916d38d7c12af3a71e09f5e82b819a7e156e4992118bb160ee63ede1  server/NOTICE
+56c89049984cec6fa66a5bea6b905c63aa46fbd627ba12bbfabac1a6b466d321  server/README.md
+88e4dea8cc92f467a03bdd45a5fb968e402517116b09e24edcea85bb15f121e5  server/benchmarks/DECISION_BENCHMARK_SELECTION.md
+7fea6b9dc44e34ce448f1d80a9f2d273b76ab9b0ac1859c032fa4faa4f458501  server/benchmarks/METRIC_VALIDATION.md
+bb88f12ef294680e389147d781bf1f8680ea0a5ca9ed554730d8ea29f1c25eb9  server/benchmarks/PUBLIC_DATASETS.md
+e844e0de19ef528bc28ec09339072e1eab4df05bfc4ec56a220cfc7528d2d481  server/benchmarks/README.md
+3e5beed774bb0bcbfb2fcf24ba9554212c6ae112937d308040989465ab0c5784  server/benchmarks/data/jevbench-easy/LICENSE
+396422e29055bba4073f4a9e5a7163cc724f13e970a34ef58e8ea1fb439b9b38  server/benchmarks/data/jevbench-easy/THIRD-PARTY.md
+e186c6dd90b3a7a177ef15f2fd3673d7bbfce95340bfdc6062dc15bfa0e01a6a  server/benchmarks/data/jevbench-easy/public.jsonl
+d43b510898b867e3633ac9a50940f2cf237c114241c3d8e4f26437ff75b64da7  server/benchmarks/data/jevbench-easy/public.manifest.json
+3e5beed774bb0bcbfb2fcf24ba9554212c6ae112937d308040989465ab0c5784  server/benchmarks/data/jevbench-hard/LICENSE
+396422e29055bba4073f4a9e5a7163cc724f13e970a34ef58e8ea1fb439b9b38  server/benchmarks/data/jevbench-hard/THIRD-PARTY.md
+6c93c82edf10d08b70ee0c3addcf7c1ea8fd19957e94fb8021969ff4c0027f95  server/benchmarks/data/jevbench-hard/public.jsonl
+6a1589498ef3ac71798ea8681fbb6a42d8c0a3a329a972818f71f246d1f88909  server/benchmarks/data/jevbench-hard/public.manifest.json
+3e5beed774bb0bcbfb2fcf24ba9554212c6ae112937d308040989465ab0c5784  server/benchmarks/data/jevbench-original/LICENSE
+396422e29055bba4073f4a9e5a7163cc724f13e970a34ef58e8ea1fb439b9b38  server/benchmarks/data/jevbench-original/THIRD-PARTY.md
+ad453e78624b1c150d5eb3f0f47c5c14caa79542329b4f7e1bcc6199e2eb7007  server/benchmarks/data/jevbench-original/public.jsonl
+89fe50f5cc0654b37f81833ed91f171cdbd0d4c1da3622ac4736949fbea23e95  server/benchmarks/data/jevbench-original/public.manifest.json
+5434d64a4bf45a6693287b72edd9d823a907b639a8c3a18fd40cc744073c144b  server/benchmarks/run_matrix.py
+fee797d1a2ff128a5d61dd436e8252d63bc726b8ce4ba72641a1ffd9196135a6  server/benchmarks/verify_metrics.py
+2544280dd3e669985bf57dfc61c2212bcf16d27230a79cea76f9f4e822476429  server/examples/request.json
+7a14fe0f5bf73aca7a4930423c310c0a3b1f9b546d07abc8d3fd5d9f38c9a5d7  server/examples/smoke.py
+e589f268969c87f82ddd2c81c0432c6335c08a9f9f2537ce68ddd5708f1e6f9e  server/jev_adapter/__init__.py
+c79719983fcf12ad2baa020acb1be00e3b1e4d4b66e7dcaa0a8022f140975f6a  server/jev_adapter/__main__.py
+e2f84bb7867934533f8f2a87034cad76af695ca1bbdbf096c545fe60b790c6fa  server/jev_adapter/backend.py
+2cd67e71c1a601fc9333c266d909197031b7c0dc746b9a2d5475a5fb0c085e95  server/jev_adapter/benchmarks/__init__.py
+44104bc2d6431c05345cd697dd2c377cc6ba9fd57f2177df7b548490cf5f53fc  server/jev_adapter/benchmarks/compare.py
+a504e8051886086934751bffc948e588ebe0ad92c613327014b627be454d3e8c  server/jev_adapter/benchmarks/data.py
+a66e6ec289b4c32f79ac990261650870be6b923e47753b9c89ff9e58fe2f982f  server/jev_adapter/benchmarks/jevbench.py
+1b78c52f723cb3a3e614d1be4ca531e7595e0a6690e704d6f1bb3045dd948bf0  server/jev_adapter/benchmarks/metrics.py
+9c51ce87269c587f1353b9dd32d886f9f0a51613a996721a7bb81695af445c02  server/jev_adapter/benchmarks/prepare.py
+0fef99cd01164c5548de530c9ab93255e2a95d17757c12e8e3d23016afd1a03c  server/jev_adapter/benchmarks/run.py
+86e113a3ee9d15d9be93ba5e964d5f5d51171619e7856cc7dde812c6c90f4a37  server/jev_adapter/native_tokenizer.py
+ad7dee34b9c15e83ce66b3b8b07266e0b8ce81da7c8ba692a8293dbf078d0e44  server/jev_adapter/protocol.py
+3eb21e967cbc8dd7dec2481553fce6974be66276d196eb791f0292bc95c2c8f0  server/jev_adapter/server.py
+bd5e5a13fb264c1e528132c3d33bb2b85c31bc45bb0d025d8151ea47216dbff4  server/jev_adapter/service.py
+8b025570f9678aae6717c817a94e807c21a2c0fe1eb82708a611d3f67be8c3e9  server/jev_adapter/sglang.py
+159eb400efb830c762a36f81e51ba79888570284ffa70326301adfeee50b5bd3  server/pyproject.toml
+f5bf6e9b916ff0d24a2551b4f9f24ed353e7343f5be343b141d38c4c04bd1101  server/tests/test_benchmark_data.py
+c197a90d57c7516eb615f7f0626f4c3e95b2fd0773d74e8f32601251804fca66  server/tests/test_benchmark_matrix.py
+56a48c36d322d7131978809397da79d0f9abbb6801ab6176d27ed2f9dfb26063  server/tests/test_benchmark_run.py
+c492b41c07050d9b76c6c9d7d21bbfcdac6ec137346cf995df3f624cb1e345c8  server/tests/test_disconnect_cleanup.py
+83769ec3f35f5034787ffa8160a5ea7c9c7e23200b303d8b908500d71e3fe13f  server/tests/test_http_integration.py
+746cc73422976ec22b460a74a7c70d937c0ba4fd9fb7151fa6745641dbca5616  server/tests/test_jevbench.py
+ade6b76bacf23ae244d9790312f297a36557d6cd8075b81ce36e928359b742da  server/tests/test_main.py
+83084f098020374a4abfa3d4402e74e357d45170ca840843e58dba13162e9986  server/tests/test_native_tokenizer.py
+0114da8966dafa951308ed58563053918b2b2f7b3393dd483c7183e3374b444d  server/tests/test_protocol.py
+200c35bf53fd869132a69cb8adfd32a1f8e7235d0a7194b19675e46ac39c936b  server/tests/test_service.py
+50af7340ac40f3179403b3c53779f673c4ff1be888a015c2f8852731b5f69f0b  server/tests/test_sglang.py
+0a5c981e8c5c6f8886ee007a6d4543a0be6b221cb9ca32a8709384a4c6fc8cbb  special_tokens_map.json
+600bb27946565481ecf51ba8aee252e49b9a68507866080ac9c30185bb312843  tekken.json
+d5f6046775b112f0e2d456ee9dba450684ab964fe5c4e231599bdc6773028135  tokenizer.json
+f59f7294e4f26383d0ea93840fe21cf197784be0842a8301a0343e8c34ed0d6d  tokenizer_config.json

SYSTEM_PROMPT.txt

@@ -0,0 +1,29 @@
+You are Ministral-3-3B-Instruct-2512, a Large Language Model (LLM) created by Mistral AI, a French startup headquartered in Paris.
+You power an AI assistant called Le Chat.
+Your knowledge base was last updated on 2023-10-01.
+The current date is {today}.
+
+When you're not sure about some information or when the user's request requires up-to-date or specific data, you must use the available tools to fetch the information. Do not hesitate to use tools whenever they can provide a more accurate or complete response. If no relevant tools are available, then clearly state that you don't have the information and avoid making up anything.
+If the user's question is not clear, ambiguous, or does not provide enough context for you to accurately answer the question, you do not try to answer it right away and you rather ask the user to clarify their request (e.g. "What are some good restaurants around me?" => "Where are you?" or "When is the next flight to Tokyo" => "Where do you travel from?").
+You are always very attentive to dates, in particular you try to resolve dates (e.g. "yesterday" is {yesterday}) and when asked about information at specific dates, you discard information that is at another date.
+You follow these instructions in all languages, and always respond to the user in the language they use or request.
+Next sections describe the capabilities that you have.
+
+# WEB BROWSING INSTRUCTIONS
+
+You cannot perform any web search or access internet to open URLs, links etc. If it seems like the user is expecting you to do so, you clarify the situation and ask the user to copy paste the text directly in the chat.
+
+# MULTI-MODAL INSTRUCTIONS
+
+You have the ability to read images, but you cannot generate images. You also cannot transcribe audio files or videos.
+You cannot read nor transcribe audio files or videos.
+
+# TOOL CALLING INSTRUCTIONS
+
+You may have access to tools that you can use to fetch information or perform actions. You must use these tools in the following situations:
+
+1. When the request requires up-to-date information.
+2. When the request requires specific data that you do not have in your knowledge base.
+3. When the request involves actions that you cannot perform without tools.
+
+Always prioritize using tools to provide the most accurate and helpful response. If tools are not available, inform the user that you cannot perform the requested action at the moment.

chat_template.jinja

@@ -0,0 +1,216 @@
+{#- Default system message if no system prompt is passed. #}
+{%- set default_system_message = 'You are Ministral-3-3B-Instruct-2512, a Large Language Model (LLM) created by Mistral AI, a French startup headquartered in Paris.
+You power an AI assistant called Le Chat.
+Your knowledge base was last updated on 2023-10-01.
+The current date is {today}.
+
+When you\'re not sure about some information or when the user\'s request requires up-to-date or specific data, you must use the available tools to fetch the information. Do not hesitate to use tools whenever they can provide a more accurate or complete response. If no relevant tools are available, then clearly state that you don\'t have the information and avoid making up anything.
+If the user\'s question is not clear, ambiguous, or does not provide enough context for you to accurately answer the question, you do not try to answer it right away and you rather ask the user to clarify their request (e.g. "What are some good restaurants around me?" => "Where are you?" or "When is the next flight to Tokyo" => "Where do you travel from?").
+You are always very attentive to dates, in particular you try to resolve dates (e.g. "yesterday" is {yesterday}) and when asked about information at specific dates, you discard information that is at another date.
+You follow these instructions in all languages, and always respond to the user in the language they use or request.
+Next sections describe the capabilities that you have.
+
+# WEB BROWSING INSTRUCTIONS
+
+You cannot perform any web search or access internet to open URLs, links etc. If it seems like the user is expecting you to do so, you clarify the situation and ask the user to copy paste the text directly in the chat.
+
+# MULTI-MODAL INSTRUCTIONS
+
+You have the ability to read images, but you cannot generate images. You also cannot read nor transcribe audio files or videos.
+
+# TOOL CALLING INSTRUCTIONS
+
+You may have access to tools that you can use to fetch information or perform actions. You must use these tools in the following situations:
+
+1. When the request requires up-to-date information.
+2. When the request requires specific data that you do not have in your knowledge base.
+3. When the request involves actions that you cannot perform without tools.
+
+Always prioritize using tools to provide the most accurate and helpful response. If tools are not available, inform the user that you cannot perform the requested action at the moment.' %}
+
+{#- Begin of sequence token. #}
+{{- bos_token }}
+
+
+{#- Handle system prompt if it exists. #}
+{%- set loop_messages = messages %}
+{%- if messages[0]['role'] != 'system' and default_system_message != '' %}
+    {{- '[SYSTEM_PROMPT]' + default_system_message + '[/SYSTEM_PROMPT]' }}
+{%- endif %}
+
+
+{#- Tools definition #}
+{%- set available_tools = '' %}
+{%- set has_tools = false %}
+{%- if tools is defined and tools is not none and tools|length > 0 %}
+    {%- set has_tools = true %}
+    {%- set available_tools = '[AVAILABLE_TOOLS]' + (tools| tojson) + '[/AVAILABLE_TOOLS]' %}
+{%- endif %}
+
+{#- Macros #}
+{%- macro render_content(content, context_name, supported_types_desc, support_images) -%}
+    {%- if content is string -%}
+        {{- content -}}
+    {%- elif content -%}
+        {%- for block in content -%}
+            {%- if block['type'] == 'text' -%}
+                {{- block['text'] -}}
+            {%- elif support_images and block['type'] in ['image', 'image_url'] -%}
+                {{- '[IMG]' -}}
+            {%- else -%}
+                {{- raise_exception('Only ' + supported_types_desc + ' chunks are supported in ' + context_name + '.') -}}
+            {%- endif -%}
+        {%- endfor -%}
+    {%- else -%}
+        {{- raise_exception(context_name + ' must have non-empty content.') -}}
+    {%- endif -%}
+{%- endmacro -%}
+
+{#- Aggregate consecutive messages with the same role except system and tool. #}
+{#- A sentinel message is appended so the last group gets flushed inside the loop. #}
+{%- set ns_agg = namespace(messages=[], current_group=[], current_role=none) %}
+{%- for message in loop_messages + [{'role': '__sentinel__'}] %}
+    {%- if message['role'] != ns_agg.current_role or message['role'] == 'system' or message['role'] == 'tool' %}
+        {%- if ns_agg.current_role == 'tool' %}
+            {%- set ns_agg.messages = ns_agg.messages + ns_agg.current_group %}
+        {%- elif ns_agg.current_role is not none %}
+            {%- set ns_c = namespace(text_parts=[], chunks=[], has_non_text=false, tool_calls=[]) %}
+            {%- for msg in ns_agg.current_group %}
+                {%- if msg['content'] is string %}
+                    {%- set ns_c.text_parts = ns_c.text_parts + [msg['content']] %}
+                {%- elif msg['content'] is not none %}
+                    {%- for block in msg['content'] %}
+                        {%- if block['type'] == 'text' %}
+                            {%- set ns_c.text_parts = ns_c.text_parts + [block['text']] %}
+                        {%- else %}
+                            {%- if ns_c.text_parts | length > 0 %}
+                                {%- set ns_c.chunks = ns_c.chunks + [{'type': 'text', 'text': ns_c.text_parts | join('\n\n')}] %}
+                                {%- set ns_c.text_parts = [] %}
+                            {%- endif %}
+                            {%- set ns_c.chunks = ns_c.chunks + [block] %}
+                            {%- set ns_c.has_non_text = true %}
+                        {%- endif %}
+                    {%- endfor %}
+                {%- endif %}
+                {%- if msg['tool_calls'] is defined and msg['tool_calls'] is not none %}
+                    {%- set ns_c.tool_calls = ns_c.tool_calls + msg['tool_calls'] | list %}
+                {%- endif %}
+            {%- endfor %}
+            {%- if ns_c.has_non_text %}
+                {%- if ns_c.text_parts | length > 0 %}
+                    {%- set ns_c.chunks = ns_c.chunks + [{'type': 'text', 'text': ns_c.text_parts | join('\n\n')}] %}
+                {%- endif %}
+                {%- set merged_content = ns_c.chunks %}
+            {%- else %}
+                {%- set merged_content = ns_c.text_parts | join('\n\n') %}
+            {%- endif %}
+            {%- if ns_c.tool_calls | length > 0 %}
+                {%- set ns_agg.messages = ns_agg.messages + [{'role': ns_agg.current_role, 'content': merged_content, 'tool_calls': ns_c.tool_calls}] %}
+            {%- else %}
+                {%- set ns_agg.messages = ns_agg.messages + [{'role': ns_agg.current_role, 'content': merged_content}] %}
+            {%- endif %}
+        {%- endif %}
+        {%- if message['role'] != '__sentinel__' %}
+            {%- set ns_agg.current_group = [message] %}
+            {%- set ns_agg.current_role = message['role'] %}
+        {%- endif %}
+    {%- else %}
+        {%- set ns_agg.current_group = ns_agg.current_group + [message] %}
+    {%- endif %}
+{%- endfor %}
+{%- set loop_messages = ns_agg.messages %}
+
+{#- Validates message ordering. #}
+{%- set ns = namespace(available_tools_emitted=false) %}
+{%- if loop_messages | length > 0 and loop_messages[0]['role'] not in ['user', 'system'] %}
+    {{- raise_exception('Conversation must start with a user or system message, got ' + loop_messages[0]['role'] + '.') }}
+{%- endif %}
+{%- set ns_order = namespace(previous_role=none) %}
+{%- for message in loop_messages %}
+    {%- set current_role = message['role'] %}
+    {%- if ns_order.previous_role is not none %}
+        {%- if ns_order.previous_role == 'system' %}
+            {%- if current_role not in ['user', 'assistant', 'system'] %}
+                {{- raise_exception('Unexpected role \'' + current_role + '\' after role \'' + ns_order.previous_role + '\'') }}
+            {%- endif %}
+        {%- elif ns_order.previous_role == 'user' %}
+            {%- if current_role not in ['assistant', 'system', 'user'] %}
+                {{- raise_exception('Unexpected role \'' + current_role + '\' after role \'' + ns_order.previous_role + '\'') }}
+            {%- endif %}
+        {%- elif ns_order.previous_role == 'assistant' %}
+            {%- if current_role not in ['assistant', 'user', 'tool'] %}
+                {{- raise_exception('Unexpected role \'' + current_role + '\' after role \'' + ns_order.previous_role + '\'') }}
+            {%- endif %}
+        {%- elif ns_order.previous_role == 'tool' %}
+            {%- if current_role not in ['assistant', 'tool', 'user'] %}
+                {{- raise_exception('Unexpected role \'' + current_role + '\' after role \'' + ns_order.previous_role + '\'') }}
+            {%- endif %}
+        {%- endif %}
+    {%- endif %}
+    {%- set ns_order.previous_role = current_role %}
+{%- endfor %}
+
+{#- Handle conversation messages. #}
+{%- for message in loop_messages %}
+    {#- User messages supports text, image and image_url content. #}
+    {%- if message['role'] == 'user' %}
+        {%- if not ns.available_tools_emitted %}
+            {{- available_tools }}
+            {%- set ns.available_tools_emitted = true %}
+        {%- endif %}
+        {%- if message['content'] is not string and message['content'] %}
+            {#- When content has exactly one image and one text block, put image first. #}
+            {%- if message['content'] | length == 2 and message['content'][0]['type'] == 'text' and message['content'][1]['type'] in ['image', 'image_url'] %}
+                {%- set blocks = [message['content'][1], message['content'][0]] %}
+            {%- else %}
+                {%- set blocks = message['content'] %}
+            {%- endif %}
+            {%- set user_content = blocks %}
+        {%- else %}
+            {%- set user_content = message['content'] %}
+        {%- endif %}
+        {{- '[INST]' -}}
+        {{- render_content(user_content, 'user message content', supported_types_desc='text, image and image_url', support_images=true) -}}
+        {{- '[/INST]' }}
+
+    {#- Assistant messages supports text content. #}
+    {%- elif message['role'] == 'assistant' %}
+        {%- if (message['content'] is none or message['content'] == '' or message['content']|length == 0) and (message['tool_calls'] is not defined or message['tool_calls'] is none or message['tool_calls']|length == 0) %}
+            {{- raise_exception('Assistant message must have a string or a list of chunks in content or a list of tool calls.') }}
+        {%- endif %}
+
+        {%- if message['content'] %}
+            {{- render_content(message['content'], 'assistant message contents', supported_types_desc='text', support_images=false) -}}
+        {%- endif %}
+
+        {%- if message['tool_calls'] is defined and message['tool_calls'] is not none and message['tool_calls']|length > 0 %}
+            {%- for tool in message['tool_calls'] %}
+                {{- '[TOOL_CALLS]' }}
+                {%- set name = tool['function']['name'] %}
+                {%- set arguments = tool['function']['arguments'] %}
+                {%- if arguments is not string %}
+                    {%- set arguments = arguments|tojson|safe %}
+                {%- elif arguments == '' %}
+                    {%- set arguments = '{}' %}
+                {%- endif %}
+                {{- name + '[ARGS]' + arguments }}
+            {%- endfor %}
+        {%- endif %}
+
+        {{- eos_token }}
+
+    {#- Tool messages only supports text content. #}
+    {%- elif message['role'] == 'tool' %}
+        {{- '[TOOL_RESULTS]' + message['content']|string + '[/TOOL_RESULTS]' }}
+
+    {#- System messages. #}
+    {%- elif message['role'] == 'system' %}
+        {{- '[SYSTEM_PROMPT]' -}}
+        {{- render_content(message['content'], 'system message contents', supported_types_desc='text', support_images=false) -}}
+        {{- '[/SYSTEM_PROMPT]' -}}
+
+    {#- Raise exception for unsupported roles. #}
+    {%- else %}
+        {{- raise_exception('Only user, assistant, system and tool roles are supported, got ' + message['role'] + '.') }}
+    {%- endif %}
+{%- endfor %}

config.json

@@ -0,0 +1,67 @@
+{
+  "architectures": [
+    "Mistral3ForConditionalGeneration"
+  ],
+  "dtype": "bfloat16",
+  "image_token_index": 10,
+  "model_type": "mistral3",
+  "multimodal_projector_bias": false,
+  "projector_hidden_act": "gelu",
+  "spatial_merge_size": 2,
+  "text_config": {
+    "attention_dropout": 0.0,
+    "bos_token_id": 1,
+    "dtype": "bfloat16",
+    "eos_token_id": 2,
+    "head_dim": 128,
+    "hidden_act": "silu",
+    "hidden_size": 3072,
+    "initializer_range": 0.02,
+    "intermediate_size": 9216,
+    "max_position_embeddings": 262144,
+    "model_type": "ministral3",
+    "num_attention_heads": 32,
+    "num_hidden_layers": 26,
+    "num_key_value_heads": 8,
+    "pad_token_id": 11,
+    "rms_norm_eps": 1e-05,
+    "rope_parameters": {
+      "beta_fast": 32.0,
+      "beta_slow": 1.0,
+      "factor": 16.0,
+      "llama_4_scaling_beta": 0.1,
+      "mscale": 1.0,
+      "mscale_all_dim": 1.0,
+      "original_max_position_embeddings": 16384,
+      "rope_theta": 1000000.0,
+      "rope_type": "yarn",
+      "type": "yarn"
+    },
+    "sliding_window": null,
+    "tie_word_embeddings": true,
+    "use_cache": true,
+    "vocab_size": 131072
+  },
+  "tie_word_embeddings": true,
+  "transformers_version": "5.12.1",
+  "vision_config": {
+    "attention_dropout": 0.0,
+    "dtype": "bfloat16",
+    "head_dim": 64,
+    "hidden_act": "silu",
+    "hidden_size": 1024,
+    "image_size": 1540,
+    "initializer_range": 0.02,
+    "intermediate_size": 4096,
+    "model_type": "pixtral",
+    "num_attention_heads": 16,
+    "num_channels": 3,
+    "num_hidden_layers": 24,
+    "patch_size": 14,
+    "rope_parameters": {
+      "rope_theta": 10000.0,
+      "rope_type": "default"
+    }
+  },
+  "vision_feature_layer": -1
+}

docs/BENCHMARKS.md

@@ -0,0 +1,356 @@
+# Standard One — full benchmark report
+
+This report covers both released systems, **Standard One 8B** (`StandardThinking/StandardOne-8B` / `-LoRA`) and
+**Standard One 3B** (`StandardThinking/StandardOne-3B` / `-LoRA`). It is our own measurement, not an official JevBench
+leaderboard score. The only comparisons in this report are: the untuned
+base model (accuracy, see § Decision quality vs. the untuned base), Qwen-based models served on the
+same H200 stack (speed only, untuned, no accuracy comparison implied), and one authorized peer
+comparison against TypeSafe Jev 1.13 (see § Measured against Jev 1.13).
+
+## Contents
+
+- [Suite name key](#suite-name-key)
+- [Scope](#scope)
+- [Measurement paths: offline HF runner vs. served endpoint](#measurement-paths-offline-hf-runner-vs-served-endpoint)
+- [Text suites](#text-suites)
+- [Served parity](#served-parity)
+- [Public classification and decision suites](#public-classification-and-decision-suites)
+- [Latency and throughput](#latency-and-throughput)
+- [Measured against Jev 1.13](#measured-against-jev-113)
+- [Calibration](#calibration)
+- [Decision quality vs. the untuned base](#decision-quality-vs-the-untuned-base)
+- [Data provenance and licences](#data-provenance-and-licences)
+- [Training data provenance](#training-data-provenance)
+- [Public-overlap audit and disclosure](#public-overlap-audit-and-disclosure)
+- [Evidence file hashes](#evidence-file-hashes)
+
+## Suite name key
+
+The model cards use readable names for four internal evaluation suites; this report keeps the original
+identifiers so figures trace back to `release-manifest.json` and `evidence/`. Mapping: `judge-proxy-v1`
+= judge proxy (600 questions: routing + answer adequacy); `internal-eval-v2-realistic` = realistic
+transfer set (600); `distprob-val-v2` = stated-distribution probability (1,036);
+`internal-eval-hardproxy-v1` = hard proxy (600). JevBench's own public-tier names (easy / standard /
+hard) are used as-is in both places.
+
+## Scope
+
+- Both candidates: merged BF16 weights, LoRA r=16/α=32 on `q/k/v/o/gate/up/down` of the language model
+  only (vision tower + projector excluded), same three-stage training recipe on the **identical final
+  training mixture** (29 cohorts, 359,497 rows) — see `MODEL_CARD_8B.md` / `MODEL_CARD_3B.md` §
+  *Training* for the recipe and per-cohort breakdown, and `release-manifest.json` for the exact
+  per-cohort row counts and hashes. The 3B is a smaller base/adapter pair, not a reduced-data variant.
+  **Image input is supported** (the Pixtral vision tower and projector are retained; images are passed
+  as data URLs), and image-side data was included in the final training stage — this report does not
+  detail image or content-moderation training or results further.
+- Release serving configuration for both: native wording, **no system prompt**
+  (`--prompt-wording native --native-system-prompt none`), single option order, server-side default
+  temperature **T = 1.35 (8B)** / **T = 1.45 (3B)** — both refit directly on served probabilities
+  (see § Calibration). See `QUICKSTART.md` for the exact commands.
+- What is **not** covered here: any official JevBench sealed-set score (we have none yet), any additional peer-system comparison beyond Jev 1.13, and image/content-moderation
+  training detail beyond the one sentence above.
+
+## Measurement paths: offline HF runner vs. served endpoint
+
+Two different measurement paths appear throughout this report and are not interchangeable:
+
+1. **Offline HF runner**: `transformers`-based scoring harness, computes label-logit probabilities
+   directly from a loaded checkpoint (adapter or merged), no network serving stack involved.
+2. **Served endpoint**: stock SGLang 0.5.20 + `jev-adapter`, `POST /v1/systemone`, the same path an
+   external caller would use — see `QUICKSTART.md` for how to reproduce this endpoint.
+
+**These two paths do not agree exactly.** Served probabilities differ measurably from offline-runner
+probabilities for both candidates: mean total-variation distance ≈0.07 on the hard tier, with 8–12 of
+111 argmax label changes, even though the rendered prompts were verified byte-identical between the two
+paths. **Treat served numbers as authoritative and offline numbers as indicative.**
+
+Temperature is **final and served-path-fit** for both candidates: T = 1.35 (8B) and T = 1.45 (3B) were
+refit directly on served probabilities from the running endpoint, on
+`internal-eval-hardproxy-calib-v1` + `distprob-calib-v2`, no system prompt. This supersedes an earlier
+offline-harness fit (T = 1.5 / T = 1.3).
+
+Most of the offline text-suite table below reflects the **with-system-prompt** configuration, while the
+release ships **without** the system prompt. Where we independently re-measured the no-system-prompt
+configuration (hard, original, judge-proxy-v1, internal-eval-v2-realistic, distprob-val-v2, easy), both
+columns are shown.
+
+## Text suites
+
+### Standard One 8B
+
+| Suite | n | With system prompt | No system prompt (shipped) | Notes |
+|---|---|---|---|---|
+| JevBench public easy | 48 | 100.00 % | **100.00 %** | |
+| JevBench public original | 72 | 98.61 % | **97.22 %** | |
+| JevBench public hard | 111 | 56.76 % | **54.95 %** | never used for training/T-fitting/selection |
+| judge-proxy-v1 (routing + adequacy) | 600 | 89.33 % | **88.67 %** | leaderboard judge text not public; proxy only |
+| internal-eval-v2-realistic | 600 | 88.83 % | **88.83 %** | unchanged with/without system prompt |
+| distprob-val-v2 | 1,036 | 81.27 % | **81.08 %** | |
+| internal-eval-v1 | 600 | 78.50 % | not separately re-measured | |
+| internal-eval-hardproxy-v1 | 600 | 48.67 % | not separately re-measured | hard-tier proxy |
+| games-harness-eval-v1 | 360 | 97.78 % | not separately re-measured | othello/chess/tic-tac-toe/snake/graph, text |
+| tetris-drought-eval-v1 (text) | 120 | 67.50 % | not separately re-measured | |
+
+### Standard One 3B
+
+| Suite | n | With system prompt | No system prompt (shipped) | Notes |
+|---|---|---|---|---|
+| JevBench public easy | 48 | 100.00 % | **100.00 %** | |
+| JevBench public original | 72 | 90.28 % | **91.67 %** | one of the few rows where dropping the prompt *helps* |
+| JevBench public hard | 111 | 52.25 % | **49.55 %** | never used for training/T-fitting/selection |
+| judge-proxy-v1 (routing + adequacy) | 600 | 86.83 % | **85.67 %** | |
+| internal-eval-v2-realistic | 600 | 88.83 % | **88.67 %** | |
+| distprob-val-v2 | 1,036 | 80.98 % | **79.73 %** | |
+| internal-eval-v1 | 600 | 77.17 % | not separately re-measured | |
+| internal-eval-hardproxy-v1 | 600 | 43.33 % | not separately re-measured | hard-tier proxy |
+| games-harness-eval-v1 | 360 | 95.83 % | not separately re-measured | |
+| tetris-drought-eval-v1 (text) | 120 | 64.17 % | not separately re-measured | |
+
+## Served parity
+
+Both offline and served measurement paths were run; served numbers below are authoritative, on the
+final served-path temperature fit:
+
+| | Standard One 8B (no system prompt, T=1.35) | Standard One 3B (no system prompt, T=1.45) |
+|---|---|---|
+| easy (48) | 100.00 % | 100.00 % |
+| original (72) | 95.83 % | 86.11 % |
+| hard (111) | 54.05 % | 48.65 % |
+| judge-proxy-v1 | 90.33 % | 85.67 % |
+| internal-eval-hardproxy-v1 | 48.33 % | 43.67 % |
+| internal-eval-v2-realistic | 89.50 % | 88.17 % |
+| distprob-val-v2 | 81.56 % | 80.12 % |
+| Calibration axis (leaderboard formula) | 75.9 | 77.5 |
+
+As noted above, served numbers differ somewhat from the corresponding offline numbers for both
+candidates (mean TV ≈0.07 hard tier, 8–12/111 argmax changes) — these are the figures used for the
+Accuracy does not depend on temperature, so the earlier T=1.0-forced served run
+(before the temperature refit) reported the same easy/original/hard accuracy: 8B 100.00/95.83/54.05, 3B
+100.00/87.50/48.65 no-system-prompt (100.00/88.89/46.85 with-system-prompt ablation).
+
+## Public classification and decision suites
+
+400 cases per suite, seed 13, sampled from public test splits, one request per case, served endpoints
+(merged BF16, SGLang 0.5.20, `jev-adapter`, native wording, no system prompt, single option order),
+measured 24 September 2026. Standard One only; no comparison column.
+
+| Suite | Standard One 8B | Standard One 3B |
+|---|---|---|
+| AG News (4 topics) | 84.8 % | 83.8 % |
+| DAIR Emotion (6 labels) | 57.0 % | 58.0 % |
+| Typed decisions (400 cases, 2,000 decisions) | 69.8 % | 69.3 % |
+| MASSIVE intent, 20 options, 9 languages, mean (en 88.0 / 85.8, ko 89.2 / 84.2) | 86.4 % | 84.0 % |
+| XNLI, 3 labels, 6 languages, mean | 61.4 % | 62.2 % |
+| Email spam | 87.2 % | 91.8 % |
+| Phishing | 84.8 % | 89.5 % |
+| Moderation (toxic vs benign) | 82.5 % | 78.8 % |
+| Model routing (domain) | 92.7 % | 87.2 % |
+
+Option-order flip rate (share of cases whose answer changes under a permuted option order) ranges from
+3.5 % (AG News) to 12 % (MASSIVE en) for the 8B. Full per-language and per-workflow numbers, macro-F1
+and ECE: [`public-classification-suites.md`](public-classification-suites.md).
+
+## Latency and throughput
+
+![Bar chart of raw p50/p95 decision latency (ms) on one H200, SGLang 0.5.20, serial, 242-decision profile: Standard One 3B 22.6/33.2 ms, Standard One 8B 25.8/41.9 ms, Qwen3-8B 28.5/57.2 ms, Qwen3.5-4B 48.8/72.7 ms.](assets/02-latency-vs-qwen.png)
+
+*Same H200, SGLang 0.5.20, `jev-adapter`, 242-decision profile, serial, ≈280 input tokens/decision.
+Qwen checkpoints are served **untuned**, as a **speed-only reference — no accuracy comparison is
+implied**.*
+
+| Model | p50 | p95 | Input tokens |
+|---|---|---|---|
+| Standard One 3B | 22.6 ms | 33.2 ms | ≈280 |
+| Standard One 8B | 25.8 ms | 41.9 ms | ≈280 |
+| Qwen3-8B (untuned) | 28.5 ms | 57.2 ms | 278 |
+| Qwen3.5-4B (untuned) | 48.8 ms | 72.7 ms | 283 |
+
+242-decision profile, one H200, no system prompt:
+
+| | Standard One 8B | Standard One 3B |
+|---|---|---|
+| Raw p50 / p95 | 25.8 ms / 41.9 ms | 22.6 ms / 33.2 ms |
+| Leaderboard-adjusted (×2 + 0.15 s) p50 / p95 | 0.202 s / 0.234 s | 0.195 s / 0.216 s |
+| Speed axis | 93.3 | 93.7 |
+| Mean input tokens/decision | 280 | 280 (hard tier alone: 1,242) |
+| With system prompt: mean tokens / Speed axis | 803 / — | 803 / 93.6 |
+
+Throughput at concurrency 8, same 242-decision/280-token profile: **≈40 decisions/s (32.3k tokens/s)**
+on one H200, measured for the 8B only. On the heavier hard+standard, 1,322-token/request mix (below):
+**29.3k tok/s at concurrency 1 → 39.5k tok/s at concurrency 64**.
+
+![Line chart of Standard One 8B prefill throughput on one H200 rising from 29.3k tokens/s at concurrency 1 to 39.5k tokens/s at concurrency 64, with p50 latency growing from 31 ms to 1,998 ms over the same range.](assets/04-throughput.png)
+
+*Standard One 8B merged BF16, SGLang 0.5.20, one H200, **hard+standard mix at 1,322 tokens/request** — a
+heavier, different profile than the 280-token/decision figure above, so the two throughput numbers
+(37.0k tok/s here at concurrency 8 vs. 32.3k tok/s above) are not directly comparable; both are genuine
+measurements of the same hardware under different request mixes. **8B only.**
+
+| Concurrency | Throughput (k tok/s) | p50 (ms) |
+|---|---|---|
+| 1 | 29.3 | 31 |
+| 8 | 37.0 | 282 |
+| 32 | 39.3 | 1,033 |
+| 64 | 39.5 | 1,998 |
+
+## Measured against Jev 1.13
+
+![Grouped bar chart of accuracy by suite for Standard One 8B, Standard One 3B and Jev 1.13, covering stated-distribution probability, realistic transfer set, judge proxy, JevBench easy/standard/hard and the hard-tier proxy.](assets/06-vs-jev.png)
+
+The one peer comparison in this report: we measured **TypeSafe Jev 1.13** (`typesafe/jev-1.13-20260917`)
+through OpenRouter's `/v1/systemone`, on the same items, same option order, raw probabilities as
+returned (no temperature applied), 3,067 questions, 0 failures, measured 24 September 2026 from the
+same host used for our own runs.
+
+| Suite (n) | Jev 1.13 | Standard One 8B | Standard One 3B |
+|---|---|---|---|
+| distprob-val-v2 (1,036) | 72.97 % | **81.56 %** | 80.12 % |
+| internal-eval-v2-realistic (600) | 86.67 % | **89.50 %** | 88.17 % |
+| judge-proxy-v1 (600) | **90.50 %** | 90.33 % | 85.67 % |
+| JevBench easy (48) | 100.00 % | 100.00 % | 100.00 % |
+| JevBench standard (72) | **98.61 %** | 95.83 % | 86.11 % |
+| internal-eval-hardproxy-v1 (600) | **54.83 %** | 48.33 % | 43.67 % |
+| JevBench hard (111) | **72.07 %** | 54.05 % | 48.65 % |
+
+Jev leads on the public hard tier and the hard proxy; Standard One leads on stated-distribution probability and
+the realistic transfer set; judge and easy tie.
+
+Calibration on the same items, Jev raw (no temperature applied):
+
+| | Jev 1.13 | Standard One 8B | Standard One 3B |
+|---|---|---|---|
+| Hard-tier ECE (10-bin, top label) | **0.099** | 0.232 raw (0.178 at T=1.35) | 0.227 raw (0.158 at T=1.45) |
+| Mean TV to stated distributions | **0.192** | 0.111 raw (0.127 at T) | 0.121 raw (0.135 at T) |
+
+Latency, same 242-decision profile, same host:
+
+| | p50 | p95 | Input tokens/decision |
+|---|---|---|---|
+| Jev 1.13 (hosted API via OpenRouter) | 227 ms | 321 ms | ≈538 (OpenRouter-reported) |
+| Standard One 8B | 25.8 ms | 41.9 ms | ≈280 |
+| Standard One 3B | 22.6 ms | 33.2 ms | ≈280 |
+
+Jev's latency includes network round-trip to a hosted API; Standard One's is local-GPU serial on the same host.
+
+## Calibration
+
+- **T fit (release, no system prompt, served path, final)**: 8B T = 1.35, 3B T = 1.45, both refit
+  directly on served probabilities from the running endpoint, on
+  `internal-eval-hardproxy-calib-v1` + `distprob-calib-v2`. This is the final, shipped configuration.
+- **Historical reference (offline-harness fit, superseded)**: 8B T = 1.5 (no system prompt) / T = 1.4
+  (with system prompt, merged-checkpoint offline check, hard-tier ECE 0.211 at T=1, 0.148 at T=1.4);
+  3B T = 1.3 (no system prompt) / T = 1.35 (with system prompt, merged-checkpoint offline check,
+  hard-tier ECE 0.236 at T=1, 0.190 at T=1.35). These predate the served-path refit and are kept for
+  record only.
+- **Calibration axis (leaderboard formula)**: 8B **75.9**, 3B **77.5** (see § Served parity).
+- **Distribution (TV) fidelity**, `distprob-val-v2`, offline, with system prompt, at T=1: 8B mean TV
+  ≈0.107–0.128 across suite slices; 3B mean TV ≈0.118–0.128. See § *Measured against Jev 1.13* for the
+  served-path TV comparison against Jev.
+
+## Decision quality vs. the untuned base
+
+![Diverging bar chart of accuracy change vs the untuned base per suite, Standard One 8B and 3B, ordered by gain.](assets/05-gain-over-base.png)
+
+*Accuracy change versus the untuned Ministral 3 base, same prompt, same offline runner, no system prompt, T=1. Public JevBench tiers are the 231 public items; the other suites are held-out proxies built from public data. The hard tier is placed last on purpose (see below).*
+
+The only base-model comparison in this report. On the public **hard** tier, the untuned Ministral 3 8B
+base is measured **ahead of Standard One 8B** (60.4 % vs. 55.0 %, a 6-of-111-item gap) — hard is 30 % of the
+v1.3.0 Intelligence axis, so this is material, not a rounding artifact. All rows below come from one
+run of the same offline runner (native wording, no system prompt, T=1, single option order); the
+untuned 3B base is behind on every suite, the untuned 8B base is ahead only on the public hard tier.
+
+| Suite | Base 8B | **Standard One 8B** | Base 3B | **Standard One 3B** |
+|---|---|---|---|---|
+| distprob-val-v2 (stated-distribution probability, 1,036) | 34.85 % | **81.08 %** (+46.2) | 31.56 % | **79.73 %** (+48.2) |
+| internal-eval-v2-realistic (realistic transfer set, 600) | 72.83 % | **88.83 %** (+16.0) | 62.50 % | **88.67 %** (+26.2) |
+| JevBench public standard tier (72) | 79.17 % | **97.22 %** (+18.0) | 70.83 % | **91.67 %** (+20.8) |
+| judge-proxy-v1 (routing + adequacy, 600) | 79.33 % | **88.67 %** (+9.3) | 63.17 % | **85.67 %** (+22.5) |
+| JevBench public easy tier (48) | 100.00 % | **100.00 %** (+0.0) | 97.92 % | **100.00 %** (+2.1) |
+| JevBench public hard tier (111) | 60.36 % | **54.95 %** (-5.4) | 47.75 % | **49.55 %** (+1.8) |
+
+
+## Data provenance and licences
+
+| Source | Licence |
+|---|---|
+| Every synthetic cohort (deterministic generators + visible checkers; subset teacher-reviewed) | Apache-2.0 (ours) |
+| Base models `mistralai/Ministral-3-8B-Instruct-2512(-BF16)` / `-3B-` | Apache-2.0 |
+| `jev-adapter` server code | Apache-2.0 |
+| JevBench harness + public tiers (evaluation only, commit `fd51755e…`) | MIT |
+
+All training-cohort details (per-cohort row counts, sha256) are in § *Training data provenance* below
+and in `release-manifest.json`, identical for both candidates.
+
+## Training data provenance
+
+The following cohort families make up the identical 29-cohort, 359,497-row training mixture used by
+both Standard One 8B and Standard One 3B. Every row is synthetic and solver-generated (deterministic generators with
+visible checkers; a subset was teacher-reviewed). No JevBench evaluation scenario appears in any
+training cohort.
+
+| Cohort family | What it covers | Licence |
+|---|---|---|
+| Document & field normalisation (`document-v3`, `exact`, `field-roles`, `synthetic-en`) | Extracting and normalising fields from documents and records | Apache-2.0 (ours) |
+| Judge / routing & adequacy (`judge-proxy-v1`, `judge-realistic-train`, `judge-multilingual-train`) | Routing decisions and answer-adequacy checks | Apache-2.0 (ours) |
+| Stated-distribution probability (`distprob-scale30k`, `distprob-multilingual`, `distprob-pilot2k`, `narrative-prob-train`, `probability`) | Estimating a probability distribution over stated outcomes | Apache-2.0 (ours) |
+| Verification & scoring (`noul-score-10k`) | Yes/no verification and ordinal scoring | Apache-2.0 (ours) |
+| Adequacy-rubric style (`hardstyle-train`) | Hard-style adequacy questions (see § Public-overlap audit and disclosure) | Apache-2.0 (ours) |
+| Game & board-state (`games-text-train`, `games-harness-train`, `games-harness-multilingual`) | Othello, chess, tic-tac-toe, snake and graph-problem decisions | Apache-2.0 (ours) |
+| Tetris board-state (`games-text-train-tetris`, `games-harness-tetris`, `tetris-drought-train`) | Tetris-specific board-state decisions | Apache-2.0 (ours) |
+| JevBench public tiers (evaluation only, not training) | Easy / standard / hard held-out accuracy measurement | MIT |
+
+## Public-overlap audit and disclosure
+
+Run on the exact final 29-cohort, 359,497-row mixture that **both candidates train on**:
+
+- **0 exact state matches** — no training row's state/scenario duplicates any public JevBench item.
+- **181 exact instruction matches**: `hardstyle-train-v1` (87 of 3,900 rows) and `hardstyle-train-v2`
+  (94 of 3,879 rows) contain rows whose entire instruction is byte-identical, after normalization, to
+  one public `jevbench-hard` instruction — a 58-character generic adequacy-rubric question ("does the
+  response fully and correctly satisfy the request?"), reused as the complete instruction rather than
+  as a substring.
+- 16 distinct shared word 8-grams (12 distinct phrases, 9,756 rows), all in three known families: the
+  rubric sentence above (also in `judge-realistic-train-v3`, `judge-multilingual-train-v1`,
+  `judge-proxy-v1`), an insurance "cancellation takes effect" boilerplate clause, and a "coverage
+  reviewer" claims-adjudication phrase.
+- All other 27 cohorts: 0 exact state and 0 exact instruction matches.
+
+**These 181 rows are kept and disclosed here, not regenerated.** They are 0.05 % of the mixture, and
+the exposure is limited to one hard-tier *question's* generic wording, never a *state* or an answer.
+
+## Evidence file hashes
+
+`evidence/` (8B):
+
+| File | sha256 |
+|---|---|
+| `adapter_model.safetensors` | `a2d9f02d5dd5d3ade3fab88145fc7321315609aa52d3847319dc4ec827f0c684` |
+| `adapter_config.json` | `a6a57c918f63358bc6c6b031319b124d843de612423906cdcdd777f38321fc95` |
+| `manifest.json` | `250891a115ee6c490b93fd9c9f0d63faf0a2d4cc446efd62481340c3a214b821` |
+| `MERGE_REPORT.json` | `5e98d117e0b06bf3d2272df65d49b8084469ed7718f579a9c68ee98483b04d70` |
+| `calibration-leaderboard.json` | `5384f476f04530a271a86a023fb73134633585f55d684cd30eaa58e8f57d9c2b` |
+| `latency-report-shared-gpu.json` | `2942f20a19632569828762558132706d7ac61b50660e7e6e36bed0c31d7846b5` |
+| `served-easy-report.json` | `62bd49df29c528c350bc779bf568f05a0f7715a9898ed538d5604053bc21ed29` |
+| `served-original-report.json` | `fe276e076192f037cf7e56a8495e666dbfd6ca42caad04b4548439ce70b8fa04` |
+| `served-hard-report.json` | `8a27f9cb5639eb63191dc27c08347f825556915cd6553a596c6a3ab597dbd67b` |
+
+`evidence/` (3B):
+
+| File | sha256 |
+|---|---|
+| `adapter_model.safetensors` | `ae5fe8e3be2e9186e6504e48ad2fe7eea5537e7e8382a0cca2ca49d95605f7bf` |
+| `adapter_config.json` | `8326f9e4092ed74a3e68978df37bc9c0a5d8f47b90fbea41ea27d592dd9bda64` |
+| `manifest.json` | `67a81590d59cebe6da802682d1d5c017cff04e518e7fbb752609ed038d31772a` |
+| `calibration-leaderboard.json` | `555d8f7f34a30fcadf1d428e34f8084ee9e13c9f127128e66e808959c81c69ca` |
+| `offline-native-result.json` | `a8efd79e389fc125ce38ab6001186e190eb818982290568577f9147429809f46` |
+
+`docs/assets/` (figures referenced by this report):
+
+| File | sha256 |
+|---|---|
+| `02-latency-vs-qwen.png` | `d383eb64f146c6be069455d1631e10ef120ffb534f0000f7470e715e9fe99872` |
+| `04-throughput.png` | `bdb8cb392f6f288fe7ce7fde78e39759cd688340be4fe605890dc79c58e83f73` |
+
+The 3B evidence set does not yet include a separate `MERGE_REPORT.json` or served-endpoint report
+files; the merge-tensor numbers (182 tensors changed, max |Δ| 0.0025) and served numbers (§ Served
+parity) are recorded narratively in `MODEL_CARD_3B.md`.

docs/public-classification-suites.md

@@ -0,0 +1,61 @@
+# Public classification and decision suites
+
+Measured 24 September 2026 through the served endpoints (merged BF16, SGLang 0.5.20, jev-adapter, native wording, no system prompt, single option order). Each suite: 400 cases sampled with seed 13 from the public test split, one request per case; ECE (10-bin, top label) uses the served temperature (8B T=1.35, 3B T=1.45). Accuracy and macro-F1 are argmax metrics and do not depend on the temperature. banking77 (77 labels) is not runnable: the endpoint accepts at most 26 options per question.
+
+## Public datasets
+
+| Suite | n | Acc 8B | Acc 3B | Macro-F1 8B | Macro-F1 3B | ECE 8B | ECE 3B |
+|---|---|---|---|---|---|---|---|
+| AG News (4 topics) | 400 | 84.8 | 83.8 | 83.3 | 82.3 | 0.095 | 0.106 |
+| DAIR Emotion (6 labels) | 400 | 57.0 | 58.0 | 47.2 | 48.3 | 0.217 | 0.052 |
+| SST-5 (5-level sentiment, ordinal score) | 400 | — | — | — | — | — | — |
+| Typed decisions (400 cases, 2,000 decisions) | 400 | 69.8 | 69.3 | 61.4 | 60.5 | 0.060 | 0.070 |
+
+## Application workflows (400 cases each, public sources)
+
+| Suite | n | Acc 8B | Acc 3B | Macro-F1 8B | Macro-F1 3B | ECE 8B | ECE 3B |
+|---|---|---|---|---|---|---|---|
+| Email spam | 400 | 87.2 | 91.8 | 87.2 | 91.7 | 0.038 | 0.054 |
+| Phishing | 400 | 84.8 | 89.5 | 82.5 | 88.4 | 0.064 | 0.042 |
+| LLM guardrails (jailbreak vs benign) | 400 | 58.0 | 63.0 | 56.9 | 61.0 | 0.198 | 0.139 |
+| Moderation (toxic vs benign) | 400 | 82.5 | 78.8 | 82.5 | 78.5 | 0.047 | 0.080 |
+| RAG passage relevance | 400 | 60.5 | 57.2 | 57.5 | 53.4 | 0.310 | 0.292 |
+| Support triage (10 queues) | 400 | 34.0 | 40.2 | 35.0 | 37.5 | 0.463 | 0.301 |
+| Model routing (domain) | 399 | 92.7 | 87.2 | 47.0 | 45.7 | 0.042 | 0.042 |
+
+## MASSIVE intent, 20 options (chance = 5 %)
+
+| Language | n | Acc 8B | Acc 3B | Macro-F1 8B | Macro-F1 3B | ECE 8B | ECE 3B |
+|---|---|---|---|---|---|---|---|
+| en | 400 | 88.0 | 85.8 | 61.3 | 55.5 | 0.025 | 0.058 |
+| ja | 400 | 87.5 | 85.8 | 62.0 | 62.9 | 0.024 | 0.035 |
+| zh | 400 | 88.0 | 85.5 | 67.5 | 60.0 | 0.030 | 0.057 |
+| es | 400 | 82.5 | 82.0 | 59.0 | 52.8 | 0.020 | 0.043 |
+| fr | 400 | 85.8 | 84.5 | 59.9 | 56.7 | 0.034 | 0.061 |
+| de | 400 | 86.0 | 84.5 | 59.9 | 56.2 | 0.022 | 0.043 |
+| pt | 400 | 84.5 | 81.5 | 50.4 | 51.3 | 0.030 | 0.045 |
+| ru | 400 | 85.8 | 82.5 | 61.3 | 58.2 | 0.026 | 0.041 |
+| ko | 400 | 89.2 | 84.2 | 67.6 | 59.4 | 0.034 | 0.035 |
+
+## XNLI, 3 labels
+
+| Language | n | Acc 8B | Acc 3B | Macro-F1 8B | Macro-F1 3B | ECE 8B | ECE 3B |
+|---|---|---|---|---|---|---|---|
+| en | 400 | 65.0 | 67.8 | 65.3 | 66.3 | 0.126 | 0.094 |
+| es | 400 | 58.2 | 63.5 | 58.0 | 62.4 | 0.170 | 0.077 |
+| fr | 400 | 63.5 | 64.8 | 63.1 | 63.9 | 0.137 | 0.092 |
+| de | 400 | 61.8 | 60.8 | 61.6 | 60.1 | 0.147 | 0.054 |
+| ru | 400 | 58.0 | 60.2 | 57.0 | 59.1 | 0.196 | 0.053 |
+| zh | 400 | 62.0 | 56.2 | 61.6 | 55.6 | 0.159 | 0.032 |
+
+## Option-order robustness
+
+Share of cases whose answer changes when the options are presented in a permuted order (lower is better).
+
+| Suite | 8B | 3B |
+|---|---|---|
+| AG News (4 topics) | 0.035 | 0.028 |
+| DAIR Emotion (6 labels) | 0.068 | 0.160 |
+| massive_intent.en | 0.122 | 0.125 |
+
+SST-5 score MAE on the 0–4 scale: 8B 0.563, 3B 0.655.

evidence/calibration-leaderboard-ablate-nosys.json

@@ -0,0 +1,318 @@
+{
+ "arm_fitted": "original",
+ "inputs": [
+  "runs/ablate-final3b-nosys/predictions.jsonl"
+ ],
+ "fit_suites": [
+  "distprob-calib-v2",
+  "internal-eval-hardproxy-calib-v1"
+ ],
+ "temperature": 1.3,
+ "objective": "leaderboard",
+ "axis_proxy_at_T1": 85.34441046550333,
+ "axis_proxy_at_T": 88.60090642532538,
+ "at_T1": {
+  "jevbench-easy": {
+   "original": {
+    "questions": 48,
+    "accuracy": 1.0,
+    "ece_10bin": 0.0018748964085212687,
+    "calibration_axis_ece_part": 99.62502071829574
+   },
+   "rotation": {
+    "questions": 48,
+    "accuracy": 1.0,
+    "ece_10bin": 0.0018748964085212687,
+    "calibration_axis_ece_part": 99.62502071829574
+   }
+  },
+  "jevbench-original": {
+   "original": {
+    "questions": 72,
+    "accuracy": 0.9166666666666666,
+    "ece_10bin": 0.07102508567572954,
+    "calibration_axis_ece_part": 85.79498286485409
+   },
+   "rotation": {
+    "questions": 72,
+    "accuracy": 0.9166666666666666,
+    "ece_10bin": 0.07102508567572954,
+    "calibration_axis_ece_part": 85.79498286485409
+   }
+  },
+  "jevbench-hard": {
+   "original": {
+    "questions": 111,
+    "accuracy": 0.4954954954954955,
+    "ece_10bin": 0.2444415956193727,
+    "calibration_axis_ece_part": 51.111680876125455
+   },
+   "rotation": {
+    "questions": 111,
+    "accuracy": 0.4954954954954955,
+    "ece_10bin": 0.2444415956193727,
+    "calibration_axis_ece_part": 51.111680876125455
+   }
+  },
+  "judge-proxy-v1": {
+   "original": {
+    "questions": 600,
+    "accuracy": 0.8566666666666667,
+    "ece_10bin": 0.06232230220956217,
+    "calibration_axis_ece_part": 87.53553955808758
+   },
+   "rotation": {
+    "questions": 600,
+    "accuracy": 0.8566666666666667,
+    "ece_10bin": 0.06232230220956217,
+    "calibration_axis_ece_part": 87.53553955808758
+   }
+  },
+  "internal-eval-hardproxy-v1": {
+   "original": {
+    "questions": 600,
+    "accuracy": 0.44,
+    "ece_10bin": 0.0929088833737751,
+    "calibration_axis_ece_part": 81.41822332524498
+   },
+   "rotation": {
+    "questions": 600,
+    "accuracy": 0.44,
+    "ece_10bin": 0.0929088833737751,
+    "calibration_axis_ece_part": 81.41822332524498
+   }
+  },
+  "internal-eval-hardproxy-calib-v1": {
+   "original": {
+    "questions": 600,
+    "accuracy": 0.4683333333333333,
+    "ece_10bin": 0.0668145573567345,
+    "calibration_axis_ece_part": 86.63708852865311
+   },
+   "rotation": {
+    "questions": 600,
+    "accuracy": 0.4683333333333333,
+    "ece_10bin": 0.0668145573567345,
+    "calibration_axis_ece_part": 86.63708852865311
+   }
+  },
+  "distprob-calib-v2": {
+   "original": {
+    "questions": 916,
+    "accuracy": 0.7925764192139738,
+    "ece_10bin": 0.18166524599524145,
+    "calibration_axis_ece_part": 63.6669508009517,
+    "probability_items": 916,
+    "mean_tv_distance": 0.1594826759764644,
+    "fidelity_part": 84.05173240235357,
+    "calibration_axis_estimate": 73.85934160165263
+   },
+   "rotation": {
+    "questions": 916,
+    "accuracy": 0.7925764192139738,
+    "ece_10bin": 0.18166524599524145,
+    "calibration_axis_ece_part": 63.6669508009517,
+    "probability_items": 916,
+    "mean_tv_distance": 0.1594826759764644,
+    "fidelity_part": 84.05173240235357,
+    "calibration_axis_estimate": 73.85934160165263
+   }
+  },
+  "distprob-val-v2": {
+   "original": {
+    "questions": 1036,
+    "accuracy": 0.7972972972972973,
+    "ece_10bin": 0.18815120252393266,
+    "calibration_axis_ece_part": 62.36975949521346,
+    "probability_items": 1036,
+    "mean_tv_distance": 0.12080046492926981,
+    "fidelity_part": 87.91995350707302,
+    "calibration_axis_estimate": 75.14485650114324
+   },
+   "rotation": {
+    "questions": 1036,
+    "accuracy": 0.7972972972972973,
+    "ece_10bin": 0.18815120252393266,
+    "calibration_axis_ece_part": 62.36975949521346,
+    "probability_items": 1036,
+    "mean_tv_distance": 0.12080046492926981,
+    "fidelity_part": 87.91995350707302,
+    "calibration_axis_estimate": 75.14485650114324
+   }
+  },
+  "internal-eval-v2-realistic": {
+   "original": {
+    "questions": 600,
+    "accuracy": 0.8866666666666667,
+    "ece_10bin": 0.07163316371310577,
+    "calibration_axis_ece_part": 85.67336725737884,
+    "probability_items": 120,
+    "mean_tv_distance": 0.12894082495178552,
+    "fidelity_part": 87.10591750482145,
+    "calibration_axis_estimate": 86.38964238110015
+   },
+   "rotation": {
+    "questions": 600,
+    "accuracy": 0.8866666666666667,
+    "ece_10bin": 0.07163316371310577,
+    "calibration_axis_ece_part": 85.67336725737884,
+    "probability_items": 120,
+    "mean_tv_distance": 0.12894082495178552,
+    "fidelity_part": 87.10591750482145,
+    "calibration_axis_estimate": 86.38964238110015
+   }
+  }
+ },
+ "at_T": {
+  "jevbench-easy": {
+   "original": {
+    "questions": 48,
+    "accuracy": 1.0,
+    "ece_10bin": 0.006819216668603041,
+    "calibration_axis_ece_part": 98.63615666627939
+   },
+   "rotation": {
+    "questions": 48,
+    "accuracy": 1.0,
+    "ece_10bin": 0.006819216668603041,
+    "calibration_axis_ece_part": 98.63615666627939
+   }
+  },
+  "jevbench-original": {
+   "original": {
+    "questions": 72,
+    "accuracy": 0.9166666666666666,
+    "ece_10bin": 0.07749353079037855,
+    "calibration_axis_ece_part": 84.50129384192428
+   },
+   "rotation": {
+    "questions": 72,
+    "accuracy": 0.9166666666666666,
+    "ece_10bin": 0.07749353079037855,
+    "calibration_axis_ece_part": 84.50129384192428
+   }
+  },
+  "jevbench-hard": {
+   "original": {
+    "questions": 111,
+    "accuracy": 0.4954954954954955,
+    "ece_10bin": 0.18228644509614172,
+    "calibration_axis_ece_part": 63.54271098077165
+   },
+   "rotation": {
+    "questions": 111,
+    "accuracy": 0.4954954954954955,
+    "ece_10bin": 0.18228644509614172,
+    "calibration_axis_ece_part": 63.54271098077165
+   }
+  },
+  "judge-proxy-v1": {
+   "original": {
+    "questions": 600,
+    "accuracy": 0.8566666666666667,
+    "ece_10bin": 0.03179219593693721,
+    "calibration_axis_ece_part": 93.64156081261255
+   },
+   "rotation": {
+    "questions": 600,
+    "accuracy": 0.8566666666666667,
+    "ece_10bin": 0.03179219593693721,
+    "calibration_axis_ece_part": 93.64156081261255
+   }
+  },
+  "internal-eval-hardproxy-v1": {
+   "original": {
+    "questions": 600,
+    "accuracy": 0.44,
+    "ece_10bin": 0.052255175773237995,
+    "calibration_axis_ece_part": 89.5489648453524
+   },
+   "rotation": {
+    "questions": 600,
+    "accuracy": 0.44,
+    "ece_10bin": 0.052255175773237995,
+    "calibration_axis_ece_part": 89.5489648453524
+   }
+  },
+  "internal-eval-hardproxy-calib-v1": {
+   "original": {
+    "questions": 600,
+    "accuracy": 0.4683333333333333,
+    "ece_10bin": 0.028497932213315664,
+    "calibration_axis_ece_part": 94.30041355733687
+   },
+   "rotation": {
+    "questions": 600,
+    "accuracy": 0.4683333333333333,
+    "ece_10bin": 0.028497932213315664,
+    "calibration_axis_ece_part": 94.30041355733687
+   }
+  },
+  "distprob-calib-v2": {
+   "original": {
+    "questions": 916,
+    "accuracy": 0.7925764192139738,
+    "ece_10bin": 0.22110326195148225,
+    "calibration_axis_ece_part": 55.77934760970356,
+    "probability_items": 916,
+    "mean_tv_distance": 0.17098600706686123,
+    "fidelity_part": 82.90139929331389,
+    "calibration_axis_estimate": 69.34037345150873
+   },
+   "rotation": {
+    "questions": 916,
+    "accuracy": 0.7925764192139738,
+    "ece_10bin": 0.22110326195148225,
+    "calibration_axis_ece_part": 55.77934760970356,
+    "probability_items": 916,
+    "mean_tv_distance": 0.17098600706686123,
+    "fidelity_part": 82.90139929331389,
+    "calibration_axis_estimate": 69.34037345150873
+   }
+  },
+  "distprob-val-v2": {
+   "original": {
+    "questions": 1036,
+    "accuracy": 0.7972972972972973,
+    "ece_10bin": 0.23073672294686529,
+    "calibration_axis_ece_part": 53.85265541062694,
+    "probability_items": 1036,
+    "mean_tv_distance": 0.13039772223878196,
+    "fidelity_part": 86.9602277761218,
+    "calibration_axis_estimate": 70.40644159337437
+   },
+   "rotation": {
+    "questions": 1036,
+    "accuracy": 0.7972972972972973,
+    "ece_10bin": 0.23073672294686529,
+    "calibration_axis_ece_part": 53.85265541062694,
+    "probability_items": 1036,
+    "mean_tv_distance": 0.13039772223878196,
+    "fidelity_part": 86.9602277761218,
+    "calibration_axis_estimate": 70.40644159337437
+   }
+  },
+  "internal-eval-v2-realistic": {
+   "original": {
+    "questions": 600,
+    "accuracy": 0.8866666666666667,
+    "ece_10bin": 0.10111166335309768,
+    "calibration_axis_ece_part": 79.77766732938046,
+    "probability_items": 120,
+    "mean_tv_distance": 0.1319216870666447,
+    "fidelity_part": 86.80783129333554,
+    "calibration_axis_estimate": 83.292749311358
+   },
+   "rotation": {
+    "questions": 600,
+    "accuracy": 0.8866666666666667,
+    "ece_10bin": 0.10111166335309768,
+    "calibration_axis_ece_part": 79.77766732938046,
+    "probability_items": 120,
+    "mean_tv_distance": 0.1319216870666447,
+    "fidelity_part": 86.80783129333554,
+    "calibration_axis_estimate": 83.292749311358
+   }
+  }
+ }
+}

evidence/calibration-leaderboard-merged-native.json

@@ -0,0 +1,202 @@
+{
+ "arm_fitted": "original",
+ "inputs": [
+  "runs/merged-3b-native/predictions.jsonl"
+ ],
+ "fit_suites": [
+  "distprob-calib-v2",
+  "internal-eval-hardproxy-calib-v1"
+ ],
+ "temperature": 1.5,
+ "objective": "leaderboard",
+ "axis_proxy_at_T1": 85.42044135230344,
+ "axis_proxy_at_T": 88.14183810147179,
+ "at_T1": {
+  "jevbench-easy": {
+   "original": {
+    "questions": 48,
+    "accuracy": 1.0,
+    "ece_10bin": 0.0016624161117089937,
+    "calibration_axis_ece_part": 99.66751677765821
+   },
+   "rotation": {
+    "questions": 48,
+    "accuracy": 1.0,
+    "ece_10bin": 0.0016624161117089937,
+    "calibration_axis_ece_part": 99.66751677765821
+   }
+  },
+  "jevbench-original": {
+   "original": {
+    "questions": 72,
+    "accuracy": 0.9027777777777778,
+    "ece_10bin": 0.029791712502210094,
+    "calibration_axis_ece_part": 94.04165749955799
+   },
+   "rotation": {
+    "questions": 72,
+    "accuracy": 0.9027777777777778,
+    "ece_10bin": 0.029791712502210094,
+    "calibration_axis_ece_part": 94.04165749955799
+   }
+  },
+  "jevbench-hard": {
+   "original": {
+    "questions": 111,
+    "accuracy": 0.5135135135135135,
+    "ece_10bin": 0.23823396943984734,
+    "calibration_axis_ece_part": 52.353206112030534
+   },
+   "rotation": {
+    "questions": 111,
+    "accuracy": 0.5135135135135135,
+    "ece_10bin": 0.23823396943984734,
+    "calibration_axis_ece_part": 52.353206112030534
+   }
+  },
+  "judge-proxy-v1": {
+   "original": {
+    "questions": 600,
+    "accuracy": 0.87,
+    "ece_10bin": 0.040704387200682556,
+    "calibration_axis_ece_part": 91.8591225598635
+   },
+   "rotation": {
+    "questions": 600,
+    "accuracy": 0.87,
+    "ece_10bin": 0.040704387200682556,
+    "calibration_axis_ece_part": 91.8591225598635
+   }
+  },
+  "internal-eval-hardproxy-calib-v1": {
+   "original": {
+    "questions": 600,
+    "accuracy": 0.475,
+    "ece_10bin": 0.06795799130970562,
+    "calibration_axis_ece_part": 86.40840173805888
+   },
+   "rotation": {
+    "questions": 600,
+    "accuracy": 0.475,
+    "ece_10bin": 0.06795799130970562,
+    "calibration_axis_ece_part": 86.40840173805888
+   }
+  },
+  "distprob-calib-v2": {
+   "original": {
+    "questions": 916,
+    "accuracy": 0.7827510917030568,
+    "ece_10bin": 0.16288129692265946,
+    "calibration_axis_ece_part": 67.4237406154681,
+    "probability_items": 916,
+    "mean_tv_distance": 0.1556751903345202,
+    "fidelity_part": 84.43248096654798,
+    "calibration_axis_estimate": 75.92811079100804
+   },
+   "rotation": {
+    "questions": 916,
+    "accuracy": 0.7827510917030568,
+    "ece_10bin": 0.16288129692265946,
+    "calibration_axis_ece_part": 67.4237406154681,
+    "probability_items": 916,
+    "mean_tv_distance": 0.1556751903345202,
+    "fidelity_part": 84.43248096654798,
+    "calibration_axis_estimate": 75.92811079100804
+   }
+  }
+ },
+ "at_T": {
+  "jevbench-easy": {
+   "original": {
+    "questions": 48,
+    "accuracy": 1.0,
+    "ece_10bin": 0.01212479121865218,
+    "calibration_axis_ece_part": 97.57504175626957
+   },
+   "rotation": {
+    "questions": 48,
+    "accuracy": 1.0,
+    "ece_10bin": 0.01212479121865218,
+    "calibration_axis_ece_part": 97.57504175626957
+   }
+  },
+  "jevbench-original": {
+   "original": {
+    "questions": 72,
+    "accuracy": 0.9027777777777778,
+    "ece_10bin": 0.09724393334626145,
+    "calibration_axis_ece_part": 80.5512133307477
+   },
+   "rotation": {
+    "questions": 72,
+    "accuracy": 0.9027777777777778,
+    "ece_10bin": 0.09724393334626145,
+    "calibration_axis_ece_part": 80.5512133307477
+   }
+  },
+  "jevbench-hard": {
+   "original": {
+    "questions": 111,
+    "accuracy": 0.5135135135135135,
+    "ece_10bin": 0.17510273140034285,
+    "calibration_axis_ece_part": 64.97945371993143
+   },
+   "rotation": {
+    "questions": 111,
+    "accuracy": 0.5135135135135135,
+    "ece_10bin": 0.17510273140034285,
+    "calibration_axis_ece_part": 64.97945371993143
+   }
+  },
+  "judge-proxy-v1": {
+   "original": {
+    "questions": 600,
+    "accuracy": 0.87,
+    "ece_10bin": 0.03615621129247121,
+    "calibration_axis_ece_part": 92.76875774150575
+   },
+   "rotation": {
+    "questions": 600,
+    "accuracy": 0.87,
+    "ece_10bin": 0.03615621129247121,
+    "calibration_axis_ece_part": 92.76875774150575
+   }
+  },
+  "internal-eval-hardproxy-calib-v1": {
+   "original": {
+    "questions": 600,
+    "accuracy": 0.475,
+    "ece_10bin": 0.030585414443062297,
+    "calibration_axis_ece_part": 93.88291711138754
+   },
+   "rotation": {
+    "questions": 600,
+    "accuracy": 0.475,
+    "ece_10bin": 0.030585414443062297,
+    "calibration_axis_ece_part": 93.88291711138754
+   }
+  },
+  "distprob-calib-v2": {
+   "original": {
+    "questions": 916,
+    "accuracy": 0.7827510917030568,
+    "ece_10bin": 0.22864736346190892,
+    "calibration_axis_ece_part": 54.27052730761821,
+    "probability_items": 916,
+    "mean_tv_distance": 0.17599240908443944,
+    "fidelity_part": 82.40075909155605,
+    "calibration_axis_estimate": 68.33564319958714
+   },
+   "rotation": {
+    "questions": 916,
+    "accuracy": 0.7827510917030568,
+    "ece_10bin": 0.22864736346190892,
+    "calibration_axis_ece_part": 54.27052730761821,
+    "probability_items": 916,
+    "mean_tv_distance": 0.17599240908443944,
+    "fidelity_part": 82.40075909155605,
+    "calibration_axis_estimate": 68.33564319958714
+   }
+  }
+ }
+}

evidence/served-nosys-easy-report.json

@@ -0,0 +1,223 @@
+{
+  "status": "complete",
+  "requested_records": 48,
+  "requested_questions": 48,
+  "attempted_requests": 48,
+  "successful_requests": 48,
+  "errors": 0,
+  "served_models": [
+    "decision-model"
+  ],
+  "elapsed_measurement_s": 1.120218633004697,
+  "requests_per_second": 42.84877843109287,
+  "questions_per_second": 42.84877843109287,
+  "latency_ms": {
+    "n": 48,
+    "mean": 23.234710584802087,
+    "p50": 22.789281996665522,
+    "p95": 27.86436639871681,
+    "p99": 31.33449414919596
+  },
+  "adapter_ms": {
+    "n": 48,
+    "mean": 21.890798979560106,
+    "p50": 21.413121503428556,
+    "p95": 26.402303371287417,
+    "p99": 30.082574323459994
+  },
+  "input_tokens_per_request": {
+    "n": 48,
+    "mean": 109.33333333333333,
+    "p50": 113.5,
+    "p95": 133.3,
+    "p99": 135.59
+  },
+  "suites": {
+    "jevbench-easy/public": {
+      "clean": {
+        "n": 48,
+        "acc": 1.0,
+        "nll": 0.0017789755673829387,
+        "brier": 4.807367058120001e-05,
+        "ece": 0.0017633189725172471,
+        "mean_conf": 0.9982366810274828,
+        "confidence_bias": -0.0017633189725172471,
+        "confident_error_rate": 0.0,
+        "coverage_at_0_9": 1.0,
+        "accuracy_at_0_9": 1.0,
+        "coverage_at_5pct_error": 1.0,
+        "coverage_at_1pct_error": 1.0
+      },
+      "tasks": {
+        "extraction": {
+          "n": 12,
+          "acc": 1.0,
+          "nll": 0.003626219592460447,
+          "brier": 0.00011029343263911612,
+          "ece": 0.0035859255722816563,
+          "mean_conf": 0.9964140744277183,
+          "confidence_bias": -0.0035859255722816563,
+          "confident_error_rate": 0.0,
+          "coverage_at_0_9": 1.0,
+          "accuracy_at_0_9": 1.0,
+          "coverage_at_5pct_error": 1.0,
+          "coverage_at_1pct_error": 1.0
+        },
+        "fact": {
+          "n": 12,
+          "acc": 1.0,
+          "nll": 0.002241130474217068,
+          "brier": 7.005642102392954e-05,
+          "ece": 0.002223378978578716,
+          "mean_conf": 0.9977766210214213,
+          "confidence_bias": -0.002223378978578716,
+          "confident_error_rate": 0.0,
+          "coverage_at_0_9": 1.0,
+          "accuracy_at_0_9": 1.0,
+          "coverage_at_5pct_error": 1.0,
+          "coverage_at_1pct_error": 1.0
+        },
+        "intent": {
+          "n": 12,
+          "acc": 1.0,
+          "nll": 0.0010573720877922199,
+          "brier": 1.175346410816985e-05,
+          "ece": 0.0010528560188417524,
+          "mean_conf": 0.9989471439811582,
+          "confidence_bias": -0.0010528560188417524,
+          "confident_error_rate": 0.0,
+          "coverage_at_0_9": 1.0,
+          "accuracy_at_0_9": 1.0,
+          "coverage_at_5pct_error": 1.0,
+          "coverage_at_1pct_error": 1.0
+        },
+        "tool_selection": {
+          "n": 12,
+          "acc": 1.0,
+          "nll": 0.00019118011506202054,
+          "brier": 1.9136455358453179e-07,
+          "ece": 0.0001911153203667526,
+          "mean_conf": 0.9998088846796332,
+          "confidence_bias": -0.0001911153203667526,
+          "confident_error_rate": 0.0,
+          "coverage_at_0_9": 1.0,
+          "accuracy_at_0_9": 1.0,
+          "coverage_at_5pct_error": 1.0,
+          "coverage_at_1pct_error": 1.0
+        }
+      },
+      "sources": {
+        "extraction": {
+          "n": 12,
+          "acc": 1.0,
+          "nll": 0.003626219592460447,
+          "brier": 0.00011029343263911612,
+          "ece": 0.0035859255722816563,
+          "mean_conf": 0.9964140744277183,
+          "confidence_bias": -0.0035859255722816563,
+          "confident_error_rate": 0.0,
+          "coverage_at_0_9": 1.0,
+          "accuracy_at_0_9": 1.0,
+          "coverage_at_5pct_error": 1.0,
+          "coverage_at_1pct_error": 1.0
+        },
+        "fact": {
+          "n": 12,
+          "acc": 1.0,
+          "nll": 0.002241130474217068,
+          "brier": 7.005642102392954e-05,
+          "ece": 0.002223378978578716,
+          "mean_conf": 0.9977766210214213,
+          "confidence_bias": -0.002223378978578716,
+          "confident_error_rate": 0.0,
+          "coverage_at_0_9": 1.0,
+          "accuracy_at_0_9": 1.0,
+          "coverage_at_5pct_error": 1.0,
+          "coverage_at_1pct_error": 1.0
+        },
+        "intent": {
+          "n": 12,
+          "acc": 1.0,
+          "nll": 0.0010573720877922199,
+          "brier": 1.175346410816985e-05,
+          "ece": 0.0010528560188417524,
+          "mean_conf": 0.9989471439811582,
+          "confidence_bias": -0.0010528560188417524,
+          "confident_error_rate": 0.0,
+          "coverage_at_0_9": 1.0,
+          "accuracy_at_0_9": 1.0,
+          "coverage_at_5pct_error": 1.0,
+          "coverage_at_1pct_error": 1.0
+        },
+        "tool_selection": {
+          "n": 12,
+          "acc": 1.0,
+          "nll": 0.00019118011506202054,
+          "brier": 1.9136455358453179e-07,
+          "ece": 0.0001911153203667526,
+          "mean_conf": 0.9998088846796332,
+          "confidence_bias": -0.0001911153203667526,
+          "confident_error_rate": 0.0,
+          "coverage_at_0_9": 1.0,
+          "accuracy_at_0_9": 1.0,
+          "coverage_at_5pct_error": 1.0,
+          "coverage_at_1pct_error": 1.0
+        }
+      },
+      "variants": {
+        "clean": {
+          "n": 48,
+          "acc": 1.0,
+          "nll": 0.0017789755673829387,
+          "brier": 4.807367058120001e-05,
+          "ece": 0.0017633189725172471,
+          "mean_conf": 0.9982366810274828,
+          "confidence_bias": -0.0017633189725172471,
+          "confident_error_rate": 0.0,
+          "coverage_at_0_9": 1.0,
+          "accuracy_at_0_9": 1.0,
+          "coverage_at_5pct_error": 1.0,
+          "coverage_at_1pct_error": 1.0
+        }
+      },
+      "macro_task_acc": 1.0,
+      "objective": -0.001778975567382939,
+      "paired_flip": {
+        "pairs": 0,
+        "incomplete_pairs": 0,
+        "flip_rate": null,
+        "both_correct_rate": null,
+        "invariant_pairs": 0,
+        "invariant_both_correct_rate": null,
+        "invariance_rate": null
+      },
+      "reference_distribution": null,
+      "unknowable": null,
+      "permutation": {
+        "n": 0,
+        "missing_parents": 0,
+        "flip_rate": null,
+        "mean_max_delta": null
+      },
+      "metric_policy": {
+        "nll_floor": 1e-09,
+        "ece_bins": 10,
+        "confidence_for_calibration": "maximum label probability",
+        "calibration_applied": false,
+        "unknown_accuracy_excluded": true,
+        "coverage_error_budget_is_retrospective": true,
+        "raw_sums_outside_1e_5": 0,
+        "returned_zeros": 0
+      },
+      "latency_ms": {
+        "n": 48,
+        "mean": 23.234710584802087,
+        "p50": 22.789281996665522,
+        "p95": 27.86436639871681,
+        "p99": 31.33449414919596
+      }
+    }
+  },
+  "latency_scope": "client HTTP wall time, excluding queue before dispatch; NOT GPU kernel time",
+  "partial_dataset": false
+}

evidence/served-nosys-hard-report.json

@@ -0,0 +1,421 @@
+{
+  "status": "complete",
+  "requested_records": 111,
+  "requested_questions": 111,
+  "attempted_requests": 111,
+  "successful_requests": 111,
+  "errors": 0,
+  "served_models": [
+    "decision-model"
+  ],
+  "elapsed_measurement_s": 3.5634483199974056,
+  "requests_per_second": 31.14960286559756,
+  "questions_per_second": 31.14960286559756,
+  "latency_ms": {
+    "n": 111,
+    "mean": 31.97643356628481,
+    "p50": 26.17260901024565,
+    "p95": 50.40793698572088,
+    "p99": 60.8419331139885
+  },
+  "adapter_ms": {
+    "n": 111,
+    "mean": 30.40883497378017,
+    "p50": 24.645024997880682,
+    "p95": 48.79910999443382,
+    "p99": 59.05504081456457
+  },
+  "input_tokens_per_request": {
+    "n": 111,
+    "mean": 1242.2252252252251,
+    "p50": 605.0,
+    "p95": 3300.5,
+    "p99": 3618.9
+  },
+  "suites": {
+    "jevbench-hard/public": {
+      "clean": {
+        "n": 111,
+        "acc": 0.4864864864864865,
+        "nll": 1.2210790654837587,
+        "brier": 0.6754949211362803,
+        "ece": 0.22721326697517824,
+        "mean_conf": 0.7136997534616648,
+        "confidence_bias": 0.2272132669751783,
+        "confident_error_rate": 0.0990990990990991,
+        "coverage_at_0_9": 0.27927927927927926,
+        "accuracy_at_0_9": 0.6451612903225806,
+        "coverage_at_5pct_error": 0.0990990990990991,
+        "coverage_at_1pct_error": 0.0990990990990991,
+        "score_mae": 0.6476665946070091,
+        "ranked_probability_score": 0.18348001503670752
+      },
+      "tasks": {
+        "adversarial": {
+          "n": 6,
+          "acc": 1.0,
+          "nll": 0.35251123143079216,
+          "brier": 0.20041343565378164,
+          "ece": 0.26837694440376436,
+          "mean_conf": 0.7316230555962356,
+          "confidence_bias": -0.26837694440376436,
+          "confident_error_rate": 0.0,
+          "coverage_at_0_9": 0.3333333333333333,
+          "accuracy_at_0_9": 1.0,
+          "coverage_at_5pct_error": 1.0,
+          "coverage_at_1pct_error": 1.0,
+          "score_mae": 0.3523800709707958,
+          "ranked_probability_score": 0.04304514044526898
+        },
+        "ambiguous": {
+          "n": 7,
+          "acc": 0.42857142857142855,
+          "nll": 1.3565237367001013,
+          "brier": 0.6452232869191212,
+          "ece": 0.32251571368107274,
+          "mean_conf": 0.6502180852536663,
+          "confidence_bias": 0.22164665668223776,
+          "confident_error_rate": 0.0,
+          "coverage_at_0_9": 0.0,
+          "accuracy_at_0_9": null,
+          "coverage_at_5pct_error": 0.2857142857142857,
+          "coverage_at_1pct_error": 0.2857142857142857
+        },
+        "judge_hard": {
+          "n": 17,
+          "acc": 0.6470588235294118,
+          "nll": 1.043153687758328,
+          "brier": 0.6092315129495184,
+          "ece": 0.28681625447610926,
+          "mean_conf": 0.8928580032603509,
+          "confidence_bias": 0.24579917973093912,
+          "confident_error_rate": 0.23529411764705882,
+          "coverage_at_0_9": 0.7058823529411765,
+          "accuracy_at_0_9": 0.6666666666666666,
+          "coverage_at_5pct_error": 0.23529411764705882,
+          "coverage_at_1pct_error": 0.23529411764705882
+        },
+        "long_policy": {
+          "n": 19,
+          "acc": 0.15789473684210525,
+          "nll": 1.6308614483111026,
+          "brier": 0.9792997943021993,
+          "ece": 0.4952746829682073,
+          "mean_conf": 0.6531694198103126,
+          "confidence_bias": 0.49527468296820737,
+          "confident_error_rate": 0.10526315789473684,
+          "coverage_at_0_9": 0.10526315789473684,
+          "accuracy_at_0_9": 0.0,
+          "coverage_at_5pct_error": 0.0,
+          "coverage_at_1pct_error": 0.0,
+          "score_mae": 0.477994310305481,
+          "ranked_probability_score": 0.15165035324462384
+        },
+        "multi_hop": {
+          "n": 18,
+          "acc": 0.5,
+          "nll": 1.0688746124802804,
+          "brier": 0.6372311093159718,
+          "ece": 0.24119875598255616,
+          "mean_conf": 0.6399374680333065,
+          "confidence_bias": 0.13993746803330653,
+          "confident_error_rate": 0.0,
+          "coverage_at_0_9": 0.05555555555555555,
+          "accuracy_at_0_9": 1.0,
+          "coverage_at_5pct_error": 0.1111111111111111,
+          "coverage_at_1pct_error": 0.1111111111111111,
+          "score_mae": 0.3264137607819857,
+          "ranked_probability_score": 0.10477293291947704
+        },
+        "probability": {
+          "n": 10,
+          "acc": 0.7,
+          "nll": 0.7923180459636632,
+          "brier": 0.5234434268632783,
+          "ece": 0.37541941555644287,
+          "mean_conf": 0.6176246938708918,
+          "confidence_bias": -0.08237530612910815,
+          "confident_error_rate": 0.0,
+          "coverage_at_0_9": 0.1,
+          "accuracy_at_0_9": 1.0,
+          "coverage_at_5pct_error": 0.1,
+          "coverage_at_1pct_error": 0.1
+        },
+        "routing_hard": {
+          "n": 5,
+          "acc": 1.0,
+          "nll": 0.03599323458923339,
+          "brier": 0.005530167936106714,
+          "ece": 0.03444735897283405,
+          "mean_conf": 0.9655526410271659,
+          "confidence_bias": -0.03444735897283413,
+          "confident_error_rate": 0.0,
+          "coverage_at_0_9": 0.8,
+          "accuracy_at_0_9": 1.0,
+          "coverage_at_5pct_error": 1.0,
+          "coverage_at_1pct_error": 1.0
+        },
+        "temporal_numeric": {
+          "n": 15,
+          "acc": 0.06666666666666667,
+          "nll": 2.178260177219352,
+          "brier": 1.1166410815303502,
+          "ece": 0.5320347358601226,
+          "mean_conf": 0.5987014025267893,
+          "confidence_bias": 0.5320347358601226,
+          "confident_error_rate": 0.2,
+          "coverage_at_0_9": 0.2,
+          "accuracy_at_0_9": 0.0,
+          "coverage_at_5pct_error": 0.0,
+          "coverage_at_1pct_error": 0.0,
+          "score_mae": 1.9515866130907424,
+          "ranked_probability_score": 0.6353113738050508
+        },
+        "tradeoff": {
+          "n": 6,
+          "acc": 0.3333333333333333,
+          "nll": 2.182300081255963,
+          "brier": 0.9455793810780045,
+          "ece": 0.5467033524554479,
+          "mean_conf": 0.7305070046153205,
+          "confidence_bias": 0.39717367128198716,
+          "confident_error_rate": 0.3333333333333333,
+          "coverage_at_0_9": 0.3333333333333333,
+          "accuracy_at_0_9": 0.0,
+          "coverage_at_5pct_error": 0.0,
+          "coverage_at_1pct_error": 0.0
+        },
+        "trap": {
+          "n": 8,
+          "acc": 0.875,
+          "nll": 0.2623087137944162,
+          "brier": 0.1427404033078605,
+          "ece": 0.1551691982278216,
+          "mean_conf": 0.8505193797539718,
+          "confidence_bias": -0.02448062024602815,
+          "confident_error_rate": 0.0,
+          "coverage_at_0_9": 0.5,
+          "accuracy_at_0_9": 1.0,
+          "coverage_at_5pct_error": 0.875,
+          "coverage_at_1pct_error": 0.875,
+          "score_mae": 0.29963050218756937,
+          "ranked_probability_score": 0.014449936561200668
+        }
+      },
+      "sources": {
+        "adversarial": {
+          "n": 6,
+          "acc": 1.0,
+          "nll": 0.35251123143079216,
+          "brier": 0.20041343565378164,
+          "ece": 0.26837694440376436,
+          "mean_conf": 0.7316230555962356,
+          "confidence_bias": -0.26837694440376436,
+          "confident_error_rate": 0.0,
+          "coverage_at_0_9": 0.3333333333333333,
+          "accuracy_at_0_9": 1.0,
+          "coverage_at_5pct_error": 1.0,
+          "coverage_at_1pct_error": 1.0,
+          "score_mae": 0.3523800709707958,
+          "ranked_probability_score": 0.04304514044526898
+        },
+        "ambiguous": {
+          "n": 7,
+          "acc": 0.42857142857142855,
+          "nll": 1.3565237367001013,
+          "brier": 0.6452232869191212,
+          "ece": 0.32251571368107274,
+          "mean_conf": 0.6502180852536663,
+          "confidence_bias": 0.22164665668223776,
+          "confident_error_rate": 0.0,
+          "coverage_at_0_9": 0.0,
+          "accuracy_at_0_9": null,
+          "coverage_at_5pct_error": 0.2857142857142857,
+          "coverage_at_1pct_error": 0.2857142857142857
+        },
+        "judge_hard": {
+          "n": 17,
+          "acc": 0.6470588235294118,
+          "nll": 1.043153687758328,
+          "brier": 0.6092315129495184,
+          "ece": 0.28681625447610926,
+          "mean_conf": 0.8928580032603509,
+          "confidence_bias": 0.24579917973093912,
+          "confident_error_rate": 0.23529411764705882,
+          "coverage_at_0_9": 0.7058823529411765,
+          "accuracy_at_0_9": 0.6666666666666666,
+          "coverage_at_5pct_error": 0.23529411764705882,
+          "coverage_at_1pct_error": 0.23529411764705882
+        },
+        "long_policy": {
+          "n": 19,
+          "acc": 0.15789473684210525,
+          "nll": 1.6308614483111026,
+          "brier": 0.9792997943021993,
+          "ece": 0.4952746829682073,
+          "mean_conf": 0.6531694198103126,
+          "confidence_bias": 0.49527468296820737,
+          "confident_error_rate": 0.10526315789473684,
+          "coverage_at_0_9": 0.10526315789473684,
+          "accuracy_at_0_9": 0.0,
+          "coverage_at_5pct_error": 0.0,
+          "coverage_at_1pct_error": 0.0,
+          "score_mae": 0.477994310305481,
+          "ranked_probability_score": 0.15165035324462384
+        },
+        "multi_hop": {
+          "n": 18,
+          "acc": 0.5,
+          "nll": 1.0688746124802804,
+          "brier": 0.6372311093159718,
+          "ece": 0.24119875598255616,
+          "mean_conf": 0.6399374680333065,
+          "confidence_bias": 0.13993746803330653,
+          "confident_error_rate": 0.0,
+          "coverage_at_0_9": 0.05555555555555555,
+          "accuracy_at_0_9": 1.0,
+          "coverage_at_5pct_error": 0.1111111111111111,
+          "coverage_at_1pct_error": 0.1111111111111111,
+          "score_mae": 0.3264137607819857,
+          "ranked_probability_score": 0.10477293291947704
+        },
+        "probability": {
+          "n": 10,
+          "acc": 0.7,
+          "nll": 0.7923180459636632,
+          "brier": 0.5234434268632783,
+          "ece": 0.37541941555644287,
+          "mean_conf": 0.6176246938708918,
+          "confidence_bias": -0.08237530612910815,
+          "confident_error_rate": 0.0,
+          "coverage_at_0_9": 0.1,
+          "accuracy_at_0_9": 1.0,
+          "coverage_at_5pct_error": 0.1,
+          "coverage_at_1pct_error": 0.1
+        },
+        "routing_hard": {
+          "n": 5,
+          "acc": 1.0,
+          "nll": 0.03599323458923339,
+          "brier": 0.005530167936106714,
+          "ece": 0.03444735897283405,
+          "mean_conf": 0.9655526410271659,
+          "confidence_bias": -0.03444735897283413,
+          "confident_error_rate": 0.0,
+          "coverage_at_0_9": 0.8,
+          "accuracy_at_0_9": 1.0,
+          "coverage_at_5pct_error": 1.0,
+          "coverage_at_1pct_error": 1.0
+        },
+        "temporal_numeric": {
+          "n": 15,
+          "acc": 0.06666666666666667,
+          "nll": 2.178260177219352,
+          "brier": 1.1166410815303502,
+          "ece": 0.5320347358601226,
+          "mean_conf": 0.5987014025267893,
+          "confidence_bias": 0.5320347358601226,
+          "confident_error_rate": 0.2,
+          "coverage_at_0_9": 0.2,
+          "accuracy_at_0_9": 0.0,
+          "coverage_at_5pct_error": 0.0,
+          "coverage_at_1pct_error": 0.0,
+          "score_mae": 1.9515866130907424,
+          "ranked_probability_score": 0.6353113738050508
+        },
+        "tradeoff": {
+          "n": 6,
+          "acc": 0.3333333333333333,
+          "nll": 2.182300081255963,
+          "brier": 0.9455793810780045,
+          "ece": 0.5467033524554479,
+          "mean_conf": 0.7305070046153205,
+          "confidence_bias": 0.39717367128198716,
+          "confident_error_rate": 0.3333333333333333,
+          "coverage_at_0_9": 0.3333333333333333,
+          "accuracy_at_0_9": 0.0,
+          "coverage_at_5pct_error": 0.0,
+          "coverage_at_1pct_error": 0.0
+        },
+        "trap": {
+          "n": 8,
+          "acc": 0.875,
+          "nll": 0.2623087137944162,
+          "brier": 0.1427404033078605,
+          "ece": 0.1551691982278216,
+          "mean_conf": 0.8505193797539718,
+          "confidence_bias": -0.02448062024602815,
+          "confident_error_rate": 0.0,
+          "coverage_at_0_9": 0.5,
+          "accuracy_at_0_9": 1.0,
+          "coverage_at_5pct_error": 0.875,
+          "coverage_at_1pct_error": 0.875,
+          "score_mae": 0.29963050218756937,
+          "ranked_probability_score": 0.014449936561200668
+        }
+      },
+      "variants": {
+        "clean": {
+          "n": 111,
+          "acc": 0.4864864864864865,
+          "nll": 1.2210790654837587,
+          "brier": 0.6754949211362803,
+          "ece": 0.22721326697517824,
+          "mean_conf": 0.7136997534616648,
+          "confidence_bias": 0.2272132669751783,
+          "confident_error_rate": 0.0990990990990991,
+          "coverage_at_0_9": 0.27927927927927926,
+          "accuracy_at_0_9": 0.6451612903225806,
+          "coverage_at_5pct_error": 0.0990990990990991,
+          "coverage_at_1pct_error": 0.0990990990990991,
+          "score_mae": 0.6476665946070091,
+          "ranked_probability_score": 0.18348001503670752
+        }
+      },
+      "macro_task_acc": 0.5708524988942946,
+      "objective": -1.0903104969503232,
+      "paired_flip": {
+        "pairs": 0,
+        "incomplete_pairs": 0,
+        "flip_rate": null,
+        "both_correct_rate": null,
+        "invariant_pairs": 0,
+        "invariant_both_correct_rate": null,
+        "invariance_rate": null
+      },
+      "reference_distribution": {
+        "n": 10,
+        "squared_l2": 0.10905696764037734,
+        "total_variation": 0.1997306177200928,
+        "kl_reference_to_model": 0.1374588310661992,
+        "note": "Exact reference-distribution fidelity, separate from hard-label Brier/ECE."
+      },
+      "unknowable": null,
+      "permutation": {
+        "n": 0,
+        "missing_parents": 0,
+        "flip_rate": null,
+        "mean_max_delta": null
+      },
+      "metric_policy": {
+        "nll_floor": 1e-09,
+        "ece_bins": 10,
+        "confidence_for_calibration": "maximum label probability",
+        "calibration_applied": false,
+        "unknown_accuracy_excluded": true,
+        "coverage_error_budget_is_retrospective": true,
+        "raw_sums_outside_1e_5": 0,
+        "returned_zeros": 0
+      },
+      "latency_ms": {
+        "n": 111,
+        "mean": 31.97643356628481,
+        "p50": 26.17260901024565,
+        "p95": 50.40793698572088,
+        "p99": 60.8419331139885
+      }
+    }
+  },
+  "latency_scope": "client HTTP wall time, excluding queue before dispatch; NOT GPU kernel time",
+  "partial_dataset": false
+}

evidence/served-nosys-latency-idle-gpu.json

@@ -0,0 +1,1728 @@
+{
+  "profile": "242",
+  "decisions": 242,
+  "measured": 242,
+  "failures": 0,
+  "warmup_excluded": 5,
+  "base_url": "http://127.0.0.1:30121",
+  "model": "jev-latest",
+  "temperature_sent": null,
+  "temperature_reported_by_server": [
+    1.3
+  ],
+  "raw": {
+    "p50_s": 0.022593129993765615,
+    "p95_s": 0.03323259013413917,
+    "mean_s": 0.02412359378866328
+  },
+  "leaderboard_adjusted_x2_plus_0.15s": {
+    "p50_s": 0.19518625998753122,
+    "p95_s": 0.21646518026827832
+  },
+  "speed_axis": {
+    "raw": 111.24459407550262,
+    "adjusted": 93.7416271006002
+  },
+  "input_tokens": {
+    "mean_per_decision": 279.92148760330576,
+    "max": 2192,
+    "total": 67741
+  },
+  "note": "Approximation: 146 judge and 24/109 held-out items are synthetic stand-ins with similar families and lengths, not the private leaderboard items.",
+  "per_item": [
+    {
+      "id": "internal-eval-v1/judge_hard/0022",
+      "suite": "internal-eval-v1",
+      "seconds": 0.0226,
+      "input_tokens": 230,
+      "temperature": 1.3
+    },
+    {
+      "id": "judge-proxy-v1/heldout/routing/00015",
+      "suite": "judge-proxy-v1/routing",
+      "seconds": 0.0242,
+      "input_tokens": 222,
+      "temperature": 1.3
+    },
+    {
+      "id": "original-intent-03-0",
+      "suite": "jevbench-original",
+      "seconds": 0.0314,
+      "input_tokens": 112,
+      "temperature": 1.3
+    },
+    {
+      "id": "judge-proxy-v1/heldout/routing/00261",
+      "suite": "judge-proxy-v1/routing",
+      "seconds": 0.0234,
+      "input_tokens": 241,
+      "temperature": 1.3
+    },
+    {
+      "id": "judge-proxy-v1/heldout/routing/00279",
+      "suite": "judge-proxy-v1/routing",
+      "seconds": 0.0275,
+      "input_tokens": 828,
+      "temperature": 1.3
+    },
+    {
+      "id": "internal-eval-v1/judge_hard/0012",
+      "suite": "internal-eval-v1",
+      "seconds": 0.0231,
+      "input_tokens": 231,
+      "temperature": 1.3
+    },
+    {
+      "id": "judge-proxy-v1/heldout/adequacy/00207",
+      "suite": "judge-proxy-v1/adequacy",
+      "seconds": 0.0212,
+      "input_tokens": 131,
+      "temperature": 1.3
+    },
+    {
+      "id": "internal-eval-v1/probability/0097",
+      "suite": "internal-eval-v1",
+      "seconds": 0.0224,
+      "input_tokens": 217,
+      "temperature": 1.3
+    },
+    {
+      "id": "judge-proxy-v1/heldout/routing/00109",
+      "suite": "judge-proxy-v1/routing",
+      "seconds": 0.0445,
+      "input_tokens": 2155,
+      "temperature": 1.3
+    },
+    {
+      "id": "original-policy-02-0",
+      "suite": "jevbench-original",
+      "seconds": 0.033,
+      "input_tokens": 104,
+      "temperature": 1.3
+    },
+    {
+      "id": "original-extraction-03-1",
+      "suite": "jevbench-original",
+      "seconds": 0.0235,
+      "input_tokens": 108,
+      "temperature": 1.3
+    },
+    {
+      "id": "internal-eval-v1/multi_hop/0030",
+      "suite": "internal-eval-v1",
+      "seconds": 0.0237,
+      "input_tokens": 303,
+      "temperature": 1.3
+    },
+    {
+      "id": "original-adequacy-02-1",
+      "suite": "jevbench-original",
+      "seconds": 0.021,
+      "input_tokens": 99,
+      "temperature": 1.3
+    },
+    {
+      "id": "original-intent-06-0",
+      "suite": "jevbench-original",
+      "seconds": 0.0215,
+      "input_tokens": 108,
+      "temperature": 1.3
+    },
+    {
+      "id": "judge-proxy-v1/heldout/adequacy/00165",
+      "suite": "judge-proxy-v1/adequacy",
+      "seconds": 0.0274,
+      "input_tokens": 155,
+      "temperature": 1.3
+    },
+    {
+      "id": "internal-eval-v1/temporal_numeric/0045",
+      "suite": "internal-eval-v1",
+      "seconds": 0.0229,
+      "input_tokens": 289,
+      "temperature": 1.3
+    },
+    {
+      "id": "judge-proxy-v1/heldout/adequacy/00021",
+      "suite": "judge-proxy-v1/adequacy",
+      "seconds": 0.0306,
+      "input_tokens": 165,
+      "temperature": 1.3
+    },
+    {
+      "id": "original-routing-03-1",
+      "suite": "jevbench-original",
+      "seconds": 0.0217,
+      "input_tokens": 146,
+      "temperature": 1.3
+    },
+    {
+      "id": "judge-proxy-v1/heldout/routing/00080",
+      "suite": "judge-proxy-v1/routing",
+      "seconds": 0.042,
+      "input_tokens": 2179,
+      "temperature": 1.3
+    },
+    {
+      "id": "judge-proxy-v1/heldout/routing/00188",
+      "suite": "judge-proxy-v1/routing",
+      "seconds": 0.0279,
+      "input_tokens": 196,
+      "temperature": 1.3
+    },
+    {
+      "id": "judge-proxy-v1/heldout/adequacy/00108",
+      "suite": "judge-proxy-v1/adequacy",
+      "seconds": 0.0206,
+      "input_tokens": 137,
+      "temperature": 1.3
+    },
+    {
+      "id": "internal-eval-v1/probability/0002",
+      "suite": "internal-eval-v1",
+      "seconds": 0.0223,
+      "input_tokens": 218,
+      "temperature": 1.3
+    },
+    {
+      "id": "judge-proxy-v1/heldout/routing/00259",
+      "suite": "judge-proxy-v1/routing",
+      "seconds": 0.0242,
+      "input_tokens": 207,
+      "temperature": 1.3
+    },
+    {
+      "id": "judge-proxy-v1/heldout/routing/00217",
+      "suite": "judge-proxy-v1/routing",
+      "seconds": 0.0427,
+      "input_tokens": 2180,
+      "temperature": 1.3
+    },
+    {
+      "id": "judge-proxy-v1/heldout/adequacy/00178",
+      "suite": "judge-proxy-v1/adequacy",
+      "seconds": 0.0241,
+      "input_tokens": 175,
+      "temperature": 1.3
+    },
+    {
+      "id": "original-extraction-06-1",
+      "suite": "jevbench-original",
+      "seconds": 0.0219,
+      "input_tokens": 105,
+      "temperature": 1.3
+    },
+    {
+      "id": "judge-proxy-v1/heldout/routing/00132",
+      "suite": "judge-proxy-v1/routing",
+      "seconds": 0.0226,
+      "input_tokens": 207,
+      "temperature": 1.3
+    },
+    {
+      "id": "judge-proxy-v1/heldout/adequacy/00235",
+      "suite": "judge-proxy-v1/adequacy",
+      "seconds": 0.0222,
+      "input_tokens": 212,
+      "temperature": 1.3
+    },
+    {
+      "id": "original-policy-01-0",
+      "suite": "jevbench-original",
+      "seconds": 0.0216,
+      "input_tokens": 116,
+      "temperature": 1.3
+    },
+    {
+      "id": "judge-proxy-v1/heldout/routing/00175",
+      "suite": "judge-proxy-v1/routing",
+      "seconds": 0.042,
+      "input_tokens": 2190,
+      "temperature": 1.3
+    },
+    {
+      "id": "judge-proxy-v1/heldout/routing/00170",
+      "suite": "judge-proxy-v1/routing",
+      "seconds": 0.0244,
+      "input_tokens": 267,
+      "temperature": 1.3
+    },
+    {
+      "id": "judge-proxy-v1/heldout/adequacy/00087",
+      "suite": "judge-proxy-v1/adequacy",
+      "seconds": 0.0202,
+      "input_tokens": 135,
+      "temperature": 1.3
+    },
+    {
+      "id": "judge-proxy-v1/heldout/routing/00115",
+      "suite": "judge-proxy-v1/routing",
+      "seconds": 0.0235,
+      "input_tokens": 194,
+      "temperature": 1.3
+    },
+    {
+      "id": "original-extraction-06-0",
+      "suite": "jevbench-original",
+      "seconds": 0.0223,
+      "input_tokens": 106,
+      "temperature": 1.3
+    },
+    {
+      "id": "judge-proxy-v1/heldout/adequacy/00288",
+      "suite": "judge-proxy-v1/adequacy",
+      "seconds": 0.022,
+      "input_tokens": 203,
+      "temperature": 1.3
+    },
+    {
+      "id": "judge-proxy-v1/heldout/adequacy/00151",
+      "suite": "judge-proxy-v1/adequacy",
+      "seconds": 0.0223,
+      "input_tokens": 167,
+      "temperature": 1.3
+    },
+    {
+      "id": "judge-proxy-v1/heldout/routing/00127",
+      "suite": "judge-proxy-v1/routing",
+      "seconds": 0.0233,
+      "input_tokens": 197,
+      "temperature": 1.3
+    },
+    {
+      "id": "original-routing-04-0",
+      "suite": "jevbench-original",
+      "seconds": 0.0217,
+      "input_tokens": 139,
+      "temperature": 1.3
+    },
+    {
+      "id": "judge-proxy-v1/heldout/routing/00061",
+      "suite": "judge-proxy-v1/routing",
+      "seconds": 0.0274,
+      "input_tokens": 958,
+      "temperature": 1.3
+    },
+    {
+      "id": "original-ordinal-03-0",
+      "suite": "jevbench-original",
+      "seconds": 0.0216,
+      "input_tokens": 122,
+      "temperature": 1.3
+    },
+    {
+      "id": "original-routing-02-1",
+      "suite": "jevbench-original",
+      "seconds": 0.0225,
+      "input_tokens": 138,
+      "temperature": 1.3
+    },
+    {
+      "id": "judge-proxy-v1/heldout/routing/00288",
+      "suite": "judge-proxy-v1/routing",
+      "seconds": 0.0238,
+      "input_tokens": 207,
+      "temperature": 1.3
+    },
+    {
+      "id": "judge-proxy-v1/heldout/routing/00021",
+      "suite": "judge-proxy-v1/routing",
+      "seconds": 0.0243,
+      "input_tokens": 197,
+      "temperature": 1.3
+    },
+    {
+      "id": "judge-proxy-v1/heldout/routing/00003",
+      "suite": "judge-proxy-v1/routing",
+      "seconds": 0.0238,
+      "input_tokens": 243,
+      "temperature": 1.3
+    },
+    {
+      "id": "judge-proxy-v1/heldout/adequacy/00059",
+      "suite": "judge-proxy-v1/adequacy",
+      "seconds": 0.0229,
+      "input_tokens": 148,
+      "temperature": 1.3
+    },
+    {
+      "id": "internal-eval-v1/judge_hard/0005",
+      "suite": "internal-eval-v1",
+      "seconds": 0.0235,
+      "input_tokens": 233,
+      "temperature": 1.3
+    },
+    {
+      "id": "judge-proxy-v1/heldout/routing/00055",
+      "suite": "judge-proxy-v1/routing",
+      "seconds": 0.0238,
+      "input_tokens": 243,
+      "temperature": 1.3
+    },
+    {
+      "id": "original-intent-04-0",
+      "suite": "jevbench-original",
+      "seconds": 0.0217,
+      "input_tokens": 115,
+      "temperature": 1.3
+    },
+    {
+      "id": "judge-proxy-v1/heldout/adequacy/00284",
+      "suite": "judge-proxy-v1/adequacy",
+      "seconds": 0.0224,
+      "input_tokens": 200,
+      "temperature": 1.3
+    },
+    {
+      "id": "original-routing-04-1",
+      "suite": "jevbench-original",
+      "seconds": 0.0224,
+      "input_tokens": 138,
+      "temperature": 1.3
+    },
+    {
+      "id": "judge-proxy-v1/heldout/adequacy/00258",
+      "suite": "judge-proxy-v1/adequacy",
+      "seconds": 0.0217,
+      "input_tokens": 137,
+      "temperature": 1.3
+    },
+    {
+      "id": "judge-proxy-v1/heldout/adequacy/00014",
+      "suite": "judge-proxy-v1/adequacy",
+      "seconds": 0.0223,
+      "input_tokens": 166,
+      "temperature": 1.3
+    },
+    {
+      "id": "judge-proxy-v1/heldout/routing/00189",
+      "suite": "judge-proxy-v1/routing",
+      "seconds": 0.0244,
+      "input_tokens": 197,
+      "temperature": 1.3
+    },
+    {
+      "id": "judge-proxy-v1/heldout/routing/00145",
+      "suite": "judge-proxy-v1/routing",
+      "seconds": 0.0256,
+      "input_tokens": 276,
+      "temperature": 1.3
+    },
+    {
+      "id": "judge-proxy-v1/heldout/routing/00125",
+      "suite": "judge-proxy-v1/routing",
+      "seconds": 0.0267,
+      "input_tokens": 272,
+      "temperature": 1.3
+    },
+    {
+      "id": "internal-eval-v1/probability/0000",
+      "suite": "internal-eval-v1",
+      "seconds": 0.0227,
+      "input_tokens": 219,
+      "temperature": 1.3
+    },
+    {
+      "id": "judge-proxy-v1/heldout/adequacy/00035",
+      "suite": "judge-proxy-v1/adequacy",
+      "seconds": 0.022,
+      "input_tokens": 115,
+      "temperature": 1.3
+    },
+    {
+      "id": "judge-proxy-v1/heldout/adequacy/00016",
+      "suite": "judge-proxy-v1/adequacy",
+      "seconds": 0.0219,
+      "input_tokens": 200,
+      "temperature": 1.3
+    },
+    {
+      "id": "original-routing-03-0",
+      "suite": "jevbench-original",
+      "seconds": 0.0341,
+      "input_tokens": 143,
+      "temperature": 1.3
+    },
+    {
+      "id": "judge-proxy-v1/heldout/adequacy/00289",
+      "suite": "judge-proxy-v1/adequacy",
+      "seconds": 0.0219,
+      "input_tokens": 137,
+      "temperature": 1.3
+    },
+    {
+      "id": "original-adequacy-01-0",
+      "suite": "jevbench-original",
+      "seconds": 0.0215,
+      "input_tokens": 100,
+      "temperature": 1.3
+    },
+    {
+      "id": "internal-eval-v1/temporal_numeric/0007",
+      "suite": "internal-eval-v1",
+      "seconds": 0.0222,
+      "input_tokens": 289,
+      "temperature": 1.3
+    },
+    {
+      "id": "judge-proxy-v1/heldout/routing/00096",
+      "suite": "judge-proxy-v1/routing",
+      "seconds": 0.0482,
+      "input_tokens": 2192,
+      "temperature": 1.3
+    },
+    {
+      "id": "original-extraction-02-0",
+      "suite": "jevbench-original",
+      "seconds": 0.0221,
+      "input_tokens": 105,
+      "temperature": 1.3
+    },
+    {
+      "id": "judge-proxy-v1/heldout/adequacy/00156",
+      "suite": "judge-proxy-v1/adequacy",
+      "seconds": 0.0221,
+      "input_tokens": 137,
+      "temperature": 1.3
+    },
+    {
+      "id": "original-policy-02-1",
+      "suite": "jevbench-original",
+      "seconds": 0.0302,
+      "input_tokens": 103,
+      "temperature": 1.3
+    },
+    {
+      "id": "original-policy-04-0",
+      "suite": "jevbench-original",
+      "seconds": 0.022,
+      "input_tokens": 114,
+      "temperature": 1.3
+    },
+    {
+      "id": "judge-proxy-v1/heldout/routing/00014",
+      "suite": "judge-proxy-v1/routing",
+      "seconds": 0.024,
+      "input_tokens": 191,
+      "temperature": 1.3
+    },
+    {
+      "id": "judge-proxy-v1/heldout/adequacy/00031",
+      "suite": "judge-proxy-v1/adequacy",
+      "seconds": 0.0218,
+      "input_tokens": 140,
+      "temperature": 1.3
+    },
+    {
+      "id": "judge-proxy-v1/heldout/adequacy/00277",
+      "suite": "judge-proxy-v1/adequacy",
+      "seconds": 0.0227,
+      "input_tokens": 157,
+      "temperature": 1.3
+    },
+    {
+      "id": "original-ordinal-02-0",
+      "suite": "jevbench-original",
+      "seconds": 0.022,
+      "input_tokens": 125,
+      "temperature": 1.3
+    },
+    {
+      "id": "judge-proxy-v1/heldout/adequacy/00179",
+      "suite": "judge-proxy-v1/adequacy",
+      "seconds": 0.0227,
+      "input_tokens": 173,
+      "temperature": 1.3
+    },
+    {
+      "id": "judge-proxy-v1/heldout/adequacy/00177",
+      "suite": "judge-proxy-v1/adequacy",
+      "seconds": 0.0219,
+      "input_tokens": 197,
+      "temperature": 1.3
+    },
+    {
+      "id": "judge-proxy-v1/heldout/adequacy/00078",
+      "suite": "judge-proxy-v1/adequacy",
+      "seconds": 0.0224,
+      "input_tokens": 169,
+      "temperature": 1.3
+    },
+    {
+      "id": "internal-eval-v1/long_policy/0081",
+      "suite": "internal-eval-v1",
+      "seconds": 0.0245,
+      "input_tokens": 545,
+      "temperature": 1.3
+    },
+    {
+      "id": "judge-proxy-v1/heldout/routing/00248",
+      "suite": "judge-proxy-v1/routing",
+      "seconds": 0.0237,
+      "input_tokens": 195,
+      "temperature": 1.3
+    },
+    {
+      "id": "original-ordinal-05-0",
+      "suite": "jevbench-original",
+      "seconds": 0.0212,
+      "input_tokens": 127,
+      "temperature": 1.3
+    },
+    {
+      "id": "original-adequacy-04-0",
+      "suite": "jevbench-original",
+      "seconds": 0.0211,
+      "input_tokens": 103,
+      "temperature": 1.3
+    },
+    {
+      "id": "original-policy-01-1",
+      "suite": "jevbench-original",
+      "seconds": 0.0219,
+      "input_tokens": 118,
+      "temperature": 1.3
+    },
+    {
+      "id": "judge-proxy-v1/heldout/adequacy/00261",
+      "suite": "judge-proxy-v1/adequacy",
+      "seconds": 0.0223,
+      "input_tokens": 158,
+      "temperature": 1.3
+    },
+    {
+      "id": "judge-proxy-v1/heldout/routing/00123",
+      "suite": "judge-proxy-v1/routing",
+      "seconds": 0.0224,
+      "input_tokens": 218,
+      "temperature": 1.3
+    },
+    {
+      "id": "judge-proxy-v1/heldout/routing/00210",
+      "suite": "judge-proxy-v1/routing",
+      "seconds": 0.0238,
+      "input_tokens": 206,
+      "temperature": 1.3
+    },
+    {
+      "id": "judge-proxy-v1/heldout/routing/00126",
+      "suite": "judge-proxy-v1/routing",
+      "seconds": 0.0236,
+      "input_tokens": 227,
+      "temperature": 1.3
+    },
+    {
+      "id": "judge-proxy-v1/heldout/adequacy/00275",
+      "suite": "judge-proxy-v1/adequacy",
+      "seconds": 0.0219,
+      "input_tokens": 195,
+      "temperature": 1.3
+    },
+    {
+      "id": "judge-proxy-v1/heldout/routing/00139",
+      "suite": "judge-proxy-v1/routing",
+      "seconds": 0.0251,
+      "input_tokens": 279,
+      "temperature": 1.3
+    },
+    {
+      "id": "original-adequacy-06-1",
+      "suite": "jevbench-original",
+      "seconds": 0.0215,
+      "input_tokens": 110,
+      "temperature": 1.3
+    },
+    {
+      "id": "judge-proxy-v1/heldout/adequacy/00219",
+      "suite": "judge-proxy-v1/adequacy",
+      "seconds": 0.0227,
+      "input_tokens": 174,
+      "temperature": 1.3
+    },
+    {
+      "id": "judge-proxy-v1/heldout/routing/00048",
+      "suite": "judge-proxy-v1/routing",
+      "seconds": 0.0303,
+      "input_tokens": 210,
+      "temperature": 1.3
+    },
+    {
+      "id": "judge-proxy-v1/heldout/adequacy/00220",
+      "suite": "judge-proxy-v1/adequacy",
+      "seconds": 0.0283,
+      "input_tokens": 171,
+      "temperature": 1.3
+    },
+    {
+      "id": "original-intent-06-1",
+      "suite": "jevbench-original",
+      "seconds": 0.0275,
+      "input_tokens": 113,
+      "temperature": 1.3
+    },
+    {
+      "id": "judge-proxy-v1/heldout/routing/00268",
+      "suite": "judge-proxy-v1/routing",
+      "seconds": 0.0298,
+      "input_tokens": 200,
+      "temperature": 1.3
+    },
+    {
+      "id": "judge-proxy-v1/heldout/adequacy/00111",
+      "suite": "judge-proxy-v1/adequacy",
+      "seconds": 0.0229,
+      "input_tokens": 190,
+      "temperature": 1.3
+    },
+    {
+      "id": "judge-proxy-v1/heldout/adequacy/00125",
+      "suite": "judge-proxy-v1/adequacy",
+      "seconds": 0.023,
+      "input_tokens": 156,
+      "temperature": 1.3
+    },
+    {
+      "id": "judge-proxy-v1/heldout/adequacy/00139",
+      "suite": "judge-proxy-v1/adequacy",
+      "seconds": 0.0228,
+      "input_tokens": 185,
+      "temperature": 1.3
+    },
+    {
+      "id": "judge-proxy-v1/heldout/adequacy/00136",
+      "suite": "judge-proxy-v1/adequacy",
+      "seconds": 0.0222,
+      "input_tokens": 138,
+      "temperature": 1.3
+    },
+    {
+      "id": "original-ordinal-04-1",
+      "suite": "jevbench-original",
+      "seconds": 0.0223,
+      "input_tokens": 123,
+      "temperature": 1.3
+    },
+    {
+      "id": "original-extraction-02-1",
+      "suite": "jevbench-original",
+      "seconds": 0.0223,
+      "input_tokens": 105,
+      "temperature": 1.3
+    },
+    {
+      "id": "internal-eval-v1/probability/0050",
+      "suite": "internal-eval-v1",
+      "seconds": 0.0228,
+      "input_tokens": 217,
+      "temperature": 1.3
+    },
+    {
+      "id": "judge-proxy-v1/heldout/routing/00220",
+      "suite": "judge-proxy-v1/routing",
+      "seconds": 0.0286,
+      "input_tokens": 192,
+      "temperature": 1.3
+    },
+    {
+      "id": "judge-proxy-v1/heldout/adequacy/00063",
+      "suite": "judge-proxy-v1/adequacy",
+      "seconds": 0.0225,
+      "input_tokens": 200,
+      "temperature": 1.3
+    },
+    {
+      "id": "judge-proxy-v1/heldout/adequacy/00144",
+      "suite": "judge-proxy-v1/adequacy",
+      "seconds": 0.0253,
+      "input_tokens": 183,
+      "temperature": 1.3
+    },
+    {
+      "id": "judge-proxy-v1/heldout/routing/00271",
+      "suite": "judge-proxy-v1/routing",
+      "seconds": 0.0237,
+      "input_tokens": 235,
+      "temperature": 1.3
+    },
+    {
+      "id": "original-ordinal-03-1",
+      "suite": "jevbench-original",
+      "seconds": 0.022,
+      "input_tokens": 124,
+      "temperature": 1.3
+    },
+    {
+      "id": "judge-proxy-v1/heldout/adequacy/00155",
+      "suite": "judge-proxy-v1/adequacy",
+      "seconds": 0.0217,
+      "input_tokens": 182,
+      "temperature": 1.3
+    },
+    {
+      "id": "internal-eval-v1/ambiguous/0089",
+      "suite": "internal-eval-v1",
+      "seconds": 0.022,
+      "input_tokens": 244,
+      "temperature": 1.3
+    },
+    {
+      "id": "judge-proxy-v1/heldout/adequacy/00047",
+      "suite": "judge-proxy-v1/adequacy",
+      "seconds": 0.0215,
+      "input_tokens": 140,
+      "temperature": 1.3
+    },
+    {
+      "id": "judge-proxy-v1/heldout/routing/00028",
+      "suite": "judge-proxy-v1/routing",
+      "seconds": 0.0299,
+      "input_tokens": 1028,
+      "temperature": 1.3
+    },
+    {
+      "id": "judge-proxy-v1/heldout/adequacy/00210",
+      "suite": "judge-proxy-v1/adequacy",
+      "seconds": 0.0221,
+      "input_tokens": 177,
+      "temperature": 1.3
+    },
+    {
+      "id": "original-intent-05-1",
+      "suite": "jevbench-original",
+      "seconds": 0.0217,
+      "input_tokens": 110,
+      "temperature": 1.3
+    },
+    {
+      "id": "judge-proxy-v1/heldout/routing/00124",
+      "suite": "judge-proxy-v1/routing",
+      "seconds": 0.0237,
+      "input_tokens": 209,
+      "temperature": 1.3
+    },
+    {
+      "id": "internal-eval-v1/judge_hard/0069",
+      "suite": "internal-eval-v1",
+      "seconds": 0.0219,
+      "input_tokens": 230,
+      "temperature": 1.3
+    },
+    {
+      "id": "judge-proxy-v1/heldout/routing/00016",
+      "suite": "judge-proxy-v1/routing",
+      "seconds": 0.0296,
+      "input_tokens": 1212,
+      "temperature": 1.3
+    },
+    {
+      "id": "judge-proxy-v1/heldout/adequacy/00282",
+      "suite": "judge-proxy-v1/adequacy",
+      "seconds": 0.0221,
+      "input_tokens": 205,
+      "temperature": 1.3
+    },
+    {
+      "id": "original-extraction-05-0",
+      "suite": "jevbench-original",
+      "seconds": 0.0218,
+      "input_tokens": 106,
+      "temperature": 1.3
+    },
+    {
+      "id": "internal-eval-v1/judge_hard/0080",
+      "suite": "internal-eval-v1",
+      "seconds": 0.0221,
+      "input_tokens": 230,
+      "temperature": 1.3
+    },
+    {
+      "id": "original-policy-06-1",
+      "suite": "jevbench-original",
+      "seconds": 0.0211,
+      "input_tokens": 108,
+      "temperature": 1.3
+    },
+    {
+      "id": "judge-proxy-v1/heldout/routing/00013",
+      "suite": "judge-proxy-v1/routing",
+      "seconds": 0.0241,
+      "input_tokens": 229,
+      "temperature": 1.3
+    },
+    {
+      "id": "judge-proxy-v1/heldout/adequacy/00109",
+      "suite": "judge-proxy-v1/adequacy",
+      "seconds": 0.0207,
+      "input_tokens": 148,
+      "temperature": 1.3
+    },
+    {
+      "id": "judge-proxy-v1/heldout/adequacy/00013",
+      "suite": "judge-proxy-v1/adequacy",
+      "seconds": 0.021,
+      "input_tokens": 155,
+      "temperature": 1.3
+    },
+    {
+      "id": "internal-eval-v1/long_policy/0020",
+      "suite": "internal-eval-v1",
+      "seconds": 0.0244,
+      "input_tokens": 512,
+      "temperature": 1.3
+    },
+    {
+      "id": "original-ordinal-06-0",
+      "suite": "jevbench-original",
+      "seconds": 0.0218,
+      "input_tokens": 127,
+      "temperature": 1.3
+    },
+    {
+      "id": "judge-proxy-v1/heldout/routing/00229",
+      "suite": "judge-proxy-v1/routing",
+      "seconds": 0.031,
+      "input_tokens": 1350,
+      "temperature": 1.3
+    },
+    {
+      "id": "original-routing-06-0",
+      "suite": "jevbench-original",
+      "seconds": 0.0213,
+      "input_tokens": 140,
+      "temperature": 1.3
+    },
+    {
+      "id": "judge-proxy-v1/heldout/adequacy/00130",
+      "suite": "judge-proxy-v1/adequacy",
+      "seconds": 0.0214,
+      "input_tokens": 166,
+      "temperature": 1.3
+    },
+    {
+      "id": "original-ordinal-01-1",
+      "suite": "jevbench-original",
+      "seconds": 0.022,
+      "input_tokens": 124,
+      "temperature": 1.3
+    },
+    {
+      "id": "original-intent-01-1",
+      "suite": "jevbench-original",
+      "seconds": 0.0219,
+      "input_tokens": 112,
+      "temperature": 1.3
+    },
+    {
+      "id": "judge-proxy-v1/heldout/routing/00299",
+      "suite": "judge-proxy-v1/routing",
+      "seconds": 0.0239,
+      "input_tokens": 231,
+      "temperature": 1.3
+    },
+    {
+      "id": "original-ordinal-05-1",
+      "suite": "jevbench-original",
+      "seconds": 0.0213,
+      "input_tokens": 126,
+      "temperature": 1.3
+    },
+    {
+      "id": "judge-proxy-v1/heldout/adequacy/00018",
+      "suite": "judge-proxy-v1/adequacy",
+      "seconds": 0.022,
+      "input_tokens": 164,
+      "temperature": 1.3
+    },
+    {
+      "id": "judge-proxy-v1/heldout/routing/00277",
+      "suite": "judge-proxy-v1/routing",
+      "seconds": 0.0251,
+      "input_tokens": 201,
+      "temperature": 1.3
+    },
+    {
+      "id": "judge-proxy-v1/heldout/routing/00149",
+      "suite": "judge-proxy-v1/routing",
+      "seconds": 0.0244,
+      "input_tokens": 255,
+      "temperature": 1.3
+    },
+    {
+      "id": "original-adequacy-03-0",
+      "suite": "jevbench-original",
+      "seconds": 0.0211,
+      "input_tokens": 99,
+      "temperature": 1.3
+    },
+    {
+      "id": "original-adequacy-05-1",
+      "suite": "jevbench-original",
+      "seconds": 0.0206,
+      "input_tokens": 93,
+      "temperature": 1.3
+    },
+    {
+      "id": "judge-proxy-v1/heldout/routing/00007",
+      "suite": "judge-proxy-v1/routing",
+      "seconds": 0.0332,
+      "input_tokens": 200,
+      "temperature": 1.3
+    },
+    {
+      "id": "original-policy-03-1",
+      "suite": "jevbench-original",
+      "seconds": 0.0287,
+      "input_tokens": 106,
+      "temperature": 1.3
+    },
+    {
+      "id": "judge-proxy-v1/heldout/routing/00289",
+      "suite": "judge-proxy-v1/routing",
+      "seconds": 0.0286,
+      "input_tokens": 235,
+      "temperature": 1.3
+    },
+    {
+      "id": "judge-proxy-v1/heldout/adequacy/00120",
+      "suite": "judge-proxy-v1/adequacy",
+      "seconds": 0.0275,
+      "input_tokens": 200,
+      "temperature": 1.3
+    },
+    {
+      "id": "judge-proxy-v1/heldout/adequacy/00175",
+      "suite": "judge-proxy-v1/adequacy",
+      "seconds": 0.0269,
+      "input_tokens": 121,
+      "temperature": 1.3
+    },
+    {
+      "id": "judge-proxy-v1/heldout/routing/00029",
+      "suite": "judge-proxy-v1/routing",
+      "seconds": 0.0237,
+      "input_tokens": 194,
+      "temperature": 1.3
+    },
+    {
+      "id": "judge-proxy-v1/heldout/adequacy/00048",
+      "suite": "judge-proxy-v1/adequacy",
+      "seconds": 0.021,
+      "input_tokens": 191,
+      "temperature": 1.3
+    },
+    {
+      "id": "judge-proxy-v1/heldout/routing/00225",
+      "suite": "judge-proxy-v1/routing",
+      "seconds": 0.0288,
+      "input_tokens": 925,
+      "temperature": 1.3
+    },
+    {
+      "id": "judge-proxy-v1/heldout/adequacy/00118",
+      "suite": "judge-proxy-v1/adequacy",
+      "seconds": 0.023,
+      "input_tokens": 142,
+      "temperature": 1.3
+    },
+    {
+      "id": "original-ordinal-02-1",
+      "suite": "jevbench-original",
+      "seconds": 0.0224,
+      "input_tokens": 126,
+      "temperature": 1.3
+    },
+    {
+      "id": "judge-proxy-v1/heldout/routing/00107",
+      "suite": "judge-proxy-v1/routing",
+      "seconds": 0.0229,
+      "input_tokens": 196,
+      "temperature": 1.3
+    },
+    {
+      "id": "original-extraction-04-0",
+      "suite": "jevbench-original",
+      "seconds": 0.0214,
+      "input_tokens": 105,
+      "temperature": 1.3
+    },
+    {
+      "id": "judge-proxy-v1/heldout/routing/00265",
+      "suite": "judge-proxy-v1/routing",
+      "seconds": 0.0242,
+      "input_tokens": 199,
+      "temperature": 1.3
+    },
+    {
+      "id": "original-ordinal-04-0",
+      "suite": "jevbench-original",
+      "seconds": 0.0218,
+      "input_tokens": 123,
+      "temperature": 1.3
+    },
+    {
+      "id": "judge-proxy-v1/heldout/routing/00165",
+      "suite": "judge-proxy-v1/routing",
+      "seconds": 0.0236,
+      "input_tokens": 217,
+      "temperature": 1.3
+    },
+    {
+      "id": "judge-proxy-v1/heldout/adequacy/00256",
+      "suite": "judge-proxy-v1/adequacy",
+      "seconds": 0.0212,
+      "input_tokens": 185,
+      "temperature": 1.3
+    },
+    {
+      "id": "judge-proxy-v1/heldout/routing/00128",
+      "suite": "judge-proxy-v1/routing",
+      "seconds": 0.0237,
+      "input_tokens": 193,
+      "temperature": 1.3
+    },
+    {
+      "id": "judge-proxy-v1/heldout/routing/00075",
+      "suite": "judge-proxy-v1/routing",
+      "seconds": 0.024,
+      "input_tokens": 305,
+      "temperature": 1.3
+    },
+    {
+      "id": "original-policy-04-1",
+      "suite": "jevbench-original",
+      "seconds": 0.0213,
+      "input_tokens": 111,
+      "temperature": 1.3
+    },
+    {
+      "id": "judge-proxy-v1/heldout/routing/00273",
+      "suite": "judge-proxy-v1/routing",
+      "seconds": 0.0236,
+      "input_tokens": 223,
+      "temperature": 1.3
+    },
+    {
+      "id": "internal-eval-v1/probability/0006",
+      "suite": "internal-eval-v1",
+      "seconds": 0.0224,
+      "input_tokens": 220,
+      "temperature": 1.3
+    },
+    {
+      "id": "judge-proxy-v1/heldout/routing/00176",
+      "suite": "judge-proxy-v1/routing",
+      "seconds": 0.0237,
+      "input_tokens": 279,
+      "temperature": 1.3
+    },
+    {
+      "id": "judge-proxy-v1/heldout/adequacy/00195",
+      "suite": "judge-proxy-v1/adequacy",
+      "seconds": 0.0216,
+      "input_tokens": 136,
+      "temperature": 1.3
+    },
+    {
+      "id": "judge-proxy-v1/heldout/routing/00242",
+      "suite": "judge-proxy-v1/routing",
+      "seconds": 0.023,
+      "input_tokens": 245,
+      "temperature": 1.3
+    },
+    {
+      "id": "original-routing-02-0",
+      "suite": "jevbench-original",
+      "seconds": 0.0219,
+      "input_tokens": 144,
+      "temperature": 1.3
+    },
+    {
+      "id": "judge-proxy-v1/heldout/routing/00297",
+      "suite": "judge-proxy-v1/routing",
+      "seconds": 0.0228,
+      "input_tokens": 197,
+      "temperature": 1.3
+    },
+    {
+      "id": "internal-eval-v1/probability/0030",
+      "suite": "internal-eval-v1",
+      "seconds": 0.0216,
+      "input_tokens": 219,
+      "temperature": 1.3
+    },
+    {
+      "id": "judge-proxy-v1/heldout/adequacy/00005",
+      "suite": "judge-proxy-v1/adequacy",
+      "seconds": 0.0212,
+      "input_tokens": 135,
+      "temperature": 1.3
+    },
+    {
+      "id": "judge-proxy-v1/heldout/routing/00034",
+      "suite": "judge-proxy-v1/routing",
+      "seconds": 0.042,
+      "input_tokens": 2184,
+      "temperature": 1.3
+    },
+    {
+      "id": "judge-proxy-v1/heldout/routing/00151",
+      "suite": "judge-proxy-v1/routing",
+      "seconds": 0.0229,
+      "input_tokens": 206,
+      "temperature": 1.3
+    },
+    {
+      "id": "judge-proxy-v1/heldout/routing/00286",
+      "suite": "judge-proxy-v1/routing",
+      "seconds": 0.0227,
+      "input_tokens": 308,
+      "temperature": 1.3
+    },
+    {
+      "id": "original-adequacy-05-0",
+      "suite": "jevbench-original",
+      "seconds": 0.0199,
+      "input_tokens": 93,
+      "temperature": 1.3
+    },
+    {
+      "id": "original-intent-02-0",
+      "suite": "jevbench-original",
+      "seconds": 0.0209,
+      "input_tokens": 111,
+      "temperature": 1.3
+    },
+    {
+      "id": "judge-proxy-v1/heldout/routing/00168",
+      "suite": "judge-proxy-v1/routing",
+      "seconds": 0.0226,
+      "input_tokens": 202,
+      "temperature": 1.3
+    },
+    {
+      "id": "judge-proxy-v1/heldout/routing/00215",
+      "suite": "judge-proxy-v1/routing",
+      "seconds": 0.0228,
+      "input_tokens": 229,
+      "temperature": 1.3
+    },
+    {
+      "id": "original-adequacy-02-0",
+      "suite": "jevbench-original",
+      "seconds": 0.0209,
+      "input_tokens": 98,
+      "temperature": 1.3
+    },
+    {
+      "id": "judge-proxy-v1/heldout/routing/00030",
+      "suite": "judge-proxy-v1/routing",
+      "seconds": 0.0237,
+      "input_tokens": 219,
+      "temperature": 1.3
+    },
+    {
+      "id": "judge-proxy-v1/heldout/adequacy/00299",
+      "suite": "judge-proxy-v1/adequacy",
+      "seconds": 0.0212,
+      "input_tokens": 149,
+      "temperature": 1.3
+    },
+    {
+      "id": "original-extraction-01-0",
+      "suite": "jevbench-original",
+      "seconds": 0.0203,
+      "input_tokens": 103,
+      "temperature": 1.3
+    },
+    {
+      "id": "judge-proxy-v1/heldout/adequacy/00237",
+      "suite": "judge-proxy-v1/adequacy",
+      "seconds": 0.0214,
+      "input_tokens": 174,
+      "temperature": 1.3
+    },
+    {
+      "id": "original-adequacy-01-1",
+      "suite": "jevbench-original",
+      "seconds": 0.0214,
+      "input_tokens": 96,
+      "temperature": 1.3
+    },
+    {
+      "id": "judge-proxy-v1/heldout/adequacy/00116",
+      "suite": "judge-proxy-v1/adequacy",
+      "seconds": 0.0221,
+      "input_tokens": 202,
+      "temperature": 1.3
+    },
+    {
+      "id": "original-policy-05-1",
+      "suite": "jevbench-original",
+      "seconds": 0.0217,
+      "input_tokens": 109,
+      "temperature": 1.3
+    },
+    {
+      "id": "judge-proxy-v1/heldout/routing/00207",
+      "suite": "judge-proxy-v1/routing",
+      "seconds": 0.0235,
+      "input_tokens": 197,
+      "temperature": 1.3
+    },
+    {
+      "id": "original-ordinal-01-0",
+      "suite": "jevbench-original",
+      "seconds": 0.0213,
+      "input_tokens": 121,
+      "temperature": 1.3
+    },
+    {
+      "id": "original-intent-01-0",
+      "suite": "jevbench-original",
+      "seconds": 0.0233,
+      "input_tokens": 110,
+      "temperature": 1.3
+    },
+    {
+      "id": "internal-eval-v1/judge_hard/0057",
+      "suite": "internal-eval-v1",
+      "seconds": 0.024,
+      "input_tokens": 228,
+      "temperature": 1.3
+    },
+    {
+      "id": "original-adequacy-06-0",
+      "suite": "jevbench-original",
+      "seconds": 0.0222,
+      "input_tokens": 110,
+      "temperature": 1.3
+    },
+    {
+      "id": "judge-proxy-v1/heldout/routing/00041",
+      "suite": "judge-proxy-v1/routing",
+      "seconds": 0.025,
+      "input_tokens": 191,
+      "temperature": 1.3
+    },
+    {
+      "id": "original-routing-01-1",
+      "suite": "jevbench-original",
+      "seconds": 0.023,
+      "input_tokens": 139,
+      "temperature": 1.3
+    },
+    {
+      "id": "original-routing-01-0",
+      "suite": "jevbench-original",
+      "seconds": 0.0229,
+      "input_tokens": 143,
+      "temperature": 1.3
+    },
+    {
+      "id": "original-policy-06-0",
+      "suite": "jevbench-original",
+      "seconds": 0.0218,
+      "input_tokens": 119,
+      "temperature": 1.3
+    },
+    {
+      "id": "internal-eval-v1/multi_hop/0070",
+      "suite": "internal-eval-v1",
+      "seconds": 0.0231,
+      "input_tokens": 290,
+      "temperature": 1.3
+    },
+    {
+      "id": "judge-proxy-v1/heldout/adequacy/00072",
+      "suite": "judge-proxy-v1/adequacy",
+      "seconds": 0.0212,
+      "input_tokens": 164,
+      "temperature": 1.3
+    },
+    {
+      "id": "internal-eval-v1/judge_hard/0098",
+      "suite": "internal-eval-v1",
+      "seconds": 0.0235,
+      "input_tokens": 226,
+      "temperature": 1.3
+    },
+    {
+      "id": "judge-proxy-v1/heldout/routing/00196",
+      "suite": "judge-proxy-v1/routing",
+      "seconds": 0.0415,
+      "input_tokens": 2172,
+      "temperature": 1.3
+    },
+    {
+      "id": "original-intent-05-0",
+      "suite": "jevbench-original",
+      "seconds": 0.0217,
+      "input_tokens": 110,
+      "temperature": 1.3
+    },
+    {
+      "id": "judge-proxy-v1/heldout/routing/00076",
+      "suite": "judge-proxy-v1/routing",
+      "seconds": 0.0231,
+      "input_tokens": 230,
+      "temperature": 1.3
+    },
+    {
+      "id": "internal-eval-v1/probability/0051",
+      "suite": "internal-eval-v1",
+      "seconds": 0.0231,
+      "input_tokens": 219,
+      "temperature": 1.3
+    },
+    {
+      "id": "judge-proxy-v1/heldout/routing/00038",
+      "suite": "judge-proxy-v1/routing",
+      "seconds": 0.0426,
+      "input_tokens": 2170,
+      "temperature": 1.3
+    },
+    {
+      "id": "original-adequacy-04-1",
+      "suite": "jevbench-original",
+      "seconds": 0.0213,
+      "input_tokens": 101,
+      "temperature": 1.3
+    },
+    {
+      "id": "original-extraction-01-1",
+      "suite": "jevbench-original",
+      "seconds": 0.0225,
+      "input_tokens": 105,
+      "temperature": 1.3
+    },
+    {
+      "id": "original-extraction-04-1",
+      "suite": "jevbench-original",
+      "seconds": 0.022,
+      "input_tokens": 103,
+      "temperature": 1.3
+    },
+    {
+      "id": "judge-proxy-v1/heldout/routing/00174",
+      "suite": "judge-proxy-v1/routing",
+      "seconds": 0.0247,
+      "input_tokens": 199,
+      "temperature": 1.3
+    },
+    {
+      "id": "judge-proxy-v1/heldout/routing/00155",
+      "suite": "judge-proxy-v1/routing",
+      "seconds": 0.0302,
+      "input_tokens": 1173,
+      "temperature": 1.3
+    },
+    {
+      "id": "judge-proxy-v1/heldout/routing/00088",
+      "suite": "judge-proxy-v1/routing",
+      "seconds": 0.0252,
+      "input_tokens": 633,
+      "temperature": 1.3
+    },
+    {
+      "id": "original-extraction-05-1",
+      "suite": "jevbench-original",
+      "seconds": 0.021,
+      "input_tokens": 102,
+      "temperature": 1.3
+    },
+    {
+      "id": "original-adequacy-03-1",
+      "suite": "jevbench-original",
+      "seconds": 0.022,
+      "input_tokens": 103,
+      "temperature": 1.3
+    },
+    {
+      "id": "original-routing-05-0",
+      "suite": "jevbench-original",
+      "seconds": 0.0231,
+      "input_tokens": 140,
+      "temperature": 1.3
+    },
+    {
+      "id": "judge-proxy-v1/heldout/adequacy/00069",
+      "suite": "judge-proxy-v1/adequacy",
+      "seconds": 0.0223,
+      "input_tokens": 172,
+      "temperature": 1.3
+    },
+    {
+      "id": "judge-proxy-v1/heldout/adequacy/00088",
+      "suite": "judge-proxy-v1/adequacy",
+      "seconds": 0.0208,
+      "input_tokens": 169,
+      "temperature": 1.3
+    },
+    {
+      "id": "judge-proxy-v1/heldout/routing/00264",
+      "suite": "judge-proxy-v1/routing",
+      "seconds": 0.0234,
+      "input_tokens": 195,
+      "temperature": 1.3
+    },
+    {
+      "id": "judge-proxy-v1/heldout/routing/00285",
+      "suite": "judge-proxy-v1/routing",
+      "seconds": 0.0466,
+      "input_tokens": 2171,
+      "temperature": 1.3
+    },
+    {
+      "id": "judge-proxy-v1/heldout/adequacy/00298",
+      "suite": "judge-proxy-v1/adequacy",
+      "seconds": 0.0216,
+      "input_tokens": 181,
+      "temperature": 1.3
+    },
+    {
+      "id": "original-routing-06-1",
+      "suite": "jevbench-original",
+      "seconds": 0.0214,
+      "input_tokens": 138,
+      "temperature": 1.3
+    },
+    {
+      "id": "judge-proxy-v1/heldout/adequacy/00092",
+      "suite": "judge-proxy-v1/adequacy",
+      "seconds": 0.0346,
+      "input_tokens": 176,
+      "temperature": 1.3
+    },
+    {
+      "id": "original-policy-03-0",
+      "suite": "jevbench-original",
+      "seconds": 0.0205,
+      "input_tokens": 106,
+      "temperature": 1.3
+    },
+    {
+      "id": "judge-proxy-v1/heldout/adequacy/00242",
+      "suite": "judge-proxy-v1/adequacy",
+      "seconds": 0.0205,
+      "input_tokens": 183,
+      "temperature": 1.3
+    },
+    {
+      "id": "judge-proxy-v1/heldout/adequacy/00251",
+      "suite": "judge-proxy-v1/adequacy",
+      "seconds": 0.0214,
+      "input_tokens": 186,
+      "temperature": 1.3
+    },
+    {
+      "id": "judge-proxy-v1/heldout/adequacy/00184",
+      "suite": "judge-proxy-v1/adequacy",
+      "seconds": 0.0216,
+      "input_tokens": 187,
+      "temperature": 1.3
+    },
+    {
+      "id": "internal-eval-v1/temporal_numeric/0040",
+      "suite": "internal-eval-v1",
+      "seconds": 0.0215,
+      "input_tokens": 290,
+      "temperature": 1.3
+    },
+    {
+      "id": "judge-proxy-v1/heldout/routing/00230",
+      "suite": "judge-proxy-v1/routing",
+      "seconds": 0.0299,
+      "input_tokens": 613,
+      "temperature": 1.3
+    },
+    {
+      "id": "judge-proxy-v1/heldout/adequacy/00127",
+      "suite": "judge-proxy-v1/adequacy",
+      "seconds": 0.0227,
+      "input_tokens": 168,
+      "temperature": 1.3
+    },
+    {
+      "id": "internal-eval-v1/probability/0003",
+      "suite": "internal-eval-v1",
+      "seconds": 0.0347,
+      "input_tokens": 219,
+      "temperature": 1.3
+    },
+    {
+      "id": "judge-proxy-v1/heldout/adequacy/00176",
+      "suite": "judge-proxy-v1/adequacy",
+      "seconds": 0.0243,
+      "input_tokens": 180,
+      "temperature": 1.3
+    },
+    {
+      "id": "original-routing-05-1",
+      "suite": "jevbench-original",
+      "seconds": 0.0225,
+      "input_tokens": 140,
+      "temperature": 1.3
+    },
+    {
+      "id": "judge-proxy-v1/heldout/adequacy/00260",
+      "suite": "judge-proxy-v1/adequacy",
+      "seconds": 0.0219,
+      "input_tokens": 191,
+      "temperature": 1.3
+    },
+    {
+      "id": "judge-proxy-v1/heldout/routing/00042",
+      "suite": "judge-proxy-v1/routing",
+      "seconds": 0.0245,
+      "input_tokens": 267,
+      "temperature": 1.3
+    },
+    {
+      "id": "judge-proxy-v1/heldout/adequacy/00056",
+      "suite": "judge-proxy-v1/adequacy",
+      "seconds": 0.0222,
+      "input_tokens": 159,
+      "temperature": 1.3
+    },
+    {
+      "id": "judge-proxy-v1/heldout/adequacy/00052",
+      "suite": "judge-proxy-v1/adequacy",
+      "seconds": 0.0216,
+      "input_tokens": 164,
+      "temperature": 1.3
+    },
+    {
+      "id": "judge-proxy-v1/heldout/routing/00097",
+      "suite": "judge-proxy-v1/routing",
+      "seconds": 0.0237,
+      "input_tokens": 217,
+      "temperature": 1.3
+    },
+    {
+      "id": "judge-proxy-v1/heldout/adequacy/00044",
+      "suite": "judge-proxy-v1/adequacy",
+      "seconds": 0.0211,
+      "input_tokens": 135,
+      "temperature": 1.3
+    },
+    {
+      "id": "judge-proxy-v1/heldout/adequacy/00146",
+      "suite": "judge-proxy-v1/adequacy",
+      "seconds": 0.0211,
+      "input_tokens": 192,
+      "temperature": 1.3
+    },
+    {
+      "id": "judge-proxy-v1/heldout/routing/00260",
+      "suite": "judge-proxy-v1/routing",
+      "seconds": 0.0243,
+      "input_tokens": 231,
+      "temperature": 1.3
+    },
+    {
+      "id": "original-intent-04-1",
+      "suite": "jevbench-original",
+      "seconds": 0.0227,
+      "input_tokens": 110,
+      "temperature": 1.3
+    },
+    {
+      "id": "original-intent-03-1",
+      "suite": "jevbench-original",
+      "seconds": 0.0226,
+      "input_tokens": 110,
+      "temperature": 1.3
+    },
+    {
+      "id": "original-intent-02-1",
+      "suite": "jevbench-original",
+      "seconds": 0.0223,
+      "input_tokens": 110,
+      "temperature": 1.3
+    },
+    {
+      "id": "judge-proxy-v1/heldout/routing/00031",
+      "suite": "judge-proxy-v1/routing",
+      "seconds": 0.0254,
+      "input_tokens": 221,
+      "temperature": 1.3
+    },
+    {
+      "id": "judge-proxy-v1/heldout/adequacy/00171",
+      "suite": "judge-proxy-v1/adequacy",
+      "seconds": 0.0218,
+      "input_tokens": 171,
+      "temperature": 1.3
+    },
+    {
+      "id": "internal-eval-v1/ambiguous/0091",
+      "suite": "internal-eval-v1",
+      "seconds": 0.0222,
+      "input_tokens": 233,
+      "temperature": 1.3
+    },
+    {
+      "id": "original-policy-05-0",
+      "suite": "jevbench-original",
+      "seconds": 0.0215,
+      "input_tokens": 116,
+      "temperature": 1.3
+    },
+    {
+      "id": "judge-proxy-v1/heldout/routing/00252",
+      "suite": "judge-proxy-v1/routing",
+      "seconds": 0.0237,
+      "input_tokens": 210,
+      "temperature": 1.3
+    },
+    {
+      "id": "judge-proxy-v1/heldout/routing/00291",
+      "suite": "judge-proxy-v1/routing",
+      "seconds": 0.0235,
+      "input_tokens": 210,
+      "temperature": 1.3
+    },
+    {
+      "id": "judge-proxy-v1/heldout/adequacy/00214",
+      "suite": "judge-proxy-v1/adequacy",
+      "seconds": 0.0214,
+      "input_tokens": 200,
+      "temperature": 1.3
+    },
+    {
+      "id": "judge-proxy-v1/heldout/adequacy/00033",
+      "suite": "judge-proxy-v1/adequacy",
+      "seconds": 0.0226,
+      "input_tokens": 154,
+      "temperature": 1.3
+    },
+    {
+      "id": "judge-proxy-v1/heldout/routing/00213",
+      "suite": "judge-proxy-v1/routing",
+      "seconds": 0.0265,
+      "input_tokens": 270,
+      "temperature": 1.3
+    },
+    {
+      "id": "judge-proxy-v1/heldout/routing/00104",
+      "suite": "judge-proxy-v1/routing",
+      "seconds": 0.0239,
+      "input_tokens": 219,
+      "temperature": 1.3
+    },
+    {
+      "id": "original-extraction-03-0",
+      "suite": "jevbench-original",
+      "seconds": 0.0212,
+      "input_tokens": 109,
+      "temperature": 1.3
+    },
+    {
+      "id": "original-ordinal-06-1",
+      "suite": "jevbench-original",
+      "seconds": 0.0214,
+      "input_tokens": 135,
+      "temperature": 1.3
+    }
+  ]
+}

evidence/served-nosys-original-report.json

@@ -0,0 +1,287 @@
+{
+  "status": "complete",
+  "requested_records": 72,
+  "requested_questions": 72,
+  "attempted_requests": 72,
+  "successful_requests": 72,
+  "errors": 0,
+  "served_models": [
+    "decision-model"
+  ],
+  "elapsed_measurement_s": 1.6658977939805482,
+  "requests_per_second": 43.219938378068775,
+  "questions_per_second": 43.219938378068775,
+  "latency_ms": {
+    "n": 72,
+    "mean": 23.027374291511176,
+    "p50": 22.36310299485922,
+    "p95": 30.51040534919594,
+    "p99": 34.34268011507812
+  },
+  "adapter_ms": {
+    "n": 72,
+    "mean": 21.671233003467528,
+    "p50": 21.02683151315432,
+    "p95": 29.079331248067323,
+    "p99": 31.649900618358537
+  },
+  "input_tokens_per_request": {
+    "n": 72,
+    "mean": 115.54166666666667,
+    "p50": 110.0,
+    "p95": 141.35000000000002,
+    "p99": 144.58
+  },
+  "suites": {
+    "jevbench-original/public": {
+      "clean": {
+        "n": 72,
+        "acc": 0.875,
+        "nll": 0.2811701595983037,
+        "brier": 0.16998368696190747,
+        "ece": 0.027585444231134813,
+        "mean_conf": 0.9003661785243816,
+        "confidence_bias": 0.02536617852438161,
+        "confident_error_rate": 0.013888888888888888,
+        "coverage_at_0_9": 0.6666666666666666,
+        "accuracy_at_0_9": 0.9791666666666666,
+        "coverage_at_5pct_error": 0.8333333333333334,
+        "coverage_at_1pct_error": 0.5277777777777778,
+        "score_mae": 0.14433068031614896,
+        "ranked_probability_score": 0.021378525778158384
+      },
+      "tasks": {
+        "adequacy": {
+          "n": 12,
+          "acc": 0.6666666666666666,
+          "nll": 0.7816227650376891,
+          "brier": 0.5076781498841229,
+          "ece": 0.19641208292198364,
+          "mean_conf": 0.8096085306199567,
+          "confidence_bias": 0.14294186395329012,
+          "confident_error_rate": 0.08333333333333333,
+          "coverage_at_0_9": 0.3333333333333333,
+          "accuracy_at_0_9": 0.75,
+          "coverage_at_5pct_error": 0.25,
+          "coverage_at_1pct_error": 0.25
+        },
+        "extraction": {
+          "n": 12,
+          "acc": 1.0,
+          "nll": 0.0542116119441933,
+          "brier": 0.01493325495130398,
+          "ece": 0.0495188270475546,
+          "mean_conf": 0.9504811729524453,
+          "confidence_bias": -0.049518827047554725,
+          "confident_error_rate": 0.0,
+          "coverage_at_0_9": 0.8333333333333334,
+          "accuracy_at_0_9": 1.0,
+          "coverage_at_5pct_error": 1.0,
+          "coverage_at_1pct_error": 1.0
+        },
+        "intent": {
+          "n": 12,
+          "acc": 0.8333333333333334,
+          "nll": 0.30572276294355083,
+          "brier": 0.20045464895299747,
+          "ece": 0.14304785668742614,
+          "mean_conf": 0.9322074612496651,
+          "confidence_bias": 0.09887412791633177,
+          "confident_error_rate": 0.0,
+          "coverage_at_0_9": 0.75,
+          "accuracy_at_0_9": 1.0,
+          "coverage_at_5pct_error": 0.8333333333333334,
+          "coverage_at_1pct_error": 0.8333333333333334
+        },
+        "ordinal": {
+          "n": 12,
+          "acc": 0.9166666666666666,
+          "nll": 0.20756268320968063,
+          "brier": 0.10919592930395382,
+          "ece": 0.12099966786008741,
+          "mean_conf": 0.9077797081312969,
+          "confidence_bias": -0.008886958535369693,
+          "confident_error_rate": 0.0,
+          "coverage_at_0_9": 0.6666666666666666,
+          "accuracy_at_0_9": 1.0,
+          "coverage_at_5pct_error": 0.9166666666666666,
+          "coverage_at_1pct_error": 0.9166666666666666,
+          "score_mae": 0.14433068031614896,
+          "ranked_probability_score": 0.021378525778158384
+        },
+        "policy": {
+          "n": 12,
+          "acc": 0.9166666666666666,
+          "nll": 0.10467945507562453,
+          "brier": 0.05772104532732034,
+          "ece": 0.08135385206374438,
+          "mean_conf": 0.9290088980838886,
+          "confidence_bias": 0.012342231417222016,
+          "confident_error_rate": 0.0,
+          "coverage_at_0_9": 0.8333333333333334,
+          "accuracy_at_0_9": 1.0,
+          "coverage_at_5pct_error": 0.9166666666666666,
+          "coverage_at_1pct_error": 0.9166666666666666
+        },
+        "routing": {
+          "n": 12,
+          "acc": 0.9166666666666666,
+          "nll": 0.23322167937908408,
+          "brier": 0.12991909335174637,
+          "ece": 0.1168843091364116,
+          "mean_conf": 0.8731113001090374,
+          "confidence_bias": -0.04355536655762926,
+          "confident_error_rate": 0.0,
+          "coverage_at_0_9": 0.5833333333333334,
+          "accuracy_at_0_9": 1.0,
+          "coverage_at_5pct_error": 0.75,
+          "coverage_at_1pct_error": 0.75
+        }
+      },
+      "sources": {
+        "adequacy": {
+          "n": 12,
+          "acc": 0.6666666666666666,
+          "nll": 0.7816227650376891,
+          "brier": 0.5076781498841229,
+          "ece": 0.19641208292198364,
+          "mean_conf": 0.8096085306199567,
+          "confidence_bias": 0.14294186395329012,
+          "confident_error_rate": 0.08333333333333333,
+          "coverage_at_0_9": 0.3333333333333333,
+          "accuracy_at_0_9": 0.75,
+          "coverage_at_5pct_error": 0.25,
+          "coverage_at_1pct_error": 0.25
+        },
+        "extraction": {
+          "n": 12,
+          "acc": 1.0,
+          "nll": 0.0542116119441933,
+          "brier": 0.01493325495130398,
+          "ece": 0.0495188270475546,
+          "mean_conf": 0.9504811729524453,
+          "confidence_bias": -0.049518827047554725,
+          "confident_error_rate": 0.0,
+          "coverage_at_0_9": 0.8333333333333334,
+          "accuracy_at_0_9": 1.0,
+          "coverage_at_5pct_error": 1.0,
+          "coverage_at_1pct_error": 1.0
+        },
+        "intent": {
+          "n": 12,
+          "acc": 0.8333333333333334,
+          "nll": 0.30572276294355083,
+          "brier": 0.20045464895299747,
+          "ece": 0.14304785668742614,
+          "mean_conf": 0.9322074612496651,
+          "confidence_bias": 0.09887412791633177,
+          "confident_error_rate": 0.0,
+          "coverage_at_0_9": 0.75,
+          "accuracy_at_0_9": 1.0,
+          "coverage_at_5pct_error": 0.8333333333333334,
+          "coverage_at_1pct_error": 0.8333333333333334
+        },
+        "ordinal": {
+          "n": 12,
+          "acc": 0.9166666666666666,
+          "nll": 0.20756268320968063,
+          "brier": 0.10919592930395382,
+          "ece": 0.12099966786008741,
+          "mean_conf": 0.9077797081312969,
+          "confidence_bias": -0.008886958535369693,
+          "confident_error_rate": 0.0,
+          "coverage_at_0_9": 0.6666666666666666,
+          "accuracy_at_0_9": 1.0,
+          "coverage_at_5pct_error": 0.9166666666666666,
+          "coverage_at_1pct_error": 0.9166666666666666,
+          "score_mae": 0.14433068031614896,
+          "ranked_probability_score": 0.021378525778158384
+        },
+        "policy": {
+          "n": 12,
+          "acc": 0.9166666666666666,
+          "nll": 0.10467945507562453,
+          "brier": 0.05772104532732034,
+          "ece": 0.08135385206374438,
+          "mean_conf": 0.9290088980838886,
+          "confidence_bias": 0.012342231417222016,
+          "confident_error_rate": 0.0,
+          "coverage_at_0_9": 0.8333333333333334,
+          "accuracy_at_0_9": 1.0,
+          "coverage_at_5pct_error": 0.9166666666666666,
+          "coverage_at_1pct_error": 0.9166666666666666
+        },
+        "routing": {
+          "n": 12,
+          "acc": 0.9166666666666666,
+          "nll": 0.23322167937908408,
+          "brier": 0.12991909335174637,
+          "ece": 0.1168843091364116,
+          "mean_conf": 0.8731113001090374,
+          "confidence_bias": -0.04355536655762926,
+          "confident_error_rate": 0.0,
+          "coverage_at_0_9": 0.5833333333333334,
+          "accuracy_at_0_9": 1.0,
+          "coverage_at_5pct_error": 0.75,
+          "coverage_at_1pct_error": 0.75
+        }
+      },
+      "variants": {
+        "clean": {
+          "n": 72,
+          "acc": 0.875,
+          "nll": 0.2811701595983037,
+          "brier": 0.16998368696190747,
+          "ece": 0.027585444231134813,
+          "mean_conf": 0.9003661785243816,
+          "confidence_bias": 0.02536617852438161,
+          "confident_error_rate": 0.013888888888888888,
+          "coverage_at_0_9": 0.6666666666666666,
+          "accuracy_at_0_9": 0.9791666666666666,
+          "coverage_at_5pct_error": 0.8333333333333334,
+          "coverage_at_1pct_error": 0.5277777777777778,
+          "score_mae": 0.14433068031614896,
+          "ranked_probability_score": 0.021378525778158384
+        }
+      },
+      "macro_task_acc": 0.875,
+      "objective": -0.28117015959830377,
+      "paired_flip": {
+        "pairs": 0,
+        "incomplete_pairs": 0,
+        "flip_rate": null,
+        "both_correct_rate": null,
+        "invariant_pairs": 0,
+        "invariant_both_correct_rate": null,
+        "invariance_rate": null
+      },
+      "reference_distribution": null,
+      "unknowable": null,
+      "permutation": {
+        "n": 0,
+        "missing_parents": 0,
+        "flip_rate": null,
+        "mean_max_delta": null
+      },
+      "metric_policy": {
+        "nll_floor": 1e-09,
+        "ece_bins": 10,
+        "confidence_for_calibration": "maximum label probability",
+        "calibration_applied": false,
+        "unknown_accuracy_excluded": true,
+        "coverage_error_budget_is_retrospective": true,
+        "raw_sums_outside_1e_5": 0,
+        "returned_zeros": 0
+      },
+      "latency_ms": {
+        "n": 72,
+        "mean": 23.027374291511176,
+        "p50": 22.36310299485922,
+        "p95": 30.51040534919594,
+        "p99": 34.34268011507812
+      }
+    }
+  },
+  "latency_scope": "client HTTP wall time, excluding queue before dispatch; NOT GPU kernel time",
+  "partial_dataset": false
+}

generation_config.json

@@ -0,0 +1,7 @@
+{
+  "bos_token_id": 1,
+  "eos_token_id": 2,
+  "max_length": 262144,
+  "pad_token_id": 11,
+  "transformers_version": "5.0.0.dev0"
+}

model.safetensors.index.json

@@ -0,0 +1,466 @@
+{
+  "metadata": {
+    "total_parameters": 3849090048,
+    "total_size": 7698180096
+  },
+  "weight_map": {
+    "language_model.model.embed_tokens.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.0.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.0.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.0.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.0.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.0.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.0.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.0.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.0.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.0.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.1.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.1.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.1.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.1.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.1.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.1.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.1.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.1.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.1.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.10.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.10.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.10.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.10.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.10.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.10.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.10.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.10.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.10.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.11.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.11.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.11.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.11.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.11.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.11.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.11.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.11.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.11.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.12.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.12.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.12.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.12.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.12.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.12.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.12.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.12.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.12.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.13.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.13.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.13.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.13.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.13.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.13.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.13.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.13.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.13.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.14.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.14.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.14.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.14.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.14.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.14.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.14.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.14.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.14.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.15.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.15.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.15.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.15.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.15.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.15.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.15.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.15.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.15.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.16.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.16.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.16.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.16.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.16.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.16.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.16.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.16.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.16.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.17.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.17.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.17.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.17.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.17.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.17.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.17.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.17.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.17.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.18.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.18.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "language_model.model.layers.18.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "language_model.model.layers.18.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "language_model.model.layers.18.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "language_model.model.layers.18.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "language_model.model.layers.18.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "language_model.model.layers.18.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "language_model.model.layers.18.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "language_model.model.layers.19.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "language_model.model.layers.19.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "language_model.model.layers.19.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "language_model.model.layers.19.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "language_model.model.layers.19.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "language_model.model.layers.19.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "language_model.model.layers.19.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "language_model.model.layers.19.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "language_model.model.layers.19.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "language_model.model.layers.2.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.2.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.2.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.2.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.2.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.2.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.2.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.2.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.2.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.20.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "language_model.model.layers.20.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "language_model.model.layers.20.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "language_model.model.layers.20.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "language_model.model.layers.20.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "language_model.model.layers.20.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "language_model.model.layers.20.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "language_model.model.layers.20.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "language_model.model.layers.20.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "language_model.model.layers.21.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "language_model.model.layers.21.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "language_model.model.layers.21.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "language_model.model.layers.21.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "language_model.model.layers.21.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "language_model.model.layers.21.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "language_model.model.layers.21.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "language_model.model.layers.21.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "language_model.model.layers.21.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "language_model.model.layers.22.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "language_model.model.layers.22.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "language_model.model.layers.22.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "language_model.model.layers.22.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "language_model.model.layers.22.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "language_model.model.layers.22.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "language_model.model.layers.22.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "language_model.model.layers.22.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "language_model.model.layers.22.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "language_model.model.layers.23.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "language_model.model.layers.23.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "language_model.model.layers.23.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "language_model.model.layers.23.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "language_model.model.layers.23.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "language_model.model.layers.23.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "language_model.model.layers.23.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "language_model.model.layers.23.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "language_model.model.layers.23.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "language_model.model.layers.24.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "language_model.model.layers.24.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "language_model.model.layers.24.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "language_model.model.layers.24.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "language_model.model.layers.24.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "language_model.model.layers.24.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "language_model.model.layers.24.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "language_model.model.layers.24.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "language_model.model.layers.24.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "language_model.model.layers.25.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "language_model.model.layers.25.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "language_model.model.layers.25.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "language_model.model.layers.25.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "language_model.model.layers.25.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "language_model.model.layers.25.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "language_model.model.layers.25.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "language_model.model.layers.25.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "language_model.model.layers.25.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "language_model.model.layers.3.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.3.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.3.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.3.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.3.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.3.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.3.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.3.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.3.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.4.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.4.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.4.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.4.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.4.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.4.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.4.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.4.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.4.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.5.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.5.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.5.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.5.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.5.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.5.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.5.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.5.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.5.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.6.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.6.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.6.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.6.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.6.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.6.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.6.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.6.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.6.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.7.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.7.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.7.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.7.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.7.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.7.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.7.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.7.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.7.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.8.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.8.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.8.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.8.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.8.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.8.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.8.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.8.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.8.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.9.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.9.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.9.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.9.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.9.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.9.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.9.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.9.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.9.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.norm.weight": "model-00002-of-00002.safetensors",
+    "multi_modal_projector.linear_1.weight": "model-00002-of-00002.safetensors",
+    "multi_modal_projector.linear_2.weight": "model-00002-of-00002.safetensors",
+    "multi_modal_projector.norm.weight": "model-00002-of-00002.safetensors",
+    "multi_modal_projector.patch_merger.merging_layer.weight": "model-00002-of-00002.safetensors",
+    "vision_tower.ln_pre.weight": "model-00002-of-00002.safetensors",
+    "vision_tower.patch_conv.weight": "model-00002-of-00002.safetensors",
+    "vision_tower.transformer.layers.0.attention.k_proj.weight": "model-00002-of-00002.safetensors",
+    "vision_tower.transformer.layers.0.attention.o_proj.weight": "model-00002-of-00002.safetensors",
+    "vision_tower.transformer.layers.0.attention.q_proj.weight": "model-00002-of-00002.safetensors",
+    "vision_tower.transformer.layers.0.attention.v_proj.weight": "model-00002-of-00002.safetensors",
+    "vision_tower.transformer.layers.0.attention_norm.weight": "model-00002-of-00002.safetensors",
+    "vision_tower.transformer.layers.0.feed_forward.down_proj.weight": "model-00002-of-00002.safetensors",
+    "vision_tower.transformer.layers.0.feed_forward.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "vision_tower.transformer.layers.0.feed_forward.up_proj.weight": "model-00002-of-00002.safetensors",
+    "vision_tower.transformer.layers.0.ffn_norm.weight": "model-00002-of-00002.safetensors",
+    "vision_tower.transformer.layers.1.attention.k_proj.weight": "model-00002-of-00002.safetensors",
+    "vision_tower.transformer.layers.1.attention.o_proj.weight": "model-00002-of-00002.safetensors",
+    "vision_tower.transformer.layers.1.attention.q_proj.weight": "model-00002-of-00002.safetensors",
+    "vision_tower.transformer.layers.1.attention.v_proj.weight": "model-00002-of-00002.safetensors",
+    "vision_tower.transformer.layers.1.attention_norm.weight": "model-00002-of-00002.safetensors",
+    "vision_tower.transformer.layers.1.feed_forward.down_proj.weight": "model-00002-of-00002.safetensors",
+    "vision_tower.transformer.layers.1.feed_forward.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "vision_tower.transformer.layers.1.feed_forward.up_proj.weight": "model-00002-of-00002.safetensors",
+    "vision_tower.transformer.layers.1.ffn_norm.weight": "model-00002-of-00002.safetensors",
+    "vision_tower.transformer.layers.10.attention.k_proj.weight": "model-00002-of-00002.safetensors",
+    "vision_tower.transformer.layers.10.attention.o_proj.weight": "model-00002-of-00002.safetensors",
+    "vision_tower.transformer.layers.10.attention.q_proj.weight": "model-00002-of-00002.safetensors",
+    "vision_tower.transformer.layers.10.attention.v_proj.weight": "model-00002-of-00002.safetensors",
+    "vision_tower.transformer.layers.10.attention_norm.weight": "model-00002-of-00002.safetensors",
+    "vision_tower.transformer.layers.10.feed_forward.down_proj.weight": "model-00002-of-00002.safetensors",
+    "vision_tower.transformer.layers.10.feed_forward.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "vision_tower.transformer.layers.10.feed_forward.up_proj.weight": "model-00002-of-00002.safetensors",
+    "vision_tower.transformer.layers.10.ffn_norm.weight": "model-00002-of-00002.safetensors",
+    "vision_tower.transformer.layers.11.attention.k_proj.weight": "model-00002-of-00002.safetensors",
+    "vision_tower.transformer.layers.11.attention.o_proj.weight": "model-00002-of-00002.safetensors",
+    "vision_tower.transformer.layers.11.attention.q_proj.weight": "model-00002-of-00002.safetensors",
+    "vision_tower.transformer.layers.11.attention.v_proj.weight": "model-00002-of-00002.safetensors",
+    "vision_tower.transformer.layers.11.attention_norm.weight": "model-00002-of-00002.safetensors",
+    "vision_tower.transformer.layers.11.feed_forward.down_proj.weight": "model-00002-of-00002.safetensors",
+    "vision_tower.transformer.layers.11.feed_forward.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "vision_tower.transformer.layers.11.feed_forward.up_proj.weight": "model-00002-of-00002.safetensors",
+    "vision_tower.transformer.layers.11.ffn_norm.weight": "model-00002-of-00002.safetensors",
+    "vision_tower.transformer.layers.12.attention.k_proj.weight": "model-00002-of-00002.safetensors",
+    "vision_tower.transformer.layers.12.attention.o_proj.weight": "model-00002-of-00002.safetensors",
+    "vision_tower.transformer.layers.12.attention.q_proj.weight": "model-00002-of-00002.safetensors",
+    "vision_tower.transformer.layers.12.attention.v_proj.weight": "model-00002-of-00002.safetensors",
+    "vision_tower.transformer.layers.12.attention_norm.weight": "model-00002-of-00002.safetensors",
+    "vision_tower.transformer.layers.12.feed_forward.down_proj.weight": "model-00002-of-00002.safetensors",
+    "vision_tower.transformer.layers.12.feed_forward.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "vision_tower.transformer.layers.12.feed_forward.up_proj.weight": "model-00002-of-00002.safetensors",
+    "vision_tower.transformer.layers.12.ffn_norm.weight": "model-00002-of-00002.safetensors",
+    "vision_tower.transformer.layers.13.attention.k_proj.weight": "model-00002-of-00002.safetensors",
+    "vision_tower.transformer.layers.13.attention.o_proj.weight": "model-00002-of-00002.safetensors",
+    "vision_tower.transformer.layers.13.attention.q_proj.weight": "model-00002-of-00002.safetensors",
+    "vision_tower.transformer.layers.13.attention.v_proj.weight": "model-00002-of-00002.safetensors",
+    "vision_tower.transformer.layers.13.attention_norm.weight": "model-00002-of-00002.safetensors",
+    "vision_tower.transformer.layers.13.feed_forward.down_proj.weight": "model-00002-of-00002.safetensors",
+    "vision_tower.transformer.layers.13.feed_forward.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "vision_tower.transformer.layers.13.feed_forward.up_proj.weight": "model-00002-of-00002.safetensors",
+    "vision_tower.transformer.layers.13.ffn_norm.weight": "model-00002-of-00002.safetensors",
+    "vision_tower.transformer.layers.14.attention.k_proj.weight": "model-00002-of-00002.safetensors",
+    "vision_tower.transformer.layers.14.attention.o_proj.weight": "model-00002-of-00002.safetensors",
+    "vision_tower.transformer.layers.14.attention.q_proj.weight": "model-00002-of-00002.safetensors",
+    "vision_tower.transformer.layers.14.attention.v_proj.weight": "model-00002-of-00002.safetensors",
+    "vision_tower.transformer.layers.14.attention_norm.weight": "model-00002-of-00002.safetensors",
+    "vision_tower.transformer.layers.14.feed_forward.down_proj.weight": "model-00002-of-00002.safetensors",
+    "vision_tower.transformer.layers.14.feed_forward.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "vision_tower.transformer.layers.14.feed_forward.up_proj.weight": "model-00002-of-00002.safetensors",
+    "vision_tower.transformer.layers.14.ffn_norm.weight": "model-00002-of-00002.safetensors",
+    "vision_tower.transformer.layers.15.attention.k_proj.weight": "model-00002-of-00002.safetensors",
+    "vision_tower.transformer.layers.15.attention.o_proj.weight": "model-00002-of-00002.safetensors",
+    "vision_tower.transformer.layers.15.attention.q_proj.weight": "model-00002-of-00002.safetensors",
+    "vision_tower.transformer.layers.15.attention.v_proj.weight": "model-00002-of-00002.safetensors",
+    "vision_tower.transformer.layers.15.attention_norm.weight": "model-00002-of-00002.safetensors",
+    "vision_tower.transformer.layers.15.feed_forward.down_proj.weight": "model-00002-of-00002.safetensors",
+    "vision_tower.transformer.layers.15.feed_forward.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "vision_tower.transformer.layers.15.feed_forward.up_proj.weight": "model-00002-of-00002.safetensors",
+    "vision_tower.transformer.layers.15.ffn_norm.weight": "model-00002-of-00002.safetensors",
+    "vision_tower.transformer.layers.16.attention.k_proj.weight": "model-00002-of-00002.safetensors",
+    "vision_tower.transformer.layers.16.attention.o_proj.weight": "model-00002-of-00002.safetensors",
+    "vision_tower.transformer.layers.16.attention.q_proj.weight": "model-00002-of-00002.safetensors",
+    "vision_tower.transformer.layers.16.attention.v_proj.weight": "model-00002-of-00002.safetensors",
+    "vision_tower.transformer.layers.16.attention_norm.weight": "model-00002-of-00002.safetensors",
+    "vision_tower.transformer.layers.16.feed_forward.down_proj.weight": "model-00002-of-00002.safetensors",
+    "vision_tower.transformer.layers.16.feed_forward.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "vision_tower.transformer.layers.16.feed_forward.up_proj.weight": "model-00002-of-00002.safetensors",
+    "vision_tower.transformer.layers.16.ffn_norm.weight": "model-00002-of-00002.safetensors",
+    "vision_tower.transformer.layers.17.attention.k_proj.weight": "model-00002-of-00002.safetensors",
+    "vision_tower.transformer.layers.17.attention.o_proj.weight": "model-00002-of-00002.safetensors",
+    "vision_tower.transformer.layers.17.attention.q_proj.weight": "model-00002-of-00002.safetensors",
+    "vision_tower.transformer.layers.17.attention.v_proj.weight": "model-00002-of-00002.safetensors",
+    "vision_tower.transformer.layers.17.attention_norm.weight": "model-00002-of-00002.safetensors",
+    "vision_tower.transformer.layers.17.feed_forward.down_proj.weight": "model-00002-of-00002.safetensors",
+    "vision_tower.transformer.layers.17.feed_forward.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "vision_tower.transformer.layers.17.feed_forward.up_proj.weight": "model-00002-of-00002.safetensors",
+    "vision_tower.transformer.layers.17.ffn_norm.weight": "model-00002-of-00002.safetensors",
+    "vision_tower.transformer.layers.18.attention.k_proj.weight": "model-00002-of-00002.safetensors",
+    "vision_tower.transformer.layers.18.attention.o_proj.weight": "model-00002-of-00002.safetensors",
+    "vision_tower.transformer.layers.18.attention.q_proj.weight": "model-00002-of-00002.safetensors",
+    "vision_tower.transformer.layers.18.attention.v_proj.weight": "model-00002-of-00002.safetensors",
+    "vision_tower.transformer.layers.18.attention_norm.weight": "model-00002-of-00002.safetensors",
+    "vision_tower.transformer.layers.18.feed_forward.down_proj.weight": "model-00002-of-00002.safetensors",
+    "vision_tower.transformer.layers.18.feed_forward.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "vision_tower.transformer.layers.18.feed_forward.up_proj.weight": "model-00002-of-00002.safetensors",
+    "vision_tower.transformer.layers.18.ffn_norm.weight": "model-00002-of-00002.safetensors",
+    "vision_tower.transformer.layers.19.attention.k_proj.weight": "model-00002-of-00002.safetensors",
+    "vision_tower.transformer.layers.19.attention.o_proj.weight": "model-00002-of-00002.safetensors",
+    "vision_tower.transformer.layers.19.attention.q_proj.weight": "model-00002-of-00002.safetensors",
+    "vision_tower.transformer.layers.19.attention.v_proj.weight": "model-00002-of-00002.safetensors",
+    "vision_tower.transformer.layers.19.attention_norm.weight": "model-00002-of-00002.safetensors",
+    "vision_tower.transformer.layers.19.feed_forward.down_proj.weight": "model-00002-of-00002.safetensors",
+    "vision_tower.transformer.layers.19.feed_forward.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "vision_tower.transformer.layers.19.feed_forward.up_proj.weight": "model-00002-of-00002.safetensors",
+    "vision_tower.transformer.layers.19.ffn_norm.weight": "model-00002-of-00002.safetensors",
+    "vision_tower.transformer.layers.2.attention.k_proj.weight": "model-00002-of-00002.safetensors",
+    "vision_tower.transformer.layers.2.attention.o_proj.weight": "model-00002-of-00002.safetensors",
+    "vision_tower.transformer.layers.2.attention.q_proj.weight": "model-00002-of-00002.safetensors",
+    "vision_tower.transformer.layers.2.attention.v_proj.weight": "model-00002-of-00002.safetensors",
+    "vision_tower.transformer.layers.2.attention_norm.weight": "model-00002-of-00002.safetensors",
+    "vision_tower.transformer.layers.2.feed_forward.down_proj.weight": "model-00002-of-00002.safetensors",
+    "vision_tower.transformer.layers.2.feed_forward.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "vision_tower.transformer.layers.2.feed_forward.up_proj.weight": "model-00002-of-00002.safetensors",
+    "vision_tower.transformer.layers.2.ffn_norm.weight": "model-00002-of-00002.safetensors",
+    "vision_tower.transformer.layers.20.attention.k_proj.weight": "model-00002-of-00002.safetensors",
+    "vision_tower.transformer.layers.20.attention.o_proj.weight": "model-00002-of-00002.safetensors",
+    "vision_tower.transformer.layers.20.attention.q_proj.weight": "model-00002-of-00002.safetensors",
+    "vision_tower.transformer.layers.20.attention.v_proj.weight": "model-00002-of-00002.safetensors",
+    "vision_tower.transformer.layers.20.attention_norm.weight": "model-00002-of-00002.safetensors",
+    "vision_tower.transformer.layers.20.feed_forward.down_proj.weight": "model-00002-of-00002.safetensors",
+    "vision_tower.transformer.layers.20.feed_forward.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "vision_tower.transformer.layers.20.feed_forward.up_proj.weight": "model-00002-of-00002.safetensors",
+    "vision_tower.transformer.layers.20.ffn_norm.weight": "model-00002-of-00002.safetensors",
+    "vision_tower.transformer.layers.21.attention.k_proj.weight": "model-00002-of-00002.safetensors",
+    "vision_tower.transformer.layers.21.attention.o_proj.weight": "model-00002-of-00002.safetensors",
+    "vision_tower.transformer.layers.21.attention.q_proj.weight": "model-00002-of-00002.safetensors",
+    "vision_tower.transformer.layers.21.attention.v_proj.weight": "model-00002-of-00002.safetensors",
+    "vision_tower.transformer.layers.21.attention_norm.weight": "model-00002-of-00002.safetensors",
+    "vision_tower.transformer.layers.21.feed_forward.down_proj.weight": "model-00002-of-00002.safetensors",
+    "vision_tower.transformer.layers.21.feed_forward.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "vision_tower.transformer.layers.21.feed_forward.up_proj.weight": "model-00002-of-00002.safetensors",
+    "vision_tower.transformer.layers.21.ffn_norm.weight": "model-00002-of-00002.safetensors",
+    "vision_tower.transformer.layers.22.attention.k_proj.weight": "model-00002-of-00002.safetensors",
+    "vision_tower.transformer.layers.22.attention.o_proj.weight": "model-00002-of-00002.safetensors",
+    "vision_tower.transformer.layers.22.attention.q_proj.weight": "model-00002-of-00002.safetensors",
+    "vision_tower.transformer.layers.22.attention.v_proj.weight": "model-00002-of-00002.safetensors",
+    "vision_tower.transformer.layers.22.attention_norm.weight": "model-00002-of-00002.safetensors",
+    "vision_tower.transformer.layers.22.feed_forward.down_proj.weight": "model-00002-of-00002.safetensors",
+    "vision_tower.transformer.layers.22.feed_forward.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "vision_tower.transformer.layers.22.feed_forward.up_proj.weight": "model-00002-of-00002.safetensors",
+    "vision_tower.transformer.layers.22.ffn_norm.weight": "model-00002-of-00002.safetensors",
+    "vision_tower.transformer.layers.23.attention.k_proj.weight": "model-00002-of-00002.safetensors",
+    "vision_tower.transformer.layers.23.attention.o_proj.weight": "model-00002-of-00002.safetensors",
+    "vision_tower.transformer.layers.23.attention.q_proj.weight": "model-00002-of-00002.safetensors",
+    "vision_tower.transformer.layers.23.attention.v_proj.weight": "model-00002-of-00002.safetensors",
+    "vision_tower.transformer.layers.23.attention_norm.weight": "model-00002-of-00002.safetensors",
+    "vision_tower.transformer.layers.23.feed_forward.down_proj.weight": "model-00002-of-00002.safetensors",
+    "vision_tower.transformer.layers.23.feed_forward.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "vision_tower.transformer.layers.23.feed_forward.up_proj.weight": "model-00002-of-00002.safetensors",
+    "vision_tower.transformer.layers.23.ffn_norm.weight": "model-00002-of-00002.safetensors",
+    "vision_tower.transformer.layers.3.attention.k_proj.weight": "model-00002-of-00002.safetensors",
+    "vision_tower.transformer.layers.3.attention.o_proj.weight": "model-00002-of-00002.safetensors",
+    "vision_tower.transformer.layers.3.attention.q_proj.weight": "model-00002-of-00002.safetensors",
+    "vision_tower.transformer.layers.3.attention.v_proj.weight": "model-00002-of-00002.safetensors",
+    "vision_tower.transformer.layers.3.attention_norm.weight": "model-00002-of-00002.safetensors",
+    "vision_tower.transformer.layers.3.feed_forward.down_proj.weight": "model-00002-of-00002.safetensors",
+    "vision_tower.transformer.layers.3.feed_forward.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "vision_tower.transformer.layers.3.feed_forward.up_proj.weight": "model-00002-of-00002.safetensors",
+    "vision_tower.transformer.layers.3.ffn_norm.weight": "model-00002-of-00002.safetensors",
+    "vision_tower.transformer.layers.4.attention.k_proj.weight": "model-00002-of-00002.safetensors",
+    "vision_tower.transformer.layers.4.attention.o_proj.weight": "model-00002-of-00002.safetensors",
+    "vision_tower.transformer.layers.4.attention.q_proj.weight": "model-00002-of-00002.safetensors",
+    "vision_tower.transformer.layers.4.attention.v_proj.weight": "model-00002-of-00002.safetensors",
+    "vision_tower.transformer.layers.4.attention_norm.weight": "model-00002-of-00002.safetensors",
+    "vision_tower.transformer.layers.4.feed_forward.down_proj.weight": "model-00002-of-00002.safetensors",
+    "vision_tower.transformer.layers.4.feed_forward.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "vision_tower.transformer.layers.4.feed_forward.up_proj.weight": "model-00002-of-00002.safetensors",
+    "vision_tower.transformer.layers.4.ffn_norm.weight": "model-00002-of-00002.safetensors",
+    "vision_tower.transformer.layers.5.attention.k_proj.weight": "model-00002-of-00002.safetensors",
+    "vision_tower.transformer.layers.5.attention.o_proj.weight": "model-00002-of-00002.safetensors",
+    "vision_tower.transformer.layers.5.attention.q_proj.weight": "model-00002-of-00002.safetensors",
+    "vision_tower.transformer.layers.5.attention.v_proj.weight": "model-00002-of-00002.safetensors",
+    "vision_tower.transformer.layers.5.attention_norm.weight": "model-00002-of-00002.safetensors",
+    "vision_tower.transformer.layers.5.feed_forward.down_proj.weight": "model-00002-of-00002.safetensors",
+    "vision_tower.transformer.layers.5.feed_forward.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "vision_tower.transformer.layers.5.feed_forward.up_proj.weight": "model-00002-of-00002.safetensors",
+    "vision_tower.transformer.layers.5.ffn_norm.weight": "model-00002-of-00002.safetensors",
+    "vision_tower.transformer.layers.6.attention.k_proj.weight": "model-00002-of-00002.safetensors",
+    "vision_tower.transformer.layers.6.attention.o_proj.weight": "model-00002-of-00002.safetensors",
+    "vision_tower.transformer.layers.6.attention.q_proj.weight": "model-00002-of-00002.safetensors",
+    "vision_tower.transformer.layers.6.attention.v_proj.weight": "model-00002-of-00002.safetensors",
+    "vision_tower.transformer.layers.6.attention_norm.weight": "model-00002-of-00002.safetensors",
+    "vision_tower.transformer.layers.6.feed_forward.down_proj.weight": "model-00002-of-00002.safetensors",
+    "vision_tower.transformer.layers.6.feed_forward.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "vision_tower.transformer.layers.6.feed_forward.up_proj.weight": "model-00002-of-00002.safetensors",
+    "vision_tower.transformer.layers.6.ffn_norm.weight": "model-00002-of-00002.safetensors",
+    "vision_tower.transformer.layers.7.attention.k_proj.weight": "model-00002-of-00002.safetensors",
+    "vision_tower.transformer.layers.7.attention.o_proj.weight": "model-00002-of-00002.safetensors",
+    "vision_tower.transformer.layers.7.attention.q_proj.weight": "model-00002-of-00002.safetensors",
+    "vision_tower.transformer.layers.7.attention.v_proj.weight": "model-00002-of-00002.safetensors",
+    "vision_tower.transformer.layers.7.attention_norm.weight": "model-00002-of-00002.safetensors",
+    "vision_tower.transformer.layers.7.feed_forward.down_proj.weight": "model-00002-of-00002.safetensors",
+    "vision_tower.transformer.layers.7.feed_forward.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "vision_tower.transformer.layers.7.feed_forward.up_proj.weight": "model-00002-of-00002.safetensors",
+    "vision_tower.transformer.layers.7.ffn_norm.weight": "model-00002-of-00002.safetensors",
+    "vision_tower.transformer.layers.8.attention.k_proj.weight": "model-00002-of-00002.safetensors",
+    "vision_tower.transformer.layers.8.attention.o_proj.weight": "model-00002-of-00002.safetensors",
+    "vision_tower.transformer.layers.8.attention.q_proj.weight": "model-00002-of-00002.safetensors",
+    "vision_tower.transformer.layers.8.attention.v_proj.weight": "model-00002-of-00002.safetensors",
+    "vision_tower.transformer.layers.8.attention_norm.weight": "model-00002-of-00002.safetensors",
+    "vision_tower.transformer.layers.8.feed_forward.down_proj.weight": "model-00002-of-00002.safetensors",
+    "vision_tower.transformer.layers.8.feed_forward.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "vision_tower.transformer.layers.8.feed_forward.up_proj.weight": "model-00002-of-00002.safetensors",
+    "vision_tower.transformer.layers.8.ffn_norm.weight": "model-00002-of-00002.safetensors",
+    "vision_tower.transformer.layers.9.attention.k_proj.weight": "model-00002-of-00002.safetensors",
+    "vision_tower.transformer.layers.9.attention.o_proj.weight": "model-00002-of-00002.safetensors",
+    "vision_tower.transformer.layers.9.attention.q_proj.weight": "model-00002-of-00002.safetensors",
+    "vision_tower.transformer.layers.9.attention.v_proj.weight": "model-00002-of-00002.safetensors",
+    "vision_tower.transformer.layers.9.attention_norm.weight": "model-00002-of-00002.safetensors",
+    "vision_tower.transformer.layers.9.feed_forward.down_proj.weight": "model-00002-of-00002.safetensors",
+    "vision_tower.transformer.layers.9.feed_forward.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "vision_tower.transformer.layers.9.feed_forward.up_proj.weight": "model-00002-of-00002.safetensors",
+    "vision_tower.transformer.layers.9.ffn_norm.weight": "model-00002-of-00002.safetensors"
+  }
+}

params.json

@@ -0,0 +1,47 @@
+{
+  "dim": 3072,
+  "n_layers": 26,
+  "head_dim": 128,
+  "hidden_dim": 9216,
+  "n_heads": 32,
+  "n_kv_heads": 8,
+  "rope_theta": 1000000.0,
+  "norm_eps": 1e-05,
+  "vocab_size": 131072,
+  "tied_embeddings": true,
+  "max_position_embeddings": 262144,
+  "llama_4_scaling": {
+    "original_max_position_embeddings": 16384,
+    "beta": 0.1
+  },
+  "q_lora_rank": null,
+  "qk_rope_head_dim": null,
+  "qk_nope_head_dim": null,
+  "kv_lora_rank": null,
+  "v_head_dim": null,
+  "yarn": {
+    "original_max_position_embeddings": 16384,
+    "factor": 16,
+    "apply_scale": false,
+    "beta": 32,
+    "alpha": 1
+  },
+  "vision_encoder": {
+    "image_token_id": 10,
+    "intermediate_size": 4096,
+    "num_hidden_layers": 24,
+    "num_attention_heads": 16,
+    "mm_projector_id": "patch_merge",
+    "spatial_merge_size": 2,
+    "hidden_size": 1024,
+    "num_channels": 3,
+    "image_size": 1540,
+    "max_image_size": 1540,
+    "patch_size": 14,
+    "rope_theta": 10000.0,
+    "add_pre_mm_projector_layer_norm": true,
+    "adapter_bias": false,
+    "image_break_token_id": 12,
+    "image_end_token_id": 13
+  }
+}

processor_config.json

@@ -0,0 +1,42 @@
+{
+  "image_break_token": "[IMG_BREAK]",
+  "image_end_token": "[IMG_END]",
+  "image_processor": {
+    "crop_size": null,
+    "data_format": "channels_first",
+    "device": null,
+    "disable_grouping": null,
+    "do_center_crop": null,
+    "do_convert_rgb": true,
+    "do_normalize": true,
+    "do_pad": null,
+    "do_rescale": true,
+    "do_resize": true,
+    "image_mean": [
+      0.48145466,
+      0.4578275,
+      0.40821073
+    ],
+    "image_processor_type": "PixtralImageProcessorFast",
+    "image_seq_length": null,
+    "image_std": [
+      0.26862954,
+      0.26130258,
+      0.27577711
+    ],
+    "input_data_format": null,
+    "pad_size": null,
+    "patch_size": 14,
+    "processor_class": "PixtralProcessor",
+    "resample": 3,
+    "rescale_factor": 0.00392156862745098,
+    "return_tensors": null,
+    "size": {
+      "longest_edge": 1540
+    }
+  },
+  "image_token": "[IMG]",
+  "patch_size": 14,
+  "processor_class": "PixtralProcessor",
+  "spatial_merge_size": 2
+}

release-manifest.json

@@ -0,0 +1,24 @@
+{
+  "release": "StandardOne-3B",
+  "base_model": {
+    "repo": "mistralai/Ministral-3-3B-Instruct-2512-BF16",
+    "revision": "b6d637bef2393152b3da2b2fde72eecdee30557e"
+  },
+  "adapter": {
+    "source_run": "final2-3b-long",
+    "adapter_model_sha256": "ae5fe8e3be2e9186e6504e48ad2fe7eea5537e7e8382a0cca2ca49d95605f7bf"
+  },
+  "merge_report_summary": {
+    "changed_params": 182,
+    "unchanged_params": 276,
+    "changed_outside_lm_projections": [],
+    "max_abs_delta": 0.002506256103515625
+  },
+  "serving": {
+    "wording": "native",
+    "system_prompt": null,
+    "temperature": 1.45,
+    "served_model_name": "standard-one-3b",
+    "engine": "SGLang 0.5.20"
+  }
+}

server/.gitignore

@@ -0,0 +1,11 @@
+.venv/
+__pycache__/
+*.py[cod]
+.pytest_cache/
+.ruff_cache/
+*.egg-info/
+build/
+dist/
+.env
+benchmarks/data/
+benchmarks/results/

server/LICENSE

@@ -0,0 +1,201 @@
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright 2023-2024 SGLang Team
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

server/NOTICE

@@ -0,0 +1,19 @@
+Jev adapter
+
+The initial protocol helpers and their unit tests were extracted from the
+System One extension in the local sglang-jev experiment, commit
+633f05e (Add prefill-only System One decisions and visual game evaluation).
+That repository was based on SGLang upstream commit
+0024efa0de38794ee309ba10ab00ebb891a3d050.
+
+The accompanying Apache License 2.0 is retained from that source tree.
+SGLang project: https://github.com/sgl-project/sglang
+
+This repository is an independent API adapter. It does not include the
+proprietary Jev model and does not bundle the SGLang inference engine.
+
+Benchmark metric definitions and dataset schema mapping follow Jared Palmer's
+Apache-2.0 KEV project, pinned at 4f8110a3f8620cc3a182ae9a708e4398492c4b1a:
+https://github.com/jaredpalmer/kev
+The benchmark runner is an independent implementation. Downloaded benchmark
+data retains upstream provenance and any underlying dataset license terms.

server/README.md

@@ -0,0 +1,196 @@
+# Jev adapter
+
+기본 추론 엔진을 별도 프로세스로 실행하고, 그 앞에서 `POST /v1/systemone` 응답을 만드는 독립 Python 서비스입니다. 모델 학습, 엔진 fork, SGLang Python 패키지, CUDA, 모델 가중치가 어댑터에 필요하지 않습니다. 현재 구현한 백엔드는 **SGLang HTTP API**입니다.
+
+```text
+클라이언트 → jev-adapter :30120 → 기본 SGLang :30000 → GPU 모델
+            질문/응답 변환       템플릿·토큰화·비전 처리·프리필
+```
+
+모델은 선택지 라벨의 다음 토큰 logprob를 반환하고, 어댑터가 확률 정규화와 최종 JSON 구성을 담당합니다. 엔진 내부 객체나 비공개 Python 함수를 호출하지 않습니다. 상용 Jev 모델이나 학습 결과를 복제한 것은 아닙니다.
+
+## 실행
+
+엔진 환경에서 모델을 실행합니다. 어댑터와 같은 컴퓨터일 필요는 없습니다. 아래는 모델의 served name을 `decision-model`로 정한 예시입니다. `MODEL_PATH`는 엔진이 지원하는 실제 모델 경로 또는 Hugging Face ID로 설정합니다.
+
+```bash
+python -m sglang.launch_server \
+  --model-path "$MODEL_PATH" \
+  --served-model-name decision-model \
+  --host 127.0.0.1 --port 30000
+```
+
+어댑터는 이 레포의 별도 가상 환경에서 실행합니다. 엔진을 다른 호스트에서 실행하면 `--engine-url`만 바꾸면 됩니다.
+
+```bash
+python3 -m venv .venv
+.venv/bin/python -m pip install -e '.[dev]'
+.venv/bin/jev-adapter \
+  --engine-url http://127.0.0.1:30000 \
+  --model decision-model \
+  --port 30120
+```
+
+`python -m jev_adapter`도 같은 CLI입니다. 기본 listen 주소는 `127.0.0.1`이며, `--host`로 변경합니다. `--max-concurrency` 기본값은 32, 엔진 HTTP timeout은 `--timeout 120`입니다.
+
+`--default-temperature T` (환경 변수 `JEV_DEFAULT_TEMPERATURE`)는 요청이 `options.temperature`를 생략했을 때만 적용되는 서버 기본 온도입니다. 요청이 온도를 명시하면 요청 값이 우선하고, `options.temperature_scaling: false`이면 기본값과 무관하게 스케일링하지 않습니다. 지정하지 않으면 기존과 같이 1.0입니다. 실제 적용된 온도는 응답의 `metadata.temperature`로 확인합니다.
+
+`SGLANG_API_KEY`는 엔진에 보낼 키, `JEV_API_KEY`는 클라이언트가 어댑터에 보낼 키입니다. 설정했다면 각각 `Authorization: Bearer ...` 인증을 사용합니다. 키를 설정하지 않은 어댑터는 인증 없이 동작합니다.
+
+시작할 때 엔진 정보를 확인하고, 엔진 토크나이저로 라벨을 검증합니다. 엔진을 먼저 실행해야 합니다. 모델/토크나이저를 교체하면 어댑터도 재시작합니다.
+
+Ministral의 공식 `mistral-common` 토크나이저는 제어 토큰을 문자열로 변환했다가 다시
+인코딩하면 토큰 ID가 달라질 수 있습니다. SGLang 0.5.20에서 이 문제가 확인되어,
+이 모델의 텍스트 요청은 선택형 공식 토크나이저 경로를 사용합니다.
+
+```bash
+.venv/bin/python -m pip install -e '.[native-tokenizer]'
+.venv/bin/jev-adapter --model decision-model \
+  --tokenizer-model mistralai/Ministral-3-3B-Instruct-2512-BF16 \
+  --tokenizer-revision b6d637bef2393152b3da2b2fde72eecdee30557e
+```
+
+이 경로는 CPU에서 토크나이저 파일만 로드하고 공식 `apply_chat_template(tokenize=True)`
+결과를 엔진에 전달합니다. 각 assistant 라벨의 토큰 경계도 공식 continuation으로
+검증합니다. 이미지 입력은 지원하지 않으며 기본 HTTP 경로로 자동 전환하지 않습니다.
+
+### 프롬프트 문구 (`--prompt-wording`)
+
+`--prompt-wording {served,native}` (환경 변수 `JEV_PROMPT_WORDING`)는 어떤 문구로
+프롬프트를 렌더링할지 정하는 서버 전역 설정입니다. 기본값 `served`는 기존과 동일하게
+어댑터 자체 문구(`Context:/.../Options: A: name: description`)를 사용합니다.
+
+`native`는 jevbench-hard 러너 자신의 오프라인 채점 문구(`run_suites_rotation.py`의
+`State:/Question:/Options: A. name: description`)를 그대로 렌더링합니다. 러너 자체
+ablation에서 이 문구 차이만으로 jevbench-hard 정확도가 약 +6점 높게 측정되었습니다
+(native 61.3 대 served 55.0; 동일 토크나이저 계열, rotation 없는 canonical 순서 기준).
+옵션 **순서** 자체는 바꾸지 않습니다 — 요청이 준 순서(`choice`/`score`는 criteria
+순서, `noul`은 항상 true 다음 false)를 그대로 쓰고 문구만 바꿉니다. 그래서
+`--tokenizer-model`/`--tokenizer-revision`이 가리키는 토크나이저의 앞쪽 라벨이
+정확히 대문자 `A`..`Z`(최대 26개 선택지)가 아니면 해당 요청은 거부됩니다 — 대부분의
+영문 어휘 토크나이저에서는 자동으로 성립합니다.
+
+`--tokenizer-model`/`--tokenizer-revision`도 함께 주면, 어댑터는 같은 pin을
+`fix_mistral_regex=True`로 별도 로드해 HF chat template이 ��동 주입하는 모델
+기본 시스템 프롬프트 텍스트를 추출하고, 이를 명시적 system 메시지로 앞에 붙입니다.
+Ministral처럼 서빙 토크나이저가 `mistral-common` 백엔드로 해석되는 경우
+(`native_tokenizer.py` 문서 참고) 이 백엔드는 기본 시스템 프롬프트를 자동으로
+주입하지 않기 때문입니다 — 직접 추출한 텍스트가 없으면 시스템 메시지 없이 문구만
+`native`로 서빙합니다(시작 시 경고 로그). `/opt/.../SYSTEM_PROMPT.txt` 같은 정적
+문서 파일은 실제 chat template이 렌더링하는 기본 텍스트와 다를 수 있어(확인된 사례:
+문장 하나가 중복/누락) 신뢰하지 않고, 항상 `apply_chat_template`을 직접 렌더링해서
+추출합니다.
+
+이 기본 시스템 프롬프트 자동 추출은 `--native-system-prompt {auto,none}`
+(환경 변수 `JEV_NATIVE_SYSTEM_PROMPT`)로 끌 수 있습니다. 기본값 `auto`는 위에서
+설명한 기존 동작 그대로이고, `none`은 추출 자체를 건너뛰고 시스템 메시지 없이
+`native` 문구만 렌더링하며 누락 경고 로그도 남기지 않습니다. 이 옵션을 둔 이유는
+Ministral 기본 시스템 프롬프트가 531토큰으로, 전형적인 요청 하나의 약 80%에 달할
+만큼 커서 — 프리필 비용과 지연시간에 영향을 주는데, jevbench-hard 정확도 이득은
+문구(`native` vs `served`) 자체에서 오지 시스템 프롬프트 유무에서 오는지는 별도
+질문이기 때문입니다. `--prompt-wording served`에는 아무 영향이 없습니다.
+
+```bash
+.venv/bin/jev-adapter --model decision-model \
+  --tokenizer-model mistralai/Ministral-3-8B-Instruct-2512-BF16 \
+  --tokenizer-revision f6fae9795746f63c9be8344932f01275f3c63734 \
+  --prompt-wording native --default-temperature 2.4
+```
+
+CPU 전용 검증(`jevbench-original` 40개 + `internal-eval-v1` 40개 전체, `jevbench-hard`
+5개는 구조만) 결과, 요청 옵션 순서를 고정하면 어댑터가 렌더링한 `native` 문구와
+러너의 `format_prompt()` 출력은 **텍스트와 토큰 ID가 모두 완전히 일치**했고
+(`mistral-common`이 명시 system 메시지를 렌더링한 토큰 ID와 HF 템플릿의 자동 주입
+토큰 ID도 이 표본에서는 차이가 없었습니다), 라벨 토큰 ID도 일치했습니다. 다만
+`jevbench-original`의 40개 중 24개, 표본 `jevbench-hard` 5개 중 5개에서
+데이터셋이 기록한 "정답 채점 순서"(`expected.labels`)가 어댑터가 실제로 쓰는 옵션
+순서(criteria 순서 / noul 고정 순서)와 달랐습니다 — 이는 문구와 무관한, 옵션 **순서**
+자체의 사전 존재 차이이며 어댑터가 요청 순서를 그대로 따르는 한 코드로 없앨 수
+없습니다. 러너의 native 61.3점 자체도 이 순서로 측정된 값이라, jev-adapter로
+`--prompt-wording native`를 서빙했을 때 실제로 회수되는 점수는 이 순서 효과만큼
+61.3보다 낮을 수 있습니다 — 문구 효과 자체는 위에서 텍스트/토큰 단위로 확인했습니다.
+
+```bash
+.venv/bin/python examples/smoke.py
+.venv/bin/python examples/smoke.py --image /path/to/screenshot.png
+```
+
+기존 평가 클라이언트에서도 base URL을 `http://127.0.0.1:30120`으로 바꾸면 됩니다. `/v1/models`는 실제 모델명을 반환하고, `/health`는 어댑터 프로세스의 생존 여부만 확인합니다.
+
+## 요청과 응답
+
+전체 예시는 [examples/request.json](examples/request.json)을 참고하세요. 모델에는 실제 served name 또는 기본 별칭 `jev-latest`를 사용할 수 있습니다.
+
+```json
+{
+  "model": "jev-latest",
+  "state": "중복 결제를 환불해 주세요.",
+  "questions": {
+    "route": {
+      "type": "choice",
+      "instructions": "담당 부서는?",
+      "criteria": {"billing": "결제 및 환불", "technical": "기술 지원"}
+    },
+    "refund": {"type": "noul", "instructions": "환불 요청인가?"},
+    "urgency": {"type": "score", "instructions": "긴급도", "criteria": ["낮음", "보통", "높음"]}
+  },
+  "options": {"temperature": 1.0, "permutations": 1}
+}
+```
+
+| 유형 | 결과 |
+| --- | --- |
+| `choice` | 선택지 확률, 최대 확률의 `choice`, 엔트로피 기반 `confidence` |
+| `noul` | 참일 선택지의 확률인 `noul` |
+| `score` | 0부터 시작하는 등급 인덱스의 기대값 `score`, 확률, `legend`, `confidence` |
+
+`state`와 설명은 문자열 또는 JSON 객체/배열을 받을 수 있습니다. `images`는 HTTP(S) URL 또는 `data:image/...;base64,...` 목록이며, 이미지 바이트 처리와 비전 인코더 실행은 엔진이 담당합니다.
+
+`options.permutations`는 선택지 순서를 회전한 뒤 원래 순서로 복원하여 확률을 평균합니다. **질문 수 × permutations만큼 프리필 평가**가 발생합니다. `return_logprobs: true`는 각 평가의 원래 vocabulary logprob를 제공합니다. `assistant_prefix`를 설정하면 실제 응답 경계에서 라벨이 ���일 토큰인지 다시 검증합니다.
+
+확률은 지정한 라벨 안에서 정규화한 값입니다. `confidence = 1 - entropy / log(선택지 수)`는 실제 정답률의 보정값이 아닙니다. 학습하지 않은 모델의 판단 정확도는 별도로 평가해야 합니다.
+
+## 엔진 API와 지원 범위
+
+사용하는 공개 API는 `/model_info`, `/server_info`, `/v1/tokenize`, `/v1/detokenize`, `/generate`입니다. SGLang upstream [`0024efa0de38794ee309ba10ab00ebb891a3d050`](https://github.com/sgl-project/sglang/tree/0024efa0de38794ee309ba10ab00ebb891a3d050)의 구현을 기준으로 연결 규약을 확인했습니다. 오래된 릴리스에 필요한 API가 없으면 엔진 버전을 올려야 합니다.
+
+일반 텍스트는 템플릿을 적용한 token IDs를 엔진에 전달합니다. 이미지 요청은 Qwen VL/Qwen3.5, Mistral3/Pixtral 계열의 일반적인 텍스트 기반 멀티모달 경로를 대상으로 합니다. 이미지가 포함된 프롬프트의 특수 토큰과 원본 이미지를 함께 전달합니다. 토큰만으로 프롬프트를 표현하는 다른 VLM 계열은 명시적으로 거부합니다. 모델마다 이미지 경로의 GPU 통합 검증이 필요합니다.
+
+`/generate`에는 `max_new_tokens: 0`, `return_logprob: true`, `token_ids_logprob: [...]`를 설정합니다. 모든 라벨의 유한한 logprob와 `completion_tokens == 0`을 확인합니다. 누락된 확률이나 토큰 사용량을 임의로 보완하지 않습니다. speculative decoding/MTP를 사용하지 않는 일반 generation 서버가 필요합니다.
+
+현재의 엄격한 프롬프트 검증은 평가마다 템플릿 토큰화, detokenize, 라벨 경계 일괄 검증, generate의 **HTTP 호출 4회**를 사용합니다. 이 왕복과 CPU 처리 비용은 GPU 프리필 시간에 추가됩니다. `metadata.adapter_elapsed_ms`는 이 작업과 대기 시간을 포함하고, 외부 HTTP 응답 직렬화/전송 시간은 제외합니다. `smoke.py`는 클라이언트에서 전체 왕복 시간을 표시합니다. GPU 간 지연 비교 수치로 대신 사용할 수 없습니다.
+
+선택형 공식 토크나이저 경로는 초기 라벨 검증 이후 평가마다 CPU 토큰화와 엔진
+`/generate` 호출 1회를 사용합니다. 두 경로의 HTTP 지연에는 서로 다른 CPU 처리와
+왕복 횟수가 포함되므로 순수 모델 연산 성능처럼 비교하면 안 됩니다.
+
+클라이언트 연결이 끊어지면 어댑터의 대기 작업과 엔진 HTTP 요청을 취소합니다. 이후 정리는 기본 엔진의 disconnect 처리에 의존합니다. 이미 실행 중인 GPU 연산을 즉시 중단한다는 보장은 없습니다.
+
+vLLM 백엔드는 아직 구현하지 않았습니다. 추가 시 `ScoringBackend` 인터페이스를 구현하면 질문/응답 규약은 그대로 재사용할 수 있습니다.
+
+## 검증과 코드
+
+```bash
+.venv/bin/python -m pytest
+.venv/bin/ruff check .
+```
+
+테스트는 실제 프로토콜/서비스를 사용하고 추론 서버 HTTP 응답만 모의합니다. 확률 정규화, 순서 복원, 혼합 질문, 이미지 전달, 오류, 취소와 동시 실행 제한을 검증합니다. 이 테스트 통과는 NVIDIA GPU에서의 정확도·성능 또는 모델별 이미지 처리 성공을 뜻하지 않습니다.
+
+2026-09-21 검증: 독립 Python 3.12 가상환경에서 **97개 테스트와 74개 하위 케이스 통과**, Ruff 통과. 기본 어댑터 환경에는 SGLang, vLLM, PyTorch, Transformers, MLX가 필요하지 않습니다. H200의 공식 SGLang 0.5.20 wheel로 네 모델 각각 4,006개 요청의 전체 벤치를 완료했으며, Ministral은 선택형 CPU 토크나이저 환경을 사용합니다.
+
+2026-09-23 `--prompt-wording native` 추가 검증: 같은 가상환경에서 **136개 테스트와 80개 하위 케이스 통과**, Ruff 통과. 운영 서버(CPU 전용, GPU 미사용)에서 `jevbench-original`/`internal-eval-v1`/`jevbench-hard` 실제 레코드로 어댑터의 `native` 문구·토큰 ID 렌더링을 jevbench-hard 러너의 `run_suites_rotation.py` 계약과 직접 비교했습니다(위 "프롬프트 문구" 절 참고).
+
+| 파일 | 역할 |
+| --- | --- |
+| `jev_adapter/protocol.py` | 요청 스키마, 프롬프트, 확률 및 응답 계산 |
+| `jev_adapter/service.py` | 질문별 평가 실행, 병렬 수 제한, 결과 조합 |
+| `jev_adapter/sglang.py` | 공개 SGLang HTTP API 클라이언트 |
+| `jev_adapter/server.py` | 독립 FastAPI 서버 및 인증 |
+| `jev_adapter/backend.py` | 다른 추론 엔진을 연결할 최소 인터페이스 |
+
+이 레포는 엔진 소스 트리나 형제 디렉터리를 import하지 않습니다. 기존 `sglang-jev` 실험 레포와 벤치마크 결과는 원래 위치에 남아 있습니다. 초기 프로토콜 코드의 출처는 [NOTICE](NOTICE)를 참고하세요.
+
+## H200 학습 전 벤치마크
+
+[benchmarks/README.md](benchmarks/README.md)에 H200 한 장에서 Ministral 3 3B, Qwen3.5-4B, Qwen3.6-27B, Qwen3.6-35B-A3B를 순서대로 평가하는 절차가 ��습니다. 기본 비교는 BF16이며 KEV 공개 개발 세트의 정확도·확률 품질·HTTP 지연시간을 기록합니다. 데이터와 모델 리비전을 고정했고 H200 실측을 완료했습니다. KEV 외 공개 평가셋 조사는 [benchmarks/PUBLIC_DATASETS.md](benchmarks/PUBLIC_DATASETS.md)를 참고하세요.

server/jev_adapter/__init__.py

@@ -0,0 +1,3 @@
+"""Engine-independent next-token decision API."""
+
+__version__ = "0.1.0"

server/jev_adapter/__main__.py

@@ -0,0 +1,145 @@
+"""CLI for the independent adapter process."""
+
+import argparse
+import logging
+import math
+import os
+
+logger = logging.getLogger(__name__)
+
+
+def positive_temperature(text: str) -> float:
+    """argparse type for --default-temperature: a finite number above zero."""
+    try:
+        value = float(text)
+    except ValueError:
+        raise argparse.ArgumentTypeError(f"not a number: {text!r}") from None
+    if not math.isfinite(value) or value <= 0:
+        raise argparse.ArgumentTypeError("must be finite and greater than zero")
+    return value
+
+
+def main():
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--engine-url", default="http://127.0.0.1:30000")
+    parser.add_argument("--model", required=True, help="Engine's served model name")
+    parser.add_argument("--alias", default="jev-latest")
+    parser.add_argument("--host", default="127.0.0.1")
+    parser.add_argument("--port", type=int, default=30120)
+    parser.add_argument("--timeout", type=float, default=120.0)
+    parser.add_argument("--max-concurrency", type=int, default=32)
+    parser.add_argument(
+        "--tokenizer-model", help="Optional official text tokenizer repo"
+    )
+    parser.add_argument("--tokenizer-revision", help="Immutable tokenizer revision")
+    parser.add_argument(
+        "--default-temperature",
+        type=positive_temperature,
+        default=os.environ.get("JEV_DEFAULT_TEMPERATURE", "1.0"),
+        help="Temperature applied when a request omits options.temperature "
+        "(label logits are divided by T before softmax; an explicit request "
+        "value wins). Env: JEV_DEFAULT_TEMPERATURE. Default 1.0.",
+    )
+    parser.add_argument(
+        "--prompt-wording",
+        choices=("served", "native"),
+        default=os.environ.get("JEV_PROMPT_WORDING", "served"),
+        help="'served' (default, unchanged behaviour): this adapter's own "
+        "prompt text (Context:/.../Options: A: name: description). 'native': "
+        "the jevbench-hard runner's own contract instead (run_suites_rotation.py: "
+        "State:/Question:/Options: A. name: description), which the runner's "
+        "own ablation measured as worth about +6 points on jevbench-hard by "
+        "wording alone. Requires the engine tokenizer's first labels to be "
+        "canonical single-token A-Z (true for essentially all "
+        "English-vocabulary tokenizers; jev_adapter.protocol.build_prompt "
+        "raises per-request otherwise). With --tokenizer-model/"
+        "--tokenizer-revision also set, the adapter additionally extracts the "
+        "model's default system-prompt text from a dedicated HF-template load "
+        "of that same pin and prepends it explicitly, since a mistral-common "
+        "native tokenizer (the default for Ministral) never auto-injects it. "
+        "See --native-system-prompt to disable this. Env: JEV_PROMPT_WORDING.",
+    )
+    parser.add_argument(
+        "--native-system-prompt",
+        choices=("auto", "none"),
+        default=os.environ.get("JEV_NATIVE_SYSTEM_PROMPT", "auto"),
+        help="Only takes effect with --prompt-wording native; ignored "
+        "otherwise. 'auto' (default, unchanged behaviour): with "
+        "--tokenizer-model/--tokenizer-revision also set, extract the "
+        "model's default system-prompt text from the pinned tokenizer's chat "
+        "template (see --prompt-wording) and prepend it as an explicit "
+        "system message; warns at startup if no tokenizer pin is given or "
+        "the template defines no default system prompt. 'none': render the "
+        "native wording with no system message at all -- skips the "
+        "extraction entirely and never warns about a missing prompt (useful "
+        "when the extra system-prompt tokens are not wanted, e.g. the "
+        "Ministral default is 531 tokens, about 80%% of a typical request). "
+        "Env: JEV_NATIVE_SYSTEM_PROMPT.",
+    )
+    # argparse runs the type converter on the string default too, so a bad
+    # JEV_DEFAULT_TEMPERATURE fails at startup with the same message.
+    args = parser.parse_args()
+    if args.timeout <= 0 or args.max_concurrency < 1:
+        parser.error("timeout and max-concurrency must be positive")
+    if bool(args.tokenizer_model) != bool(args.tokenizer_revision):
+        parser.error("tokenizer-model and tokenizer-revision must be supplied together")
+
+    import uvicorn
+
+    from .server import create_app
+    from .sglang import SGLangBackend
+
+    native_tokenizer = None
+    native_system_prompt = None
+    if args.tokenizer_model:
+        from .native_tokenizer import NativeTokenizer
+
+        native_tokenizer = NativeTokenizer.from_pretrained(
+            args.tokenizer_model, args.tokenizer_revision
+        )
+        if args.prompt_wording == "native" and args.native_system_prompt == "auto":
+            native_system_prompt = NativeTokenizer.native_default_system_prompt(
+                args.tokenizer_model, args.tokenizer_revision
+            )
+    if (
+        args.prompt_wording == "native"
+        and args.native_system_prompt == "auto"
+        and native_system_prompt is None
+    ):
+        logger.warning(
+            "prompt-wording=native without a default system prompt (pass "
+            "--tokenizer-model/--tokenizer-revision to extract and prepend the "
+            "model's own default system message, or the model's template "
+            "defines none); serving the native wording text without it."
+        )
+    logger.info(
+        "prompt_wording=%s native_system_prompt=%s (%s)",
+        args.prompt_wording,
+        args.native_system_prompt,
+        "system message attached"
+        if native_system_prompt
+        else "no system message attached",
+    )
+    backend = SGLangBackend(
+        args.engine_url,
+        args.model,
+        api_key=os.environ.get("SGLANG_API_KEY"),
+        timeout=args.timeout,
+        max_connections=args.max_concurrency,
+        native_tokenizer=native_tokenizer,
+        prompt_wording=args.prompt_wording,
+        native_system_prompt=native_system_prompt,
+    )
+    app = create_app(
+        backend,
+        api_key=os.environ.get("JEV_API_KEY"),
+        max_concurrency=args.max_concurrency,
+        alias=args.alias,
+        default_temperature=args.default_temperature,
+        prompt_wording=args.prompt_wording,
+    )
+    uvicorn.run(app, host=args.host, port=args.port)
+
+
+if __name__ == "__main__":
+    main()

server/jev_adapter/backend.py

@@ -0,0 +1,35 @@
+"""The small contract between response formatting and inference transport."""
+
+from dataclasses import dataclass
+from typing import Protocol
+
+
+class AdapterError(Exception):
+    def __init__(self, code: str, message: str, field: str = "", status: int = 422):
+        super().__init__(message)
+        self.code, self.field, self.status = code, field, status
+
+
+@dataclass(frozen=True)
+class ScoringResult:
+    logprobs: tuple[float, ...]
+    input_tokens: int
+
+
+class ScoringBackend(Protocol):
+    model: str
+
+    async def start(self) -> None: ...
+
+    async def close(self) -> None: ...
+
+    def labels(self, count: int) -> tuple[tuple[str, ...], tuple[int, ...]]: ...
+
+    async def evaluate(
+        self,
+        prompt: str,
+        images: list[str],
+        labels: tuple[str, ...],
+        token_ids: tuple[int, ...],
+        assistant_prefix: str | None,
+    ) -> ScoringResult: ...

server/jev_adapter/native_tokenizer.py

@@ -0,0 +1,169 @@
+"""Optional CPU chat tokenization that preserves native control-token IDs.
+
+This path is useful when an engine's string-based chat API cannot roundtrip a
+tokenizer's control tokens. It loads tokenizer assets, never model weights.
+"""
+
+from typing import Any
+
+from .backend import AdapterError
+
+
+class NativeTokenizer:
+    def __init__(self, tokenizer: Any):
+        self.tokenizer = tokenizer
+
+    @classmethod
+    def from_pretrained(cls, model: str, revision: str) -> "NativeTokenizer":
+        if not model.strip() or not revision.strip():
+            raise ValueError("Native tokenizer model and pinned revision are required")
+        # Keep the default HTTP-only installation free of Transformers.
+        from transformers import AutoTokenizer
+
+        return cls(
+            AutoTokenizer.from_pretrained(
+                model,
+                revision=revision,
+                trust_remote_code=False,
+            )
+        )
+
+    def _chat(self, messages: list[dict], *, continuation: bool) -> list[int]:
+        try:
+            tokens = self.tokenizer.apply_chat_template(
+                messages,
+                tokenize=True,
+                add_generation_prompt=not continuation,
+                continue_final_message=continuation,
+                return_dict=False,
+                enable_thinking=False,
+            )
+        except Exception as error:
+            raise AdapterError(
+                "unsupported_native_tokenizer",
+                "The native tokenizer could not prepare an open assistant message.",
+            ) from error
+        if (
+            not isinstance(tokens, list)
+            or not tokens
+            or any(type(token) is not int or token < 0 for token in tokens)
+        ):
+            raise AdapterError(
+                "unsupported_native_tokenizer",
+                "The native tokenizer returned invalid chat token IDs.",
+            )
+        return tokens
+
+    def prepare(
+        self,
+        prompt: str,
+        labels: tuple[str, ...],
+        token_ids: tuple[int, ...],
+        assistant_prefix: str | None,
+        system_prompt: str | None = None,
+    ) -> list[int]:
+        """Return native prompt IDs only when every answer continues by one token.
+
+        Full native conversations are tokenized for both the base and each
+        continuation. Decoding and re-encoding control tokens is never used.
+        Comparing against engine-selected IDs also detects tokenizer mismatch.
+
+        ``system_prompt``, when given, is prepended as an explicit system
+        message rather than relying on the tokenizer's own auto-injection: a
+        mistral-common-backed tokenizer never auto-injects a default system
+        prompt (unlike the HF Jinja chat template), so wording="native"
+        serving supplies the runner's default system text explicitly here
+        (see ``extract_default_system_prompt``/``native_default_system_prompt``).
+        This can still render to different token IDs than the HF template's
+        own auto-injection; that gap is measured, not assumed away.
+        """
+        if (
+            len(labels) < 2
+            or len(labels) != len(token_ids)
+            or len(set(labels)) != len(labels)
+            or len(set(token_ids)) != len(token_ids)
+            or any(not isinstance(label, str) or not label for label in labels)
+            or any(type(token) is not int or token < 0 for token in token_ids)
+        ):
+            raise AdapterError(
+                "invalid_labels", "Distinct label/token pairs are required."
+            )
+        messages = (
+            [{"role": "system", "content": system_prompt}] if system_prompt else []
+        ) + [{"role": "user", "content": prompt}]
+        base = self._chat(messages, continuation=False)
+        prefix = assistant_prefix or ""
+        final_ids = base
+        if prefix:
+            final_ids = self._chat(
+                messages + [{"role": "assistant", "content": prefix}],
+                continuation=True,
+            )
+            if final_ids[: len(base)] != base:
+                raise AdapterError(
+                    "invalid_label_boundary",
+                    "The assistant prefix changes the native chat prompt boundary.",
+                    "options.assistant_prefix",
+                )
+        for label, token_id in zip(labels, token_ids):
+            continued = self._chat(
+                messages + [{"role": "assistant", "content": prefix + label}],
+                continuation=True,
+            )
+            if continued != final_ids + [token_id]:
+                raise AdapterError(
+                    "invalid_label_boundary",
+                    "A native assistant label does not continue by its engine token ID; "
+                    "use matching tokenizers and a compatible assistant_prefix.",
+                    "options.assistant_prefix",
+                )
+        return final_ids
+
+    @classmethod
+    def native_default_system_prompt(cls, model: str, revision: str) -> "str | None":
+        """The jevbench-hard runner's default system-prompt text for ``model``
+        @``revision``: a dedicated HF fast-tokenizer load with the runner's own
+        ``fix_mistral_regex=True`` (for ``mistralai/*`` models), independent of
+        whichever tokenizer class plain ``from_pretrained`` resolves to for
+        serving -- for Ministral that is a mistral-common backend (see the
+        module docstring and ``extract_default_system_prompt``), which never
+        performs this auto-injection, so probing the serving tokenizer itself
+        would incorrectly report "no default system prompt".
+
+        Returns None when the model's template defines no default system
+        message. Requires transformers; call only for wording="native".
+        """
+        if not model.strip() or not revision.strip():
+            raise ValueError("Native tokenizer model and pinned revision are required")
+        from transformers import AutoTokenizer
+
+        kwargs: dict[str, Any] = {"revision": revision, "trust_remote_code": False}
+        if model.startswith("mistralai/"):
+            kwargs["fix_mistral_regex"] = True
+        tokenizer = AutoTokenizer.from_pretrained(model, **kwargs)
+        return extract_default_system_prompt(tokenizer)
+
+
+def extract_default_system_prompt(tokenizer: Any) -> "str | None":
+    """Return the text a chat template auto-injects as a default system
+    message, or None if rendering a system-free conversation injects none.
+
+    Works by rendering one throwaway user turn and looking for the literal
+    ``[SYSTEM_PROMPT]``/``[/SYSTEM_PROMPT]`` markers Mistral's HF Jinja
+    template wraps its auto-injected default system message in. A
+    mistral-common-backed tokenizer implements the same ``apply_chat_template``
+    surface but never performs this injection (see run_suites_ablate.py's
+    native_mc contract), so calling this on one correctly returns None --
+    that is the documented tokenizer-path difference wording="native" serving
+    has to work around by supplying the text explicitly instead of relying on
+    auto-injection (see ``NativeTokenizer.prepare``'s ``system_prompt``).
+    """
+    rendered = tokenizer.apply_chat_template(
+        [{"role": "user", "content": ""}],
+        tokenize=False,
+        add_generation_prompt=True,
+        enable_thinking=False,
+    )
+    if not isinstance(rendered, str) or "[SYSTEM_PROMPT]" not in rendered:
+        return None
+    return rendered.split("[SYSTEM_PROMPT]", 1)[1].split("[/SYSTEM_PROMPT]", 1)[0]

server/jev_adapter/protocol.py

@@ -0,0 +1,386 @@
+"""Validated requests and pure scoring helpers for the System One compatible API.
+
+This is next-token classification with an existing language model, not the
+proprietary Jev model. Confidence follows the llama.cpp fork's formula,
+``1 - entropy / log(number of options)``; it is not a calibration guarantee.
+"""
+
+import json
+import math
+import string
+from collections.abc import Sequence
+from dataclasses import dataclass
+from numbers import Real
+from typing import Annotated, Literal
+from urllib.parse import urlsplit
+
+from pydantic import (
+    BaseModel,
+    ConfigDict,
+    Field,
+    JsonValue,
+    StrictBool,
+    StrictStr,
+    field_validator,
+    model_validator,
+)
+
+StructuredText = StrictStr | dict[StrictStr, JsonValue] | list[JsonValue]
+Description = StructuredText | None
+QuestionType = Literal["choice", "noul", "score"]
+PromptWording = Literal["served", "native"]
+
+# The jevbench-hard runner's own contract (run_suites_rotation.py: PREFIX,
+# format_prompt, CONTRACT), copied verbatim. Rendering ``build_prompt`` with
+# wording="native" reproduces this text exactly; the runner measured this
+# wording alone as worth about +6 points on jevbench-hard over the served
+# wording below (native 61.3 vs served 55.0 Hard accuracy, same tokenizer
+# family). See run_suites_ablate.py for the full effect decomposition.
+NATIVE_PROMPT_PREFIX = (
+    "Read the state and question. Choose the single best option using the "
+    "supplied information. Respond with exactly one option letter and no "
+    "other text."
+)
+NATIVE_PROMPT_CONTRACT = "jev-public-hard-native-letter-v1"
+_NATIVE_DISPLAY = string.ascii_uppercase
+
+
+class _SystemOneModel(BaseModel):
+    model_config = ConfigDict(extra="forbid")
+
+    @model_validator(mode="after")
+    def validate_finite_json(self):
+        # JsonValue permits floats; JSON descriptions must not contain NaN/Inf.
+        try:
+            json.dumps(self.model_dump(), allow_nan=False)
+        except (ValueError, TypeError) as exc:
+            raise ValueError("all structured values must be finite JSON") from exc
+        return self
+
+
+class ChoiceQuestion(_SystemOneModel):
+    type: Literal["choice"]
+    instructions: StructuredText
+    criteria: dict[StrictStr, Description] = Field(min_length=2, max_length=255)
+
+    @field_validator("criteria")
+    @classmethod
+    def validate_names(cls, criteria):
+        if any(not name.strip() for name in criteria):
+            raise ValueError("option names must not be empty")
+        return criteria
+
+
+class NoulQuestion(_SystemOneModel):
+    type: Literal["noul"]
+    instructions: StructuredText
+    criteria: dict[StrictStr, Description] | None = None
+
+    @field_validator("criteria")
+    @classmethod
+    def validate_boolean_criteria(cls, criteria):
+        if criteria is not None and set(criteria) != {"true", "false"}:
+            raise ValueError('noul criteria must contain exactly "true" and "false"')
+        return criteria
+
+
+class ScoreQuestion(_SystemOneModel):
+    type: Literal["score"]
+    instructions: StructuredText
+    criteria: list[Description] = Field(min_length=2, max_length=10)
+
+
+SystemOneQuestion = Annotated[
+    ChoiceQuestion | NoulQuestion | ScoreQuestion, Field(discriminator="type")
+]
+
+
+class SystemOneOptions(_SystemOneModel):
+    temperature: float = Field(default=1.0, gt=0, allow_inf_nan=False, strict=True)
+    temperature_scaling: StrictBool = True
+    permutations: int = Field(default=1, ge=1, le=16, strict=True)
+    return_logprobs: StrictBool = False
+    assistant_prefix: StrictStr | None = None
+
+
+class SystemOneRequest(_SystemOneModel):
+    model: StrictStr = Field(min_length=1)
+    state: StructuredText
+    questions: dict[StrictStr, SystemOneQuestion] = Field(min_length=1, max_length=256)
+    options: SystemOneOptions = Field(default_factory=SystemOneOptions)
+    images: list[StrictStr] = Field(default_factory=list)
+
+    @field_validator("model")
+    @classmethod
+    def validate_model(cls, model):
+        if not model.strip():
+            raise ValueError("model must not be empty")
+        return model
+
+    @field_validator("questions")
+    @classmethod
+    def validate_question_names(cls, questions):
+        if any(not name.strip() for name in questions):
+            raise ValueError("question names must not be empty")
+        return questions
+
+    @field_validator("images")
+    @classmethod
+    def validate_images(cls, images):
+        for image in images:
+            if image.startswith("data:image/"):
+                if "," not in image or not image.split(",", 1)[1]:
+                    raise ValueError("image data URLs must have a nonempty payload")
+            else:
+                parsed = urlsplit(image)
+                if parsed.scheme not in ("http", "https") or not parsed.netloc:
+                    raise ValueError("images must be image data URLs or HTTP(S) URLs")
+        return images
+
+
+@dataclass(frozen=True)
+class QuestionPlan:
+    type: QuestionType
+    instructions: StructuredText
+    option_names: tuple[str, ...]
+    descriptions: tuple[Description, ...]
+
+
+def plan_question(question: SystemOneQuestion) -> QuestionPlan:
+    """Keep caller option order; boolean options always map to true, false."""
+    if isinstance(question, ChoiceQuestion):
+        names = tuple(question.criteria)
+        descriptions = tuple(question.criteria.values())
+    elif isinstance(question, NoulQuestion):
+        names = ("true", "false")
+        criteria = question.criteria or {}
+        descriptions = tuple(criteria.get(name) for name in names)
+    elif isinstance(question, ScoreQuestion):
+        names = tuple(str(index) for index in range(len(question.criteria)))
+        descriptions = tuple(question.criteria)
+    else:
+        raise TypeError("question must be a validated System One question")
+    return QuestionPlan(question.type, question.instructions, names, descriptions)
+
+
+def render_json_text(value: StructuredText) -> str:
+    """Render structured input deterministically without changing plain text."""
+    if isinstance(value, str):
+        return value
+    return json.dumps(value, ensure_ascii=False, sort_keys=True, allow_nan=False)
+
+
+def render_native_state_text(value: StructuredText) -> str:
+    """Render ``state`` exactly as the jevbench-hard runner's ``format_prompt``
+    does: plain strings verbatim, everything else as a sorted, indented JSON
+    dump. Only ``state`` gets this treatment in the native contract -- unlike
+    the served wording, the runner interpolates ``instructions`` and option
+    descriptions directly (``str()``), which ``_build_native_prompt`` mirrors."""
+    if isinstance(value, str):
+        return value
+    return json.dumps(
+        value, sort_keys=True, ensure_ascii=False, indent=2, allow_nan=False
+    )
+
+
+def rotation_order(option_count: int, rotation: int) -> tuple[int, ...]:
+    """Map each displayed label position to its original option index."""
+    if option_count < 2 or rotation < 0:
+        raise ValueError("at least two options and a nonnegative rotation are required")
+    return tuple(
+        (position + rotation) % option_count for position in range(option_count)
+    )
+
+
+def _validate_order(order: Sequence[int], option_count: int) -> None:
+    if (
+        len(order) != option_count
+        or any(isinstance(index, bool) or not isinstance(index, int) for index in order)
+        or set(order) != set(range(option_count))
+    ):
+        raise ValueError("option order must be a permutation of all original indices")
+
+
+def build_prompt(
+    state: StructuredText,
+    plan: QuestionPlan,
+    labels: Sequence[str],
+    order: Sequence[int] | None = None,
+    *,
+    wording: PromptWording = "served",
+) -> str:
+    """Build user content; the serving layer applies a non-thinking chat template.
+
+    Labels must already have been verified as distinct single-token symbols in
+    the assistant context by the serving layer. The shared state is first to
+    make prefix reuse possible across questions and option rotations.
+
+    ``wording="served"`` (default, unchanged) renders this adapter's own
+    contract (``Context:/.../Options: A: name: description``).
+
+    ``wording="native"`` renders the jevbench-hard runner's own contract
+    instead (run_suites_rotation.py: ``PREFIX``/``State:``/``Question:``/
+    ``Options: A. name: description``), which the runner's own ablation
+    measured as worth about +6 points on jevbench-hard by wording alone. It
+    requires canonical, single-token ``A``..``Z`` labels (at most 26 options)
+    in the given order, because that is the letter-token contract the native
+    runner assumes; a backend whose tokenizer's first labels are not exactly
+    those letters cannot serve this wording and ``build_prompt`` raises.
+    """
+    option_count = len(plan.option_names)
+    if len(labels) != option_count or len(set(labels)) != option_count:
+        raise ValueError("exactly one distinct label per option is required")
+    if any(
+        not isinstance(label, str) or not label or "\n" in label for label in labels
+    ):
+        raise ValueError("labels must be nonempty single-line strings")
+    order = tuple(range(option_count)) if order is None else order
+    _validate_order(order, option_count)
+    if wording == "native":
+        return _build_native_prompt(state, plan, labels, order)
+    if wording != "served":
+        raise ValueError('wording must be "served" or "native"')
+    lines = []
+    for label, index in zip(labels, order):
+        name = plan.option_names[index]
+        if plan.type == "noul":
+            name = "yes" if name == "true" else "no"
+        description = plan.descriptions[index]
+        text = (
+            name if description is None else f"{name}: {render_json_text(description)}"
+        )
+        lines.append(f"{label}: {text}")
+    return (
+        f"Context:\n{render_json_text(state)}\n\n"
+        "Answer the question with only the label of the best option "
+        "(the complete label before the colon), nothing else.\n"
+        f"Question: {render_json_text(plan.instructions)}\nOptions:\n"
+        + "\n".join(lines)
+    )
+
+
+def _build_native_prompt(
+    state: StructuredText,
+    plan: QuestionPlan,
+    labels: Sequence[str],
+    order: Sequence[int],
+) -> str:
+    """``build_prompt(..., wording="native")``'s body: a byte-for-byte port of
+    run_suites_rotation.py's ``option_texts`` + ``format_prompt`` for the
+    normalized KEV question schema this adapter already validates into
+    (``plan.type``/``plan.option_names``/``plan.descriptions``).
+
+    Ported quirks, kept intentionally rather than "fixed", because parity with
+    the runner's actual output is the point: only ``state`` gets a JSON dump
+    when non-string (``instructions`` and descriptions are interpolated with
+    plain ``str()``, exactly like the runner's f-strings), and a *falsy*
+    description (not just ``None``) is omitted, matching ``if description``
+    in the runner's ``format_prompt``.
+    """
+    option_count = len(plan.option_names)
+    if tuple(labels) != tuple(_NATIVE_DISPLAY[:option_count]):
+        raise ValueError(
+            "native prompt wording requires canonical single-token A-Z labels "
+            "in order (at most 26 options); this backend's labels are "
+            f"{tuple(labels)!r}"
+        )
+    lines = []
+    for position, index in enumerate(order):
+        name = plan.option_names[index]
+        if plan.type == "noul":
+            name = "yes" if name == "true" else "no"
+        description = plan.descriptions[index]
+        line = f"{_NATIVE_DISPLAY[position]}. {name}"
+        if description:
+            line += f": {description}"
+        lines.append(line)
+    return (
+        f"{NATIVE_PROMPT_PREFIX}\n\n"
+        f"State:\n{render_native_state_text(state)}\n\n"
+        f"Question:\n{plan.instructions}\n\nOptions:\n" + "\n".join(lines)
+    )
+
+
+def probabilities_from_logprobs(
+    logprobs: Sequence[float], temperature: float = 1.0
+) -> list[float]:
+    """Normalize selected vocabulary logprobs, equivalent to label-logit softmax.
+
+    Subtract the maximum *before* scaling to stay stable even with tiny positive
+    temperatures or very negative logprobs. Missing/nonfinite engine values are
+    errors, rather than fabricated probabilities.
+    """
+    if (
+        isinstance(temperature, bool)
+        or not isinstance(temperature, Real)
+        or not math.isfinite(temperature)
+        or temperature <= 0
+    ):
+        raise ValueError("temperature must be finite and greater than zero")
+    if len(logprobs) < 2:
+        raise ValueError("at least two label logprobs are required")
+    if any(
+        isinstance(value, bool)
+        or not isinstance(value, Real)
+        or not math.isfinite(value)
+        for value in logprobs
+    ):
+        raise ValueError("every label must have a finite numeric logprob")
+    maximum = max(logprobs)
+    weights = [math.exp((value - maximum) / temperature) for value in logprobs]
+    total = math.fsum(weights)
+    return [weight / total for weight in weights]
+
+
+def reduce_probabilities(
+    plan: QuestionPlan,
+    logprob_vectors: Sequence[Sequence[float]],
+    orders: Sequence[Sequence[int]],
+    options: SystemOneOptions,
+) -> dict:
+    """Scale, undo rotations, then average probabilities across evaluations.
+
+    Returned ``logprobs`` (when requested) is a list of canonical option maps,
+    one per evaluation. These are unmodified vocabulary-normalized logprobs,
+    not raw logits and not the final label distribution. Keeping each evaluation
+    separately permits fitting temperature before permutation averaging.
+    """
+    if not logprob_vectors or len(logprob_vectors) != len(orders):
+        raise ValueError("each logprob vector must have a matching option order")
+    option_count = len(plan.option_names)
+    temperature = options.temperature if options.temperature_scaling else 1.0
+    columns: list[list[float]] = [[] for _ in range(option_count)]
+    raw_rows = []
+    for vector, order in zip(logprob_vectors, orders):
+        if len(vector) != option_count:
+            raise ValueError("engine must return one logprob for every label")
+        _validate_order(order, option_count)
+        probabilities = probabilities_from_logprobs(vector, temperature)
+        raw = {}
+        for position, original in enumerate(order):
+            columns[original].append(probabilities[position])
+            raw[plan.option_names[original]] = float(vector[position])
+        raw_rows.append({name: raw[name] for name in plan.option_names})
+    averaged = [math.fsum(column) / len(logprob_vectors) for column in columns]
+    # Account for floating-point summation drift before entropy and expectation.
+    total = math.fsum(averaged)
+    averaged = [probability / total for probability in averaged]
+    answer = {"type": plan.type}
+    if plan.type == "noul":
+        answer["noul"] = averaged[0]
+    else:
+        answer["probabilities"] = dict(zip(plan.option_names, averaged))
+        entropy = -math.fsum(p * math.log(p) for p in averaged if p > 0)
+        answer["confidence"] = min(
+            1.0, max(0.0, 1.0 - entropy / math.log(option_count))
+        )
+        if plan.type == "choice":
+            answer["choice"] = plan.option_names[
+                max(range(option_count), key=averaged.__getitem__)
+            ]
+        else:
+            answer["score"] = math.fsum(index * p for index, p in enumerate(averaged))
+            answer["legend"] = dict(zip(plan.option_names, plan.descriptions))
+    if options.return_logprobs:
+        answer["logprobs"] = raw_rows
+    return answer

server/jev_adapter/server.py

@@ -0,0 +1,140 @@
+"""Standalone HTTP process; only the backend client talks to an inference server."""
+
+import asyncio
+import logging
+import secrets
+from contextlib import asynccontextmanager
+
+from fastapi import Depends, FastAPI, Request
+from fastapi.exceptions import RequestValidationError
+from fastapi.responses import JSONResponse
+
+from .backend import AdapterError, ScoringBackend
+from .protocol import SystemOneRequest
+from .service import SystemOneService
+
+logger = logging.getLogger(__name__)
+
+
+def create_app(
+    backend: ScoringBackend,
+    *,
+    api_key: str | None = None,
+    max_concurrency: int = 32,
+    alias: str = "jev-latest",
+    default_temperature: float = 1.0,
+    prompt_wording: str = "served",
+) -> FastAPI:
+    service = SystemOneService(
+        backend,
+        max_concurrency=max_concurrency,
+        alias=alias,
+        default_temperature=default_temperature,
+        prompt_wording=prompt_wording,
+    )
+
+    @asynccontextmanager
+    async def lifespan(app):
+        try:
+            await backend.start()
+            yield
+        finally:
+            await backend.close()
+
+    app = FastAPI(title="Jev adapter", version="0.1.0", lifespan=lifespan)
+
+    async def authorize(request: Request):
+        if api_key and not secrets.compare_digest(
+            request.headers.get("authorization", "").encode("utf-8"),
+            f"Bearer {api_key}".encode(),
+        ):
+            raise AdapterError(
+                "unauthorized", "A valid bearer token is required.", status=401
+            )
+
+    @app.exception_handler(AdapterError)
+    async def adapter_error(request, exc):
+        return JSONResponse(
+            status_code=exc.status,
+            content={
+                "error": {
+                    "code": exc.code,
+                    "message": str(exc),
+                    "field": exc.field,
+                }
+            },
+        )
+
+    @app.exception_handler(RequestValidationError)
+    async def validation_error(request, exc):
+        error = exc.errors()[0]
+        return JSONResponse(
+            status_code=422,
+            content={
+                "error": {
+                    "code": "invalid_request",
+                    "message": error["msg"],
+                    "field": ".".join(str(p) for p in error["loc"] if p != "body"),
+                }
+            },
+        )
+
+    @app.get("/health")
+    async def health():
+        # Process liveness, not an inference/GPU readiness probe.
+        return {"status": "ok"}
+
+    @app.get("/v1/models", dependencies=[Depends(authorize)])
+    async def models():
+        return {
+            "object": "list",
+            "data": [
+                {
+                    "id": backend.model,
+                    "object": "model",
+                    "owned_by": "inference-engine",
+                },
+            ],
+        }
+
+    @app.post("/v1/systemone", dependencies=[Depends(authorize)])
+    async def systemone(body: SystemOneRequest, request: Request):
+        work = asyncio.create_task(service.score(body))
+        stop_watching = False
+
+        async def disconnected():
+            while not stop_watching:
+                if await request.is_disconnected():
+                    return
+                if not stop_watching:
+                    await asyncio.sleep(0.05)
+
+        watcher = asyncio.create_task(disconnected())
+        try:
+            done, _ = await asyncio.wait(
+                (work, watcher),
+                return_when=asyncio.FIRST_COMPLETED,
+            )
+            if work in done:
+                return await work
+            raise AdapterError(
+                "client_disconnected", "Client disconnected.", status=499
+            )
+        except AdapterError:
+            raise
+        except Exception:
+            logger.exception("Decision scoring failed")
+            raise AdapterError(
+                "scoring_failed",
+                "Could not compute a complete decision.",
+                status=500,
+            ) from None
+        finally:
+            # A disconnect probe's own cancellation scope can absorb cancel().
+            # Stop explicitly as well, including before another polling sleep.
+            stop_watching = True
+            work.cancel()
+            watcher.cancel()
+            await asyncio.gather(work, watcher, return_exceptions=True)
+
+    return app

server/jev_adapter/service.py

@@ -0,0 +1,122 @@
+"""Build questions and construct responses independently of the inference engine."""
+
+import asyncio
+import math
+import time
+from numbers import Real
+
+from .backend import AdapterError, ScoringBackend
+from .protocol import (
+    SystemOneRequest,
+    build_prompt,
+    plan_question,
+    reduce_probabilities,
+    rotation_order,
+)
+
+
+class SystemOneService:
+    def __init__(
+        self,
+        backend: ScoringBackend,
+        *,
+        max_concurrency: int = 32,
+        alias: str = "jev-latest",
+        default_temperature: float = 1.0,
+        prompt_wording: str = "served",
+    ):
+        if max_concurrency < 1:
+            raise ValueError("max_concurrency must be positive")
+        if (
+            isinstance(default_temperature, bool)
+            or not isinstance(default_temperature, Real)
+            or not math.isfinite(default_temperature)
+            or default_temperature <= 0
+        ):
+            raise ValueError("default_temperature must be finite and greater than zero")
+        if prompt_wording not in ("served", "native"):
+            raise ValueError('prompt_wording must be "served" or "native"')
+        self.backend, self.alias = backend, alias
+        # Applied only when a request omits options.temperature; an explicit
+        # request value always wins, and temperature_scaling=false still
+        # disables scaling. Without this setting the request default (1.0) holds.
+        self.default_temperature = float(default_temperature)
+        # Server-wide, not per-request: which text build_prompt() renders.
+        # "native" additionally requires the backend's labels to be canonical
+        # A-Z (build_prompt raises otherwise) and relies on the backend to
+        # prepend a matching system message where feasible (see sglang.py).
+        self.prompt_wording = prompt_wording
+        self.capacity = asyncio.Semaphore(max_concurrency)
+
+    def effective_options(self, request: SystemOneRequest):
+        """Request options with the server default filled into an omitted temperature."""
+        options = request.options
+        if "temperature" in options.model_fields_set:
+            return options
+        return options.model_copy(update={"temperature": self.default_temperature})
+
+    async def score(self, request: SystemOneRequest) -> dict:
+        if request.model not in {self.backend.model, self.alias}:
+            raise AdapterError(
+                "model_not_found",
+                "Requested model is not configured.",
+                "model",
+                404,
+            )
+        started = time.perf_counter()
+        options = self.effective_options(request)
+        jobs, plans = [], {}
+        # Validate and prepare all questions before scheduling any inference.
+        for key, question in request.questions.items():
+            plan = plans[key] = plan_question(question)
+            labels, token_ids = self.backend.labels(len(plan.option_names))
+            for rotation in range(options.permutations):
+                order = rotation_order(len(labels), rotation)
+                prompt = build_prompt(
+                    request.state, plan, labels, order, wording=self.prompt_wording
+                )
+                jobs.append((key, order, prompt, labels, token_ids))
+
+        async def evaluate(job):
+            key, order, prompt, labels, token_ids = job
+            async with self.capacity:
+                result = await self.backend.evaluate(
+                    prompt,
+                    request.images,
+                    labels,
+                    token_ids,
+                    options.assistant_prefix,
+                )
+            return key, order, result
+
+        tasks = [asyncio.create_task(evaluate(job)) for job in jobs]
+        try:
+            results = await asyncio.gather(*tasks)
+        except BaseException:
+            for task in tasks:
+                task.cancel()
+            await asyncio.gather(*tasks, return_exceptions=True)
+            raise
+        grouped = {key: ([], []) for key in plans}
+        input_tokens = 0
+        for key, order, result in results:
+            vectors, orders = grouped[key]
+            vectors.append(result.logprobs)
+            orders.append(order)
+            input_tokens += result.input_tokens
+        return {
+            "model": self.backend.model,
+            "answers": {
+                key: reduce_probabilities(plan, *grouped[key], options)
+                for key, plan in plans.items()
+            },
+            "usage": {"input_tokens": input_tokens, "output_tokens": 0},
+            "metadata": {
+                "confidence_method": "1 - normalized_entropy",
+                "temperature": (
+                    options.temperature if options.temperature_scaling else 1.0
+                ),
+                "evaluations": len(jobs),
+                "adapter_elapsed_ms": (time.perf_counter() - started) * 1000,
+            },
+        }

server/jev_adapter/sglang.py

@@ -0,0 +1,510 @@
+"""Prefill-only scoring through stock SGLang HTTP APIs.
+
+Chat tokenization uses the inference server by default. An optional native CPU
+tokenizer preserves control-token IDs for engines with a lossy string chat API.
+No SGLang, CUDA, or model weights are imported into this process.
+"""
+
+import asyncio
+import math
+import string
+import uuid
+from typing import Any
+
+import httpx
+
+from .backend import AdapterError, ScoringResult
+
+
+def _label_candidates() -> list[str]:
+    letters = string.ascii_uppercase + string.ascii_lowercase + string.digits
+    return (
+        list(letters)
+        + [f"{value:03d}" for value in range(1000)]
+        + [
+            first + second
+            for alphabet in (string.ascii_uppercase, string.ascii_lowercase)
+            for first in alphabet
+            for second in alphabet
+        ]
+    )
+
+
+_IMAGE_MODEL_TYPES = {
+    "qwen2_vl",
+    "qwen2_5_vl",
+    "qwen3_vl",
+    "qwen3_vl_moe",
+    "qwen3_5",
+    "qwen3_5_moe",
+    "mistral3",
+    "pixtral",
+}
+_IMAGE_ARCHITECTURES = {
+    "Qwen2VLForConditionalGeneration",
+    "Qwen2_5_VLForConditionalGeneration",
+    "Qwen3VLForConditionalGeneration",
+    "Qwen3VLMoeForConditionalGeneration",
+    "Qwen3_5ForConditionalGeneration",
+    "Qwen3_5MoeForConditionalGeneration",
+    "Mistral3ForConditionalGeneration",
+    "PixtralForConditionalGeneration",
+}
+
+
+def _bad_response(message: str) -> AdapterError:
+    return AdapterError("invalid_engine_response", message, status=502)
+
+
+def _token_list(value: Any) -> bool:
+    return isinstance(value, list) and all(
+        type(token) is int and token >= 0 for token in value
+    )
+
+
+class SGLangBackend:
+    def __init__(
+        self,
+        base_url: str,
+        model: str,
+        api_key: str | None = None,
+        timeout: float = 120,
+        max_connections: int = 32,
+        client: httpx.AsyncClient | None = None,
+        native_tokenizer: Any | None = None,
+        prompt_wording: str = "served",
+        native_system_prompt: str | None = None,
+    ):
+        if not math.isfinite(timeout) or timeout <= 0 or max_connections < 1:
+            raise ValueError("timeout and max_connections must be positive")
+        if prompt_wording not in ("served", "native"):
+            raise ValueError('prompt_wording must be "served" or "native"')
+        url = httpx.URL(base_url)
+        if (
+            url.scheme not in ("http", "https")
+            or not url.host
+            or url.query
+            or url.fragment
+        ):
+            raise ValueError(
+                "base_url must be an HTTP(S) origin or path without query or fragment"
+            )
+        self.base_url = base_url.rstrip("/")
+        self.model = model
+        self.timeout = timeout
+        self._headers = {"Authorization": f"Bearer {api_key}"} if api_key else {}
+        self._owns_client = client is None
+        self._client = client or httpx.AsyncClient(
+            limits=httpx.Limits(max_connections=max_connections),
+            timeout=timeout,
+        )
+        self._startup_lock = asyncio.Lock()
+        self._label_strings: tuple[str, ...] = ()
+        self._label_ids: tuple[int, ...] = ()
+        self._image_capable = False
+        self._image_format_supported = False
+        self._ready = False
+        self._native_tokenizer = native_tokenizer
+        self._prompt_wording = prompt_wording
+        # Only used when prompt_wording == "native": the runner's default
+        # system-prompt text, prepended explicitly as a system message (see
+        # native_tokenizer.extract_default_system_prompt/NativeTokenizer.prepare).
+        # None means "wording changed, but no system message is added" --
+        # either the wording is "served", or no such text was supplied/found.
+        self._native_system_prompt = native_system_prompt if prompt_wording == "native" else None
+
+    async def _request(
+        self, method: str, path: str, payload: dict | None = None
+    ) -> dict:
+        try:
+            response = await self._client.request(
+                method,
+                self.base_url + path,
+                json=payload,
+                headers=self._headers,
+                timeout=self.timeout,
+            )
+        except httpx.TimeoutException as error:
+            raise AdapterError(
+                "engine_timeout", "The inference engine timed out.", status=504
+            ) from error
+        except httpx.HTTPError as error:
+            raise AdapterError(
+                "engine_unavailable",
+                "Could not reach the inference engine.",
+                status=502,
+            ) from error
+        if response.status_code == 404:
+            raise AdapterError(
+                "engine_endpoint_missing",
+                f"The inference engine does not provide {path}.",
+                status=502,
+            )
+        if response.status_code in (400, 422):
+            raise AdapterError(
+                "engine_rejected_request",
+                "The inference engine rejected the scoring input.",
+                status=422,
+            )
+        if not response.is_success:
+            raise AdapterError(
+                "engine_http_error",
+                f"The inference engine returned HTTP {response.status_code}.",
+                status=502,
+            )
+        try:
+            result = response.json()
+        except ValueError as error:
+            raise _bad_response(
+                "The inference engine returned invalid JSON."
+            ) from error
+        if not isinstance(result, dict) or "error" in result:
+            raise _bad_response("The inference engine returned an unexpected response.")
+        return result
+
+    async def _information(self, path: str, legacy: str) -> dict:
+        try:
+            return await self._request("GET", path)
+        except AdapterError as error:
+            if error.code != "engine_endpoint_missing":
+                raise
+            return await self._request("GET", legacy)
+
+    async def start(self) -> None:
+        async with self._startup_lock:
+            if self._ready:
+                return
+            info = await self._information("/model_info", "/get_model_info")
+            if info.get("is_generation") is not True:
+                raise AdapterError(
+                    "unsupported_model", "A generation model is required."
+                )
+            served = info.get("served_model_name")
+            if not isinstance(served, str) or not served:
+                raise _bad_response(
+                    "The inference engine did not identify its served model."
+                )
+            if self.model not in (served, info.get("model_path")):
+                raise AdapterError(
+                    "model_not_found",
+                    "Configured model is not served by the inference engine.",
+                    "model",
+                    404,
+                )
+            config = await self._information("/server_info", "/get_server_info")
+            if config.get("skip_tokenizer_init"):
+                raise AdapterError(
+                    "tokenizer_unavailable",
+                    "The inference engine must initialize its tokenizer.",
+                )
+            if config.get("speculative_algorithm") is not None:
+                raise AdapterError(
+                    "speculation_not_supported",
+                    "Disable speculative decoding and MTP for prefill-only scoring.",
+                )
+            self.model = served
+            self._image_capable = info.get("has_image_understanding") is True
+            architectures = info.get("architectures") or []
+            self._image_format_supported = info.get(
+                "model_type"
+            ) in _IMAGE_MODEL_TYPES or (
+                isinstance(architectures, list)
+                and any(
+                    arch in _IMAGE_ARCHITECTURES
+                    for arch in architectures
+                    if isinstance(arch, str)
+                )
+            )
+            candidates = _label_candidates()
+            tokenized = await self._request(
+                "POST",
+                "/v1/tokenize",
+                {
+                    "model": self.model,
+                    "prompt": candidates,
+                    "add_special_tokens": False,
+                },
+            )
+            batches = tokenized.get("tokens")
+            if (
+                not isinstance(batches, list)
+                or len(batches) != len(candidates)
+                or not all(_token_list(ids) for ids in batches)
+            ):
+                raise _bad_response(
+                    "The inference engine returned malformed label tokenization."
+                )
+            singles = [
+                (label, ids[0])
+                for label, ids in zip(candidates, batches)
+                if len(ids) == 1
+            ]
+            if not singles:
+                raise AdapterError(
+                    "labels_unavailable",
+                    "The tokenizer has no usable single-token labels.",
+                )
+            decoded = await self._request(
+                "POST",
+                "/v1/detokenize",
+                {
+                    "model": self.model,
+                    "tokens": [[token] for _, token in singles],
+                    "skip_special_tokens": True,
+                },
+            )
+            texts = decoded.get("text")
+            if (
+                not isinstance(texts, list)
+                or len(texts) != len(singles)
+                or not all(isinstance(item, str) for item in texts)
+            ):
+                raise _bad_response(
+                    "The inference engine returned malformed label detokenization."
+                )
+            valid: list[tuple[str, int]] = []
+            seen: set[int] = set()
+            for (label, token), decoded_label in zip(singles, texts):
+                if label == decoded_label and token not in seen:
+                    valid.append((label, token))
+                    seen.add(token)
+                    if len(valid) == 255:
+                        break
+            if len(valid) < 2:
+                raise AdapterError(
+                    "labels_unavailable",
+                    "At least two distinct, non-special single-token labels are required.",
+                )
+            self._label_strings, self._label_ids = tuple(zip(*valid))
+            self._ready = True
+
+    async def close(self) -> None:
+        if self._owns_client:
+            await self._client.aclose()
+
+    def labels(self, count: int) -> tuple[tuple[str, ...], tuple[int, ...]]:
+        if not self._ready:
+            raise AdapterError(
+                "engine_not_ready", "The scoring backend has not started.", status=503
+            )
+        if type(count) is not int or count < 2 or count > len(self._label_strings):
+            raise AdapterError(
+                "labels_unavailable",
+                f"This tokenizer supports 2 to {len(self._label_strings)} answer labels.",
+                "questions",
+            )
+        return self._label_strings[:count], self._label_ids[:count]
+
+    async def evaluate(
+        self,
+        prompt: str,
+        images: list[str],
+        labels: tuple[str, ...],
+        token_ids: tuple[int, ...],
+        assistant_prefix: str | None,
+    ) -> ScoringResult:
+        if not self._ready:
+            raise AdapterError(
+                "engine_not_ready", "The scoring backend has not started.", status=503
+            )
+        if (
+            len(labels) < 2
+            or len(labels) != len(token_ids)
+            or len(set(token_ids)) != len(token_ids)
+        ):
+            raise AdapterError(
+                "invalid_labels", "Distinct label/token pairs are required."
+            )
+        known = dict(zip(self._label_strings, self._label_ids))
+        if any(known.get(label) != token for label, token in zip(labels, token_ids)):
+            raise AdapterError(
+                "invalid_labels", "Labels must come from this backend's tokenizer."
+            )
+        if images and self._native_tokenizer is not None:
+            raise AdapterError(
+                "native_images_not_supported",
+                "Native tokenizer scoring currently accepts text only.",
+                "images",
+            )
+        if images and not self._image_capable:
+            raise AdapterError(
+                "images_not_supported",
+                "The loaded model does not support images.",
+                "images",
+            )
+        if images and not self._image_format_supported:
+            raise AdapterError(
+                "image_model_not_supported",
+                "Image scoring currently supports Qwen VL/Qwen3.5 and Mistral3/Pixtral chat formats.",
+                "images",
+            )
+        if self._native_tokenizer is not None:
+            final_ids = await asyncio.to_thread(
+                self._native_tokenizer.prepare,
+                prompt,
+                labels,
+                token_ids,
+                assistant_prefix,
+                self._native_system_prompt,
+            )
+            if not _token_list(final_ids) or not final_ids:
+                raise _bad_response("The native tokenizer returned invalid prompt IDs.")
+            final_text = None
+        else:
+            final_ids, final_text = await self._prepare_http(
+                prompt,
+                images,
+                labels,
+                token_ids,
+                assistant_prefix,
+                self._native_system_prompt,
+            )
+        payload = {
+            "sampling_params": {
+                "max_new_tokens": 0,
+                "temperature": 1,
+                "top_p": 1,
+                "top_k": -1,
+            },
+            "return_logprob": True,
+            "logprob_start_len": -1,
+            "token_ids_logprob": list(token_ids),
+            "top_logprobs_num": 0,
+            "return_text_in_logprobs": False,
+            "stream": False,
+            "rid": "jev-" + uuid.uuid4().hex,
+        }
+        if images:
+            payload.update(text=final_text, image_data=images)
+        else:
+            payload["input_ids"] = final_ids
+        result = await self._request("POST", "/generate", payload)
+        meta = result.get("meta_info")
+        if not isinstance(meta, dict):
+            raise _bad_response("The inference engine omitted scoring metadata.")
+        if (
+            type(meta.get("completion_tokens")) is not int
+            or meta["completion_tokens"] != 0
+        ):
+            raise _bad_response(
+                "The inference engine did not honor zero-output scoring."
+            )
+        input_tokens = meta.get("prompt_tokens")
+        if type(input_tokens) is not int or input_tokens < 1:
+            raise _bad_response(
+                "The inference engine returned invalid input-token usage."
+            )
+        positions = meta.get("output_token_ids_logprobs")
+        if (
+            not isinstance(positions, list)
+            or len(positions) != 1
+            or not isinstance(positions[0], list)
+        ):
+            raise _bad_response(
+                "The inference engine omitted the requested next-token logprobs."
+            )
+        values: dict[int, float] = {}
+        for entry in positions[0]:
+            if not isinstance(entry, (list, tuple)) or len(entry) < 2:
+                raise _bad_response("The inference engine returned malformed logprobs.")
+            value, token = entry[:2]
+            if (
+                type(token) is not int
+                or type(value) not in (int, float)
+                or not math.isfinite(value)
+                or value > 0
+                or token in values
+            ):
+                raise _bad_response("The inference engine returned invalid logprobs.")
+            values[token] = float(value)
+        if set(values) != set(token_ids):
+            raise _bad_response(
+                "The inference engine did not return every requested label logprob."
+            )
+        return ScoringResult(tuple(values[token] for token in token_ids), input_tokens)
+
+    async def _prepare_http(
+        self,
+        prompt: str,
+        images: list[str],
+        labels: tuple[str, ...],
+        token_ids: tuple[int, ...],
+        assistant_prefix: str | None,
+        system_prompt: str | None = None,
+    ) -> tuple[list[int], str]:
+        content: Any = prompt
+        if images:
+            content = [
+                {"type": "image_url", "image_url": {"url": image}} for image in images
+            ]
+            content.append({"type": "text", "text": prompt})
+        messages = (
+            [{"role": "system", "content": system_prompt}] if system_prompt else []
+        ) + [{"role": "user", "content": content}]
+        rendered = await self._request(
+            "POST",
+            "/v1/tokenize",
+            {
+                "model": self.model,
+                "messages": messages,
+                "chat_template_kwargs": {"enable_thinking": False},
+            },
+        )
+        prompt_ids = rendered.get("tokens")
+        if not _token_list(prompt_ids) or not prompt_ids:
+            raise _bad_response(
+                "The inference engine returned malformed chat tokenization."
+            )
+        decoded = await self._request(
+            "POST",
+            "/v1/detokenize",
+            {
+                "model": self.model,
+                "tokens": prompt_ids,
+                "skip_special_tokens": False,
+            },
+        )
+        rendered_text = decoded.get("text")
+        if not isinstance(rendered_text, str) or not rendered_text:
+            raise _bad_response(
+                "The inference engine returned malformed chat detokenization."
+            )
+        final_text = rendered_text + (assistant_prefix or "")
+        # Check the complete template roundtrip and the actual assistant boundary.
+        # Checking labels in isolation can score the wrong tokens for BPE tokenizers.
+        prompts = [rendered_text, final_text] + [final_text + label for label in labels]
+        checked = await self._request(
+            "POST",
+            "/v1/tokenize",
+            {
+                "model": self.model,
+                "prompt": prompts,
+                "add_special_tokens": False,
+            },
+        )
+        variants = checked.get("tokens")
+        if (
+            not isinstance(variants, list)
+            or len(variants) != len(prompts)
+            or not all(_token_list(ids) for ids in variants)
+        ):
+            raise _bad_response(
+                "The inference engine returned malformed boundary tokenization."
+            )
+        if variants[0] != prompt_ids:
+            raise AdapterError(
+                "unsupported_tokenizer_roundtrip",
+                "The rendered chat prompt cannot be losslessly tokenized through this HTTP adapter.",
+            )
+        final_ids = variants[1]
+        if not final_ids:
+            raise _bad_response("The formatted scoring prompt is empty.")
+        if any(
+            ids != final_ids + [token] for ids, token in zip(variants[2:], token_ids)
+        ):
+            raise AdapterError(
+                "invalid_label_boundary",
+                "Labels are not single tokens at the assistant boundary; configure a compatible assistant_prefix.",
+                "options.assistant_prefix",
+            )
+        return final_ids, final_text

server/pyproject.toml

@@ -0,0 +1,34 @@
+[build-system]
+requires = ["setuptools>=68"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "jev-adapter"
+version = "0.1.0"
+description = "Independent next-token decision API over an unmodified inference server"
+readme = "README.md"
+requires-python = ">=3.11"
+license = {file = "LICENSE"}
+dependencies = [
+    "fastapi>=0.115,<1",
+    "httpx>=0.27,<1",
+    "pydantic>=2.10,<3",
+    "uvicorn>=0.30,<1",
+]
+
+[project.optional-dependencies]
+dev = ["pytest>=8,<10", "pytest-asyncio>=0.24,<2", "ruff>=0.12"]
+native-tokenizer = ["transformers==5.12.1", "mistral-common==1.11.7"]
+
+[project.scripts]
+jev-adapter = "jev_adapter.__main__:main"
+
+[tool.setuptools.packages.find]
+include = ["jev_adapter*"]
+
+[tool.pytest.ini_options]
+testpaths = ["tests"]
+
+[tool.ruff]
+target-version = "py311"
+line-length = 88

server/tests/test_benchmark_matrix.py

@@ -0,0 +1,101 @@
+import importlib.util
+import os
+import select
+import signal
+import subprocess
+import sys
+from concurrent.futures import ThreadPoolExecutor
+from pathlib import Path
+
+
+def test_matrix_preview_uses_four_bf16_models_without_starting(tmp_path):
+    root = Path(__file__).resolve().parents[1]
+    output = tmp_path / "does-not-exist"
+    engine_python = tmp_path / "engine-venv-python"
+    engine_python.symlink_to(sys.executable)
+    result = subprocess.run(
+        [
+            sys.executable,
+            str(root / "benchmarks/run_matrix.py"),
+            "--output",
+            str(output),
+            "--include-external",
+            "--concurrency",
+            "8",
+            "--engine-python",
+            str(engine_python),
+        ],
+        capture_output=True,
+        text=True,
+        check=True,
+    )
+    assert "Preview only" in result.stdout
+    assert "ministral3-3b-bf16" in result.stdout
+    assert "ministral3-3b-fp8" not in result.stdout
+    assert "qwen36-27b-bf16" in result.stdout
+    assert "qwen36-35b-a3b-bf16" in result.stdout
+    assert "qwen35-4b-bf16" in result.stdout
+    assert "scienthoon-v1" in result.stdout
+    assert "jevbench-hard/public.jsonl" in result.stdout
+    assert "--probe-vision" not in result.stdout
+    assert f"--engine-python {engine_python}" in result.stdout
+    assert not output.exists()
+
+
+def test_stop_owned_cleans_runtime_that_inherits_ignored_sigint(tmp_path):
+    root = Path(__file__).resolve().parents[1]
+    spec = importlib.util.spec_from_file_location(
+        "benchmark_matrix_under_test", root / "benchmarks/run_matrix.py"
+    )
+    matrix = importlib.util.module_from_spec(spec)
+    spec.loader.exec_module(matrix)
+    cleanup_marker = tmp_path / "runtime-cleaned-up"
+    child_code = """
+import signal
+import sys
+from pathlib import Path
+
+# A background launcher can inherit this disposition from its parent shell.
+signal.signal(signal.SIGINT, signal.SIG_IGN)
+
+def terminate(signum, frame):
+    raise KeyboardInterrupt
+
+signal.signal(signal.SIGTERM, terminate)
+try:
+    print("ready", flush=True)
+    while True:
+        signal.pause()
+except KeyboardInterrupt:
+    pass
+finally:
+    Path(sys.argv[1]).write_text("graceful SIGTERM cleanup")
+"""
+    process = subprocess.Popen(
+        [sys.executable, "-c", child_code, str(cleanup_marker)],
+        start_new_session=True,
+        stdout=subprocess.PIPE,
+        stderr=subprocess.PIPE,
+        text=True,
+    )
+    try:
+        readable, _, _ = select.select([process.stdout], [], [], 5)
+        assert readable, "child did not install its signal handlers"
+        assert process.stdout.readline().strip() == "ready"
+        # Bound regressions without replacing real OS signal delivery or wait().
+        with ThreadPoolExecutor(max_workers=1) as executor:
+            stopped = executor.submit(matrix.stop_owned, process)
+            try:
+                stopped.result(timeout=5)
+            finally:
+                if process.poll() is None:
+                    os.killpg(process.pid, signal.SIGKILL)
+                    process.wait(timeout=5)
+        assert process.returncode == 0
+        assert cleanup_marker.read_text() == "graceful SIGTERM cleanup"
+    finally:
+        if process.poll() is None:
+            os.killpg(process.pid, signal.SIGKILL)
+            process.wait(timeout=5)
+        process.stdout.close()
+        process.stderr.close()

server/tests/test_benchmark_run.py

@@ -0,0 +1,290 @@
+import argparse
+import hashlib
+import json
+
+import httpx
+import pytest
+
+from jev_adapter.benchmarks.compare import collect
+from jev_adapter.benchmarks.data import normalize_record
+from jev_adapter.benchmarks.metrics import metrics, prediction_rows, summarize
+from jev_adapter.benchmarks.run import evaluate, load_data, validate_launch
+
+
+def item(record_id="one", variant="clean", source="policy"):
+    return normalize_record(
+        {
+            "state": "The transaction is refundable.",
+            "questions": {
+                "route": {
+                    "type": "choice",
+                    "instructions": "Choose a team.",
+                    "criteria": {"billing": None, "support": "Technical support"},
+                    "label": "billing",
+                    "src": "route",
+                },
+                "refund": {
+                    "type": "noul",
+                    "instructions": "Refundable?",
+                    "label": True,
+                    "src": "refund",
+                },
+                "priority": {
+                    "type": "score",
+                    "instructions": "How urgent?",
+                    "criteria": ["low", "mid", "high"],
+                    "label": 2,
+                    "src": "priority",
+                },
+            },
+            "_meta": {
+                "id": record_id,
+                "source": source,
+                "variant": variant,
+                "group_id": record_id,
+            },
+        },
+        "fixture",
+        "development",
+    )
+
+
+def response():
+    return {
+        "model": "decision-model",
+        "answers": {
+            "route": {
+                "type": "choice",
+                "probabilities": {"billing": 0.8, "support": 0.2},
+            },
+            "refund": {"type": "noul", "noul": 0.9},
+            "priority": {
+                "type": "score",
+                "probabilities": {"0": 0.1, "1": 0.2, "2": 0.7},
+            },
+        },
+        "usage": {"input_tokens": 100, "output_tokens": 0},
+        "metadata": {"adapter_elapsed_ms": 2.0, "evaluations": 3},
+    }
+
+
+def test_metrics_boolean_order_score_and_calibration():
+    rows = prediction_rows(item(), response())
+    assert rows[1]["keys"] == ["false", "true"]
+    assert rows[1]["p"] == pytest.approx([0.1, 0.9])
+    result = metrics(rows)
+    assert result["n"] == 3
+    assert result["acc"] == 1
+    assert result["ece"] == pytest.approx(0.2)
+    assert result["brier"] == pytest.approx((0.08 + 0.02 + 0.14) / 3)
+    assert result["score_mae"] == pytest.approx(0.4)
+    assert result["ranked_probability_score"] == pytest.approx(0.05)
+
+
+def test_jevbench_probability_reference_and_native_tie_break():
+    example = item()
+    example["metadata"]["gold_policy"] = {"argmax_tie_break": "lexicographic_label"}
+    example["expected"]["route"].update(
+        labels=["support", "billing"],
+        label=1,
+        target=[0, 1],
+        reference_probs=[0.5, 0.5],
+    )
+    body = response()
+    body["answers"]["route"]["probabilities"] = {"support": 0.5, "billing": 0.5}
+    rows = prediction_rows(example, body)
+    assert metrics(rows)["acc"] == 1
+    report = summarize(rows)
+    assert report["reference_distribution"]["n"] == 1
+    assert report["reference_distribution"]["squared_l2"] == 0
+    assert report["reference_distribution"]["total_variation"] == 0
+    assert report["reference_distribution"]["kl_reference_to_model"] == 0
+
+
+def test_unknown_excluded_and_permutation_aligned():
+    original = item()
+    permuted = item("two", "permuted")
+    permuted["metadata"]["group_id"] = "one"
+    permuted["expected"]["route"].update(
+        labels=["support", "billing"], label=1, target=[0, 1]
+    )
+    unknown = item("three", source="unknowable")
+    rows = [
+        row
+        for i in [original, permuted, unknown]
+        for row in prediction_rows(i, response())
+    ]
+    report = summarize(rows)
+    assert report["clean"]["n"] == 3
+    assert report["unknowable"]["n"] == 3
+    assert report["permutation"]["n"] == 1
+    assert report["permutation"]["flip_rate"] == 0
+    assert "unknowable" not in report["sources"]
+
+
+@pytest.mark.parametrize("change", ["missing", "nan", "output", "type"])
+def test_invalid_predictions_rejected(change):
+    body = response()
+    if change == "missing":
+        del body["answers"]["route"]["probabilities"]["support"]
+    elif change == "nan":
+        body["answers"]["refund"]["noul"] = float("nan")
+    elif change == "type":
+        body["answers"]["refund"]["type"] = "choice"
+    else:
+        body["answers"]["route"]["probabilities"] = {"billing": 0.9, "support": 0.9}
+    with pytest.raises(ValueError):
+        prediction_rows(item(), body)
+
+
+def arguments(tmp_path, records=None):
+    data = tmp_path / "data.jsonl"
+    records = records or [item(), item("two")]
+    data.write_text("".join(json.dumps(i) + "\n" for i in records))
+    return argparse.Namespace(
+        data=[data],
+        model="decision-model",
+        allow_test=False,
+        limit=None,
+        output=tmp_path / "result",
+        concurrency=2,
+        repeats=2,
+        warmup=2,
+        seed=42,
+        cache_mode="full-prefill",
+        timeout=10,
+        engine_url="http://engine",
+        base_url="http://adapter",
+        engine_manifest=None,
+        assistant_prefix=None,
+    )
+
+
+@pytest.mark.asyncio
+async def test_runner_outputs_and_excludes_warmup_repeat_quality(tmp_path):
+    args = arguments(tmp_path)
+    seen = []
+
+    def transport(request):
+        if request.url.path == "/model_info":
+            return httpx.Response(200, json={"served_model_name": "decision-model"})
+        if request.url.path == "/server_info":
+            return httpx.Response(
+                200,
+                json={
+                    "disable_radix_cache": True,
+                    "mm_preprocess_cache_size_mb": 0,
+                    "api_key": "secret",
+                },
+            )
+        payload = json.loads(request.content)
+        seen.append(payload)
+        assert "expected" not in payload and "metadata" not in payload
+        assert all(
+            "label" not in q and "src" not in q for q in payload["questions"].values()
+        )
+        return httpx.Response(200, json=response())
+
+    async with httpx.AsyncClient(transport=httpx.MockTransport(transport)) as client:
+        result = await evaluate(args, client)
+    assert len(seen) == 6
+    assert result["status"] == "complete"
+    assert result["successful_requests"] == 4
+    assert result["suites"]["fixture/development"]["clean"]["n"] == 6
+    assert result["latency_ms"]["n"] == 4
+    assert "secret" not in (args.output / "manifest.json").read_text()
+    assert len((args.output / "predictions.jsonl").read_text().splitlines()) == 4
+    comparison = collect(args.output)
+    assert len(comparison) == 1 and comparison[0]["clean_questions"] == 6
+    assert comparison[0]["partial_dataset"] is True
+    with pytest.raises(FileExistsError):
+        await evaluate(args)
+    (args.output / "invalidated.json").write_text(
+        json.dumps({"reason": "Tokenization failed an independent canonical check"})
+    )
+    assert collect(args.output) == []
+
+
+@pytest.mark.asyncio
+async def test_runner_errors_do_not_produce_headline_or_retry(tmp_path):
+    args = arguments(tmp_path)
+    args.warmup = 0
+    calls = 0
+
+    def transport(request):
+        nonlocal calls
+        if request.url.path == "/model_info":
+            return httpx.Response(200, json={"served_model_name": "decision-model"})
+        if request.url.path == "/server_info":
+            return httpx.Response(
+                200,
+                json={"disable_radix_cache": True, "mm_preprocess_cache_size_mb": 0},
+            )
+        calls += 1
+        body = response()
+        body["usage"]["output_tokens"] = 1
+        return httpx.Response(200, json=body)
+
+    async with httpx.AsyncClient(transport=httpx.MockTransport(transport)) as client:
+        result = await evaluate(args, client)
+    assert result["status"] == "failed"
+    assert result["suites"] == {}
+    assert result["errors"] == 4 and calls == 4
+    assert (args.output / "failures.json").exists()
+    assert collect(args.output) == []
+
+
+def test_locked_split_and_data_integrity(tmp_path):
+    locked = item()
+    locked["split"] = "test"
+    args = arguments(tmp_path, [locked])
+    with pytest.raises(ValueError, match="allow-test"):
+        load_data(args.data, args.model)
+    assert len(load_data(args.data, args.model, allow_test=True)[0]) == 1
+    args.data[0].with_suffix(".manifest.json").write_text(
+        json.dumps({"data_sha256": "bad"})
+    )
+    with pytest.raises(ValueError, match="checksum"):
+        load_data(args.data, args.model, allow_test=True)
+
+
+def test_prepared_subset_metadata_is_retained(tmp_path):
+    args = arguments(tmp_path)
+    raw = args.data[0].read_bytes()
+    args.data[0].with_suffix(".manifest.json").write_text(
+        json.dumps(
+            {
+                "data_sha256": hashlib.sha256(raw).hexdigest(),
+                "selected": {"records": 2},
+                "selection": {"is_full_partition": False},
+            }
+        )
+    )
+    _, sources = load_data(args.data, args.model)
+    assert sources[0]["prepared_manifest"]["selection"]["is_full_partition"] is False
+
+
+def test_launch_cannot_label_another_checkpoint():
+    launch = {
+        "model": {
+            "repo_id": "Qwen/one",
+            "revision": "abc",
+            "dtype": "bfloat16",
+            "quantization": None,
+        },
+        "profile": "one",
+        "engine": {"revision": "def"},
+    }
+    snapshot = {
+        "server_info": {
+            "model_path": "Qwen/one",
+            "revision": "abc",
+            "dtype": "bfloat16",
+            "quantization": None,
+        },
+        "model_info": {"model_path": "Qwen/one"},
+    }
+    validate_launch(snapshot, launch)
+    snapshot["server_info"]["revision"] = "changed"
+    with pytest.raises(ValueError, match="revision"):
+        validate_launch(snapshot, launch)

server/tests/test_disconnect_cleanup.py

@@ -0,0 +1,150 @@
+"""A completed score must not wait forever for a cancelled disconnect poll."""
+
+import asyncio
+
+import pytest
+
+from jev_adapter.backend import AdapterError, ScoringResult
+from jev_adapter.protocol import SystemOneRequest
+from jev_adapter.server import create_app
+
+
+class ControlledBackend:
+    model = "test-model"
+
+    def __init__(self):
+        self.release = asyncio.Event()
+        self.entered = asyncio.Event()
+        self.cancelled = asyncio.Event()
+
+    def labels(self, count):
+        return tuple("AB"[:count]), tuple(range(count))
+
+    async def evaluate(self, prompt, images, labels, token_ids, assistant_prefix):
+        self.entered.set()
+        try:
+            await self.release.wait()
+        except asyncio.CancelledError:
+            self.cancelled.set()
+            raise
+        return ScoringResult((-0.1, -2.0), 10)
+
+
+class CancellationSwallowingRequest:
+    """Reproduce a request probe whose internal cancellation scope wins a race.
+
+    CancelledError can be suppressed within a probe, leaving its caller alive.
+    A real Request.is_disconnected uses a self-cancelling AnyIO CancelScope;
+    deliberately control that boundary here rather than rely on scheduler luck.
+    """
+
+    def __init__(self):
+        self.entered = asyncio.Event()
+        self.swallowed = asyncio.Event()
+        self.probe_task = None
+        self.calls = 0
+
+    async def is_disconnected(self):
+        self.probe_task = asyncio.current_task()
+        self.calls += 1
+        self.entered.set()
+        try:
+            await asyncio.Future()
+        except asyncio.CancelledError:
+            self.swallowed.set()
+            return False
+
+
+def endpoint_and_body(backend):
+    app = create_app(backend)
+    endpoint = next(
+        route.endpoint
+        for route in app.routes
+        if getattr(route, "path", None) == "/v1/systemone"
+    )
+    body = SystemOneRequest.model_validate(
+        {
+            "model": backend.model,
+            "state": "A billing question.",
+            "questions": {
+                "route": {
+                    "type": "choice",
+                    "instructions": "Choose the team.",
+                    "criteria": {"billing": None, "technical": None},
+                }
+            },
+        }
+    )
+    return endpoint, body
+
+
+@pytest.mark.asyncio
+async def test_completed_score_returns_when_disconnect_probe_swallows_cancel():
+    backend = ControlledBackend()
+    endpoint, body = endpoint_and_body(backend)
+    request = CancellationSwallowingRequest()
+    response = asyncio.create_task(endpoint(body, request))
+    try:
+        await asyncio.wait_for(request.entered.wait(), timeout=1)
+        await asyncio.wait_for(backend.entered.wait(), timeout=1)
+        backend.release.set()
+        await asyncio.wait_for(request.swallowed.wait(), timeout=1)
+        done, _ = await asyncio.wait({response}, timeout=0.1)
+        assert response in done, "completed inference is stuck cleaning up its watcher"
+        result = response.result()
+        assert result["answers"]["route"]["choice"] == "billing"
+        assert request.calls == 1
+        assert request.probe_task.done()
+    finally:
+        # Do not let a deliberately cancellation-resistant fake leak after red.
+        request.is_disconnected = disconnected_now
+        if request.probe_task is not None:
+            request.probe_task.cancel()
+        response.cancel()
+        await asyncio.gather(response, return_exceptions=True)
+
+
+async def disconnected_now():
+    return True
+
+
+@pytest.mark.asyncio
+async def test_client_disconnect_still_cancels_unfinished_inference():
+    backend = ControlledBackend()
+    endpoint, body = endpoint_and_body(backend)
+
+    class DisconnectedRequest:
+        async def is_disconnected(self):
+            await backend.entered.wait()
+            return True
+
+    with pytest.raises(AdapterError) as error:
+        await asyncio.wait_for(endpoint(body, DisconnectedRequest()), timeout=1)
+    assert error.value.code == "client_disconnected"
+    assert error.value.status == 499
+    assert backend.cancelled.is_set()
+
+
+@pytest.mark.asyncio
+async def test_cancelled_handler_joins_work_and_cancellation_resistant_watcher():
+    backend = ControlledBackend()
+    endpoint, body = endpoint_and_body(backend)
+    request = CancellationSwallowingRequest()
+    response = asyncio.create_task(endpoint(body, request))
+    try:
+        await asyncio.wait_for(request.entered.wait(), timeout=1)
+        await asyncio.wait_for(backend.entered.wait(), timeout=1)
+        response.cancel()
+        done, _ = await asyncio.wait({response}, timeout=0.1)
+        assert response in done, "cancelled handler did not release its child tasks"
+        with pytest.raises(asyncio.CancelledError):
+            response.result()
+        assert backend.cancelled.is_set()
+        assert request.swallowed.is_set()
+        assert request.probe_task.done()
+    finally:
+        request.is_disconnected = disconnected_now
+        if request.probe_task is not None:
+            request.probe_task.cancel()
+        response.cancel()
+        await asyncio.gather(response, return_exceptions=True)

server/tests/test_http_integration.py

@@ -0,0 +1,49 @@
+"""Full adapter stack, with only the remote engine's HTTP transport simulated."""
+
+import httpx
+import pytest
+from test_service import payload
+from test_sglang import Engine
+
+from jev_adapter.server import create_app
+from jev_adapter.sglang import SGLangBackend
+
+
+@pytest.mark.asyncio
+async def test_public_request_through_real_service_and_sglang_client():
+    engine = Engine()
+    async with httpx.AsyncClient(transport=httpx.MockTransport(engine)) as upstream:
+        backend = SGLangBackend("http://engine", "qwen", client=upstream)
+        app = create_app(backend, api_key="client-key")
+        async with (
+            app.router.lifespan_context(app),
+            httpx.AsyncClient(
+                transport=httpx.ASGITransport(app),
+                base_url="http://adapter",
+            ) as client,
+        ):
+            body = payload()
+            body["model"] = "jev-latest"
+            body["images"] = ["data:image/png;base64,YQ=="]
+            body["options"] = {"permutations": 2, "return_logprobs": True}
+            response = await client.post(
+                "/v1/systemone",
+                json=body,
+                headers={"Authorization": "Bearer client-key"},
+            )
+        assert response.status_code == 200, response.text
+        result = response.json()
+        assert result["model"] == "qwen"
+        assert set(result["answers"]) == set(body["questions"])
+        assert result["answers"]["route"]["probabilities"]["billing"] == pytest.approx(
+            0.5
+        )
+        assert result["usage"] == {"input_tokens": 426, "output_tokens": 0}
+        assert result["metadata"]["evaluations"] == 6
+        generations = [data for path, data, _ in engine.calls if path == "/generate"]
+        assert len(generations) == 6
+        assert len({data["rid"] for data in generations}) == 6
+        assert all(data["image_data"] == body["images"] for data in generations)
+        assert all(
+            data["sampling_params"]["max_new_tokens"] == 0 for data in generations
+        )

server/tests/test_jevbench.py

@@ -0,0 +1,270 @@
+"""Check JevBench conversion, reference alignment and frozen source integrity."""
+
+import copy
+import json
+import tempfile
+import unittest
+from dataclasses import replace
+from pathlib import Path
+from unittest.mock import patch
+
+import httpx
+
+from jev_adapter.benchmarks.data import sha256
+from jev_adapter.benchmarks.jevbench import (
+    JEVBENCH_COMMIT,
+    JEVBENCH_SUITES,
+    JevBenchSpec,
+    normalize_jevbench_record,
+    parse_jevbench,
+    prepare_jevbench,
+)
+
+
+def fixture(kind="choice", name="sample"):
+    questions = {
+        "choice": {
+            "type": "choice",
+            "instructions": "Choose a queue.",
+            "criteria": {"support": "Support", "billing": "Billing"},
+        },
+        "noul": {
+            "type": "noul",
+            "instructions": "Is a refund requested?",
+            "criteria": {"true": "Refund requested", "false": "No refund request"},
+        },
+        "score": {
+            "type": "score",
+            "instructions": "How urgent?",
+            "criteria": ["low", "medium", "high"],
+        },
+    }
+    return {
+        "id": name,
+        "family": "routing",
+        "group": None,
+        "split": "public",
+        "state": {"message": "Please refund my bill."},
+        "question": questions[kind],
+        "labels": {
+            "choice": ["billing", "support"],
+            "noul": ["no", "yes"],
+            "score": ["0", "1", "2"],
+        }[kind],
+        "expected": {"choice": "billing", "noul": "yes", "score": 2}[kind],
+        "provenance": {
+            "source": "Unit test fixture",
+            "license": "MIT",
+            "rationale": "SECRET_RATIONALE",
+            "surface_answer": "SECRET_SURFACE_ANSWER",
+            "label_basis": "Authored rubric",
+        },
+    }
+
+
+def source_fixture(records):
+    payload = "".join(json.dumps(row) + "\n" for row in records).encode()
+    notices = {"LICENSE": b"Fixture MIT notice", "THIRD-PARTY.md": b"Fixture notice"}
+    spec = JevBenchSpec(
+        "datasets/public/fixture.jsonl", sha256(payload), len(records), "Fixture"
+    )
+    return (
+        spec,
+        {spec.path: payload, **notices},
+        {path: sha256(content) for path, content in notices.items()},
+    )
+
+
+class TestJevBenchNormalization(unittest.TestCase):
+    def test_native_question_and_canonical_label_orders_preserved_without_gold(self):
+        raw = fixture()
+        raw["provenance"]["gold_probs"] = {"support": 0.25, "billing": 0.75}
+        original = copy.deepcopy(raw)
+        row = normalize_jevbench_record(raw, "jevbench-hard")
+        self.assertEqual(raw, original)
+        self.assertEqual(
+            row["record"],
+            {"state": raw["state"], "questions": {"decision": raw["question"]}},
+        )
+        expected = row["expected"]["decision"]
+        self.assertEqual(expected["labels"], ["billing", "support"])
+        self.assertEqual(
+            list(row["record"]["questions"]["decision"]["criteria"]),
+            ["support", "billing"],
+        )
+        self.assertEqual(expected["target"], [1, 0])
+        self.assertEqual(expected["reference_probs"], [0.75, 0.25])
+        request_text = json.dumps(row["record"])
+        for secret in (
+            "SECRET_",
+            "gold_probs",
+            "expected",
+            "provenance",
+            "label_basis",
+        ):
+            self.assertNotIn(secret, request_text)
+        self.assertEqual(row["metadata"]["provenance"], raw["provenance"])
+        self.assertEqual(row["metadata"]["group_id"], raw["id"])
+        self.assertEqual(row["split"], "public")
+
+    def test_noul_reference_mapping_and_score_hard_label(self):
+        raw = fixture("noul")
+        raw["provenance"]["gold_probs"] = {"yes": 0.8, "no": 0.2}
+        noul = normalize_jevbench_record(raw, "jevbench-hard")["expected"]["decision"]
+        self.assertEqual(noul["labels"], ["false", "true"])
+        self.assertEqual(noul["target"], [0, 1])
+        self.assertEqual(noul["reference_probs"], [0.2, 0.8])
+        score = normalize_jevbench_record(fixture("score"), "jevbench-original")
+        self.assertEqual(score["expected"]["decision"]["label"], 2)
+        self.assertNotIn("reference_probs", score["expected"]["decision"])
+
+    def test_paraphrase_group_preserved_and_null_groups_independent(self):
+        first, second = fixture(name="one"), fixture(name="two")
+        a = normalize_jevbench_record(first, "jevbench-easy")
+        b = normalize_jevbench_record(second, "jevbench-easy")
+        self.assertNotEqual(a["metadata"]["group_id"], b["metadata"]["group_id"])
+        first["group"] = second["group"] = "pair"
+        self.assertEqual(
+            normalize_jevbench_record(first, "jevbench-original")["metadata"][
+                "group_id"
+            ],
+            normalize_jevbench_record(second, "jevbench-original")["metadata"][
+                "group_id"
+            ],
+        )
+
+    def test_bad_labels_private_rows_and_malformed_references_rejected(self):
+        changes = [
+            {"split": "private"},
+            {"labels": ["billing", "absent"]},
+            {"expected": "absent"},
+            {"provenance": {"exclude_reason": "ambiguous"}},
+        ]
+        for change in changes:
+            with self.subTest(change=change), self.assertRaises(ValueError):
+                normalize_jevbench_record({**fixture(), **change}, "jevbench-hard")
+        for probs in (
+            {"billing": 0.5},
+            {"billing": 0.8, "support": 0.3},
+            {"billing": float("nan"), "support": 0.3},
+            {"billing": True, "support": 0.0},
+        ):
+            raw = fixture()
+            raw["provenance"]["gold_probs"] = probs
+            with self.subTest(probs=probs), self.assertRaises(ValueError):
+                normalize_jevbench_record(raw, "jevbench-hard")
+        for kind, gold in (("score", True), ("score", 3), ("noul", True)):
+            raw = fixture(kind)
+            raw["expected"] = gold
+            with self.subTest(kind=kind, gold=gold), self.assertRaises(ValueError):
+                normalize_jevbench_record(raw, "jevbench-hard")
+
+    def test_duplicate_and_blank_rows_rejected(self):
+        row = json.dumps(fixture()) + "\n"
+        with self.assertRaisesRegex(ValueError, "duplicate"):
+            parse_jevbench((row * 2).encode(), "jevbench-original")
+        with self.assertRaisesRegex(ValueError, "blank"):
+            parse_jevbench((row + "\n").encode(), "jevbench-original")
+
+
+class TestJevBenchPreparation(unittest.TestCase):
+    def test_pinned_download_preserves_notices_and_is_idempotent(self):
+        spec, sources, notices = source_fixture(
+            [fixture(name="one"), fixture(name="two")]
+        )
+        requests = []
+
+        def transport(request):
+            requests.append(request.url.path)
+            prefix = f"/fstandhartinger/jevbench/{JEVBENCH_COMMIT}/"
+            self.assertTrue(request.url.path.startswith(prefix))
+            return httpx.Response(
+                200, content=sources[request.url.path.removeprefix(prefix)]
+            )
+
+        with (
+            tempfile.TemporaryDirectory() as tmp,
+            patch.dict(JEVBENCH_SUITES, {"jevbench-fixture": spec}),
+            patch("jev_adapter.benchmarks.jevbench.JEVBENCH_NOTICES", notices),
+            httpx.Client(transport=httpx.MockTransport(transport)) as client,
+        ):
+            output = Path(tmp)
+            result = prepare_jevbench(
+                "jevbench-fixture", output, client=client, limit=1
+            )
+            self.assertEqual(
+                result,
+                prepare_jevbench("jevbench-fixture", output, client=client, limit=1),
+            )
+            self.assertEqual(result["full_partition"]["questions"], 2)
+            self.assertEqual(result["selected"]["questions"], 1)
+            self.assertFalse(result["selection"]["is_full_partition"])
+            directory = output / "jevbench-fixture"
+            self.assertEqual(
+                result["data_sha256"], sha256((directory / "public.jsonl").read_bytes())
+            )
+            for name in notices:
+                self.assertEqual((directory / name).read_bytes(), sources[name])
+            self.assertTrue(
+                all("private" not in path and "train" not in path for path in requests)
+            )
+            with self.assertRaises(FileExistsError):
+                prepare_jevbench("jevbench-fixture", output, client=client)
+            self.assertEqual(
+                result["data_sha256"], sha256((directory / "public.jsonl").read_bytes())
+            )
+
+    def test_bad_data_notice_or_count_fails_before_writing(self):
+        spec, sources, notices = source_fixture([fixture()])
+        for corrupt in (spec.path, "LICENSE", "THIRD-PARTY.md", "count"):
+            with (
+                self.subTest(corrupt=corrupt),
+                tempfile.TemporaryDirectory() as tmp,
+                patch.dict(
+                    JEVBENCH_SUITES,
+                    {
+                        "jevbench-fixture": replace(spec, records=2)
+                        if corrupt == "count"
+                        else spec
+                    },
+                ),
+                patch("jev_adapter.benchmarks.jevbench.JEVBENCH_NOTICES", notices),
+            ):
+                root, output = Path(tmp) / "source", Path(tmp) / "out"
+                for name, content in sources.items():
+                    path = root / name
+                    path.parent.mkdir(parents=True, exist_ok=True)
+                    path.write_bytes(content + (b" " if name == corrupt else b""))
+                with self.assertRaisesRegex(
+                    ValueError, "SHA256 mismatch|count mismatch"
+                ):
+                    prepare_jevbench(
+                        "jevbench-fixture", output, source_root=root, limit=1
+                    )
+                self.assertFalse(output.exists())
+
+    def test_changed_notice_is_not_partially_overwritten(self):
+        spec, sources, notices = source_fixture([fixture()])
+        with (
+            tempfile.TemporaryDirectory() as tmp,
+            patch.dict(JEVBENCH_SUITES, {"jevbench-fixture": spec}),
+            patch("jev_adapter.benchmarks.jevbench.JEVBENCH_NOTICES", notices),
+        ):
+            root, output = Path(tmp) / "source", Path(tmp) / "out"
+            for name, content in sources.items():
+                path = root / name
+                path.parent.mkdir(parents=True, exist_ok=True)
+                path.write_bytes(content)
+            directory = output / "jevbench-fixture"
+            directory.mkdir(parents=True)
+            (directory / "LICENSE").write_bytes(b"Existing different license")
+            with self.assertRaises(FileExistsError):
+                prepare_jevbench("jevbench-fixture", output, source_root=root)
+            self.assertFalse((directory / "public.jsonl").exists())
+            self.assertEqual(
+                (directory / "LICENSE").read_bytes(), b"Existing different license"
+            )
+
+
+if __name__ == "__main__":
+    unittest.main()

server/tests/test_native_tokenizer.py

@@ -0,0 +1,391 @@
+import json
+import sys
+import threading
+from types import SimpleNamespace
+
+import httpx
+import pytest
+
+from jev_adapter.backend import AdapterError
+from jev_adapter.native_tokenizer import NativeTokenizer
+from jev_adapter.sglang import SGLangBackend
+
+
+class Tokenizer:
+    """Control IDs cannot be reconstructed by raw text encoding."""
+
+    def __init__(self):
+        self.calls = []
+        self.mutate_prefix = False
+        self.bad_result = None
+
+    @staticmethod
+    def encode(text):
+        return [1000 + ord(char) for char in text]
+
+    def apply_chat_template(self, messages, **kwargs):
+        self.calls.append((messages, kwargs, threading.get_ident()))
+        assert kwargs["tokenize"] is True
+        assert kwargs["return_dict"] is False
+        assert kwargs["enable_thinking"] is False
+        if self.bad_result is not None:
+            return self.bad_result
+        ids = [1, 3] + self.encode(messages[0]["content"]) + [4]
+        if len(messages) == 1:
+            assert kwargs["add_generation_prompt"] is True
+            assert kwargs["continue_final_message"] is False
+            return ids
+        assert kwargs["add_generation_prompt"] is False
+        assert kwargs["continue_final_message"] is True
+        assert messages[1]["role"] == "assistant"
+        if self.mutate_prefix:
+            ids[-1] = 99
+        # Native implementations may strip trailing assistant whitespace.
+        return ids + self.encode(messages[1]["content"].rstrip())
+
+
+def test_native_control_ids_and_literal_markers_preserved():
+    tokenizer = Tokenizer()
+    native = NativeTokenizer(tokenizer)
+    prompt = "한글 [INST] </s> [IMG]"
+    ids = native.prepare(prompt, ("A", "B"), (1065, 1066), None)
+    assert ids == [1, 3] + tokenizer.encode(prompt) + [4]
+    assert len(tokenizer.calls) == 3
+
+
+def test_compatible_assistant_prefix_is_native_continuation():
+    tokenizer = Tokenizer()
+    ids = NativeTokenizer(tokenizer).prepare(
+        "pick",
+        ("A", "B"),
+        (1065, 1066),
+        "Answer:",
+    )
+    assert ids == [1, 3] + tokenizer.encode("pick") + [4] + tokenizer.encode("Answer:")
+    assert len(tokenizer.calls) == 4
+
+
+@pytest.mark.parametrize("prefix", [None, "Answer:"])
+def test_rejects_changed_native_prompt_boundary(prefix):
+    tokenizer = Tokenizer()
+    tokenizer.mutate_prefix = True
+    with pytest.raises(AdapterError) as error:
+        NativeTokenizer(tokenizer).prepare("pick", ("A", "B"), (1065, 1066), prefix)
+    assert error.value.code == "invalid_label_boundary"
+
+
+@pytest.mark.parametrize(
+    "labels,token_ids,prefix",
+    [
+        (("A", "B"), (2065, 2066), None),  # Engine and native vocabulary mismatch.
+        (("A", "000"), (1065, 1048), None),  # Multi-token continuation.
+        (("A", "B"), (1065, 1066), " "),  # Native prefix trimming changes boundary.
+    ],
+)
+def test_rejects_unsafe_label_continuations(labels, token_ids, prefix):
+    with pytest.raises(AdapterError) as error:
+        NativeTokenizer(Tokenizer()).prepare("pick", labels, token_ids, prefix)
+    assert error.value.code == "invalid_label_boundary"
+
+
+@pytest.mark.parametrize("bad_result", [[], [True], [-1], {"input_ids": [1, 2]}])
+def test_rejects_malformed_native_token_ids(bad_result):
+    tokenizer = Tokenizer()
+    tokenizer.bad_result = bad_result
+    with pytest.raises(AdapterError) as error:
+        NativeTokenizer(tokenizer).prepare("pick", ("A", "B"), (1065, 1066), None)
+    assert error.value.code == "unsupported_native_tokenizer"
+
+
+def test_loader_uses_explicit_revision_and_no_remote_code(monkeypatch):
+    calls = []
+    tokenizer = Tokenizer()
+
+    def load(model, **kwargs):
+        calls.append((model, kwargs))
+        return tokenizer
+
+    monkeypatch.setitem(
+        sys.modules,
+        "transformers",
+        SimpleNamespace(
+            AutoTokenizer=SimpleNamespace(from_pretrained=load),
+        ),
+    )
+    native = NativeTokenizer.from_pretrained("org/model", "a" * 40)
+    assert native.tokenizer is tokenizer
+    assert calls == [("org/model", {"revision": "a" * 40, "trust_remote_code": False})]
+
+
+class SystemAwareTokenizer:
+    """Like Tokenizer above, but role-aware so it can render an optional
+    leading system message instead of assuming a fixed message count/order."""
+
+    def __init__(self):
+        self.calls = []
+
+    @staticmethod
+    def encode(text):
+        return [1000 + ord(char) for char in text]
+
+    def apply_chat_template(self, messages, **kwargs):
+        self.calls.append((messages, kwargs))
+        roles = [message["role"] for message in messages]
+        assert roles[-1] in ("user", "assistant")
+        sys_text = messages[0]["content"] if roles[0] == "system" else ""
+        user_text = next(m["content"] for m in messages if m["role"] == "user")
+        ids = [1, 3] + self.encode(sys_text) + self.encode(user_text) + [4]
+        if roles[-1] == "user":
+            assert kwargs["add_generation_prompt"] is True
+            return ids
+        assert kwargs["add_generation_prompt"] is False
+        return ids + self.encode(messages[-1]["content"].rstrip())
+
+
+def test_prepare_prepends_explicit_system_message_when_given():
+    tokenizer = SystemAwareTokenizer()
+    ids = NativeTokenizer(tokenizer).prepare(
+        "pick", ("A", "B"), (1065, 1066), None, "You are helpful."
+    )
+    expected = (
+        [1, 3]
+        + tokenizer.encode("You are helpful.")
+        + tokenizer.encode("pick")
+        + [4]
+    )
+    assert ids == expected
+    for messages, _ in tokenizer.calls:
+        assert messages[0] == {"role": "system", "content": "You are helpful."}
+        assert messages[1]["role"] in ("user", "assistant")
+
+
+def test_prepare_omits_system_message_by_default():
+    tokenizer = SystemAwareTokenizer()
+    NativeTokenizer(tokenizer).prepare("pick", ("A", "B"), (1065, 1066), None)
+    for messages, _ in tokenizer.calls:
+        assert messages[0]["role"] == "user"
+        assert not any(m["role"] == "system" for m in messages)
+
+
+def test_native_system_prompt_none_renders_identically_to_plain_user_chat():
+    """--native-system-prompt none wires system_prompt=None into
+    NativeTokenizer.prepare (see __main__.py/sglang.py): no extraction, no
+    system message. The encoded messages must carry no system role and the
+    token sequence must equal a bare apply_chat_template([{"role": "user",
+    ...}]) call on the same (pinned) tokenizer -- i.e. rendering is
+    byte-for-byte what it would be if --tokenizer-model had never been
+    passed at all."""
+    tokenizer = SystemAwareTokenizer()
+    ids = NativeTokenizer(tokenizer).prepare(
+        "pick", ("A", "B"), (1065, 1066), None, system_prompt=None
+    )
+    for messages, _ in tokenizer.calls:
+        assert not any(message["role"] == "system" for message in messages)
+
+    reference_tokenizer = SystemAwareTokenizer()
+    expected = reference_tokenizer.apply_chat_template(
+        [{"role": "user", "content": "pick"}],
+        tokenize=True,
+        add_generation_prompt=True,
+        continue_final_message=False,
+        return_dict=False,
+        enable_thinking=False,
+    )
+    assert ids == expected
+
+
+def test_extract_default_system_prompt_finds_the_hf_system_block():
+    from jev_adapter.native_tokenizer import extract_default_system_prompt
+
+    class HFStyleTokenizer:
+        def apply_chat_template(self, messages, **kwargs):
+            assert messages == [{"role": "user", "content": ""}]
+            return "[SYSTEM_PROMPT]You are Ministral.[/SYSTEM_PROMPT][INST]...[/INST]"
+
+    assert extract_default_system_prompt(HFStyleTokenizer()) == "You are Ministral."
+
+
+def test_extract_default_system_prompt_returns_none_for_mistral_common_style():
+    from jev_adapter.native_tokenizer import extract_default_system_prompt
+
+    class MistralCommonStyleTokenizer:
+        """mistral-common never auto-injects a default system prompt."""
+
+        def apply_chat_template(self, messages, **kwargs):
+            return "<s>[INST]...[/INST]"
+
+    assert extract_default_system_prompt(MistralCommonStyleTokenizer()) is None
+
+
+def test_native_default_system_prompt_forces_fix_mistral_regex_for_mistralai(
+    monkeypatch,
+):
+    calls = []
+
+    def load(model, **kwargs):
+        calls.append((model, kwargs))
+
+        class Rendered:
+            @staticmethod
+            def apply_chat_template(messages, **kwargs):
+                return "[SYSTEM_PROMPT]default text[/SYSTEM_PROMPT][INST]x[/INST]"
+
+        return Rendered()
+
+    monkeypatch.setitem(
+        sys.modules,
+        "transformers",
+        SimpleNamespace(AutoTokenizer=SimpleNamespace(from_pretrained=load)),
+    )
+    result = NativeTokenizer.native_default_system_prompt(
+        "mistralai/Ministral-3-8B-Instruct-2512-BF16", "f" * 40
+    )
+    assert result == "default text"
+    assert calls == [
+        (
+            "mistralai/Ministral-3-8B-Instruct-2512-BF16",
+            {"revision": "f" * 40, "trust_remote_code": False, "fix_mistral_regex": True},
+        )
+    ]
+
+
+def test_native_default_system_prompt_omits_fix_mistral_regex_for_other_orgs(
+    monkeypatch,
+):
+    calls = []
+
+    def load(model, **kwargs):
+        calls.append((model, kwargs))
+
+        class Rendered:
+            @staticmethod
+            def apply_chat_template(messages, **kwargs):
+                return "<s>[INST]x[/INST]"
+
+        return Rendered()
+
+    monkeypatch.setitem(
+        sys.modules,
+        "transformers",
+        SimpleNamespace(AutoTokenizer=SimpleNamespace(from_pretrained=load)),
+    )
+    result = NativeTokenizer.native_default_system_prompt("org/model", "a" * 40)
+    assert result is None
+    assert calls == [("org/model", {"revision": "a" * 40, "trust_remote_code": False})]
+
+
+def test_native_default_system_prompt_requires_model_and_revision():
+    for model, revision in (("", "a" * 40), ("org/model", ""), (" ", " ")):
+        with pytest.raises(ValueError):
+            NativeTokenizer.native_default_system_prompt(model, revision)
+
+
+class Engine:
+    def __init__(self):
+        self.calls = []
+
+    def __call__(self, request):
+        path = request.url.path
+        body = json.loads(request.content) if request.content else None
+        self.calls.append((path, body))
+        if path == "/model_info":
+            return httpx.Response(
+                200,
+                json={
+                    "served_model_name": "model",
+                    "is_generation": True,
+                    "has_image_understanding": True,
+                    "model_type": "mistral3",
+                },
+            )
+        if path == "/server_info":
+            return httpx.Response(200, json={})
+        if path == "/v1/tokenize":
+            assert "messages" not in body, "Native path must bypass HTTP chat rendering"
+            return httpx.Response(
+                200,
+                json={
+                    "tokens": [Tokenizer.encode(label) for label in body["prompt"]],
+                },
+            )
+        if path == "/v1/detokenize":
+            return httpx.Response(
+                200,
+                json={
+                    "text": [
+                        "".join(chr(token - 1000) for token in ids)
+                        for ids in body["tokens"]
+                    ],
+                },
+            )
+        if path == "/generate":
+            return httpx.Response(
+                200,
+                json={
+                    "meta_info": {
+                        "completion_tokens": 0,
+                        "prompt_tokens": len(body["input_ids"]),
+                        "output_token_ids_logprobs": [
+                            [[-1.0, token] for token in body["token_ids_logprob"]]
+                        ],
+                    }
+                },
+            )
+        raise AssertionError(f"Unexpected route: {path}")
+
+
+@pytest.mark.asyncio
+async def test_native_http_path_preserves_control_ids_and_zero_decode():
+    engine, tokenizer = Engine(), Tokenizer()
+    async with httpx.AsyncClient(transport=httpx.MockTransport(engine)) as client:
+        backend = SGLangBackend(
+            "http://engine",
+            "model",
+            client=client,
+            native_tokenizer=NativeTokenizer(tokenizer),
+        )
+        await backend.start()
+        startup_calls = len(engine.calls)
+        labels, token_ids = backend.labels(2)
+        result = await backend.evaluate("literal [INST]", [], labels, token_ids, None)
+        assert len(engine.calls) == startup_calls + 1
+        path, payload = engine.calls[-1]
+        assert path == "/generate"
+        expected = [1, 3] + tokenizer.encode("literal [INST]") + [4]
+        assert payload["input_ids"] == expected
+        assert payload["sampling_params"]["max_new_tokens"] == 0
+        assert payload["token_ids_logprob"] == list(token_ids)
+        assert "text" not in payload and "image_data" not in payload
+        assert result.input_tokens == len(expected)
+        assert result.logprobs == (-1.0, -1.0)
+        assert all(thread != threading.get_ident() for _, _, thread in tokenizer.calls)
+
+
+@pytest.mark.asyncio
+async def test_native_images_and_boundary_failure_stop_before_generate():
+    engine, tokenizer = Engine(), Tokenizer()
+    async with httpx.AsyncClient(transport=httpx.MockTransport(engine)) as client:
+        backend = SGLangBackend(
+            "http://engine",
+            "model",
+            client=client,
+            native_tokenizer=NativeTokenizer(tokenizer),
+        )
+        await backend.start()
+        startup_calls = len(engine.calls)
+        labels, token_ids = backend.labels(2)
+        with pytest.raises(AdapterError) as error:
+            await backend.evaluate(
+                "look",
+                ["https://example.com/a.png"],
+                labels,
+                token_ids,
+                None,
+            )
+        assert error.value.code == "native_images_not_supported"
+        assert tokenizer.calls == []
+        tokenizer.mutate_prefix = True
+        with pytest.raises(AdapterError) as error:
+            await backend.evaluate("pick", [], labels, token_ids, None)
+        assert error.value.code == "invalid_label_boundary"
+        assert len(engine.calls) == startup_calls

server/tests/test_protocol.py

@@ -0,0 +1,433 @@
+"""CPU tests of request validation, prompt mapping, and decision probabilities."""
+
+import json
+import math
+import string
+import unittest
+
+from pydantic import ValidationError
+
+from jev_adapter.protocol import (
+    NATIVE_PROMPT_PREFIX,
+    SystemOneOptions,
+    SystemOneRequest,
+    build_prompt,
+    plan_question,
+    probabilities_from_logprobs,
+    reduce_probabilities,
+    rotation_order,
+)
+
+CANONICAL_LETTERS = tuple(string.ascii_uppercase)
+
+
+def runner_format_prompt(state, instructions, ordered):
+    """Independent, verbatim port of run_suites_rotation.py's format_prompt/
+    option_texts, used as the parity oracle for wording="native" -- kept
+    separate from jev_adapter.protocol so a bug in the adapter's own port
+    cannot also hide in the test."""
+    text = (
+        state
+        if isinstance(state, str)
+        else json.dumps(state, sort_keys=True, ensure_ascii=False, indent=2, allow_nan=False)
+    )
+    lines = []
+    for position, (name, description) in enumerate(ordered):
+        lines.append(
+            f"{CANONICAL_LETTERS[position]}. {name}"
+            + (f": {description}" if description else "")
+        )
+    return (
+        f"{NATIVE_PROMPT_PREFIX}\n\nState:\n{text}\n\nQuestion:\n{instructions}\n\nOptions:\n"
+        + "\n".join(lines)
+    )
+
+
+def request_for(question, **overrides):
+    return SystemOneRequest.model_validate(
+        {
+            "model": "test-model",
+            "state": "Customer needs help.",
+            "questions": {"routing": question},
+            **overrides,
+        }
+    )
+
+
+def choice_question(**overrides):
+    return {
+        "type": "choice",
+        "instructions": "Which team?",
+        "criteria": {"billing": "Payment issue", "technical": None},
+        **overrides,
+    }
+
+
+class TestSystemOneValidation(unittest.TestCase):
+    def test_preserves_question_and_option_order_and_structured_values(self):
+        request = request_for(
+            choice_question(criteria={"z": ["last", {"nested": True}], "a": None}),
+            state={"ticket": [7, "안녕하세요", None]},
+            questions={
+                "second": choice_question(criteria={"z": {"reason": 1}, "a": None}),
+                "first": {"type": "noul", "instructions": ["Is urgent?"]},
+            },
+        )
+        self.assertEqual(list(request.questions), ["second", "first"])
+        plan = plan_question(request.questions["second"])
+        self.assertEqual(plan.option_names, ("z", "a"))
+        self.assertEqual(plan.descriptions, ({"reason": 1}, None))
+        self.assertEqual(request.state["ticket"], [7, "안녕하세요", None])
+
+    def test_choice_accepts_255_but_rejects_256(self):
+        criteria = {str(index): None for index in range(255)}
+        request = request_for(choice_question(criteria=criteria))
+        self.assertEqual(
+            len(plan_question(request.questions["routing"]).option_names), 255
+        )
+        with self.assertRaises(ValidationError):
+            request_for(choice_question(criteria={**criteria, "255": None}))
+
+    def test_invalid_question_schemas(self):
+        cases = [
+            choice_question(type="arbitrary_json"),
+            choice_question(criteria={"only": None}),
+            choice_question(criteria={"": None, "b": None}),
+            choice_question(criteria={"a": 42, "b": None}),
+            choice_question(instructions=None),
+            choice_question(instructions=False),
+            choice_question(unrecognized=True),
+            {"type": "noul", "instructions": "Yes?", "criteria": {"true": "yes"}},
+            {
+                "type": "noul",
+                "instructions": "Yes?",
+                "criteria": {"yes": None, "no": None},
+            },
+            {"type": "score", "instructions": "Rate", "criteria": ["only"]},
+            {"type": "score", "instructions": "Rate", "criteria": [None] * 11},
+            {
+                "type": "score",
+                "instructions": "Rate",
+                "criteria": {"0": "low", "1": "high"},
+            },
+        ]
+        for question in cases:
+            with self.subTest(question=question), self.assertRaises(ValidationError):
+                request_for(question)
+
+    def test_required_fields_and_request_limits(self):
+        for updates in (
+            {"model": " "},
+            {"model": 3},
+            {"state": 42},
+            {"state": None},
+            {"questions": {}},
+            {"questions": {" ": choice_question()}},
+            {"questions": {str(i): choice_question() for i in range(257)}},
+        ):
+            with self.subTest(updates=updates), self.assertRaises(ValidationError):
+                request_for(choice_question(), **updates)
+        for field in ("model", "state", "questions"):
+            payload = request_for(choice_question()).model_dump()
+            del payload[field]
+            with self.subTest(missing=field), self.assertRaises(ValidationError):
+                SystemOneRequest.model_validate(payload)
+
+    def test_rejects_nonfinite_structured_json(self):
+        for number in (float("nan"), float("inf"), float("-inf")):
+            with self.subTest(number=number), self.assertRaises(ValidationError):
+                request_for(choice_question(), state={"nested": [number]})
+
+    def test_option_bounds_and_types(self):
+        for options in (
+            {"temperature": 0},
+            {"temperature": -1},
+            {"temperature": float("inf")},
+            {"temperature": float("nan")},
+            {"temperature": True},
+            {"temperature": "1"},
+            {"permutations": 0},
+            {"permutations": 17},
+            {"permutations": 1.5},
+            {"permutations": True},
+            {"return_logprobs": "true"},
+            {"return_logits": True},
+            {"assistant_prefix": 3},
+        ):
+            with self.subTest(options=options), self.assertRaises(ValidationError):
+                request_for(choice_question(), options=options)
+
+    def test_image_sources(self):
+        request = request_for(
+            choice_question(),
+            images=["https://example.com/frame.png", "data:image/png;base64,YQ=="],
+        )
+        self.assertEqual(len(request.images), 2)
+        for image in ("file:///tmp/frame.png", "data:image/png;base64,", "https://", 7):
+            with self.subTest(image=image), self.assertRaises(ValidationError):
+                request_for(choice_question(), images=[image])
+
+
+class TestSystemOnePrompts(unittest.TestCase):
+    def test_multicharacter_token_labels_request_the_complete_label(self):
+        plan = plan_question(request_for(choice_question()).questions["routing"])
+        prompt = build_prompt("state", plan, ["AA", "000"])
+        self.assertIn("the complete label before the colon", prompt)
+        self.assertTrue(prompt.endswith("AA: billing: Payment issue\n000: technical"))
+
+    def test_shared_prefix_and_rotation_preserve_option_meaning(self):
+        request = request_for(choice_question(), state={"b": 2, "a": "안녕"})
+        plan = plan_question(request.questions["routing"])
+        first = build_prompt(request.state, plan, ["A", "B"])
+        rotated = build_prompt(request.state, plan, ["A", "B"], rotation_order(2, 1))
+        self.assertTrue(first.startswith('Context:\n{"a": "안녕", "b": 2}\n\n'))
+        self.assertEqual(first.split("Options:\n")[0], rotated.split("Options:\n")[0])
+        self.assertTrue(first.endswith("A: billing: Payment issue\nB: technical"))
+        self.assertTrue(rotated.endswith("A: technical\nB: billing: Payment issue"))
+
+    def test_boolean_default_and_explicit_criteria(self):
+        request = request_for({"type": "noul", "instructions": "Urgent?"})
+        plan = plan_question(request.questions["routing"])
+        self.assertTrue(
+            build_prompt(request.state, plan, ["A", "B"]).endswith("A: yes\nB: no")
+        )
+        request = request_for(
+            {
+                "type": "noul",
+                "instructions": "Urgent?",
+                "criteria": {"false": "Can wait", "true": "Now"},
+            }
+        )
+        plan = plan_question(request.questions["routing"])
+        self.assertEqual(plan.option_names, ("true", "false"))
+        self.assertTrue(
+            build_prompt(request.state, plan, ["A", "B"]).endswith(
+                "A: yes: Now\nB: no: Can wait"
+            )
+        )
+
+    def test_invalid_label_and_option_mappings_fail(self):
+        plan = plan_question(request_for(choice_question()).questions["routing"])
+        for labels, order in (
+            (["A"], None),
+            (["A", "A"], None),
+            (["A", "B"], [0, 0]),
+            (["A", "B"], [0, True]),
+        ):
+            with (
+                self.subTest(labels=labels, order=order),
+                self.assertRaises(ValueError),
+            ):
+                build_prompt("state", plan, labels, order)
+
+
+class TestNativePromptWording(unittest.TestCase):
+    """wording="native" must match the jevbench-hard runner's own contract
+    (run_suites_rotation.py: PREFIX/format_prompt/option_texts) byte-for-byte;
+    ``runner_format_prompt`` above is an independent port used as the oracle."""
+
+    def test_matches_runner_for_choice_across_rotations_with_json_state(self):
+        request = request_for(
+            choice_question(
+                criteria={"billing": "Payment issue", "technical": None, "sales": ""}
+            ),
+            state={"ticket": [7, "안녕하세요", None], "z": 1},
+        )
+        plan = plan_question(request.questions["routing"])
+        labels = CANONICAL_LETTERS[:3]
+        raw_ordered = [
+            ("billing", "Payment issue"),
+            ("technical", None),
+            ("sales", ""),
+        ]
+        for rotation in range(3):
+            order = rotation_order(3, rotation)
+            mine = build_prompt(request.state, plan, labels, order, wording="native")
+            theirs = runner_format_prompt(
+                request.state,
+                plan.instructions,
+                [raw_ordered[i] for i in order],
+            )
+            with self.subTest(rotation=rotation):
+                self.assertEqual(mine, theirs)
+        self.assertTrue(mine.startswith(NATIVE_PROMPT_PREFIX + "\n\nState:\n"))
+        self.assertIn("\n\nQuestion:\n", mine)
+
+    def test_matches_runner_for_noul_and_score_with_string_state(self):
+        noul_plan = plan_question(
+            request_for(
+                {
+                    "type": "noul",
+                    "instructions": "Urgent?",
+                    "criteria": {"true": "Now", "false": "Can wait"},
+                }
+            ).questions["routing"]
+        )
+        mine = build_prompt(
+            "plain state text", noul_plan, ["A", "B"], None, wording="native"
+        )
+        theirs = runner_format_prompt(
+            "plain state text", "Urgent?", [("yes", "Now"), ("no", "Can wait")]
+        )
+        self.assertEqual(mine, theirs)
+
+        score_plan = plan_question(
+            request_for(
+                {"type": "score", "instructions": "Rate", "criteria": ["low", "medium", "high"]}
+            ).questions["routing"]
+        )
+        mine = build_prompt("s", score_plan, ["A", "B", "C"], None, wording="native")
+        theirs = runner_format_prompt(
+            "s", "Rate", [("0", "low"), ("1", "medium"), ("2", "high")]
+        )
+        self.assertEqual(mine, theirs)
+
+    def test_falsy_description_is_omitted_like_the_runner(self):
+        # The runner's `f": {description}" if description else ""` treats an
+        # empty string the same as no description at all (a truthiness check,
+        # not `is not None`) -- ported deliberately, not "fixed".
+        plan = plan_question(
+            request_for(choice_question(criteria={"a": "", "b": None})).questions[
+                "routing"
+            ]
+        )
+        prompt = build_prompt("s", plan, ["A", "B"], None, wording="native")
+        self.assertTrue(prompt.endswith("A. a\nB. b"))
+
+    def test_rejects_noncanonical_or_out_of_order_labels(self):
+        plan = plan_question(request_for(choice_question()).questions["routing"])
+        for labels in (["X", "Y"], ["B", "A"], ["A", "B", "C"]):
+            with self.subTest(labels=labels), self.assertRaises(ValueError):
+                build_prompt("s", plan, labels, None, wording="native")
+
+    def test_rejects_more_than_26_options(self):
+        criteria = {f"opt{i}": None for i in range(27)}
+        plan = plan_question(
+            request_for(choice_question(criteria=criteria)).questions["routing"]
+        )
+        with self.assertRaises(ValueError):
+            build_prompt("s", plan, CANONICAL_LETTERS[:26] + ("AA",), None, wording="native")
+
+    def test_unknown_wording_rejected(self):
+        plan = plan_question(request_for(choice_question()).questions["routing"])
+        with self.assertRaises(ValueError):
+            build_prompt("s", plan, ["A", "B"], None, wording="bogus")
+
+    def test_served_wording_is_unchanged_default(self):
+        plan = plan_question(request_for(choice_question()).questions["routing"])
+        explicit = build_prompt("s", plan, ["A", "B"], None, wording="served")
+        implicit = build_prompt("s", plan, ["A", "B"], None)
+        self.assertEqual(explicit, implicit)
+        self.assertTrue(implicit.startswith("Context:\n"))
+
+
+class TestSystemOneProbabilityReduction(unittest.TestCase):
+    def setUp(self):
+        self.plan = plan_question(request_for(choice_question()).questions["routing"])
+
+    def test_softmax_is_shift_invariant_and_extreme_safe(self):
+        expected = probabilities_from_logprobs([-1, -3], temperature=2)
+        actual = probabilities_from_logprobs([-1001, -1003], temperature=2)
+        for left, right in zip(expected, actual):
+            self.assertAlmostEqual(left, right)
+        self.assertEqual(probabilities_from_logprobs([-1e308, 1e308], 1e-300), [0, 1])
+
+    def test_temperature_scaling_can_be_disabled(self):
+        vector = [-1, -3]
+        normal = reduce_probabilities(self.plan, [vector], [[0, 1]], SystemOneOptions())
+        disabled = reduce_probabilities(
+            self.plan,
+            [vector],
+            [[0, 1]],
+            SystemOneOptions(temperature=100, temperature_scaling=False),
+        )
+        softened = probabilities_from_logprobs(vector, temperature=100)
+        self.assertEqual(normal, disabled)
+        self.assertLess(softened[0], normal["probabilities"]["billing"])
+
+    def test_rotation_is_undone_before_probability_average(self):
+        answer = reduce_probabilities(
+            self.plan,
+            [[math.log(0.8), math.log(0.2)], [math.log(0.7), math.log(0.3)]],
+            [[0, 1], [1, 0]],
+            SystemOneOptions(permutations=2, return_logprobs=True),
+        )
+        self.assertAlmostEqual(answer["probabilities"]["billing"], 0.55)
+        self.assertAlmostEqual(answer["probabilities"]["technical"], 0.45)
+        self.assertEqual(answer["choice"], "billing")
+        self.assertEqual(
+            answer["logprobs"][1],
+            {"billing": math.log(0.3), "technical": math.log(0.7)},
+        )
+
+    def test_confidence_uniform_and_certain(self):
+        uniform = reduce_probabilities(
+            self.plan, [[-5, -5]], [[0, 1]], SystemOneOptions()
+        )
+        certain = reduce_probabilities(
+            self.plan, [[0, -1000]], [[0, 1]], SystemOneOptions()
+        )
+        self.assertAlmostEqual(uniform["confidence"], 0)
+        self.assertEqual(certain["confidence"], 1)
+        self.assertEqual(uniform["choice"], "billing")  # Stable input-order tie break.
+
+    def test_score_expectation_structured_legend_and_entropy(self):
+        criteria = [{"severity": "low", "examples": ["A"]}, ["medium"], None]
+        plan = plan_question(
+            request_for(
+                {
+                    "type": "score",
+                    "instructions": {"task": "rate"},
+                    "criteria": criteria,
+                }
+            ).questions["routing"]
+        )
+        answer = reduce_probabilities(
+            plan,
+            [[math.log(0.25), math.log(0.25), math.log(0.5)]],
+            [[0, 1, 2]],
+            SystemOneOptions(),
+        )
+        self.assertAlmostEqual(answer["score"], 1.25)
+        self.assertEqual(
+            answer["legend"], {"0": criteria[0], "1": criteria[1], "2": None}
+        )
+        expected_entropy = -(0.5 * math.log(0.25) + 0.5 * math.log(0.5))
+        self.assertAlmostEqual(answer["confidence"], 1 - expected_entropy / math.log(3))
+
+    def test_noul_returns_probability_of_true_after_rotation(self):
+        plan = plan_question(
+            request_for({"type": "noul", "instructions": "Yes?"}).questions["routing"]
+        )
+        answer = reduce_probabilities(
+            plan, [[math.log(0.1), math.log(0.9)]], [[1, 0]], SystemOneOptions()
+        )
+        self.assertAlmostEqual(answer["noul"], 0.9)
+        self.assertEqual(set(answer), {"type", "noul"})
+
+    def test_missing_nonfinite_or_invalid_engine_output_fails(self):
+        for values in (
+            [None, -1],
+            [float("nan"), -1],
+            [float("inf"), -1],
+            [float("-inf"), -1],
+            ["0", -1],
+            [True, -1],
+        ):
+            with self.subTest(values=values), self.assertRaises(ValueError):
+                reduce_probabilities(self.plan, [values], [[0, 1]], SystemOneOptions())
+        for vectors, orders in (
+            ([], []),
+            ([[-1]], [[0, 1]]),
+            ([[-1, -2]], []),
+            ([[-1, -2]], [[0, 0]]),
+        ):
+            with (
+                self.subTest(vectors=vectors, orders=orders),
+                self.assertRaises(ValueError),
+            ):
+                reduce_probabilities(self.plan, vectors, orders, SystemOneOptions())
+
+
+if __name__ == "__main__":
+    unittest.main()