Publish OpenJEV E4B 1.0

.gitattributes

@@ -0,0 +1,5 @@
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.jsonl filter=lfs diff=lfs merge=lfs -text
+*.pdf filter=lfs diff=lfs merge=lfs -text

.gitignore

@@ -0,0 +1,7 @@
+__pycache__/
+*.pyc
+.venv/
+.env
+*.tmp
+.DS_Store
+eval/multimodal_smoke/work/

CHANGELOG.md

@@ -0,0 +1,9 @@
+# Changelog
+
+## OpenJEV E4B 1.0 (2026-09-27)
+
+- Packages the Experiment 1 public checkpoint with the cumulative backbone delta, LoRA adapter, decision head, calibration, loader, and release hashes.
+- Documents the frozen GPQA, GSM8K, and Chess legal choice panel, cyclic option rotation, and options-only control.
+- Includes complete untouched-Gemma choice-panel results and the row-construction audit. Gemma and OpenJEV 1.0 have complete choice-panel results, options-only controls, and paired statistics.
+- Reports five standard public tasks under their native-order protocol.
+- Adds an optional, experimental image and audio path (`multimodal=True`). The text path is unchanged. See `docs/MULTIMODAL_EXPERIMENTAL.md` for the smoke test and limits.

CITATION.cff

@@ -0,0 +1,20 @@
+cff-version: 1.2.0
+message: "If you use this model or methodology, please cite this release."
+title: "OpenJEV E4B"
+type: software
+version: "1.0"
+date-released: 2026-09-27
+authors:
+  - name: "OpenJEV contributors"
+license: Apache-2.0
+abstract: >-
+  OpenJEV E4B 1.0 is a direct-decision model built on Gemma 4 E4B.
+  The Experiment 1 report documents the method, choice-panel evaluation,
+  results with confidence intervals, and reproduction protocol.
+preferred-citation:
+  type: report
+  title: "OpenJEV E4B: Training a direct-decision readout on Gemma 4 E4B. Experiment 1 method, evaluation, and results"
+  authors:
+    - name: "OpenJEV contributors"
+  year: 2026
+  month: 9

CONTRIBUTING.md

@@ -0,0 +1,16 @@
+# Contributing
+
+Contributions that improve reproducibility, portability, evaluation, calibration,
+or documentation are welcome.
+
+Please keep changes auditable:
+
+1. Explain the experimental claim being changed.
+2. Include the exact model/data/code version used.
+3. Add or update tests when changing loading or scoring behavior.
+4. Do not replace measured results with estimates.
+5. Keep public benchmark rows out of training and checkpoint selection.
+6. Report regressions and failed experiments instead of deleting them from the record.
+
+For model-quality changes, include row-level or task-level evidence sufficient to
+compare against the frozen public 1.0 baseline.

LICENSE

@@ -0,0 +1,111 @@
+Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+1. Definitions.
+
+"License" shall mean the terms and conditions for use, reproduction, and
+distribution as defined by Sections 1 through 9 of this document.
+
+"Licensor" shall mean the copyright owner or entity authorized by the
+copyright owner that is granting the License.
+
+"Legal Entity" shall mean the union of the acting entity and all other
+entities that control, are controlled by, or are under common control with
+that entity.
+
+"You" (or "Your") shall mean an individual or Legal Entity exercising
+permissions granted by this License.
+
+"Source" form shall mean the preferred form for making modifications,
+including but not limited to software source code, documentation source,
+and configuration files.
+
+"Object" form shall mean any form resulting from mechanical transformation
+or translation of a Source form, including but not limited to compiled
+object code, generated documentation, and conversions to other media types.
+
+"Work" shall mean the work of authorship, whether in Source or Object form,
+made available under the License, as indicated by a copyright notice that
+is included in or attached to the work.
+
+"Derivative Works" shall mean any work, whether in Source or Object form,
+that is based on (or derived from) the Work and for which the editorial
+revisions, annotations, elaborations, or other modifications represent, as
+a whole, an original work of authorship.
+
+"Contribution" shall mean any work of authorship, including the original
+version of the Work and any modifications or additions to that Work or
+Derivative Works thereof, that is intentionally submitted to Licensor for
+inclusion in the Work by the copyright owner or by an individual or Legal
+Entity authorized to submit on behalf of the copyright owner.
+
+"Contributor" shall mean Licensor and any individual or Legal Entity on
+behalf of whom a Contribution has been received by Licensor and
+subsequently incorporated within the Work.
+
+2. Grant of Copyright License. Subject to the terms and conditions of this
+License, each Contributor hereby grants to You a perpetual, worldwide,
+non-exclusive, no-charge, royalty-free, irrevocable copyright license to
+reproduce, prepare Derivative Works of, publicly display, publicly perform,
+sublicense, and distribute the Work and such Derivative Works in Source or
+Object form.
+
+3. Grant of Patent License. Subject to the terms and conditions of this
+License, each Contributor hereby grants to You a perpetual, worldwide,
+non-exclusive, no-charge, royalty-free, irrevocable (except as stated in
+this section) patent license to make, have made, use, offer to sell, sell,
+import, and otherwise transfer the Work, where such license applies only to
+those patent claims licensable by such Contributor that are necessarily
+infringed by their Contribution(s) alone or by combination of their
+Contribution(s) with the Work to which such Contribution(s) was submitted.
+
+4. Redistribution. You may reproduce and distribute copies of the Work or
+Derivative Works thereof in any medium, with or without modifications, and
+in Source or Object form, provided that You meet the following conditions:
+
+(a) You must give any other recipients of the Work or Derivative Works a
+copy of this License; and
+
+(b) You must cause any modified files to carry prominent notices stating
+that You changed the files; and
+
+(c) You must retain, in the Source form of any Derivative Works that You
+distribute, all copyright, patent, trademark, and attribution notices from
+the Source form of the Work, excluding those notices that do not pertain to
+any part of the Derivative Works; and
+
+(d) If the Work includes a "NOTICE" text file as part of its distribution,
+then any Derivative Works that You distribute must include a readable copy
+of the attribution notices contained within such NOTICE file.
+
+5. Submission of Contributions. Unless You explicitly state otherwise, any
+Contribution intentionally submitted for inclusion in the Work by You to
+the Licensor shall be under the terms and conditions of this License.
+
+6. Trademarks. This License does not grant permission to use the trade
+names, trademarks, service marks, or product names of the Licensor, except
+as required for reasonable and customary use in describing the origin of
+the Work and reproducing the content of the NOTICE file.
+
+7. Disclaimer of Warranty. Unless required by applicable law or agreed to
+in writing, Licensor provides the Work (and each Contributor provides its
+Contributions) on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
+ANY KIND, either express or implied, including, without limitation, any
+warranties or conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or
+FITNESS FOR A PARTICULAR PURPOSE.
+
+8. Limitation of Liability. In no event and under no legal theory, whether
+in tort (including negligence), contract, or otherwise, unless required by
+applicable law, shall any Contributor be liable to You for damages,
+including any direct, indirect, special, incidental, or consequential
+damages arising in any way out of the use of the Work.
+
+9. Accepting Warranty or Additional Liability. While redistributing the
+Work or Derivative Works thereof, You may choose to offer support,
+warranty, indemnity, or other liability obligations and/or rights
+consistent with this License.
+
+END OF TERMS AND CONDITIONS

MANIFEST.sha256

@@ -0,0 +1,92 @@
+9bf741ee1a38f7bf5c3f39d50453ed4f011b0fdadef359b5fa4e88e592167748  .gitattributes
+05baacc8d60e5c0f2a091cdddab6fe45edeb32026517f74f04925d5a2b68e334  .gitignore
+f49dea2fe639b8cea13c17356ee9a0a66ddbb15a7256285bb3c05e09e7e6df6d  CHANGELOG.md
+9fcfc57d54d95857c287749b02d08af2398cbb3e8aba0ae96251a6d6d60ae347  CITATION.cff
+5aae8718d2c58f3915010e5efb4552446a7c9557b996e728e06d25f5939cbcf8  config.json
+53faf45e8ffd467cd1d874cb91dcedfeb36425a47c8cf7f9c976746babb79ba7  CONTRIBUTING.md
+feee7afe83109ad8d80d8ea95bea50f3c5a5e8b7f028f5ca28d452f89cb3d0f5  docs/BENCHMARK_EXPOSURE.md
+52a8ee50506d575479cdc7b24665d9610a6275e45a2398c5fb56b845ffc3d816  docs/EXPERIMENT_1_RESULTS.md
+508205b8d7e306f53ae68f3e4fa8ba562eac6a2a7fe2e53c9328d1c80cc7693a  docs/MULTIMODAL_EXPERIMENTAL.md
+f441acf38e8354254070f1d429fd9345dedbdd5ee6d30b4be278fc07dea8d504  docs/OpenJEV_E4B_Experiment_1_Whitepaper.pdf
+6305c9920344cb1371a7383f1ec49ac0c6c0fa23ddc3ba0f426cea86d9108fd6  docs/README.md
+1f9b93a9324841491e0b76a92c9c356bc948ab6d8391eda4b57e6f8280eeda09  docs/source/build_whitepaper.js
+9157ce0e76f34964cfeb12bc93e81202be4106cced28f472538c0523fa27d048  docs/source/OpenJEV_E4B_Experiment_1_Whitepaper.docx
+8782d216533d3b3ec14c1694972f966979eda0b4157d7bfcf1c3208c80ddc47e  eval/analysis/contamination_audit.json
+0308834527b91bc711b83ed3e5a444a912b40af3bfe8e735ae9b5ba10ae7f10e  eval/analysis/contamination_audit.py
+dcbb78481aff6fb33f4d73f1aed758d368e01dd476f49b16d1e0de1b8825bb0b  eval/analysis/paired_analysis.py
+e905f991f9d5685ad851207bb0430b9f40074b84066bc67439c559f3ec7b8027  eval/choice_panel/details_r7.jsonl
+8bf0ec06599e55e743fcbdaca736ff211819bc63e03af765ae62e67c26e6605a  eval/choice_panel/details_raw.jsonl
+7245eb4d52c07ff5bc31360c900970f805c0602cd9c8ce126400196f2ceaa29a  eval/choice_panel/leak_audit.json
+3f163699cd0ed731553e6cbed2cfc39f7e56379615258c9469a4e40f693fb2bf  eval/choice_panel/README.md
+a8bbb72147f51b0144f4177ed9e60a9223ef9c5979694423d93348b064bc8bf4  eval/choice_panel/rows.jsonl
+4fa296303ec511caa449db9bfb1ae860d7d89eceaf216b5a06274d775019244f  eval/choice_panel/rows.sha256
+5833b7d6fb43b909409ba15a01defa39835f73abc03ff29b33274bc14133f9f2  eval/choice_panel/run_choice_panel.py
+3c988c70b4194f9cdb61fb5f3cf0dcae0f35b85c16a6d76ded9ca7d97f5543e6  eval/choice_panel/source_run.json
+cd66c31d9d332745a371402f99ca917089bb5af9a77638bd361541dbceb1182a  eval/choice_panel/summarize.py
+492648665542917dff019f4246a038e54bd3eae21cf2e88e1e78c51564c51b15  eval/choice_panel/summary.json
+037d159f637e1c2f119aace9dd7da83a933658fe0777f11d9cb4544a2a409d93  eval/choice_panel/summary.md
+25c17b666e871b25de603f8fd701310d9151d6c6364dfcd902f5c1451de1cfba  eval/harness/benchmark_public.py
+40f3ead728cef1a45214df414e1c7cce062e4ebf4cded90c50b8a43ddcf4b503  eval/harness/requirements-benchmark.txt
+e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855  eval/harness/src/__init__.py
+4657fd261d435d508d0b63002f0563cb1f59cc6b49650b18ae278caeef2fca69  eval/harness/src/decision_head.py
+6d8acaa401dc7f5dc077833415d0215899447e7a7c626875a9ea99e7be42025e  eval/harness/src/formatting.py
+0fbf9883835e895f3e5a286dc27ff34a7de4e16da22ea48a2af52b8ae9ea9e20  eval/harness/src/metrics.py
+8c226815684ea849980ed9134aa96ebf30bb133706ff2824254a838465b158df  eval/multimodal_smoke/lines.tsv
+afc14dde53a2f4b23f9319f355ef61b7f41ef69d3c310c27906080f9b92beabb  eval/multimodal_smoke/make_audio_windows.ps1
+79cdc1f8b348e68925fd902ea6265207069b18c5c71e0c134a9c2b6d8260665e  eval/multimodal_smoke/results.json
+e799bf9ce01b0e4b07dfdc48dbeebefa2f47d79083ade8f0c144cd659fd80f36  eval/multimodal_smoke/run_smoke.py
+3287e51cb152dd175c2216100c2ba360e6cded099795dccd54072f0c63af3aaa  eval/openjev_1_0_public_benchmark.csv
+2219e8973f1423817ae75e4bc83287c777b1d9f698a7f52465b11ba9316dccf5  eval/openjev_1_0_public_benchmark.json
+9cca1e4e95851c986e0f34a5ee9fb1b795feab6471ed211ae145c176dd8a3e03  eval/raw_e4b_nf4_control_run/details.jsonl
+14b411f19b958de954b6bf119ad9be1f9db76b110f802c3a4e9e9b79a3b205b1  eval/raw_e4b_nf4_control_run/summary.csv
+2841009c608c04ed49a286d615eef8862fdd1fee77fd1acb07a03c72cb0e2822  eval/raw_e4b_nf4_control_run/summary.json
+e03337ca35d50eb49d2f6a1db72cae5e7f5163df43cae778e5bf4104929278d3  eval/raw_e4b_nf4_control_summary.json
+3dce5a53ada89fecbb215102089609e96f272769210fb78370f2ab49125f3d08  eval/raw_run/details.jsonl
+cdc1a89bb657b5b39f8b1e085d8775d11ee70b46ebc22ee9facbc70e2b0bebbf  eval/raw_run/summary.csv
+9cc1dc84eb75ac464b8b26db072e72fe2431a66819c46d6ed67673a4ee71e8a4  eval/raw_run/summary.json
+a95bbb7a0d05643ab78eb5efa4fccd5cae0753f0aa9503be2dab1f307ec5086f  examples/basic_choice.py
+9e35f6a4d74bf08a5f8ea7cdc403835b00af382f7d6884b6a1668fb09f736dec  examples/multimodal_choice.py
+d94433ab0cef6915ea4239c2bbae4f222694b8a66ad52a4acc7c1f31b7ac234b  LICENSE
+0d9268a312eddebce113670cc73b80546b03e2dd707ee58d081063dc059d47bd  model/adapter/adapter_config.json
+e1fd49fa603b046028a3891d8843a1c4e9b727a17516b6ab7ee9a1e4cc6fbebe  model/adapter/adapter_model.safetensors
+5e34c781c86c33c6b1d5ad75c27f5826537c3c822b353b4956e24f9282284779  model/backbone_delta.safetensors
+455d9f2f6e2235adf1e75b5311208bb5b75194b4e1e737ed0edf04f4c50363ad  model/calibration.json
+9a74fe26596e5accf593a4dd5f752720e05c8dd3a7baeeef491b61439c8093f7  model/decision_head.safetensors
+63496153a5024b8168086e39058b78919d09b7fcf80e9e35ffcaeff7b1bde672  model/nli_aux_head.safetensors
+737eb56c02f043f111ae1b6544e483f12b7f45bef7b33839f23b6fbfc52ea847  model/openjev_config.json
+183875cf3d65fad5abb8efdef47a51157c6421af0681571d1decc6f5072e0033  NOTICE
+4fbd8a23d3d10e0f4318980b2a07b6751b6697e02d34c0772ddb78f32d403833  openjev/__init__.py
+4657fd261d435d508d0b63002f0563cb1f59cc6b49650b18ae278caeef2fca69  openjev/decision_head.py
+6d8acaa401dc7f5dc077833415d0215899447e7a7c626875a9ea99e7be42025e  openjev/formatting.py
+fd7e199470cb6149e155991c6bd873c6e3f732e88378019dc48e83377d6afc5f  openjev/modeling.py
+f455a0508b38e2f310871bc3f0f370387c394aed42e03003b3f6d422288da5db  openjev/multimodal.py
+a86fd0cf09042589c55d87b9e451d5a92fe9147e1f9414dc3a46a6f1a4109233  provenance/checkpoint_json/arena_manifest.json
+030f5b8b1017a99402d5c226458598a958e64b9284ce41f61a31a77a2b0f3345  provenance/checkpoint_json/arena_training_meta.json
+89295fe507c532955ac762999b7c79c5e6f4b27ac5c03feaaaed543c79202100  provenance/checkpoint_json/inherited_v132/benchmark_summary.json
+50bd17c6d57741934c93321a47972fc04570cf258be43db5a297e71115333898  provenance/checkpoint_json/inherited_v132/best_checkpoint.json
+296c17091f3f71ea27e747f98890e0a3178c0937f6185b3e456f235a52f5929d  provenance/checkpoint_json/inherited_v132/calibration_console.json
+57ddc0c675278be9d11e79cebc4f99c685d6bbde6a66642bdb65868286776906  provenance/checkpoint_json/inherited_v132/data_summary.json
+5d278d737cc9595c5fea216ef04dcefcda031fc58173ad7d0645005810bd8368  provenance/checkpoint_json/inherited_v132/experiment_summary.json
+31f318bdccde051f1d5329544ec4e9679a00178e0f3460d01f65f1b28c15af74  provenance/checkpoint_json/inherited_v132/firewall_metrics.json
+5fd80e836e620ca67bf7c8df3114fc5a09887528404ab1b3f999448d120cf46c  provenance/checkpoint_json/inherited_v132/legacy_metrics.json
+3a3d1267c97c5094b748fe3b0eaac9c13ecee746ed0ebac8ca8cc583e508adf6  provenance/checkpoint_json/inherited_v132/quality_gate.json
+05cf3f301abbdbbb770052f2defea209796904b5ce5609686676f371a0f64892  provenance/checkpoint_json/inherited_v132/robustness_metrics.json
+3da3f39697a3ef2b2603db9e30397c155823075d5feab1bd89c4589c0a457e31  provenance/checkpoint_json/inherited_v132/schema_metrics.json
+a12f787c79210e2d191da1f1954bc8c2b1423407a5b8dba51066d7439f356ddc  provenance/checkpoint_json/inherited_v132/seed_firewall_metrics.json
+661145aa05e36b87a017a838fb58e1d20cd721dcaa86daa4773834d89a482f9b  provenance/checkpoint_json/inherited_v132/seed_legacy_metrics.json
+f94482f2c5698350bb9cdb024930683134a3ddd36f7736fbb0e721cea9d3c999  provenance/checkpoint_json/inherited_v132/seed_robustness_metrics.json
+b4ec424cf97d52521d9ee98c5069c4ad68a98c20eda8e4f7b3d82bddfee4c8d6  provenance/checkpoint_json/inherited_v132/seed_schema_metrics.json
+5ad8112c5fff01261ed5cf2227c4354b500336707daec51b0e1856187bd3418d  provenance/checkpoint_json/inherited_v132/seed_validation_metrics.json
+75f9197b5e2d63757a86ae7e23f36f6bb00323615e87b5c0eb6e3b9e9389f957  provenance/checkpoint_json/inherited_v132/test_metrics.json
+4f65931e9e013f30baba9d6f94174ea08920d412e9353d2f41c2c9e74e25355b  provenance/checkpoint_json/inherited_v132/training_state.json
+f6e6b675a35974289334ce6beaf03627addacf6c3534ca2890d0494abcb5ae02  provenance/checkpoint_json/v142_selection.json
+9fe5706f9d10e804b512a0390d6782cd448c31e9c5cbc03242d8075880168d42  provenance/checkpoint_json/v142_tournament.json
+ad17c564396c1c70607bf17ca2df405c0175ad3d40cd6f754d8296090da362d9  provenance/checkpoint_json/v142_train_meta.json
+1245f3c205e704c92a03ddf1859778a4af1dc269c3c26e35161d89972b2ea8ec  provenance/checkpoint_json/v151_rlcd_meta.json
+e9659f6fb8047c31e9707ca0e7b2eeb9bc3e91db01bd981e0fc381b75eb05325  provenance/checkpoint_json/v151_selection.json
+07f0d5752ed95f60d261f500d34cffba2bd55fc1e37fe47080933c1d829b8af2  provenance/README.md
+a0483f9a2babbfed89ccf5260c4395dfed13eeb0aabe3fcb5a77ebe83f975246  README.md
+3d975199013cb15b90591b35d92066cbdb0a034783c6693cfd03fe656c9c2b18  RELEASE_METADATA.json
+24991e21a564c157026ab75febaf2c5142a6f65a5d01bc323f89325cbed29969  REPRODUCIBILITY.md
+e5a17f6965d1620d20c1b6a7e660976759aedef64422a3dbdfb97fbf1acf7221  requirements.txt
+b2a8fc65c6f004d3142b921036a2ba6c0fc3e9a071a59e446587967f333a46e0  SECURITY.md

NOTICE

@@ -0,0 +1,18 @@
+OpenJEV E4B 1.0
+
+This repository contains OpenJEV-authored model deltas, decision-head weights,
+supporting code, evaluation artifacts, and documentation.
+
+Base model:
+  google/gemma-4-E4B-it
+  Copyright Google
+  Licensed under Apache License 2.0.
+  Base weights are NOT redistributed in this repository.
+
+OpenJEV is an independent research project. "Jev" and related TypeSafe names
+may be trademarks of their respective owners. This project is not affiliated
+with or endorsed by TypeSafe AI.
+
+Training datasets and public benchmark datasets retain their original
+licenses, attribution requirements, and terms. See the whitepaper and
+evaluation harness for provenance.

README.md

@@ -0,0 +1,168 @@
+---
+license: apache-2.0
+library_name: openjev
+pipeline_tag: zero-shot-classification
+base_model: google/gemma-4-E4B-it
+language:
+  - en
+tags:
+  - openjev
+  - classification
+  - zero-shot-classification
+  - decision-model
+  - listwise
+  - peft
+  - gemma4
+  - research
+---
+
+# OpenJEV E4B 1.0
+
+OpenJEV turns Gemma 4 E4B into a direct-decision classifier. A caller supplies state, a question, and a closed set of options; the model returns a probability for each option without generating text. This is the public 1.0 release of Experiment 1.
+
+OpenJEV was trained and evaluated on text. An optional, experimental path (`multimodal=True`) keeps Gemma's vision and audio encoders so a call can also take an image or an audio clip; see [Experimental: image and audio input](#experimental-image-and-audio-input).
+
+## Download and use
+
+The release is hosted at [bambamdevs/openjev-e4b](https://huggingface.co/bambamdevs/openjev-e4b). Install CUDA-enabled PyTorch 2.6 or newer first on Windows ([PyTorch instructions](https://pytorch.org/get-started/locally/)). The release contains the adapter, cumulative backbone delta, decision head, calibration, and loader. The loader downloads the pinned `google/gemma-4-E4B-it` base revision on first use. Hugging Face login is optional for public downloads.
+
+```bash
+python -m pip install --upgrade "huggingface_hub<2"
+hf auth login
+hf download bambamdevs/openjev-e4b --local-dir openjev-e4b
+cd openjev-e4b
+python -m pip install -r requirements.txt
+python -m examples.basic_choice
+```
+
+To score your own options:
+
+```python
+from openjev import OpenJEV
+
+model = OpenJEV.from_pretrained(".", device="cuda", load_mode="nf4")
+result = model.choice(
+    state="Checkout fails after a deployment; customers cannot place orders.",
+    instruction="Which team should own this incident?",
+    options=["Billing", "Technical outages", "Sales"],
+)
+print(result["probabilities"], result["selected_option"])
+```
+
+The NF4 path keeps frozen base layers quantized and restores delta-touched layers 37–41 in dense FP16. The decision head runs in FP32. `load_mode="bf16"` is available with enough memory; its public benchmarks have not been run.
+
+### Use as a classifier
+
+Put the text to classify in `state`, the question in `instruction`, and your labels in `options`. Any label set of two or more works on each call; no retraining is needed.
+
+```python
+r = model.choice(
+    state="I was charged twice for my subscription this month.",
+    instruction="Which support queue should handle this message?",
+    options=["billing", "technical support", "account access", "sales"],
+)
+r["selected_option"]  # "billing"
+r["probabilities"]    # one probability per option, in the same order as options
+```
+
+The same pattern covers sentiment, topic, intent, and entailment labels. Probabilities use the release calibration in `model/calibration.json`.
+
+### Experimental: image and audio input
+
+Load with `multimodal=True` to pass `image=` (path or PIL image) or `audio=` (`.wav` path) to `choice()`:
+
+```python
+model = OpenJEV.from_pretrained(".", device="cuda", load_mode="nf4", multimodal=True)
+r = model.choice(
+    state="",
+    audio="call.wav",
+    instruction="Which support queue should handle the spoken message?",
+    options=["billing", "technical support", "account access", "sales"],
+)
+```
+
+The adapter, backbone delta, and decision head were trained on text only, so this path is experimental. In a small smoke test on clean synthetic inputs (NF4, RTX 3060), it scored 12/12 on image shape, 12/12 on image colour, 8/8 on printed messages, 16/16 on spoken messages, and 12/12 on spoken reviews. With the media removed, the same questions scored at chance. Image calls took about 5 s and audio calls about 2 s. This is not a benchmark. Details, limits, and the rerun script are in [`docs/MULTIMODAL_EXPERIMENTAL.md`](./docs/MULTIMODAL_EXPERIMENTAL.md).
+
+## Evaluation
+
+The four choice-panel tasks use frozen rows and cyclic option rotations. For each row, probabilities are mapped back to the original option identities and averaged across rotations; accuracy is calculated from that mean. The table also reports mean accuracy for individual presentations, which exposes option-order sensitivity. A separate options-only condition removes the state and question. The five standard public tasks below were evaluated in their native option order. All model evaluations use NF4 on the Windows RTX 3060.
+
+<!-- CHOICE_RESULTS_START -->
+### Choice-panel evaluation
+
+The main score selects the largest probability after averaging each row's cyclic option rotations. Wilson 95% intervals use rows as observations. Both systems use the same frozen rows and Windows NF4 evaluation path.
+
+| Task | Rows | Rotations | Chance | Gemma base [95% CI] | OpenJEV 1.0 [95% CI] | Difference [95% CI] |
+|---|---:|---:|---:|---:|---:|---:|
+| GPQA Diamond, shuffled | 198 | 4 | 25.0% | 32.8% [26.7, 39.6] | 32.8% [26.7, 39.6] | +0.00 [-7.07, +7.07] pp |
+| Chess legal | 500 | 4 | 25.0% | 43.8% [39.5, 48.2] | 42.6% [38.3, 47.0] | -1.20 [-6.40, +3.80] pp |
+| GSM8K, 4 choices | 1,319 | 4 | 25.0% | 43.7% [41.1, 46.4] | 53.8% [51.1, 56.5] | +10.08 [+6.44, +13.72] pp |
+| GSM8K, 10 choices | 1,319 | 10 | 10.0% | 20.5% [18.5, 22.8] | 30.6% [28.2, 33.2] | +10.08 [+7.05, +13.04] pp |
+
+Mean accuracy per presentation and share of rows with the same predicted option in every rotation:
+
+| Task | Gemma per order / same answer | OpenJEV 1.0 per order / same answer |
+|---|---:|---:|
+| GPQA Diamond, shuffled | 31.3% / 10.1% | 33.7% / 23.2% |
+| Chess legal | 31.5% / 1.4% | 40.1% / 46.6% |
+| GSM8K, 4 choices | 36.7% / 6.9% | 51.5% / 43.3% |
+| GSM8K, 10 choices | 16.5% / 0.6% | 28.6% / 14.9% |
+
+### Options-only control
+
+State and question are replaced by neutral text. Four-choice tasks use four rotations; GSM8K-10 uses five evenly spaced rotations.
+
+| Task | Chance | Gemma base [95% CI] | OpenJEV 1.0 [95% CI] |
+|---|---:|---:|---:|
+| GPQA Diamond, shuffled | 25.0% | 25.8% [20.2, 32.3] | 32.3% [26.2, 39.1] |
+| Chess legal | 25.0% | 29.0% [25.2, 33.1] | 30.8% [26.9, 35.0] |
+| GSM8K, 4 choices | 25.0% | 24.7% [22.5, 27.1] | 24.3% [22.1, 26.7] |
+| GSM8K, 10 choices | 10.0% | 10.5% [9.0, 12.3] | 10.8% [9.3, 12.6] |
+
+OpenJEV 1.0 scores 32.3% on GPQA and 30.8% on Chess legal with only the options present, both above the 25% chance rate. Its GPQA full-condition score is 32.8%; this result alone does not establish use of the question. The GSM8K options-only scores are near chance. The row-construction audit passes its chance gate, but the model control reveals additional choice-only signal. Chess legal tests move legality; the GSM8K tasks test closed-choice arithmetic. Gemma uses a native answer-letter readout and OpenJEV uses a learned decision head, so comparisons measure complete systems.
+
+Evidence: [`eval/choice_panel/`](./eval/choice_panel/). The five standard public tasks below use native option order and are reported separately.
+<!-- CHOICE_RESULTS_END -->
+
+### Standard public tasks
+
+**Benchmark status: COMPLETE.** All five standard public tasks have both Gemma base NF4 control and OpenJEV 1.0 results.
+
+| Benchmark | Gemma base NF4 | OpenJEV 1.0 | OpenJEV - Gemma |
+|---|---:|---:|---:|
+| ARC-Challenge | 87.37% | 87.88% | +0.51 pp |
+| WinoGrande | 60.14% | 66.46% | +6.32 pp |
+| ARC-Easy | 95.58% | 95.62% | +0.04 pp |
+| HellaSwag | 76.51% | 89.77% | +13.26 pp |
+| MMLU | 66.19% | 64.41% | -1.78 pp |
+
+The untouched base uses native LM-head answer-label scoring; OpenJEV 1.0 uses a learned decision head. Their comparison measures the complete decision systems. ARC, HellaSwag, and MMLU have training-family exposure described in [`docs/BENCHMARK_EXPOSURE.md`](./docs/BENCHMARK_EXPOSURE.md).
+
+## Paper and evidence
+
+[Experiment 1 whitepaper](./docs/OpenJEV_E4B_Experiment_1_Whitepaper.pdf) describes the model, training lineage, evaluation protocol, results, limitations, and reproduction steps. It accompanies this release.
+
+```text
+model/                      adapter, head, cumulative backbone delta, calibration
+openjev/                    loader and scoring implementation
+eval/choice_panel/          frozen rows, audit, scoring code, row-level results
+eval/harness/               standard public-task runner
+docs/                       whitepaper, source, and result notes
+provenance/                 checkpoint metadata
+```
+
+OpenJEV 1.0 is a delta release; Google base weights are not republished. `backbone_delta.safetensors` is required to restore the model.
+
+## Training and limitations
+
+The model evolved through readout and adapter training, then an Arena curriculum with exact-oracle examples and replay across 20 decision families. The public checkpoint also updates selected decoder layers and includes 65 cumulative backbone-delta tensors. The development sequence, promotion rule, and release hashes are in the whitepaper and `RELEASE_METADATA.json`.
+
+The experiment has one training seed per stage. Internal populations changed during development. The choice-panel results measure accuracy on their stated tasks; they do not isolate the effect of the head, LoRA, backbone updates, or inference precision. GPQA and GSM8K are multiple-choice tasks, and Chess legal tests move legality rather than move quality.
+
+## Reproduce and license
+
+See [`REPRODUCIBILITY.md`](./REPRODUCIBILITY.md), [`RELEASE_METADATA.json`](./RELEASE_METADATA.json), and [`MANIFEST.sha256`](./MANIFEST.sha256). OpenJEV-authored material is Apache-2.0. The Gemma base remains under Google's terms. Training code is not part of this release.
+
+OpenJEV is independent research. It is not a TypeSafe AI product, is not affiliated with or endorsed by TypeSafe AI, and is unrelated to `AlexWortega/openjev`. Its design was inspired by System 1-style decision making: a fast, direct choice over a closed set of options, without step-by-step text generation.
+
+Generated: 2026-09-27T17:07:53.606842+00:00

RELEASE_METADATA.json

@@ -0,0 +1,39 @@
+{
+  "model_line": "openjev-e4b",
+  "release": "1.0",
+  "version": "1.0",
+  "internal_checkpoint": "v1.6 R7",
+  "accepted_arena_score": 0.024174,
+  "base_model": "google/gemma-4-E4B-it",
+  "base_revision": "ee0ef6023621cff504d758262d4e04895a5af4a2",
+  "selected_sparse_layers": [
+    37,
+    41
+  ],
+  "cumulative_backbone_delta_tensors": 65,
+  "source_checkpoint": "local benchmark/checkpoints/round-07",
+  "source_benchmark_run": "local benchmark/results/full/20260926-112930",
+  "generated_utc": "2026-09-27T17:07:53.606842+00:00",
+  "source_hashes": {
+    "checkpoints/round-07/config.json": "df245eb519b57ffbdb0d04ee7898fb96c8c01eaed5bc5b66b455e1f6610af7c3",
+    "checkpoints/round-07/decision_head.pt": "73f7f985c894580a662c5afc3e8a9ab0c4911cea963c774c1ff279e36b8f54e5",
+    "checkpoints/round-07/backbone_delta.pt": "6b47f1787bcda39392c424f4fa9834a46dc45f097ee8a9b2b6655f3e9855095e",
+    "checkpoints/round-07/adapter/adapter_config.json": "0d9268a312eddebce113670cc73b80546b03e2dd707ee58d081063dc059d47bd",
+    "checkpoints/round-07/adapter/adapter_model.safetensors": "e1fd49fa603b046028a3891d8843a1c4e9b727a17516b6ab7ee9a1e4cc6fbebe",
+    "checkpoints/round-07/nli_aux_head.pt": "46baa95ffb8e1d0d8bf9f31ef67275330be46a10a21981f811801450d991df47"
+  },
+  "benchmark_complete": true,
+  "benchmark_tasks_present": [
+    "arc_challenge",
+    "winogrande",
+    "arc_easy",
+    "hellaswag",
+    "mmlu"
+  ],
+  "benchmark_tasks_missing": [],
+  "conversion": {
+    "decision_head": "PyTorch state_dict -> safetensors via torch.load(weights_only=True)",
+    "backbone_delta": "PyTorch state_dict -> safetensors via torch.load(weights_only=True)",
+    "base_weights_redistributed": false
+  }
+}

REPRODUCIBILITY.md

@@ -0,0 +1,21 @@
+# Reproducibility contract
+
+## Model identity
+
+OpenJEV E4B 1.0 requires the `google/gemma-4-E4B-it` base at revision `ee0ef6023621cff504d758262d4e04895a5af4a2`, its LoRA adapter, hybrid pointer and span decision head, cumulative 65-tensor backbone delta for layers 37–41, calibration (`global_temperature = 1.0`), and the packing and scoring code in `openjev/`. The adapter alone does not restore the model. Every released file is hashed in `MANIFEST.sha256`; source-checkpoint hashes and the internal checkpoint identifier are in `RELEASE_METADATA.json`.
+
+## Choice-panel evaluation
+
+The frozen row file is `eval/choice_panel/rows.jsonl`, with SHA-256 in `eval/choice_panel/rows.sha256`. The construction audit is `leak_audit.json`. GPQA answers are placed by a fixed per-row shuffle. GSM8K distractors are answers to other test problems of similar magnitude. Chess legal distractors are plain legal-move strings drawn from other positions and illegal in the current position.
+
+The runner presents every row in all cyclic option rotations for the full condition. For options-only, it replaces state and question with neutral text and uses all four rotations on four-option tasks and five evenly spaced rotations on ten-option tasks. Output probabilities are mapped back to original option identities. The main accuracy is argmax of the mean mapped probabilities for each row; Wilson 95% intervals use rows as observations. Mean per-order accuracy and same-answer share are reported separately. See `eval/choice_panel/summary.json` and `docs/EXPERIMENT_1_RESULTS.md`.
+
+Untouched Gemma uses native LM-head answer-letter scoring in NF4. OpenJEV 1.0 uses the frozen base in NF4, delta-touched layers in dense FP16, LoRA, and an FP32 head. The comparison measures complete decision systems and does not isolate any one component.
+
+## Standard public tasks
+
+ARC-Easy, ARC-Challenge, WinoGrande, HellaSwag, and MMLU use the native-order runner in `eval/harness/benchmark_public.py`. Their row-level runs and summaries are included under `eval/`. ARC, HellaSwag, and MMLU have training-family exposure recorded in `docs/BENCHMARK_EXPOSURE.md`.
+
+## Training record
+
+The whitepaper describes model evolution and the Arena promotion rule. Source checkpoint records and hashes are in `provenance/` and `RELEASE_METADATA.json`. Training code is not part of this release.

SECURITY.md

@@ -0,0 +1,12 @@
+# Security
+
+Do not report secrets, authentication tokens, private training data, or personally
+identifying information in public issues.
+
+This research model returns probabilities over caller-provided choices. Consumers
+are responsible for validating inputs, enforcing authorization, bounding tool
+actions, and applying independent controls before using model outputs in
+security-sensitive or safety-critical workflows.
+
+For a security issue in release code, use the repository owner's private security
+reporting channel where available.

config.json

@@ -0,0 +1,6 @@
+{
+  "_note": "Marker for Hugging Face download counting. The model configuration is model/openjev_config.json; load with openjev.OpenJEV.from_pretrained.",
+  "model_type": "openjev",
+  "base_model": "google/gemma-4-E4B-it",
+  "openjev_config": "model/openjev_config.json"
+}

docs/BENCHMARK_EXPOSURE.md

@@ -0,0 +1,17 @@
+# Benchmark exposure and protocol
+
+Public evaluation rows were used after promotion and did not select a checkpoint. The training compilers nevertheless used other splits from some of the same dataset families.
+
+| Task | Source and evaluation protocol | Training-family exposure | Row overlap with public rows* |
+|---|---|---|---|
+| MMLU | `cais/mmlu` all/test, native order | `auxiliary_train` in v1.1 and v1.3.2 | 19 question-text / 0 question+options |
+| ARC-Easy / ARC-Challenge | `allenai/ai2_arc` test, native order | ARC train and ARC material in MMLU `auxiliary_train` | At most 9 question-text / 3 question+options |
+| HellaSwag | `Rowan/hellaswag` validation, native order | HellaSwag train | 23 context / 0 context+endings |
+| WinoGrande | XL validation, native order | No known direct source | Not measured |
+| GPQA Diamond | 198 shuffled rows; four cyclic rotations | No known direct source | Not measured |
+| GSM8K-4 / GSM8K-10 | 1,319 test questions; options drawn from other problems in a matching magnitude bucket; cyclic rotations | No known GSM8K source; Arena contains arithmetic families | Not measured |
+| Chess legal | 500 synthetic positions; one legal SAN and three position-inapplicable SAN choices; four rotations | No known chess source | Not applicable |
+
+\* Normalized-text hash matches against full upstream training splits, which are supersets of the sampled training rows. The audit output for the standard tasks is [`../eval/analysis/contamination_audit.json`](../eval/analysis/contamination_audit.json). The choice-panel option-string and answer-position audit is [`../eval/choice_panel/leak_audit.json`](../eval/choice_panel/leak_audit.json).
+
+The standard-task family exposure is distributional even where exact row overlap is small. Chess legal asks which move is legal in the shown position. The GSM8K choices test selecting a numerical answer from options; they do not measure open-ended solution generation.

docs/EXPERIMENT_1_RESULTS.md

@@ -0,0 +1,50 @@
+# Experiment 1 results — OpenJEV E4B 1.0
+
+The public evaluation has two protocols. GPQA Diamond, Chess legal, GSM8K-4, and GSM8K-10 use frozen choice-panel rows and cyclic option rotations. Five standard public tasks retain their native option order. Gemma and OpenJEV 1.0 completed the choice-panel evaluation on the Windows RTX 3060. The completed untouched-Gemma results below come from [`../eval/choice_panel/summary.json`](../eval/choice_panel/summary.json).
+
+## Choice-panel evaluation
+
+Probabilities from each cyclic rotation are mapped to original option identities and averaged per row. The main accuracy chooses the largest mean. Wilson 95% intervals treat rows as observations. Mean per-order accuracy averages individual presentations; same-answer rate is the share of rows whose chosen option never changes across rotations.
+
+| Task | Rows | Rotations | Chance | Gemma base [95% CI] | OpenJEV 1.0 [95% CI] | Difference [95% CI] |
+|---|---:|---:|---:|---:|---:|---:|
+| GPQA Diamond, shuffled | 198 | 4 | 25.0% | 32.8% [26.7, 39.6] | 32.8% [26.7, 39.6] | +0.00 [-7.07, +7.07] pp |
+| Chess legal | 500 | 4 | 25.0% | 43.8% [39.5, 48.2] | 42.6% [38.3, 47.0] | -1.20 [-6.40, +3.80] pp |
+| GSM8K, 4 choices | 1,319 | 4 | 25.0% | 43.7% [41.1, 46.4] | 53.8% [51.1, 56.5] | +10.08 [+6.44, +13.72] pp |
+| GSM8K, 10 choices | 1,319 | 10 | 10.0% | 20.5% [18.5, 22.8] | 30.6% [28.2, 33.2] | +10.08 [+7.05, +13.04] pp |
+
+| Task | Gemma per order / same answer | OpenJEV 1.0 per order / same answer |
+|---|---:|---:|
+| GPQA Diamond, shuffled | 31.3% / 10.1% | 33.7% / 23.2% |
+| Chess legal | 31.5% / 1.4% | 40.1% / 46.6% |
+| GSM8K, 4 choices | 36.7% / 6.9% | 51.5% / 43.3% |
+| GSM8K, 10 choices | 16.5% / 0.6% | 28.6% / 14.9% |
+
+Paired intervals use 2,000 row bootstrap resamples. Exact McNemar p-values and flip counts are in [`../eval/choice_panel/summary.json`](../eval/choice_panel/summary.json).
+
+## Options-only control
+
+State and question are replaced by neutral text. Ten-choice tasks use five evenly spaced rotations in this condition.
+
+| Task | Chance | Gemma base [95% CI] | OpenJEV 1.0 [95% CI] |
+|---|---:|---:|---:|
+| GPQA Diamond, shuffled | 25.0% | 25.8% [20.2, 32.3] | 32.3% [26.2, 39.1] |
+| Chess legal | 25.0% | 29.0% [25.2, 33.1] | 30.8% [26.9, 35.0] |
+| GSM8K, 4 choices | 25.0% | 24.7% [22.5, 27.1] | 24.3% [22.1, 26.7] |
+| GSM8K, 10 choices | 10.0% | 10.5% [9.0, 12.3] | 10.8% [9.3, 12.6] |
+
+The frozen-row construction audit passes its chance gate on all four tasks; see [`../eval/choice_panel/leak_audit.json`](../eval/choice_panel/leak_audit.json). OpenJEV 1.0 nevertheless scores above chance with options only on GPQA (32.3%) and Chess legal (30.8%). Its GPQA full-condition score is 32.8%, so that result alone does not establish use of the question. The GSM8K options-only scores are near chance. Chess legal measures legality rather than move quality; the GSM8K tasks do not measure open-ended solution generation.
+
+## Standard public tasks
+
+The following are single-order results on identical rows. They compare the complete native-readout Gemma system with OpenJEV 1.0. They do not isolate the training effect of the adapter or backbone delta. ARC, HellaSwag, and MMLU have training-family exposure; see [`BENCHMARK_EXPOSURE.md`](./BENCHMARK_EXPOSURE.md).
+
+| Task | Gemma base NF4 | OpenJEV 1.0 | Difference [95% CI] |
+|---|---:|---:|---:|
+| ARC-Easy | 95.58% | 95.62% | +0.04 [-0.80, +0.84] pp |
+| ARC-Challenge | 87.37% | 87.88% | +0.51 [-1.37, +2.39] pp |
+| WinoGrande | 60.14% | 66.46% | +6.31 [+3.79, +9.00] pp |
+| HellaSwag | 76.51% | 89.77% | +13.26 [+12.40, +14.06] pp |
+| MMLU | 66.19% | 64.41% | -1.77 [-2.58, -1.01] pp |
+
+The standard-task and choice-panel scores use different protocols. Cross-task averages require explicit weights and a statement of the protocol difference.

docs/MULTIMODAL_EXPERIMENTAL.md

@@ -0,0 +1,54 @@
+# Experimental image and audio input
+
+OpenJEV E4B 1.0 was trained and evaluated on text. The loader can optionally keep Gemma 4's own vision and audio encoders, so `choice()` also accepts an image or a short audio clip. This path is **experimental**: none of the adapter, backbone delta, or decision head was trained on image or audio input, and it has only passed the small smoke test below.
+
+## Use
+
+```python
+from openjev import OpenJEV
+
+model = OpenJEV.from_pretrained(".", device="cuda", load_mode="nf4", multimodal=True)
+
+model.choice(state="", image="photo.png",
+             instruction="What shape is shown in the image?",
+             options=["square", "circle", "triangle"])
+
+model.choice(state="", audio="call.wav",
+             instruction="Which support queue should handle the spoken message?",
+             options=["billing", "technical support", "account access", "sales"])
+```
+
+- `image` is a file path or a PIL image. `audio` is a PCM `.wav` path, or an array plus `sampling_rate=`; it is converted to 16 kHz mono.
+- `state` can hold extra text alongside the media. The media tokens are placed at the start of the state.
+- Without `multimodal=True`, passing `image=` or `audio=` raises an error. Text calls behave the same in both modes.
+
+## How it works
+
+The text-only loader discards Gemma's vision and audio encoders. With `multimodal=True`, it keeps them, wraps Gemma's text model in place with the OpenJEV adapter and backbone delta, and inserts Gemma's image or audio soft tokens after `State:` in the usual decision prompt. The decision head reads the same option and decision positions as for text.
+
+In NF4 mode the encoders stay in FP16, because Gemma's audio encoder cannot run with 4-bit weights. The text layers are quantized exactly as in the text-only loader.
+
+## Smoke test
+
+Run on 2026-09-27 with NF4 on an RTX 3060 12 GB. Images are generated shapes and printed messages. Audio is the same messages and short reviews spoken by the two built-in Windows voices. Every question was also asked with the media removed.
+
+| Test | With media | Media removed |
+|---|---:|---:|
+| Image: shape, 3 options | 12/12 | 4/12 |
+| Image: colour, 4 options | 12/12 | 3/12 |
+| Image: printed support message → queue, 4 options | 8/8 | 2/8 |
+| Audio: spoken support message → queue, 4 options | 16/16 | 2/8 |
+| Audio: spoken review → sentiment, 2 options | 12/12 | 3/6 |
+
+- For comparison, the same support messages given as text scored 7/8, and the reviews as text scored 6/6.
+- For text-only input, the multimodal forward path gives the same probabilities as the text path (maximum difference 0).
+- Median latency per call: about 4.9 s with an image, 2.3 s with audio, and 0.2 s for text. Peak GPU memory was 10.7 GB.
+
+Full rows are in [`eval/multimodal_smoke/results.json`](../eval/multimodal_smoke/results.json). To rerun: run `eval/multimodal_smoke/make_audio_windows.ps1`, then `python -m eval.multimodal_smoke.run_smoke` from the release root.
+
+## Limits
+
+- These inputs are clean, synthetic, and easy. The test shows that the path works end to end; it is not a benchmark, and there are no results on real photos, noisy audio, or public image or audio datasets.
+- Probabilities use the text calibration, which has not been checked for image or audio input.
+- Only NF4 on one GPU was tested; `load_mode="bf16"` with `multimodal=True` has not been run.
+- One image and one audio clip per call. Video is not supported.

docs/OpenJEV_E4B_Experiment_1_Whitepaper.pdf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f441acf38e8354254070f1d429fd9345dedbdd5ee6d30b4be278fc07dea8d504
+size 56722

docs/README.md

@@ -0,0 +1,17 @@
+# Documentation — OpenJEV E4B 1.0
+
+## Experiment 1 whitepaper
+
+**OpenJEV E4B: Training a direct-decision readout on Gemma 4 E4B. Experiment 1 method, evaluation, and results.**
+
+PDF: [`OpenJEV_E4B_Experiment_1_Whitepaper.pdf`](./OpenJEV_E4B_Experiment_1_Whitepaper.pdf)
+
+Source: [`source/OpenJEV_E4B_Experiment_1_Whitepaper.docx`](./source/OpenJEV_E4B_Experiment_1_Whitepaper.docx) and [`source/build_whitepaper.js`](./source/build_whitepaper.js).
+
+The report covers the direct-decision interface, development sequence and promotion rule, the choice-panel evaluation, five standard public tasks, limitations, release records, and reproduction steps. Gemma and OpenJEV 1.0 have complete same-row choice-panel results, including options-only controls and paired comparisons.
+
+Current results are in [`EXPERIMENT_1_RESULTS.md`](./EXPERIMENT_1_RESULTS.md). Frozen rows, the construction audit, scoring code, and row-level outputs are in [`../eval/choice_panel/`](../eval/choice_panel/).
+
+## Experimental image and audio input
+
+[`MULTIMODAL_EXPERIMENTAL.md`](./MULTIMODAL_EXPERIMENTAL.md) describes the optional `multimodal=True` loader path, its smoke test, and its limits.

docs/source/build_whitepaper.js

@@ -0,0 +1,207 @@
+// Build the Experiment 1 report from the packaged choice-panel summary.
+// Usage: node build_whitepaper.js path/to/output.docx
+const fs = require("fs");
+const path = require("path");
+const {
+  Document, Packer, Paragraph, TextRun, Table, TableRow, TableCell,
+  WidthType, BorderStyle, AlignmentType, HeadingLevel, Footer, PageNumber,
+} = require("docx");
+
+const OUT = process.argv[2] || "OpenJEV_E4B_Experiment_1_Whitepaper.docx";
+const EVAL = path.resolve(__dirname, "../../eval/choice_panel");
+const summary = JSON.parse(fs.readFileSync(path.join(EVAL, "summary.json"), "utf8"));
+const audit = JSON.parse(fs.readFileSync(path.join(EVAL, "leak_audit.json"), "utf8"));
+const rowsHash = fs.readFileSync(path.join(EVAL, "rows.sha256"), "utf8").trim().split(/\s+/)[0];
+const W = 9360;
+const body = [];
+
+function P(value, options = {}) {
+  body.push(new Paragraph({
+    children: [new TextRun({ text: value, bold: !!options.bold, size: options.size || 20 })],
+    spacing: { after: options.after === undefined ? 135 : options.after, line: 285 },
+    keepNext: !!options.keepNext,
+  }));
+}
+function H1(value) { body.push(new Paragraph({ heading: HeadingLevel.HEADING_1, text: value })); }
+function H2(value) { body.push(new Paragraph({ heading: HeadingLevel.HEADING_2, text: value })); }
+function Bullets(values) {
+  for (const value of values) body.push(new Paragraph({
+    text: "• " + value, indent: { left: 320, hanging: 200 },
+    spacing: { after: 65, line: 275 },
+  }));
+}
+function T(headers, rows, widths, fontSize = 16) {
+  const total = widths.reduce((a, b) => a + b, 0);
+  const col = widths.map(x => Math.round(W * x / total));
+  col[col.length - 1] += W - col.reduce((a, b) => a + b, 0);
+  const border = { style: BorderStyle.SINGLE, color: "D9D9D9", size: 4 };
+  const borders = { top: border, bottom: border, left: border, right: border };
+  const cell = (value, i, header) => new TableCell({
+    width: { size: col[i], type: WidthType.DXA }, borders,
+    shading: { fill: header ? "DCE6F1" : "FFFFFF" },
+    margins: { top: 90, bottom: 90, left: 95, right: 95 },
+    children: String(value).split("\n").map(line => new Paragraph({
+      children: [new TextRun({ text: line, bold: header, size: fontSize })],
+      alignment: i === 0 ? AlignmentType.LEFT : AlignmentType.CENTER,
+      spacing: { after: 0, line: 235 },
+    })),
+  });
+  body.push(new Table({
+    width: { size: W, type: WidthType.DXA }, columnWidths: col,
+    rows: [
+      new TableRow({ tableHeader: true, cantSplit: true, children: headers.map((h, i) => cell(h, i, true)) }),
+      ...rows.map(r => new TableRow({ cantSplit: true, children: r.map((v, i) => cell(v, i, false)) })),
+    ],
+  }));
+  P("", { after: 70 });
+}
+function pct(value, digits = 1) { return (100 * value).toFixed(digits) + "%"; }
+function score(model, stage, task, expected) {
+  const value = summary.tasks[model + "|" + stage + "|" + task];
+  return value && value.rows === expected ? value : null;
+}
+function fmt(value) {
+  return value ? pct(value.debiased_accuracy) + " [" + pct(value.debiased_ci95[0]) + ", " + pct(value.debiased_ci95[1]) + "]" : "In progress";
+}
+function delta(value) {
+  return value ? (value.delta_pp >= 0 ? "+" : "") + value.delta_pp.toFixed(2) + " [" +
+    value.ci95_pp[0].toFixed(2) + ", " + value.ci95_pp[1].toFixed(2) + "] pp" : "In progress";
+}
+
+const tasks = [
+  ["GPQA Diamond", "gpqa_v2", 198, 4],
+  ["Chess legal", "chess_legal_v2", 500, 4],
+  ["GSM8K 4 choices", "gsm8k_mc4_v2", 1319, 4],
+  ["GSM8K 10 choices", "gsm8k_mc10_v2", 1319, 10],
+];
+const COMPLETE = ["raw", "r7"].every(model => tasks.every(([, id, n]) =>
+  score(model, "full", id, n) && score(model, "options_only", id, n)));
+
+P("OpenJEV E4B 1.0", { bold: true, size: 46, after: 60 });
+P("Training a direct-decision readout on Gemma 4 E4B", { size: 31, after: 120 });
+P("Experiment 1 method, evaluation, and results", { size: 26, after: 320 });
+P("27 September 2026 • Public release 1.0");
+P("Technical report accompanying the bambamdevs/openjev-e4b model release. OpenJEV is independent research and is not affiliated with TypeSafe AI or the unrelated AlexWortega/openjev project.");
+
+H1("Abstract");
+P("OpenJEV E4B converts google/gemma-4-E4B-it into a closed-choice decision system. Given state, a question, and options, it returns a probability distribution from a direct head without generating text. Experiment 1 trained a LoRA adapter and hybrid decision head, then used exact-oracle Arena data, promotion gates, and selective decoder-layer updates. This report describes the checkpoint selected for public release 1.0.");
+P("The choice-panel evaluation freezes shuffled GPQA rows, GSM8K options drawn from other problems, and a Chess legal-move task. All options are evaluated in cyclic order rotations. The main score chooses the largest probability after averaging over rotations; an options-only condition and construction audit test whether choice strings reveal the answer. On untouched Gemma, rotation-averaged accuracy is 32.8% on GPQA, 43.8% on Chess legal, 43.7% on GSM8K-4, and 20.5% on GSM8K-10. The same-answer rate across orders ranges from 0.6% to 10.1%. " +
+  (COMPLETE ? "OpenJEV 1.0 reaches 53.8% on GSM8K-4 and 30.6% on GSM8K-10, gains of about 10.1 percentage points on each task. GPQA is unchanged at 32.8% and Chess legal is 42.6% versus Gemma's 43.8%. Its options-only GPQA score is 32.3%, limiting what the full GPQA result can show about use of the question. Paired comparisons follow." : "OpenJEV 1.0 scoring is in progress; this report shows only completed choice-panel results."));
+P("Five standard public tasks use native option order. They give a separate comparison between the native Gemma LM-head readout and the OpenJEV decision head. The report records family exposure and precision differences and states what the evidence can support.");
+
+H1("1 Research question and model");
+P("The research question is whether a small open base model can serve as a typed decision component: choose among caller-provided options and expose probabilities that software can inspect. A request contains State, Question, Options, and a final Decision marker. The decision query uses the hidden state at the final marker; each option uses its final-token state and a span mean.");
+P("The model extracts the text decoder from Gemma 4 E4B and attaches LoRA with rank 16, alpha 32, and zero dropout. Its hybrid head combines a normalized pointer score and a span-residual MLP, then applies softmax with released temperature 1.0. Gemma shares key/value states across layers 24–41, so the adapter and cumulative delta have no key/value projections in those layers. The public checkpoint consists of the pinned Gemma base, adapter, head, 65-tensor cumulative backbone delta, calibration, and packing code. Restoring only the adapter does not restore the model.");
+
+H2("1.1 Model evolution and promotion");
+P("Development began with a direct readout and LoRA proof of concept, expanded to a production training mixture and hard negatives, compared readout designs, and selected a proper-scoring hybrid head plus LoRA. The later Arena curriculum generated 24,000 exact-oracle examples across 20 decision families per internal training cycle and mixed them with replay. Early Arena training changed the head and LoRA; later cycles also trained selected decoder layers. Accepted base changes accumulate into the public checkpoint's 65 delta tensors across layers 37–41. Internal evaluation populations changed during development, so their percentages are not a controlled public comparison.");
+P("A challenger could be promoted only when validation accuracy, schema accuracy, firewall accuracy, order consistency, and validation log loss stayed within recorded preservation margins. The promotion score weighted gains in Arena performance, order consistency, schema, firewall, validation accuracy, and validation NLL; acceptance required all checks and a score above 0.001. The selected public checkpoint's stored score is 0.024174. The gate record preserves the score and pass flags, but not every component value.");
+
+H1("2 Evaluation protocol");
+H2("2.1 Choice-panel tasks");
+T(["Task", "Rows", "Options", "Row construction"], [
+  ["GPQA Diamond", "198", "4", "Source answers shuffled with a fixed per-row seed"],
+  ["GSM8K 4", "1,319", "4", "Gold plus distinct answers to other test problems of similar magnitude"],
+  ["GSM8K 10", "1,319", "10", "Same construction with nine distractors"],
+  ["Chess legal", "500", "4", "One legal plain-SAN move and three plain-SAN moves from other positions, illegal here"],
+], [1.4, 0.7, 1.15, 6.15]);
+P("Frozen rows are published as eval/choice_panel/rows.jsonl (SHA-256 " + rowsHash + "). GPQA comes from the 198-row Diamond mirror, GSM8K from the main test split, and Chess legal from seeded synthetic positions. Chess asks which move is legal in the position, not which is best. The GSM8K tasks are multiple-choice arithmetic, not open-ended solution generation.");
+P("Every full-condition row is presented in all cyclic option rotations: four for GPQA, Chess legal, and GSM8K-4; ten for GSM8K-10. Probabilities are mapped to original option identities before averaging. The main accuracy applies argmax to that per-row mean. Mean per-order accuracy averages individual presentations. Same-answer rate measures whether the predicted option identity is unchanged in every rotation. Options-only replaces state and question with neutral text, using four rotations for four choices and five evenly spaced rotations for ten choices.");
+P("The construction gate checks gold-position balance, fixed choice-string heuristics, and a cross-validated logistic scorer on choice-only features. The maximum heuristic score stays within its specified 95% chance band on all four tasks. That gate tests row construction; model options-only scores are reported separately.");
+T(["Task", "Chance", "Gold-position range", "Highest construction heuristic", "Gate"],
+  tasks.map(([name, id, , k]) => {
+    const x = audit[id];
+    return [name, pct(1 / k), pct(Math.min(...x.gold_position_share)) + "–" + pct(Math.max(...x.gold_position_share)), pct(x.max_heuristic), x.leak_gate];
+  }), [2.1, 1, 2, 2.2, 0.9]);
+
+H2("2.2 Systems and uncertainty");
+P("Untouched Gemma uses native LM-head answer-letter scoring in NF4. OpenJEV 1.0 uses the frozen base in NF4, delta-touched decoder modules restored in dense FP16, its LoRA adapter, and an FP32 decision head. These are complete inference systems with different readouts. Wilson 95% intervals use rows as the unit of observation. Paired differences use identical rows, row bootstrap intervals, and exact McNemar tests; rotations of one row are not independent observations. Evaluation ran on a Windows RTX 3060 with local cached weights.");
+
+H1("3 Choice-panel results");
+T(["Task", "Gemma base [95% CI]", "OpenJEV 1.0 [95% CI]", "Gemma per order / same", "OpenJEV per order / same"],
+  tasks.map(([name, id, n]) => {
+    const raw = score("raw", "full", id, n), model = score("r7", "full", id, n);
+    const order = x => x ? pct(x.rotation_mean_accuracy) + " / " + pct(x.order_consistency) : "In progress";
+    return [name, fmt(raw), fmt(model), order(raw), order(model)];
+  }), [1.6, 2.3, 2.3, 1.9, 1.9], 15);
+P("The main score uses one averaged probability vector per row. The per-order and same-answer columns show option-order sensitivity. Same-row paired differences appear in Section 3.2 when both systems are complete.");
+
+H2("3.1 Options-only control");
+T(["Task", "Chance", "Gemma base [95% CI]", "OpenJEV 1.0 [95% CI]"],
+  tasks.map(([name, id, n, k]) => [name, pct(1 / k), fmt(score("raw", "options_only", id, n)), fmt(score("r7", "options_only", id, n))]),
+  [2.1, 1, 2.7, 2.7], 15);
+P("Gemma's options-only scores are close to chance on GPQA and the two GSM8K tasks; Chess legal is 29.0%. OpenJEV 1.0 scores 32.3% on GPQA and 30.8% on Chess legal with only options, both above 25% chance. Its full GPQA score is 32.8%, so this result alone does not establish use of the question. OpenJEV's GSM8K options-only scores are near chance. The construction audit passes its gate, but model behavior reveals additional choice-only signal. This control does not explain individual full-condition decisions.");
+if (COMPLETE) {
+  H2("3.2 Paired comparisons");
+  T(["Task", "OpenJEV 1.0 − Gemma [95% CI]", "McNemar p"], tasks.map(([name, id]) => {
+    const pair = (summary.pairs || {})["raw->r7|full|" + id];
+    return [name, delta(pair), pair ? pair.mcnemar_p.toPrecision(2) : "In progress"];
+  }), [2.1, 5.1, 1.1], 15);
+  P("Intervals are paired row-bootstrap intervals with 2,000 resamples. Exact McNemar tests use discordant row-level correct/incorrect outcomes after rotation averaging. The comparison measures complete decision systems.");
+}
+
+H1("4 Standard public tasks");
+P("ARC-Easy, ARC-Challenge, WinoGrande, HellaSwag, and MMLU use the public-task runner and native option order. The table compares identical rows under that protocol. ARC, HellaSwag, and MMLU have training-family exposure. These scores should not be pooled with the rotation-averaged choice-panel scores as though they shared one protocol.");
+T(["Task", "Gemma NF4", "OpenJEV 1.0", "Difference [95% CI]"], [
+  ["ARC-Easy", "95.58%", "95.62%", "+0.04 [-0.80, +0.84] pp"],
+  ["ARC-Challenge", "87.37%", "87.88%", "+0.51 [-1.37, +2.39] pp"],
+  ["WinoGrande", "60.14%", "66.46%", "+6.31 [+3.79, +9.00] pp"],
+  ["HellaSwag", "76.51%", "89.77%", "+13.26 [+12.40, +14.06] pp"],
+  ["MMLU", "66.19%", "64.41%", "-1.77 [-2.58, -1.01] pp"],
+], [2, 1.5, 1.7, 4]);
+P("Under native order, OpenJEV and Gemma are similar on ARC-Easy and ARC-Challenge; OpenJEV is higher on WinoGrande and HellaSwag and lower on MMLU. HellaSwag and ARC have same-family training exposure. OpenJEV's WinoGrande NLL is 0.873, above the uniform binary reference ln 2 = 0.693; the released temperature is 1.0, and calibration does not transfer uniformly across tasks.");
+
+H1("5 Interpretation and next controls");
+P("Experiment 1 produces a working direct-decision model and an inspectable release record. The choice panel is the evidence for GPQA, closed-choice GSM8K, and Chess legal. Standard tasks provide separate end-to-end comparisons under native order. Current comparisons cannot attribute differences to the head, adapter, backbone updates, or inference precision separately.");
+Bullets([
+  ...(!COMPLETE ? ["Complete the OpenJEV 1.0 choice-panel run and analyze paired rows, options-only performance, and order consistency."] : []),
+  "Score a frozen Gemma backbone with a trained decision head to separate readout effects from adapter and backbone changes.",
+  "Match inference precision across controlled checkpoints before attributing a difference to selective backbone training.",
+  "Use an open-ended GSM8K reference to characterize the cost of a closed-choice interface on arithmetic generation.",
+  "Reserve a knowledge panel unused for model selection before making confirmatory preservation claims.",
+]);
+
+H1("Appendix A Release and reproduction record");
+P("The base is google/gemma-4-E4B-it at revision ee0ef6023621cff504d758262d4e04895a5af4a2. The public 1.0 release maps to internal checkpoint v1.6 R7, with stored Arena score 0.024174. It has 65 cumulative delta tensors in decoder layers 37–41 and global temperature 1.0. The repository includes the adapter, head, delta, loader, calibration, row-level evaluation artifacts, and a SHA-256 manifest. Training code is not included.");
+P("The choice-panel row source and scoring scripts are in eval/choice_panel. The standard-task harness is in eval/harness. Source-checkpoint and release-file hashes are in RELEASE_METADATA.json and MANIFEST.sha256. Training-family overlap notes are in docs/BENCHMARK_EXPOSURE.md. The restoration and scoring contract is in REPRODUCIBILITY.md.");
+P("Valid restoration requires the pinned base, adapter, head, cumulative delta, calibration, and packer. NF4 evaluation runs the frozen base in 4-bit with FP16 compute and touched decoder modules in dense FP16. Dense BF16 inference is supported by the loader but has not been publicly benchmarked.");
+
+H1("References");
+P("Google. Gemma 4 E4B-it model card. https://huggingface.co/google/gemma-4-E4B-it");
+P("Rein et al. GPQA: A Graduate-Level Google-Proof Q&A Benchmark. arXiv:2311.12022, 2023.");
+P("Cobbe et al. Training Verifiers to Solve Math Word Problems. arXiv:2110.14168, 2021.");
+P("McNemar. Note on the sampling error of the difference between correlated proportions or percentages. Psychometrika 12, 1947.");
+P("Guo et al. On Calibration of Modern Neural Networks. ICML 2017.");
+
+const doc = new Document({
+  title: "OpenJEV E4B 1.0 Experiment 1 method, evaluation, and results",
+  creator: "OpenJEV E4B",
+  description: "OpenJEV E4B 1.0 Experiment 1 technical report",
+  styles: {
+    default: { document: { run: { font: "Calibri", size: 20, color: "000000" } } },
+    paragraphStyles: [
+      { id: "Heading1", name: "Heading 1", basedOn: "Normal", next: "Normal", quickFormat: true,
+        run: { font: "Calibri", size: 29, bold: true, color: "000000" },
+        paragraph: { spacing: { before: 300, after: 120 }, outlineLevel: 0 } },
+      { id: "Heading2", name: "Heading 2", basedOn: "Normal", next: "Normal", quickFormat: true,
+        run: { font: "Calibri", size: 24, bold: true, color: "000000" },
+        paragraph: { spacing: { before: 220, after: 90 }, outlineLevel: 1 } },
+    ],
+  },
+  sections: [{
+    properties: { page: { size: { width: 12240, height: 15840 },
+      margin: { top: 1260, right: 1440, bottom: 1260, left: 1440 } } },
+    footers: { default: new Footer({ children: [new Paragraph({
+      alignment: AlignmentType.CENTER,
+      children: [new TextRun({ text: "OpenJEV E4B  •  Experiment 1  •  1.0  •  ", size: 16, color: "666666" }),
+        new TextRun({ children: [PageNumber.CURRENT], size: 16, color: "666666" })],
+    })] }) },
+    children: body,
+  }],
+});
+
+Packer.toBuffer(doc).then(buffer => {
+  fs.mkdirSync(path.dirname(path.resolve(OUT)), { recursive: true });
+  fs.writeFileSync(OUT, buffer);
+  console.log("Wrote " + OUT + " (" + buffer.length + " bytes)");
+});

eval/analysis/contamination_audit.json

@@ -0,0 +1,70 @@
+{
+  "method": "Row-level text-overlap audit between training source splits and public panel rows.",
+  "sources": {
+    "mmlu_auxiliary_train": 98370,
+    "arc_train": 3361,
+    "hellaswag_train": 39799
+  },
+  "overlap": {
+    "mmlu": {
+      "n": 14042,
+      "mmlu_auxiliary_train": {
+        "question": 19,
+        "question_and_options": 0
+      },
+      "arc_train": {
+        "question": 1,
+        "question_and_options": 0
+      },
+      "hellaswag_train": {
+        "question": 0,
+        "question_and_options": 0
+      }
+    },
+    "arc_easy": {
+      "n": 2376,
+      "mmlu_auxiliary_train": {
+        "question": 8,
+        "question_and_options": 3
+      },
+      "arc_train": {
+        "question": 7,
+        "question_and_options": 3
+      },
+      "hellaswag_train": {
+        "question": 0,
+        "question_and_options": 0
+      }
+    },
+    "arc_challenge": {
+      "n": 1172,
+      "mmlu_auxiliary_train": {
+        "question": 9,
+        "question_and_options": 0
+      },
+      "arc_train": {
+        "question": 9,
+        "question_and_options": 0
+      },
+      "hellaswag_train": {
+        "question": 0,
+        "question_and_options": 0
+      }
+    },
+    "hellaswag": {
+      "n": 10042,
+      "mmlu_auxiliary_train": {
+        "question": 0,
+        "question_and_options": 0
+      },
+      "arc_train": {
+        "question": 0,
+        "question_and_options": 0
+      },
+      "hellaswag_train": {
+        "question": 23,
+        "question_and_options": 0
+      }
+    }
+  }
+}

eval/analysis/contamination_audit.py

@@ -0,0 +1,84 @@
+"""Row-level text-overlap audit between training source splits and public panel rows.
+
+Training sources are the full upstream splits the compilers sample from
+(src/compile_datasets.py): cais/mmlu auxiliary_train, allenai/ai2_arc train,
+Rowan/hellaswag train. Using the full split is an upper bound on the sampled
+training rows. Public rows are regenerated with eval/harness/benchmark_public.py.
+
+A row matches when its normalized question text (lowercase, alphanumerics
+only, collapsed whitespace) hashes identically. "question+options" additionally
+requires the same normalized option multiset.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import json
+import re
+import sys
+from pathlib import Path
+
+from datasets import load_dataset
+
+HARNESS = Path(__file__).resolve().parents[1] / "harness"
+sys.path.insert(0, str(HARNESS))
+
+import benchmark_public as bp  # noqa: E402
+
+
+def norm(text: str) -> str:
+    return " ".join(re.sub(r"[^a-z0-9]+", " ", str(text).lower()).split())
+
+
+def h(text: str) -> str:
+    return hashlib.sha256(norm(text).encode()).hexdigest()
+
+
+def hq(question: str, options) -> str:
+    return h(question) + ":" + h("|".join(sorted(norm(o) for o in options)))
+
+
+def train_sources() -> dict[str, tuple[set[str], set[str]]]:
+    out = {}
+    aux = load_dataset("cais/mmlu", "all", split="auxiliary_train")
+    out["mmlu_auxiliary_train"] = (
+        {h(x["question"]) for x in aux},
+        {hq(x["question"], x["choices"]) for x in aux},
+    )
+    q, qo = set(), set()
+    for cfg in ("ARC-Challenge", "ARC-Easy"):
+        for x in load_dataset("allenai/ai2_arc", cfg, split="train"):
+            q.add(h(x["question"]))
+            qo.add(hq(x["question"], x["choices"]["text"]))
+    out["arc_train"] = (q, qo)
+    hs = load_dataset("Rowan/hellaswag", split="train")
+    out["hellaswag_train"] = (
+        {h(x["ctx"]) for x in hs},
+        {hq(x["ctx"], x["endings"]) for x in hs},
+    )
+    return out
+
+
+def main() -> None:
+    sources = train_sources()
+    panel = {
+        "mmlu": bp.load_mmlu(),
+        "arc_easy": bp.load_arc("ARC-Easy"),
+        "arc_challenge": bp.load_arc("ARC-Challenge"),
+        "hellaswag": bp.load_hellaswag(),
+    }
+    report = {"method": __doc__.strip().splitlines()[0], "sources": {k: len(v[0]) for k, v in sources.items()}, "overlap": {}}
+    for task, items in panel.items():
+        report["overlap"][task] = {"n": len(items)}
+        for src, (qset, qoset) in sources.items():
+            # HellaSwag panel rows carry the context in the question field.
+            q_hits = sum(h(it["question"]) in qset or h(it["state"]) in qset for it in items)
+            qo_hits = sum(hq(it["question"], it["options"]) in qoset or hq(it["state"], it["options"]) in qoset for it in items)
+            report["overlap"][task][src] = {"question": q_hits, "question_and_options": qo_hits}
+    print(json.dumps(report, indent=2))
+    if len(sys.argv) > 1:
+        Path(sys.argv[1]).write_text(json.dumps(report, indent=2), encoding="utf-8")
+
+
+if __name__ == "__main__":
+    main()

eval/analysis/paired_analysis.py

@@ -0,0 +1,165 @@
+"""Row-level paired analysis for public-benchmark runs.
+
+Usage:
+    python scripts/paired_analysis.py OUT_DIR NAME=details.jsonl [NAME=details.jsonl ...]
+
+For every run and task: accuracy with a Wilson 95% CI, an exact binomial test
+against chance, NLL against the ln(k) uniform reference, ECE (15 equal-width
+bins, as in eval/harness) with a percentile-bootstrap 95% CI, and the
+distribution of predicted positions next to the gold distribution.
+
+For every pair of runs: the paired accuracy delta with a paired-bootstrap 95%
+CI, the exact McNemar test, and the 2x2 flip matrix.
+"""
+
+from __future__ import annotations
+
+import itertools
+import json
+import math
+import sys
+from collections import Counter, defaultdict
+from pathlib import Path
+
+import numpy as np
+from scipy.stats import binomtest
+
+BOOTSTRAP = 2000
+BINS = 15
+SEED = 0
+
+
+def wilson(k: int, n: int, z: float = 1.959964) -> tuple[float, float]:
+    p = k / n
+    d = 1 + z * z / n
+    c = (p + z * z / (2 * n)) / d
+    h = z * math.sqrt(p * (1 - p) / n + z * z / (4 * n * n)) / d
+    return c - h, c + h
+
+
+def ece(conf: np.ndarray, correct: np.ndarray) -> float:
+    total = 0.0
+    for b in range(BINS):
+        lo, hi = b / BINS, (b + 1) / BINS
+        m = (conf >= lo) & ((conf < hi) if b < BINS - 1 else (conf <= hi))
+        if m.any():
+            total += m.mean() * abs(conf[m].mean() - correct[m].mean())
+    return float(total)
+
+
+def mcnemar_exact(b: int, c: int) -> float:
+    n = b + c
+    return 1.0 if n == 0 else float(binomtest(min(b, c), n, 0.5).pvalue)
+
+
+def load(path: Path) -> dict[str, dict[int, dict]]:
+    by: dict[str, dict[int, dict]] = defaultdict(dict)
+    with path.open(encoding="utf-8") as fp:
+        for line in fp:
+            r = json.loads(line)
+            by[r["task"]][int(r["index"])] = r
+    return by
+
+
+def run_stats(rows: list[dict], rng: np.random.Generator) -> dict:
+    n = len(rows)
+    k = int(rows[0]["option_count"])
+    correct = np.array([bool(r["correct"]) for r in rows], dtype=float)
+    conf = np.array([float(r["confidence"]) for r in rows])
+    nll = np.array([-math.log(max(float(r["p_gold"]), 1e-12)) for r in rows])
+    hits = int(correct.sum())
+    lo, hi = wilson(hits, n)
+    idx = rng.integers(0, n, size=(BOOTSTRAP, n))
+    eces = [ece(conf[i], correct[i]) for i in idx]
+    gold = Counter(int(r["gold"]) for r in rows)
+    pred = Counter(int(r["pred"]) for r in rows)
+    return {
+        "n": n,
+        "k": k,
+        "accuracy": hits / n,
+        "accuracy_ci95": [lo, hi],
+        "chance": 1 / k,
+        "binom_p_vs_chance_two_sided": float(binomtest(hits, n, 1 / k).pvalue),
+        "nll": float(nll.mean()),
+        "ln_k": math.log(k),
+        "nll_worse_than_uniform": bool(nll.mean() > math.log(k)),
+        "ece_15bin": ece(conf, correct),
+        "ece_15bin_ci95": [float(np.percentile(eces, 2.5)), float(np.percentile(eces, 97.5))],
+        "gold_position_share": [gold[i] / n for i in range(k)],
+        "pred_position_share": [pred[i] / n for i in range(k)],
+        "max_pred_position_share": max(pred[i] / n for i in range(k)),
+    }
+
+
+def pair_stats(a: list[dict], b: list[dict], rng: np.random.Generator) -> dict:
+    ca = np.array([bool(r["correct"]) for r in a], dtype=float)
+    cb = np.array([bool(r["correct"]) for r in b], dtype=float)
+    n = len(ca)
+    diff = cb - ca
+    idx = rng.integers(0, n, size=(BOOTSTRAP, n))
+    boots = diff[idx].mean(axis=1)
+    both = int(((ca == 1) & (cb == 1)).sum())
+    a_only = int(((ca == 1) & (cb == 0)).sum())
+    b_only = int(((ca == 0) & (cb == 1)).sum())
+    neither = int(((ca == 0) & (cb == 0)).sum())
+    return {
+        "n": n,
+        "delta_pp": 100 * float(diff.mean()),
+        "delta_pp_ci95": [100 * float(np.percentile(boots, 2.5)), 100 * float(np.percentile(boots, 97.5))],
+        "flip": {"both": both, "a_only": a_only, "b_only": b_only, "neither": neither},
+        "net_rows": b_only - a_only,
+        "mcnemar_exact_p": mcnemar_exact(a_only, b_only),
+    }
+
+
+def main() -> None:
+    out_dir = Path(sys.argv[1])
+    runs = {}
+    for arg in sys.argv[2:]:
+        name, path = arg.split("=", 1)
+        runs[name] = load(Path(path))
+    out_dir.mkdir(parents=True, exist_ok=True)
+    rng = np.random.default_rng(SEED)
+
+    tasks = sorted(set.intersection(*(set(r) for r in runs.values())), key=list(next(iter(runs.values()))).index)
+    report = {"bootstrap": BOOTSTRAP, "ece_bins": BINS, "seed": SEED, "runs": {}, "pairs": {}}
+    for name, by in runs.items():
+        report["runs"][name] = {t: run_stats([by[t][i] for i in sorted(by[t])], rng) for t in tasks}
+    for a, b in itertools.combinations(runs, 2):
+        key = f"{a}->{b}"
+        report["pairs"][key] = {}
+        for t in tasks:
+            ids = sorted(set(runs[a][t]) & set(runs[b][t]))
+            for i in ids:
+                if runs[a][t][i]["gold"] != runs[b][t][i]["gold"]:
+                    raise SystemExit(f"gold mismatch {t}:{i} between {a} and {b}")
+            report["pairs"][key][t] = pair_stats([runs[a][t][i] for i in ids], [runs[b][t][i] for i in ids], rng)
+
+    (out_dir / "paired_analysis.json").write_text(json.dumps(report, indent=2), encoding="utf-8")
+
+    lines = ["# Paired analysis", "", f"Bootstrap resamples: {BOOTSTRAP}; ECE: {BINS} equal-width bins; seed {SEED}.", ""]
+    for name, stats in report["runs"].items():
+        lines += [f"## {name}", "", "| Task | n | k | Acc | 95% CI | p vs chance | NLL | ln k | ECE (95% CI) | max pred-position share |",
+                  "|---|---:|---:|---:|---|---:|---:|---:|---|---:|"]
+        for t, s in stats.items():
+            lines.append(
+                f"| {t} | {s['n']} | {s['k']} | {100*s['accuracy']:.2f}% | [{100*s['accuracy_ci95'][0]:.1f}, {100*s['accuracy_ci95'][1]:.1f}] "
+                f"| {s['binom_p_vs_chance_two_sided']:.2g} | {s['nll']:.3f}{' (>ln k)' if s['nll_worse_than_uniform'] else ''} | {s['ln_k']:.3f} "
+                f"| {s['ece_15bin']:.3f} [{s['ece_15bin_ci95'][0]:.3f}, {s['ece_15bin_ci95'][1]:.3f}] | {100*s['max_pred_position_share']:.0f}% |"
+            )
+        lines.append("")
+    for key, stats in report["pairs"].items():
+        a, b = key.split("->")
+        lines += [f"## {a} -> {b}", "", f"| Task | Δ pp | 95% CI | {a} only | {b} only | net rows | McNemar p |", "|---|---:|---|---:|---:|---:|---:|"]
+        for t, s in stats.items():
+            lines.append(
+                f"| {t} | {s['delta_pp']:+.2f} | [{s['delta_pp_ci95'][0]:+.2f}, {s['delta_pp_ci95'][1]:+.2f}] "
+                f"| {s['flip']['a_only']} | {s['flip']['b_only']} | {s['net_rows']:+d} | {s['mcnemar_exact_p']:.2g} |"
+            )
+        lines.append("")
+    (out_dir / "paired_analysis.md").write_text("\n".join(lines), encoding="utf-8")
+    print("\n".join(lines))
+
+
+if __name__ == "__main__":
+    main()

eval/choice_panel/README.md

@@ -0,0 +1,11 @@
+# Choice-panel evaluation
+
+`rows.jsonl` contains the frozen GPQA Diamond, GSM8K-4, GSM8K-10, and Chess legal rows. `rows.sha256` hashes this published four-task subset. `source_run.json` identifies the source run and larger local row-file hash. `leak_audit.json` records answer-position and choice-string checks. All four tasks pass the specified chance-band gate.
+
+`details_raw.jsonl` contains untouched Gemma output. `details_r7.jsonl` is the OpenJEV 1.0 output under its internal checkpoint identifier. A detail file is included only when all four full-condition and options-only tasks finish. `summary.json` and `summary.md` give rotation-averaged accuracy, row-level Wilson intervals, per-order accuracy, order consistency, and paired comparisons when both systems are complete. The internal identifier maps to public release 1.0 in `RELEASE_METADATA.json`.
+
+The full condition rotates all options: four presentations per row on GPQA, Chess legal and GSM8K-4, ten on GSM8K-10. The options-only condition removes state and question and uses four or five rotations. Probabilities are mapped back to original option identities before the per-row mean is taken. The main score is the argmax of that mean; every statistical observation is a row.
+
+`run_choice_panel.py` is the internal scoring driver and `summarize.py` regenerates the summary. The driver expects the local OpenJEV benchmark bundle and untouched-Gemma benchmark bundle. Set `OPENJEV_BENCH_ROOT` and `RAW_GEMMA_BENCH_ROOT` to those directories, then run it with the same Python environment and cached Gemma weights. Released model files and checkpoint hashes are in `model/` and `RELEASE_METADATA.json`.
+
+The row generator used GPQA's Diamond mirror, GSM8K's main test split, and python-chess. GPQA answer positions were shuffled with fixed row seeds. GSM8K distractors were sampled from the final answers of other test questions in matching magnitude buckets. Chess legal choices consist of one legal plain-SAN move and three plain-SAN strings drawn from other same-side positions that are illegal in the current position. The frozen rows and their hash are the evaluation identity.

eval/choice_panel/details_r7.jsonl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e905f991f9d5685ad851207bb0430b9f40074b84066bc67439c559f3ec7b8027
+size 11162773

eval/choice_panel/details_raw.jsonl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8bf0ec06599e55e743fcbdaca736ff211819bc63e03af765ae62e67c26e6605a
+size 11187992

eval/choice_panel/leak_audit.json

@@ -0,0 +1,87 @@
+{
+  "gpqa_v2": {
+    "n": 198,
+    "k": 4,
+    "chance": 0.25,
+    "chance_upper95": 0.310314830579873,
+    "gold_position_share": [
+      0.2777777777777778,
+      0.22727272727272727,
+      0.23737373737373738,
+      0.25757575757575757
+    ],
+    "heuristics": {
+      "first_slot": 0.2777777777777778
+    },
+    "max_heuristic": 0.2777777777777778,
+    "leak_gate": "PASS"
+  },
+  "chess_legal_v2": {
+    "n": 500,
+    "k": 4,
+    "chance": 0.25,
+    "chance_upper95": 0.2879552367928327,
+    "gold_position_share": [
+      0.266,
+      0.226,
+      0.222,
+      0.286
+    ],
+    "heuristics": {
+      "san_shape": 0.25,
+      "cv_logistic_features": 0.204
+    },
+    "max_heuristic": 0.25,
+    "leak_gate": "PASS"
+  },
+  "gsm8k_mc4_v2": {
+    "n": 1319,
+    "k": 4,
+    "chance": 0.25,
+    "chance_upper95": 0.27336868688896165,
+    "gold_position_share": [
+      0.2539802880970432,
+      0.2486732373009856,
+      0.25928733889310085,
+      0.23805913570887036
+    ],
+    "heuristics": {
+      "generator_explains": 0.25037907505686113,
+      "pick_max": 0.23881728582259287,
+      "pick_min": 0.26004548900682334,
+      "pick_median": 0.2505686125852919,
+      "pick_roundest": 0.25518069244377045,
+      "cv_logistic_features": 0.24184988627748294
+    },
+    "max_heuristic": 0.26004548900682334,
+    "leak_gate": "PASS"
+  },
+  "gsm8k_mc10_v2": {
+    "n": 1319,
+    "k": 10,
+    "chance": 0.1,
+    "chance_upper95": 0.11619030119914012,
+    "gold_position_share": [
+      0.10614101592115238,
+      0.09249431387414708,
+      0.09628506444275967,
+      0.10007581501137225,
+      0.10538286580742987,
+      0.10159211523881728,
+      0.09552691432903715,
+      0.09931766489764973,
+      0.10462471569370735,
+      0.09855951478392722
+    ],
+    "heuristics": {
+      "generator_explains": 0.10595779631033612,
+      "pick_max": 0.07505686125852919,
+      "pick_min": 0.09855951478392722,
+      "pick_median": 0.09969673995451099,
+      "pick_roundest": 0.0998221957471385,
+      "cv_logistic_features": 0.10159211523881728
+    },
+    "max_heuristic": 0.10595779631033612,
+    "leak_gate": "PASS"
+  }
+}

eval/choice_panel/rows.jsonl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a8bbb72147f51b0144f4177ed9e60a9223ef9c5979694423d93348b064bc8bf4
+size 1511969

eval/choice_panel/rows.sha256

@@ -0,0 +1 @@
+a8bbb72147f51b0144f4177ed9e60a9223ef9c5979694423d93348b064bc8bf4  rows.jsonl

eval/choice_panel/run_choice_panel.py

@@ -0,0 +1,159 @@
+"""Score one model on the frozen choice-panel rows, in every option rotation.
+
+Stages
+  full          tasks with state + question, all k cyclic rotations
+  options_only  tasks with state and question removed
+                (all rotations for k <= 4, five evenly spaced for k = 10)
+
+Models (all local, NF4 on this machine's GPU; nothing is downloaded or copied
+from another host):
+  raw  untouched google/gemma-4-E4B-it, native LM-head answer-letter scoring
+  r7   OpenJEV 1.0 checkpoint (local copy; r7 is its internal identifier)
+
+Output: <out>/details_<model>.jsonl, one line per (stage, task, row, rotation),
+with probabilities mapped back to the original option order. Resumable.
+"""
+
+from __future__ import annotations
+
+import argparse
+import gc
+import hashlib
+import importlib.util
+import json
+import os
+import sys
+import time
+from pathlib import Path
+
+HERE = Path(__file__).resolve().parent
+OPENJEV_BENCH = Path(os.environ.get("OPENJEV_BENCH_ROOT", "__set_OPENJEV_BENCH_ROOT__"))
+RAW_BENCH = Path(os.environ.get("RAW_GEMMA_BENCH_ROOT", "__set_RAW_GEMMA_BENCH_ROOT__"))
+CHECKPOINTS = {"r7": OPENJEV_BENCH / "checkpoints" / "round-07"}
+
+FULL_TASKS = ["gpqa_v2", "chess_legal_v2", "gsm8k_mc4_v2", "gsm8k_mc10_v2"]
+OPTIONS_ONLY_TASKS = ["gsm8k_mc4_v2", "gsm8k_mc10_v2", "chess_legal_v2", "gpqa_v2"]
+NO_STATE = "(no context provided)"
+NO_QUESTION = "Choose the correct option."
+
+
+def _module(name, path):
+    spec = importlib.util.spec_from_file_location(name, path)
+    mod = importlib.util.module_from_spec(spec)
+    spec.loader.exec_module(mod)
+    return mod
+
+
+def load_decider(model):
+    if model == "raw":
+        if not RAW_BENCH.is_dir():
+            raise SystemExit("Set RAW_GEMMA_BENCH_ROOT to the local untouched-Gemma benchmark bundle")
+        mod = _module("raw_bench", RAW_BENCH / "scripts" / "benchmark_raw_gemma.py")
+        return mod.load_raw_gemma_decider("google/gemma-4-E4B-it", local_files_only=True)
+    if not OPENJEV_BENCH.is_dir():
+        raise SystemExit("Set OPENJEV_BENCH_ROOT to the local OpenJEV benchmark bundle")
+    sys.path.insert(0, str(OPENJEV_BENCH))   # for the bench's src/ package
+    mod = _module("openjev_bench", OPENJEV_BENCH / "scripts" / "benchmark_public.py")
+    return mod.load_v13_decider(CHECKPOINTS[model])
+
+
+def rotations(k, stage):
+    if stage == "full" or k <= 4:
+        return list(range(k))
+    return list(range(0, k, max(1, k // 5)))[:5]
+
+
+def load_rows():
+    body = (HERE / "rows" / "rows.jsonl").read_bytes()
+    expected = (HERE / "rows" / "rows.sha256").read_text().split()[0]
+    if hashlib.sha256(body).hexdigest() != expected:
+        raise SystemExit("rows.jsonl does not match rows.sha256; rebuild with build_rows.py")
+    by_task = {}
+    for line in body.decode("utf-8").split("\n"):
+        if not line:
+            continue
+        r = json.loads(line)
+        by_task.setdefault(r["task"], []).append(r)
+    return by_task, expected
+
+
+def main():
+    ap = argparse.ArgumentParser()
+    ap.add_argument("--model", required=True, choices=["raw", "r7"])
+    ap.add_argument("--out", required=True)
+    ap.add_argument("--stages", default="full,options_only")
+    ap.add_argument("--limit", type=int, default=None, help="rows per task (smoke test)")
+    args = ap.parse_args()
+
+    by_task, rows_sha = load_rows()
+    out = Path(args.out)
+    out.mkdir(parents=True, exist_ok=True)
+    details = out / f"details_{args.model}.jsonl"
+    done = set()
+    if details.exists():
+        for line in details.read_text(encoding="utf-8").split("\n"):
+            if line.strip():
+                d = json.loads(line)
+                done.add((d["stage"], d["task"], d["row"], d["rotation"]))
+
+    plan = []
+    for stage in args.stages.split(","):
+        for task in (FULL_TASKS if stage == "full" else OPTIONS_ONLY_TASKS):
+            rows = by_task[task][: args.limit] if args.limit else by_task[task]
+            for r in rows:
+                for rot in rotations(len(r["options"]), stage):
+                    if (stage, task, r["row"], rot) not in done:
+                        plan.append((stage, task, r, rot))
+    print(f"[plan] model={args.model} rows_sha256={rows_sha[:12]} todo={len(plan)} already_done={len(done)}", flush=True)
+    if not plan:
+        return
+
+    t_load = time.time()
+    decide = load_decider(args.model)
+    print(f"[load] {args.model} ready in {time.time() - t_load:.0f}s", flush=True)
+
+    t0 = time.time()
+    last_key = None
+    task_hits = task_n = 0
+
+    def task_done(key, hits, count):
+        if key and count:
+            print(f"[{args.model}] ===== DONE {key[0]}/{key[1]}: {100 * hits / count:.1f}% correct over "
+                  f"{count} presentations (run status.py for the order-debiased score) =====", flush=True)
+
+    with details.open("a", encoding="utf-8") as fp:
+        for n, (stage, task, r, rot) in enumerate(plan, 1):
+            if (stage, task) != last_key:
+                task_done(last_key, task_hits, task_n)
+                task_hits = task_n = 0
+            opts, k = r["options"], len(r["options"])
+            rotated = opts[rot:] + opts[:rot]
+            state, question = (r["state"], r["question"]) if stage == "full" else (NO_STATE, NO_QUESTION)
+            t = time.time()
+            probs, temp = decide(state, question, rotated)
+            probs = [float(x) for x in probs][:k]
+            aligned = [probs[(j - rot) % k] for j in range(k)]
+            pred = max(range(k), key=aligned.__getitem__)
+            fp.write(json.dumps({
+                "model": args.model, "stage": stage, "task": task, "row": r["row"], "rotation": rot, "k": k,
+                "gold": r["gold"], "pred": pred, "correct": pred == r["gold"], "p_gold": aligned[r["gold"]],
+                "pred_position": (pred - rot) % k, "probs": [round(x, 6) for x in aligned],
+                "temperature": temp, "latency_s": round(time.time() - t, 4),
+            }) + "\n")
+            task_hits += pred == r["gold"]
+            task_n += 1
+            if n % 250 == 0 or n == len(plan) or (stage, task) != last_key:
+                fp.flush()
+                rate = n / max(time.time() - t0, 1e-6)
+                eta = (len(plan) - n) / rate / 60 if rate else 0
+                print(f"[{args.model}] {stage}/{task}: {100 * task_hits / task_n:5.1f}% correct so far "
+                      f"({task_n} done) | model {n}/{len(plan)} | {rate:.1f} it/s | ETA {eta:.0f} min", flush=True)
+                last_key = (stage, task)
+        task_done(last_key, task_hits, task_n)
+    del decide
+    gc.collect()
+    print(f"[done] {args.model} {len(plan)} scored in {(time.time() - t0) / 60:.1f} min", flush=True)
+
+
+if __name__ == "__main__":
+    main()

eval/choice_panel/source_run.json

@@ -0,0 +1,16 @@
+{
+  "source_run_id": "20260927-0741",
+  "source_rows_sha256": "0d424d10261f0804631fd540aeb3ae06e77c91a2bc6a19148d43409ba7337ec5",
+  "published_rows_sha256": "a8bbb72147f51b0144f4177ed9e60a9223ef9c5979694423d93348b064bc8bf4",
+  "published_tasks": [
+    "gpqa_v2",
+    "chess_legal_v2",
+    "gsm8k_mc4_v2",
+    "gsm8k_mc10_v2"
+  ],
+  "complete_models": [
+    "raw",
+    "r7"
+  ],
+  "source_summary_sha256": "cd51aa0029d60f90742c1cb486004f72273256e4c454881309f4ea1d251a90c1"
+}

eval/choice_panel/summarize.py

@@ -0,0 +1,120 @@
+"""Summarize a choice-panel run: rotation-averaged and position-debiased accuracy, order
+consistency, NLL vs ln k, and paired model comparisons (exact McNemar + bootstrap CI).
+
+Usage: python summarize.py results/<run>
+Writes summary.json and summary.md into the run directory. Only rows with every
+planned rotation present are counted, so it can be run while the benchmark is live.
+"""
+
+from __future__ import annotations
+
+import json
+import math
+import random
+import sys
+from collections import defaultdict
+from pathlib import Path
+
+
+def wilson(k, n, z=1.959964):
+    if n == 0:
+        return (float("nan"), float("nan"))
+    p = k / n
+    d = 1 + z * z / n
+    c = (p + z * z / (2 * n)) / d
+    h = z * math.sqrt(p * (1 - p) / n + z * z / (4 * n * n)) / d
+    return c - h, c + h
+
+
+def mcnemar(b, c):
+    n = b + c
+    if n == 0:
+        return 1.0
+    tail = sum(math.comb(n, i) for i in range(0, min(b, c) + 1)) / 2 ** n
+    return min(1.0, 2 * tail)
+
+
+def main():
+    run = Path(sys.argv[1])
+    recs = defaultdict(lambda: defaultdict(list))  # (model, stage, task) -> row -> [records]
+    for f in sorted(run.glob("details_*.jsonl")):
+        for line in f.read_text(encoding="utf-8").split("\n"):
+            if line.strip():
+                d = json.loads(line)
+                recs[(d["model"], d["stage"], d["task"])][d["row"]].append(d)
+
+    summary, rowwise = {}, {}
+    for (model, stage, task), rows in sorted(recs.items()):
+        nrot = max(len(v) for v in rows.values())
+        complete = {r: v for r, v in rows.items() if len(v) == nrot}
+        if not complete:
+            continue
+        k = next(iter(complete.values()))[0]["k"]
+        rot_acc, deb_hits, consistent, nll = [], {}, 0, 0.0
+        pos = [0] * k
+        for r, v in complete.items():
+            gold = v[0]["gold"]
+            rot_acc.append(sum(x["correct"] for x in v) / len(v))
+            mean = [sum(x["probs"][j] for x in v) / len(v) for j in range(k)]
+            deb_hits[r] = max(range(k), key=mean.__getitem__) == gold
+            consistent += len({x["pred"] for x in v}) == 1
+            nll += -math.log(max(mean[gold], 1e-12))
+            for x in v:
+                pos[x["pred_position"]] += 1
+        n = len(complete)
+        hits = sum(deb_hits.values())
+        lo, hi = wilson(hits, n)
+        summary[f"{model}|{stage}|{task}"] = {
+            "model": model, "stage": stage, "task": task, "rows": n, "rotations": nrot, "k": k,
+            "rotation_mean_accuracy": sum(rot_acc) / n,
+            "debiased_accuracy": hits / n, "debiased_ci95": [lo, hi],
+            "order_consistency": consistent / n,
+            "nll_mean_probs": nll / n, "ln_k": math.log(k),
+            "pred_position_share": [c / sum(pos) for c in pos],
+        }
+        rowwise[(model, stage, task)] = deb_hits
+
+    pairs = {}
+    models = sorted({m for m, _, _ in rowwise})
+    for a, b in (("raw", "r7"),):
+        if a not in models or b not in models:
+            continue
+        for (m, stage, task), ha in rowwise.items():
+            if m != a or (b, stage, task) not in rowwise:
+                continue
+            hb = rowwise[(b, stage, task)]
+            ids = sorted(set(ha) & set(hb))
+            if not ids:
+                continue
+            diff = [int(hb[i]) - int(ha[i]) for i in ids]
+            rng = random.Random(0)
+            boots = sorted(sum(diff[rng.randrange(len(ids))] for _ in ids) / len(ids) for _ in range(2000))
+            b_only = sum(1 for i in ids if hb[i] and not ha[i])
+            a_only = sum(1 for i in ids if ha[i] and not hb[i])
+            pairs[f"{a}->{b}|{stage}|{task}"] = {
+                "rows": len(ids), "delta_pp": 100 * sum(diff) / len(ids),
+                "ci95_pp": [100 * boots[49], 100 * boots[1949]],
+                f"{a}_only": a_only, f"{b}_only": b_only, "mcnemar_p": mcnemar(a_only, b_only),
+            }
+
+    (run / "summary.json").write_text(json.dumps({"tasks": summary, "pairs": pairs}, indent=2), encoding="utf-8")
+    lines = ["# Choice-panel benchmark summary", "",
+             "Debiased = argmax of probabilities averaged over rotations (mapped to original option order).", "",
+             "| Model | Stage | Task | Rows | Rot. | Rotation-mean acc | Debiased acc [95% CI] | Order consistency | NLL (ln k) |",
+             "|---|---|---|---:|---:|---:|---|---:|---|"]
+    for s in summary.values():
+        lines.append(f"| {s['model']} | {s['stage']} | {s['task']} | {s['rows']} | {s['rotations']} | "
+                     f"{100*s['rotation_mean_accuracy']:.2f}% | {100*s['debiased_accuracy']:.2f}% "
+                     f"[{100*s['debiased_ci95'][0]:.1f}, {100*s['debiased_ci95'][1]:.1f}] | "
+                     f"{100*s['order_consistency']:.1f}% | {s['nll_mean_probs']:.3f} ({s['ln_k']:.3f}) |")
+    lines += ["", "| Pair | Stage | Task | Rows | Δ pp [95% CI] | McNemar p |", "|---|---|---|---:|---|---:|"]
+    for key, p in pairs.items():
+        pair, stage, task = key.split("|")
+        lines.append(f"| {pair} | {stage} | {task} | {p['rows']} | {p['delta_pp']:+.2f} "
+                     f"[{p['ci95_pp'][0]:+.2f}, {p['ci95_pp'][1]:+.2f}] | {p['mcnemar_p']:.2g} |")
+    (run / "summary.md").write_text("\n".join(lines) + "\n", encoding="utf-8")
+    print("\n".join(lines))
+
+
+if __name__ == "__main__":
+    main()

eval/choice_panel/summary.json

@@ -0,0 +1,486 @@
+{
+  "tasks": {
+    "r7|full|chess_legal_v2": {
+      "model": "r7",
+      "stage": "full",
+      "task": "chess_legal_v2",
+      "rows": 500,
+      "rotations": 4,
+      "k": 4,
+      "rotation_mean_accuracy": 0.4005,
+      "debiased_accuracy": 0.426,
+      "debiased_ci95": [
+        0.38338257803751286,
+        0.4697458243684626
+      ],
+      "order_consistency": 0.466,
+      "nll_mean_probs": 1.2895886544670165,
+      "ln_k": 1.3862943611198906,
+      "pred_position_share": [
+        0.4155,
+        0.2225,
+        0.183,
+        0.179
+      ]
+    },
+    "r7|full|gpqa_v2": {
+      "model": "r7",
+      "stage": "full",
+      "task": "gpqa_v2",
+      "rows": 198,
+      "rotations": 4,
+      "k": 4,
+      "rotation_mean_accuracy": 0.3371212121212121,
+      "debiased_accuracy": 0.3282828282828283,
+      "debiased_ci95": [
+        0.2666857032513807,
+        0.3964162165579131
+      ],
+      "order_consistency": 0.23232323232323232,
+      "nll_mean_probs": 1.3462829985829794,
+      "ln_k": 1.3862943611198906,
+      "pred_position_share": [
+        0.2032828282828283,
+        0.18686868686868688,
+        0.2777777777777778,
+        0.33207070707070707
+      ]
+    },
+    "r7|full|gsm8k_mc10_v2": {
+      "model": "r7",
+      "stage": "full",
+      "task": "gsm8k_mc10_v2",
+      "rows": 1319,
+      "rotations": 10,
+      "k": 10,
+      "rotation_mean_accuracy": 0.2858225928733891,
+      "debiased_accuracy": 0.3062926459438969,
+      "debiased_ci95": [
+        0.282008848029071,
+        0.3317014748752748
+      ],
+      "order_consistency": 0.14935557240333586,
+      "nll_mean_probs": 1.9325159750133525,
+      "ln_k": 2.302585092994046,
+      "pred_position_share": [
+        0.06739954510993176,
+        0.07816527672479151,
+        0.08695981804397271,
+        0.09833206974981046,
+        0.10151630022744504,
+        0.100303260045489,
+        0.10962850644427596,
+        0.11940864291129644,
+        0.11834723275208492,
+        0.1199393479909022
+      ]
+    },
+    "r7|full|gsm8k_mc4_v2": {
+      "model": "r7",
+      "stage": "full",
+      "task": "gsm8k_mc4_v2",
+      "rows": 1319,
+      "rotations": 4,
+      "k": 4,
+      "rotation_mean_accuracy": 0.515352539802881,
+      "debiased_accuracy": 0.5382865807429871,
+      "debiased_ci95": [
+        0.5113101453003895,
+        0.5650406519337808
+      ],
+      "order_consistency": 0.43290371493555724,
+      "nll_mean_probs": 1.096663031900488,
+      "ln_k": 1.3862943611198906,
+      "pred_position_share": [
+        0.19143290371493557,
+        0.23294162244124336,
+        0.27369219105382864,
+        0.30193328278999243
+      ]
+    },
+    "r7|options_only|chess_legal_v2": {
+      "model": "r7",
+      "stage": "options_only",
+      "task": "chess_legal_v2",
+      "rows": 500,
+      "rotations": 4,
+      "k": 4,
+      "rotation_mean_accuracy": 0.294,
+      "debiased_accuracy": 0.308,
+      "debiased_ci95": [
+        0.26912574776130455,
+        0.34980199902176684
+      ],
+      "order_consistency": 0.11,
+      "nll_mean_probs": 1.4122002220138676,
+      "ln_k": 1.3862943611198906,
+      "pred_position_share": [
+        0.154,
+        0.143,
+        0.259,
+        0.444
+      ]
+    },
+    "r7|options_only|gpqa_v2": {
+      "model": "r7",
+      "stage": "options_only",
+      "task": "gpqa_v2",
+      "rows": 198,
+      "rotations": 4,
+      "k": 4,
+      "rotation_mean_accuracy": 0.3068181818181818,
+      "debiased_accuracy": 0.32323232323232326,
+      "debiased_ci95": [
+        0.2619851056969361,
+        0.3912080470479252
+      ],
+      "order_consistency": 0.06565656565656566,
+      "nll_mean_probs": 1.4115891851171167,
+      "ln_k": 1.3862943611198906,
+      "pred_position_share": [
+        0.15025252525252525,
+        0.16540404040404041,
+        0.3068181818181818,
+        0.37752525252525254
+      ]
+    },
+    "r7|options_only|gsm8k_mc10_v2": {
+      "model": "r7",
+      "stage": "options_only",
+      "task": "gsm8k_mc10_v2",
+      "rows": 1319,
+      "rotations": 5,
+      "k": 10,
+      "rotation_mean_accuracy": 0.10947687642153177,
+      "debiased_accuracy": 0.10841546626231995,
+      "debiased_ci95": [
+        0.0927599612966865,
+        0.12634525114986767
+      ],
+      "order_consistency": 0.003032600454890068,
+      "nll_mean_probs": 2.377036472907572,
+      "ln_k": 2.302585092994046,
+      "pred_position_share": [
+        0.011372251705837756,
+        0.004397270659590599,
+        0.014859742228961335,
+        0.0357846853677028,
+        0.047611827141774074,
+        0.08809704321455648,
+        0.1420773313115997,
+        0.20272934040940105,
+        0.18832448824867323,
+        0.264746019711903
+      ]
+    },
+    "r7|options_only|gsm8k_mc4_v2": {
+      "model": "r7",
+      "stage": "options_only",
+      "task": "gsm8k_mc4_v2",
+      "rows": 1319,
+      "rotations": 4,
+      "k": 4,
+      "rotation_mean_accuracy": 0.2505686125852919,
+      "debiased_accuracy": 0.24336618650492797,
+      "debiased_ci95": [
+        0.22097520606722637,
+        0.2672476679175918
+      ],
+      "order_consistency": 0.05458680818802123,
+      "nll_mean_probs": 1.4384991214185934,
+      "ln_k": 1.3862943611198906,
+      "pred_position_share": [
+        0.24298711144806673,
+        0.07695223654283548,
+        0.22915087187263078,
+        0.45090978013646704
+      ]
+    },
+    "raw|full|chess_legal_v2": {
+      "model": "raw",
+      "stage": "full",
+      "task": "chess_legal_v2",
+      "rows": 500,
+      "rotations": 4,
+      "k": 4,
+      "rotation_mean_accuracy": 0.315,
+      "debiased_accuracy": 0.438,
+      "debiased_ci95": [
+        0.39514834375489416,
+        0.4817970744771394
+      ],
+      "order_consistency": 0.014,
+      "nll_mean_probs": 1.2759298211025412,
+      "ln_k": 1.3862943611198906,
+      "pred_position_share": [
+        0.6895,
+        0.29,
+        0.0125,
+        0.008
+      ]
+    },
+    "raw|full|gpqa_v2": {
+      "model": "raw",
+      "stage": "full",
+      "task": "gpqa_v2",
+      "rows": 198,
+      "rotations": 4,
+      "k": 4,
+      "rotation_mean_accuracy": 0.31313131313131315,
+      "debiased_accuracy": 0.3282828282828283,
+      "debiased_ci95": [
+        0.2666857032513807,
+        0.3964162165579131
+      ],
+      "order_consistency": 0.10101010101010101,
+      "nll_mean_probs": 1.554143105728764,
+      "ln_k": 1.3862943611198906,
+      "pred_position_share": [
+        0.10984848484848485,
+        0.63510101010101,
+        0.13383838383838384,
+        0.12121212121212122
+      ]
+    },
+    "raw|full|gsm8k_mc10_v2": {
+      "model": "raw",
+      "stage": "full",
+      "task": "gsm8k_mc10_v2",
+      "rows": 1319,
+      "rotations": 10,
+      "k": 10,
+      "rotation_mean_accuracy": 0.16467020470052868,
+      "debiased_accuracy": 0.20545868081880211,
+      "debiased_ci95": [
+        0.18452439827071845,
+        0.22810362696739636
+      ],
+      "order_consistency": 0.006065200909780136,
+      "nll_mean_probs": 2.160294692789794,
+      "ln_k": 2.302585092994046,
+      "pred_position_share": [
+        0.17505686125852918,
+        0.523502653525398,
+        0.05473843821076573,
+        0.054510993176648974,
+        0.013722517058377558,
+        0.03752843062926459,
+        0.019257012888551934,
+        0.06110689916603487,
+        0.030780894617134194,
+        0.02979529946929492
+      ]
+    },
+    "raw|full|gsm8k_mc4_v2": {
+      "model": "raw",
+      "stage": "full",
+      "task": "gsm8k_mc4_v2",
+      "rows": 1319,
+      "rotations": 4,
+      "k": 4,
+      "rotation_mean_accuracy": 0.36713419257012886,
+      "debiased_accuracy": 0.43745261561789234,
+      "debiased_ci95": [
+        0.4109011521947761,
+        0.4643673473731633
+      ],
+      "order_consistency": 0.06899166034874905,
+      "nll_mean_probs": 1.2742964931019587,
+      "ln_k": 1.3862943611198906,
+      "pred_position_share": [
+        0.2579605761940864,
+        0.5869977255496588,
+        0.11694465504169825,
+        0.03809704321455648
+      ]
+    },
+    "raw|options_only|chess_legal_v2": {
+      "model": "raw",
+      "stage": "options_only",
+      "task": "chess_legal_v2",
+      "rows": 500,
+      "rotations": 4,
+      "k": 4,
+      "rotation_mean_accuracy": 0.252,
+      "debiased_accuracy": 0.29,
+      "debiased_ci95": [
+        0.2519474195915272,
+        0.33125480345245706
+      ],
+      "order_consistency": 0.0,
+      "nll_mean_probs": 1.4114726259217971,
+      "ln_k": 1.3862943611198906,
+      "pred_position_share": [
+        0.0,
+        0.9815,
+        0.0075,
+        0.011
+      ]
+    },
+    "raw|options_only|gpqa_v2": {
+      "model": "raw",
+      "stage": "options_only",
+      "task": "gpqa_v2",
+      "rows": 198,
+      "rotations": 4,
+      "k": 4,
+      "rotation_mean_accuracy": 0.25757575757575757,
+      "debiased_accuracy": 0.25757575757575757,
+      "debiased_ci95": [
+        0.20168506443256926,
+        0.32269411647466895
+      ],
+      "order_consistency": 0.020202020202020204,
+      "nll_mean_probs": 1.5601990009952065,
+      "ln_k": 1.3862943611198906,
+      "pred_position_share": [
+        0.015151515151515152,
+        0.8434343434343434,
+        0.07702020202020202,
+        0.06439393939393939
+      ]
+    },
+    "raw|options_only|gsm8k_mc10_v2": {
+      "model": "raw",
+      "stage": "options_only",
+      "task": "gsm8k_mc10_v2",
+      "rows": 1319,
+      "rotations": 5,
+      "k": 10,
+      "rotation_mean_accuracy": 0.10553449583017482,
+      "debiased_accuracy": 0.10538286580742987,
+      "debiased_ci95": [
+        0.08994299481674577,
+        0.1231146297300721
+      ],
+      "order_consistency": 0.0,
+      "nll_mean_probs": 2.7989174437564013,
+      "ln_k": 2.302585092994046,
+      "pred_position_share": [
+        0.0,
+        0.13525398028809704,
+        0.0037907505686125853,
+        0.0,
+        0.0012130401819560274,
+        0.00030326004548900684,
+        0.4363912054586808,
+        0.0018195602729340408,
+        0.3905989385898408,
+        0.03062926459438969
+      ]
+    },
+    "raw|options_only|gsm8k_mc4_v2": {
+      "model": "raw",
+      "stage": "options_only",
+      "task": "gsm8k_mc4_v2",
+      "rows": 1319,
+      "rotations": 4,
+      "k": 4,
+      "rotation_mean_accuracy": 0.24981046247156938,
+      "debiased_accuracy": 0.24715693707354056,
+      "debiased_ci95": [
+        0.22463441088141858,
+        0.27114794797806996
+      ],
+      "order_consistency": 0.0,
+      "nll_mean_probs": 1.4600850953045421,
+      "ln_k": 1.3862943611198906,
+      "pred_position_share": [
+        0.0,
+        0.9474981046247157,
+        0.041319181197877176,
+        0.011182714177407127
+      ]
+    }
+  },
+  "pairs": {
+    "raw->r7|full|chess_legal_v2": {
+      "rows": 500,
+      "delta_pp": -1.2,
+      "ci95_pp": [
+        -6.4,
+        3.8
+      ],
+      "raw_only": 90,
+      "r7_only": 84,
+      "mcnemar_p": 0.7047680231195156
+    },
+    "raw->r7|full|gpqa_v2": {
+      "rows": 198,
+      "delta_pp": 0.0,
+      "ci95_pp": [
+        -7.07070707070707,
+        7.07070707070707
+      ],
+      "raw_only": 26,
+      "r7_only": 26,
+      "mcnemar_p": 1.0
+    },
+    "raw->r7|full|gsm8k_mc10_v2": {
+      "rows": 1319,
+      "delta_pp": 10.083396512509477,
+      "ci95_pp": [
+        7.050796057619409,
+        13.040181956027293
+      ],
+      "raw_only": 150,
+      "r7_only": 283,
+      "mcnemar_p": 1.625471377453816e-10
+    },
+    "raw->r7|full|gsm8k_mc4_v2": {
+      "rows": 1319,
+      "delta_pp": 10.083396512509477,
+      "ci95_pp": [
+        6.444275966641396,
+        13.722517058377559
+      ],
+      "raw_only": 226,
+      "r7_only": 359,
+      "mcnemar_p": 4.2453654446737796e-08
+    },
+    "raw->r7|options_only|chess_legal_v2": {
+      "rows": 500,
+      "delta_pp": 1.8,
+      "ci95_pp": [
+        -3.0,
+        7.199999999999999
+      ],
+      "raw_only": 79,
+      "r7_only": 88,
+      "mcnemar_p": 0.5360050623796357
+    },
+    "raw->r7|options_only|gpqa_v2": {
+      "rows": 198,
+      "delta_pp": 6.565656565656566,
+      "ci95_pp": [
+        -1.0101010101010102,
+        14.14141414141414
+      ],
+      "raw_only": 24,
+      "r7_only": 37,
+      "mcnemar_p": 0.12373144253809217
+    },
+    "raw->r7|options_only|gsm8k_mc10_v2": {
+      "rows": 1319,
+      "delta_pp": 0.3032600454890068,
+      "ci95_pp": [
+        -2.047005307050796,
+        2.577710386656558
+      ],
+      "raw_only": 119,
+      "r7_only": 123,
+      "mcnemar_p": 0.8471289384306088
+    },
+    "raw->r7|options_only|gsm8k_mc4_v2": {
+      "rows": 1319,
+      "delta_pp": -0.37907505686125853,
+      "ci95_pp": [
+        -3.790750568612585,
+        2.9567854435178167
+      ],
+      "raw_only": 261,
+      "r7_only": 256,
+      "mcnemar_p": 0.8603786002624875
+    }
+  }
+}

eval/choice_panel/summary.md

@@ -0,0 +1,22 @@
+# Choice-panel benchmark summary
+
+Main accuracy is argmax of probabilities averaged over cyclic rotations, mapped to original option identities.
+
+| Model | Stage | Task | Rows | Rotations | Debiased accuracy [95% CI] | Per-order accuracy | Same answer |
+|---|---|---|---:|---:|---|---:|---:|
+| OpenJEV 1.0 | full | chess_legal_v2 | 500 | 4 | 42.60% [38.3, 47.0] | 40.05% | 46.6% |
+| OpenJEV 1.0 | full | gpqa_v2 | 198 | 4 | 32.83% [26.7, 39.6] | 33.71% | 23.2% |
+| OpenJEV 1.0 | full | gsm8k_mc10_v2 | 1319 | 10 | 30.63% [28.2, 33.2] | 28.58% | 14.9% |
+| OpenJEV 1.0 | full | gsm8k_mc4_v2 | 1319 | 4 | 53.83% [51.1, 56.5] | 51.54% | 43.3% |
+| OpenJEV 1.0 | options_only | chess_legal_v2 | 500 | 4 | 30.80% [26.9, 35.0] | 29.40% | 11.0% |
+| OpenJEV 1.0 | options_only | gpqa_v2 | 198 | 4 | 32.32% [26.2, 39.1] | 30.68% | 6.6% |
+| OpenJEV 1.0 | options_only | gsm8k_mc10_v2 | 1319 | 5 | 10.84% [9.3, 12.6] | 10.95% | 0.3% |
+| OpenJEV 1.0 | options_only | gsm8k_mc4_v2 | 1319 | 4 | 24.34% [22.1, 26.7] | 25.06% | 5.5% |
+| Gemma base | full | chess_legal_v2 | 500 | 4 | 43.80% [39.5, 48.2] | 31.50% | 1.4% |
+| Gemma base | full | gpqa_v2 | 198 | 4 | 32.83% [26.7, 39.6] | 31.31% | 10.1% |
+| Gemma base | full | gsm8k_mc10_v2 | 1319 | 10 | 20.55% [18.5, 22.8] | 16.47% | 0.6% |
+| Gemma base | full | gsm8k_mc4_v2 | 1319 | 4 | 43.75% [41.1, 46.4] | 36.71% | 6.9% |
+| Gemma base | options_only | chess_legal_v2 | 500 | 4 | 29.00% [25.2, 33.1] | 25.20% | 0.0% |
+| Gemma base | options_only | gpqa_v2 | 198 | 4 | 25.76% [20.2, 32.3] | 25.76% | 2.0% |
+| Gemma base | options_only | gsm8k_mc10_v2 | 1319 | 5 | 10.54% [9.0, 12.3] | 10.55% | 0.0% |
+| Gemma base | options_only | gsm8k_mc4_v2 | 1319 | 4 | 24.72% [22.5, 27.1] | 24.98% | 0.0% |

eval/harness/benchmark_public.py

@@ -0,0 +1,634 @@
+#!/usr/bin/env python
+import argparse
+import csv
+import json
+import math
+import os
+import random
+import re
+import runpy
+import statistics
+import sys
+import time
+from datetime import datetime
+from pathlib import Path
+
+REPO_ROOT = Path(__file__).resolve().parents[2]
+sys.path.insert(0, str(REPO_ROOT))
+
+import numpy as np
+import torch
+from datasets import load_dataset
+
+TASK_ORDER = [
+    "arc_challenge",
+    "winogrande",
+    "arc_easy",
+    "hellaswag",
+    "mmlu",
+]
+DISPLAY = {
+    "mmlu": "MMLU",
+    "gpqa": "GPQA Diamond",
+    "arc_easy": "ARC-Easy",
+    "arc_challenge": "ARC-Challenge",
+    "winogrande": "WinoGrande",
+    "hellaswag": "HellaSwag",
+    "gsm8k_mc4": "GSM8K 4 choices",
+    "gsm8k_mc10": "GSM8K 10 choices",
+    "chess": "Chess 4 moves",
+}
+
+# Frozen measured v1.1 NF4 baseline from the exact same benchmark harness.
+V11_NF4 = {
+    "mmlu": {"accuracy": 0.6601, "nll": 0.8368, "ece": 0.0185},
+    "gpqa": {"accuracy": 0.3333, "nll": 1.3871, "ece": 0.1136},
+    "arc_easy": {"accuracy": 0.9575, "nll": 0.1487, "ece": 0.0365},
+    "arc_challenge": {"accuracy": 0.8814, "nll": 0.3334, "ece": 0.0419},
+    "winogrande": {"accuracy": 0.6417, "nll": 0.7466, "ece": 0.1704},
+    "hellaswag": {"accuracy": 0.9014, "nll": 0.2639, "ece": 0.0142},
+    "gsm8k_mc4": {"accuracy": 0.4556, "nll": 1.1917, "ece": 0.0496},
+    "gsm8k_mc10": {"accuracy": 0.2722, "nll": 1.9969, "ece": 0.0400},
+    "chess": {"accuracy": 0.3880, "nll": 1.2856, "ece": 0.0595},
+}
+
+
+def load_r7_decider(repo_dir, load_mode):
+    """Score with the public safetensors release and its published loader."""
+    from openjev import OpenJEVR7
+
+    jev = OpenJEVR7.from_pretrained(repo_dir, device="cuda", load_mode=load_mode)
+
+    def decide(state, question, options):
+        result = jev.choice(state=state, instruction=question, options=options)
+        torch.cuda.synchronize()
+        return result["probabilities"], result["temperature"]
+
+    return decide
+
+
+def item(state, question, options, gold, source_id=None):
+    return {
+        "state": state,
+        "question": str(question).strip(),
+        "options": [str(x).strip() for x in options],
+        "gold": int(gold),
+        "source_id": source_id,
+    }
+
+
+def load_mmlu():
+    ds = load_dataset("cais/mmlu", "all", split="test")
+    out = []
+    for i, ex in enumerate(ds):
+        # Matches the JEV-E4B training serialization for MMLU, but uses the
+        # official test rows used by the openjev radar.
+        out.append(item(
+            f"Subject: {ex.get('subject', 'general')}",
+            ex["question"],
+            ex["choices"],
+            int(ex["answer"]),
+            f"mmlu:test:{i}",
+        ))
+    return out
+
+
+def load_gpqa():
+    # The canonical Idavidrein/gpqa repo is gated on Hugging Face.
+    # Use a public 198-row GPQA Diamond mirror with the original columns so
+    # this benchmark does not require HF authentication.
+    ds = load_dataset("JingzeShi/gpqa_diamond", split="train")
+    out = []
+    for i, ex in enumerate(ds):
+        opts = [
+            ex["Correct Answer"],
+            ex["Incorrect Answer 1"],
+            ex["Incorrect Answer 2"],
+            ex["Incorrect Answer 3"],
+        ]
+        # Keep the same protocol as the original openjev radar loader:
+        # correct answer in slot 0.
+        out.append(item(
+            "Advanced science question",
+            ex["Question"],
+            opts,
+            0,
+            f"gpqa:diamond:{i}",
+        ))
+    return out
+
+
+def load_arc(config):
+    ds = load_dataset("allenai/ai2_arc", config, split="test")
+    out = []
+    for i, ex in enumerate(ds):
+        labels = list(ex["choices"]["label"])
+        if ex["answerKey"] not in labels:
+            continue
+        out.append(item(
+            "Science question",
+            ex["question"],
+            list(ex["choices"]["text"]),
+            labels.index(ex["answerKey"]),
+            f"arc:{config}:{ex.get('id', i)}",
+        ))
+    return out
+
+
+def load_winogrande():
+    ds = load_dataset("allenai/winogrande", "winogrande_xl", split="validation")
+    out = []
+    for i, ex in enumerate(ds):
+        out.append(item(
+            ex["sentence"],
+            "Which option best completes the blank in the sentence?",
+            [ex["option1"], ex["option2"]],
+            int(ex["answer"]) - 1,
+            f"winogrande:validation:{i}",
+        ))
+    return out
+
+
+def load_hellaswag():
+    ds = load_dataset("Rowan/hellaswag", split="validation")
+    out = []
+    for i, ex in enumerate(ds):
+        # Matches the JEV-E4B HellaSwag training shape.
+        ctx = ex.get("ctx")
+        if not ctx:
+            ctx = (ex["ctx_a"] + " " + ex["ctx_b"].capitalize()).strip()
+        out.append(item(
+            ctx,
+            "Choose the most plausible continuation.",
+            ex["endings"],
+            int(ex["label"]),
+            f"hellaswag:validation:{ex.get('ind', i)}",
+        ))
+    return out
+
+
+def extract_gsm_number(answer):
+    # GSM8K canonical final answer follows ####. Fall back to the last number.
+    if "####" in answer:
+        s = answer.rsplit("####", 1)[-1].strip().replace(",", "")
+    else:
+        nums = re.findall(r"-?\d+(?:,\d{3})*(?:\.\d+)?", answer)
+        if not nums:
+            raise ValueError(f"No numeric answer in: {answer!r}")
+        s = nums[-1].replace(",", "")
+    # Canonicalize integer-like floats.
+    v = float(s)
+    return str(int(v)) if v == int(v) else str(v)
+
+
+def load_gsm8k_mc(k, seed=0):
+    rng = random.Random(seed)
+    ds = load_dataset("openai/gsm8k", "main", split="test")
+    out = []
+    for i, ex in enumerate(ds):
+        g = extract_gsm_number(ex["answer"])
+        gv = float(g)
+        cands = set()
+
+        generators = [
+            lambda: gv + rng.choice([1, 2, 3, 5, 10]),
+            lambda: gv - rng.choice([1, 2, 3, 5, 10]),
+            lambda: gv * 2,
+            lambda: gv / 2,
+            lambda: gv + rng.choice([4, 6, 7, 8, 9, 12, 15, 20, 25, 50]),
+            lambda: gv * 10,
+            lambda: gv * rng.choice([3, 4, 5]),
+            lambda: gv - rng.choice([4, 6, 7, 8, 9, 12, 15, 20, 25, 50]),
+            lambda: abs(gv) + rng.randint(100, 999),
+        ]
+        gi = 0
+        while len(cands) < k - 1 and gi < 200:
+            v = generators[gi % len(generators)]()
+            gi += 1
+            vs = str(int(v)) if float(v) == int(v) else f"{v:.2f}"
+            if vs != g and vs not in cands and v >= 0:
+                cands.add(vs)
+
+        if len(cands) < k - 1:
+            # Deterministic emergency fallback.
+            x = 1
+            while len(cands) < k - 1:
+                v = gv + x
+                vs = str(int(v)) if float(v) == int(v) else str(v)
+                if vs != g:
+                    cands.add(vs)
+                x += 1
+
+        opts = [g] + sorted(cands, key=lambda _: rng.random())
+        rng.shuffle(opts)
+        out.append(item(
+            "Math word problem",
+            ex["question"],
+            opts,
+            opts.index(g),
+            f"gsm8k:test:{i}:k{k}",
+        ))
+    return out
+
+
+def load_chess(n=500, seed=0):
+    import chess
+
+    rng = random.Random(seed)
+    out = []
+    while len(out) < n:
+        board = chess.Board()
+        for _ in range(rng.randint(6, 40)):
+            moves = list(board.legal_moves)
+            if not moves or board.is_game_over():
+                break
+            board.push(rng.choice(moves))
+
+        legal = list(board.legal_moves)
+        if len(legal) < 2 or board.is_game_over():
+            continue
+
+        legal_san = {board.san(m) for m in legal}
+        good = board.san(rng.choice(legal))
+        bad = set()
+        tries = 0
+
+        while len(bad) < 3 and tries < 500:
+            tries += 1
+            own = [
+                s for s in chess.SQUARES
+                if board.piece_at(s) and board.piece_at(s).color == board.turn
+            ]
+            if not own:
+                break
+            sq = rng.choice(own)
+            piece = board.piece_at(sq)
+            to = rng.choice(chess.SQUARES)
+            if to == sq:
+                continue
+            dst = board.piece_at(to)
+            if dst and dst.color == board.turn:
+                continue
+
+            capture = dst is not None
+            if piece.piece_type == chess.PAWN:
+                san = (
+                    (chess.square_name(sq)[0] + "x" if capture else "")
+                    + chess.square_name(to)
+                )
+            else:
+                san = (
+                    chess.piece_symbol(piece.piece_type).upper()
+                    + ("x" if capture else "")
+                    + chess.square_name(to)
+                )
+
+            if san not in legal_san and san != good:
+                bad.add(san)
+
+        if len(bad) < 3:
+            continue
+
+        opts = [good] + sorted(bad)
+        rng.shuffle(opts)
+        pgn = chess.Board().variation_san(board.move_stack)
+        state = (
+            f"Chess position after the moves: {pgn}\n"
+            f"FEN: {board.fen()}\n"
+            f"{'White' if board.turn else 'Black'} to move."
+        )
+        out.append(item(
+            state,
+            "Which of the following moves is legal in this position?",
+            opts,
+            opts.index(good),
+            f"chess:synthetic:{len(out)}",
+        ))
+    return out
+
+
+LOADERS = {
+    "mmlu": load_mmlu,
+    "gpqa": load_gpqa,
+    "arc_easy": lambda: load_arc("ARC-Easy"),
+    "arc_challenge": lambda: load_arc("ARC-Challenge"),
+    "winogrande": load_winogrande,
+    "hellaswag": load_hellaswag,
+    "gsm8k_mc4": lambda: load_gsm8k_mc(4),
+    "gsm8k_mc10": lambda: load_gsm8k_mc(10),
+    "chess": load_chess,
+}
+
+
+def normalize_probs(ps, n):
+    arr = np.asarray([float(x) for x in ps], dtype=np.float64)
+    if arr.shape != (n,):
+        raise ValueError(f"Expected {n} probabilities, got shape {arr.shape}")
+    if not np.all(np.isfinite(arr)):
+        raise ValueError(f"Non-finite probabilities: {arr}")
+    # The local model should already return a simplex. Only repair tiny
+    # numerical drift, not model semantics.
+    if np.any(arr < -1e-6):
+        raise ValueError(f"Negative probability: {arr}")
+    arr = np.clip(arr, 0.0, None)
+    s = float(arr.sum())
+    if s <= 0:
+        raise ValueError(f"Probability sum is {s}")
+    if abs(s - 1.0) > 1e-4:
+        raise ValueError(f"Probability sum {s:.8f} is not ~1.0")
+    return arr / s
+
+
+def ece(rows, bins=15):
+    if not rows:
+        return float("nan")
+    conf = np.array([r["confidence"] for r in rows], dtype=float)
+    corr = np.array([1.0 if r["correct"] else 0.0 for r in rows], dtype=float)
+    total = len(rows)
+    val = 0.0
+    for b in range(bins):
+        lo = b / bins
+        hi = (b + 1) / bins
+        mask = (conf >= lo) & (conf < hi if b < bins - 1 else conf <= hi)
+        n = int(mask.sum())
+        if n:
+            val += (n / total) * abs(float(corr[mask].mean()) - float(conf[mask].mean()))
+    return val
+
+
+def summarize(rows):
+    if not rows:
+        return {}
+    acc = sum(1 for r in rows if r["correct"]) / len(rows)
+    nll = sum(-math.log(max(r["p_gold"], 1e-12)) for r in rows) / len(rows)
+    brier = sum(r["brier"] for r in rows) / len(rows)
+    lat = sorted(r["latency_s"] for r in rows)
+    p50 = float(np.percentile(lat, 50))
+    p95 = float(np.percentile(lat, 95))
+    return {
+        "n": len(rows),
+        "accuracy": acc,
+        "nll": nll,
+        "brier": brier,
+        "ece": ece(rows),
+        "latency_p50_ms": p50 * 1000,
+        "latency_p95_ms": p95 * 1000,
+        "mean_confidence": sum(r["confidence"] for r in rows) / len(rows),
+    }
+
+
+def run_task(task, items, decide, detail_fp, limit=None):
+    if limit:
+        items = items[:limit]
+    rows = []
+    t_task = time.perf_counter()
+    total = len(items)
+
+    print(f"\n{'='*72}")
+    print(f"{DISPLAY[task]}  n={total}")
+    print(f"{'='*72}")
+
+    for i, ex in enumerate(items, 1):
+        t0 = time.perf_counter()
+        ps, temp = decide(ex["state"], ex["question"], ex["options"])
+        latency = time.perf_counter() - t0
+        ps = normalize_probs(ps, len(ex["options"]))
+
+        pred = int(np.argmax(ps))
+        gold = ex["gold"]
+        y = np.zeros(len(ps), dtype=float)
+        y[gold] = 1.0
+
+        row = {
+            "task": task,
+            "index": i - 1,
+            "source_id": ex["source_id"],
+            "gold": gold,
+            "pred": pred,
+            "correct": pred == gold,
+            "p_gold": float(ps[gold]),
+            "confidence": float(ps[pred]),
+            "brier": float(np.sum((ps - y) ** 2)),
+            "temperature": float(temp),
+            "latency_s": latency,
+            "option_count": len(ps),
+            "probabilities": [float(x) for x in ps],
+        }
+        rows.append(row)
+        detail_fp.write(json.dumps(row, ensure_ascii=False) + "\n")
+        detail_fp.flush()
+
+        if i == 1 or i % 100 == 0 or i == total:
+            acc = sum(r["correct"] for r in rows) / len(rows)
+            elapsed = time.perf_counter() - t_task
+            rate = i / elapsed if elapsed else 0
+            eta = (total - i) / rate if rate else 0
+            print(
+                f"[{task}] {i:>6}/{total:<6} "
+                f"acc={acc:.4f} "
+                f"last={latency*1000:.0f}ms "
+                f"rate={rate:.2f}/s "
+                f"eta={eta/60:.1f}m"
+            )
+
+    return rows
+
+
+def write_csv(summary, path):
+    fields = [
+        "task", "display", "n", "accuracy", "nll", "brier", "ece",
+        "mean_confidence", "latency_p50_ms", "latency_p95_ms",
+    ]
+    with open(path, "w", newline="", encoding="utf-8") as f:
+        w = csv.DictWriter(f, fieldnames=fields)
+        w.writeheader()
+        for task in TASK_ORDER:
+            if task not in summary:
+                continue
+            r = {"task": task, "display": DISPLAY[task], **summary[task]}
+            w.writerow(r)
+
+
+def make_radar(summary, path, label):
+    try:
+        import matplotlib.pyplot as plt
+    except Exception as exc:
+        print(f"[radar] skipped: {exc}")
+        return
+
+    tasks = [t for t in TASK_ORDER if t in summary]
+    if len(tasks) < 3:
+        return
+    labels = [DISPLAY[t] for t in tasks]
+    ours = [summary[t]["accuracy"] * 100 for t in tasks]
+    base = [V11_NF4[t]["accuracy"] * 100 for t in tasks]
+    angles = np.linspace(0, 2 * np.pi, len(tasks), endpoint=False).tolist()
+    angles += angles[:1]
+    fig = plt.figure(figsize=(12, 10))
+    ax = fig.add_subplot(111, polar=True)
+    ax.set_theta_offset(np.pi / 2)
+    ax.set_theta_direction(-1)
+    ax.set_ylim(0, 100)
+    ax.set_yticks([20, 40, 60, 80, 100])
+    ax.set_yticklabels(["20%", "40%", "60%", "80%", "100%"] )
+    ax.set_xticks(angles[:-1])
+    ax.set_xticklabels(labels, fontsize=10)
+    vals = ours + ours[:1]
+    ax.plot(angles, vals, linewidth=3, marker="o", label=label)
+    ax.fill(angles, vals, alpha=0.08)
+    bvals = base + base[:1]
+    ax.plot(angles, bvals, linewidth=2, marker=".", label="JEV-E4B v1.1 NF4 measured")
+    ax.set_title(f"{label} canonical public benchmark\nexact same task loaders as v1.1 NF4 baseline", pad=28, fontsize=14, fontweight="bold")
+    ax.legend(loc="upper center", bbox_to_anchor=(0.5, -0.12), ncol=2, fontsize=9)
+    fig.tight_layout()
+    fig.savefig(path, dpi=180, bbox_inches="tight")
+    plt.close(fig)
+
+
+def write_comparison(summary, path, label):
+    out = {}
+    for task, cur in summary.items():
+        base = V11_NF4.get(task)
+        if not base:
+            continue
+        out[task] = {
+            "v1.1_nf4": base,
+            label: {k: cur[k] for k in ("accuracy", "nll", "ece", "brier") if k in cur},
+            "delta_accuracy_pp": (cur["accuracy"] - base["accuracy"]) * 100.0,
+            "delta_nll": cur["nll"] - base["nll"],
+            "delta_ece": cur["ece"] - base["ece"],
+        }
+    Path(path).write_text(json.dumps(out, indent=2), encoding="utf-8")
+
+def parse_args():
+    ap = argparse.ArgumentParser()
+    ap.add_argument("--repo", default=str(REPO_ROOT), help="Published Hugging Face repository directory")
+    ap.add_argument("--load-mode", choices=("nf4", "bf16"), default="nf4")
+    ap.add_argument("--label", default="OpenJEV E4B 1.0 public release")
+    ap.add_argument(
+        "--tasks",
+        default=",".join(TASK_ORDER),
+        help="Comma-separated tasks",
+    )
+    ap.add_argument(
+        "--limit",
+        type=int,
+        default=None,
+        help="Optional per-task cap for smoke testing. Omit for the full benchmark.",
+    )
+    ap.add_argument("--output-root", default=None)
+    ap.add_argument(
+        "--resume-latest",
+        action="store_true",
+        help="Resume the newest prior run directory by skipping tasks already present in summary.json.",
+    )
+    return ap.parse_args()
+
+
+def main():
+    args = parse_args()
+    repo_dir = Path(args.repo).resolve()
+    decide = load_r7_decider(repo_dir, args.load_mode)
+
+    tasks = [x.strip() for x in args.tasks.split(",") if x.strip()]
+    unknown = [x for x in tasks if x not in LOADERS]
+    if unknown:
+        raise SystemExit(f"Unknown tasks: {unknown}")
+
+    stamp = datetime.now().strftime("%Y%m%d-%H%M%S")
+    out_root = Path(args.output_root) if args.output_root else Path("results/full")
+    out_root.mkdir(parents=True, exist_ok=True)
+
+    out_dir = None
+    all_summary = {}
+    if args.resume_latest:
+        prior = sorted(
+            [p for p in out_root.iterdir() if p.is_dir() and (p / "summary.json").exists()],
+            key=lambda p: p.stat().st_mtime,
+            reverse=True,
+        )
+        if prior:
+            out_dir = prior[0]
+            try:
+                all_summary = json.loads((out_dir / "summary.json").read_text(encoding="utf-8"))
+            except Exception:
+                all_summary = {}
+            print(f"[resume] using prior run: {out_dir}")
+            print(f"[resume] completed tasks: {list(all_summary)}")
+
+    if out_dir is None:
+        out_dir = out_root / stamp
+        out_dir.mkdir(parents=True, exist_ok=True)
+
+    meta = {
+        "created_or_resumed_at": datetime.now().isoformat(),
+        "repo_dir": str(repo_dir),
+        "load_mode": args.load_mode,
+        "model_label": args.label,
+        "runtime": ("Gemma NF4 with adapted dense FP16 layers" if args.load_mode == "nf4"
+                    else "Gemma BF16 with adapted BF16 layers") + "; OpenJEV 1.0 adapter and FP32 decision head",
+        "network_model_endpoint": None,
+        "backbone_delta_required": True,
+        "baseline": "v1.1 NF4 exact same benchmark loaders",
+        "tasks": tasks,
+        "limit_per_task": args.limit,
+        "gpqa_source": "JingzeShi/gpqa_diamond public 198-row mirror",
+        "protocol": (
+            "openjev public benchmark rows/splits; JEV-native state/question serialization "
+            "for MMLU/ARC/HellaSwag and natural JEV serialization for GPQA/WinoGrande/GSM8K/chess"
+        ),
+    }
+    (out_dir / "run_metadata.json").write_text(json.dumps(meta, indent=2), encoding="utf-8")
+
+    detail_path = out_dir / "details.jsonl"
+
+    with detail_path.open("a", encoding="utf-8") as detail_fp:
+        for task in tasks:
+            if task in all_summary:
+                print(f"\n[resume] skipping completed task: {task}")
+                continue
+            print(f"\n[dataset] loading {task} ...")
+            rows = LOADERS[task]()
+            print(f"[dataset] {task}: {len(rows)} rows")
+            task_rows = run_task(task, rows, decide, detail_fp, limit=args.limit)
+            all_summary[task] = summarize(task_rows)
+
+            # Persist after every task.
+            (out_dir / "summary.json").write_text(
+                json.dumps(all_summary, indent=2), encoding="utf-8"
+            )
+            write_csv(all_summary, out_dir / "summary.csv")
+            write_comparison(all_summary, out_dir / "compare_v11_nf4.json", args.label)
+            make_radar(all_summary, out_dir / "radar.png", args.label)
+
+            m = all_summary[task]
+            print(
+                f"\nRESULT {DISPLAY[task]}: "
+                f"acc={m['accuracy']*100:.2f}% "
+                f"nll={m['nll']:.4f} "
+                f"brier={m['brier']:.4f} "
+                f"ece={m['ece']:.4f} "
+                f"p50={m['latency_p50_ms']:.0f}ms "
+                f"p95={m['latency_p95_ms']:.0f}ms"
+            )
+
+    print("\n\n" + "=" * 72)
+    print("FINAL")
+    print("=" * 72)
+    for task in tasks:
+        m = all_summary[task]
+        dpp = (m["accuracy"] - V11_NF4[task]["accuracy"]) * 100.0
+        print(
+            f"{DISPLAY[task]:<22} "
+            f"{m['accuracy']*100:6.2f}%  "
+            f"NLL {m['nll']:.4f}  "
+            f"ECE {m['ece']:.4f}  "
+            f"delta_vs_v11_nf4={dpp:+.2f}pp"
+        )
+    print(f"\nResults: {out_dir}")
+    print(f"Radar:   {out_dir / 'radar.png'}")
+    print("No model HTTP endpoint was used; scoring ran directly on the local Windows/WSL NVIDIA GPU.")
+
+
+if __name__ == "__main__":
+    main()

eval/harness/requirements-benchmark.txt

@@ -0,0 +1,14 @@
+transformers>=5.5.0,<5.7.0
+datasets>=4.0.0,<5.0.0
+accelerate>=1.10.0
+peft>=0.17.0
+huggingface_hub>=0.34.0
+safetensors>=0.6.2
+bitsandbytes>=0.46.0
+numpy>=2.1
+pyarrow>=20.0
+matplotlib>=3.9
+python-chess>=1.999
+pillow>=11.0
+sentencepiece>=0.2.0
+protobuf>=5.0

eval/harness/src/decision_head.py

@@ -0,0 +1,73 @@
+from __future__ import annotations
+import math
+import torch
+import torch.nn.functional as F
+from torch import nn
+
+
+class PointerDecisionHead(nn.Module):
+    def __init__(self, hidden_size: int, head_dim: int = 512, normalize: bool = True):
+        super().__init__()
+        self.normalize = normalize
+        self.q_proj = nn.Linear(hidden_size, head_dim, bias=False)
+        self.k_proj = nn.Linear(hidden_size, head_dim, bias=False)
+        self.scale = math.sqrt(head_dim)
+
+    def forward(self, decide_h: torch.Tensor, option_h: torch.Tensor, option_mask: torch.Tensor) -> torch.Tensor:
+        decide_h = decide_h.float()
+        option_h = option_h.float()
+        q = self.q_proj(decide_h)
+        k = self.k_proj(option_h)
+        if self.normalize:
+            q = F.normalize(q, dim=-1)
+            k = F.normalize(k, dim=-1)
+            logits = torch.einsum("bd,bnd->bn", q, k) * 10.0
+        else:
+            logits = torch.einsum("bd,bnd->bn", q, k) / self.scale
+        return logits.masked_fill(~option_mask, -1e9)
+
+
+class SpanDecisionHead(nn.Module):
+    """Scores the mean hidden state across each option's semantic token span."""
+    def __init__(self, hidden_size: int, head_dim: int = 384):
+        super().__init__()
+        self.q_proj = nn.Linear(hidden_size, head_dim, bias=False)
+        self.k_proj = nn.Linear(hidden_size, head_dim, bias=False)
+        self.mlp = nn.Sequential(
+            nn.Linear(head_dim * 4, head_dim),
+            nn.GELU(),
+            nn.Linear(head_dim, 1),
+        )
+
+    def forward(self, decide_h: torch.Tensor, option_mean_h: torch.Tensor, option_mask: torch.Tensor) -> torch.Tensor:
+        q = F.normalize(self.q_proj(decide_h.float()), dim=-1)
+        k = F.normalize(self.k_proj(option_mean_h.float()), dim=-1)
+        qe = q[:, None, :].expand_as(k)
+        x = torch.cat([qe, k, qe * k, (qe - k).abs()], dim=-1)
+        logits = self.mlp(x).squeeze(-1)
+        return logits.masked_fill(~option_mask, -1e9)
+
+
+class HybridDecisionHead(nn.Module):
+    """Pointer baseline plus a learned residual semantic-span scorer."""
+    def __init__(self, hidden_size: int, head_dim: int = 384, pointer_dim: int = 384, normalize: bool = True):
+        super().__init__()
+        self.pointer = PointerDecisionHead(hidden_size, pointer_dim, normalize=normalize)
+        self.span = SpanDecisionHead(hidden_size, head_dim)
+        self.logit_scale = nn.Parameter(torch.tensor(0.0))
+
+    def forward(
+        self,
+        decide_h: torch.Tensor,
+        option_last_h: torch.Tensor,
+        option_mean_h: torch.Tensor,
+        option_mask: torch.Tensor,
+    ) -> torch.Tensor:
+        base = self.pointer(decide_h, option_last_h, option_mask)
+        residual = self.span(decide_h, option_mean_h, option_mask)
+        return base + self.logit_scale.exp().clamp(max=4.0) * residual
+
+
+def brier_loss(probs: torch.Tensor, targets: torch.Tensor, mask: torch.Tensor) -> torch.Tensor:
+    diff = (probs - targets).pow(2) * mask.float()
+    return diff.sum(dim=-1).mean()

eval/harness/src/formatting.py

@@ -0,0 +1,211 @@
+from __future__ import annotations
+import json
+import torch
+
+
+def encode_piece(tok, text: str) -> list[int]:
+    return tok(text, add_special_tokens=False).input_ids
+
+
+def state_to_text(state) -> str:
+    if isinstance(state, str):
+        return state
+    return json.dumps(state, ensure_ascii=False, sort_keys=True, separators=(",", ":"))
+
+
+def _clip_head_tail(ids: list[int], limit: int) -> list[int]:
+    """Deterministically keep evidence from both ends of a long field."""
+    if limit <= 0:
+        return []
+    if len(ids) <= limit:
+        return ids
+    head = (limit + 1) // 2
+    tail = limit - head
+    return ids[:head] + (ids[-tail:] if tail else [])
+
+
+def _assemble(tok, state_ids, q_ids, option_ids, max_length: int):
+    ids = [tok.bos_token_id] if tok.bos_token_id is not None else []
+    ids += encode_piece(tok, "State:\n")
+    ids += state_ids
+    ids += encode_piece(tok, "\n\nQuestion:\n")
+    ids += q_ids
+    ids += encode_piece(tok, "\n\nOptions:\n")
+
+    option_positions = []
+    option_spans = []
+    for idx, opt_ids in enumerate(option_ids):
+        ids += encode_piece(tok, f"- [{idx}] ")
+        span_start = len(ids)
+        ids += opt_ids
+        span_end = len(ids)
+        option_spans.append((span_start, span_end))
+        # Represent each option by its final semantic token, after it has seen
+        # the full option text but before the newline delimiter.
+        option_positions.append(len(ids) - 1)
+        ids += encode_piece(tok, "\n")
+
+    ids += encode_piece(tok, "\nDecision:")
+    decide_position = len(ids) - 1
+    if len(ids) > max_length:
+        return None
+    return ids, option_positions, decide_position, option_spans
+
+
+def pack_question(tok, state, question: dict, max_length: int):
+    """Pack a decision example without dropping long states.
+
+    Priority order is structural markers/options/question first, then state.
+    Long state is head+tail truncated. If question/options themselves are huge,
+    they are bounded as a second-stage fallback. Returning None is reserved for
+    structurally impossible cases (e.g. too many options for max_length).
+    """
+    if max_length < 32:
+        return None
+
+    state_ids = encode_piece(tok, state_to_text(state))
+    q_ids = encode_piece(tok, str(question["instruction"]))
+    option_ids = [encode_piece(tok, str(opt["text"])) for opt in question["options"]]
+    if not option_ids or any(not x for x in option_ids):
+        return None
+
+    was_truncated = False
+
+    # First discover how much room remains for the state while preserving the
+    # complete question and option text.
+    probe = _assemble(tok, [], q_ids, option_ids, max_length=10**9)
+    if probe is None:
+        return None
+    structural_len = len(probe[0])
+    if structural_len <= max_length:
+        state_budget = max_length - structural_len
+        was_truncated = len(state_ids) > state_budget
+        packed = _assemble(tok, _clip_head_tail(state_ids, state_budget), q_ids, option_ids, max_length)
+    else:
+        was_truncated = True
+        # Extremely verbose question/tool schemas: cap semantic fields rather
+        # than dropping the sample. Typical training examples never hit this.
+        nopt = len(option_ids)
+        q_cap = min(len(q_ids), max(16, max_length // 8))
+        # Start modestly; then shrink until the structural representation fits.
+        opt_cap = max(8, min(96, max_length // max(8, nopt * 2)))
+        q_fit = _clip_head_tail(q_ids, q_cap)
+        opts_fit = [_clip_head_tail(x, opt_cap) for x in option_ids]
+        packed = _assemble(tok, [], q_fit, opts_fit, max_length)
+        while packed is None and (q_cap > 8 or opt_cap > 4):
+            q_cap = max(8, q_cap // 2)
+            opt_cap = max(4, opt_cap // 2)
+            q_fit = _clip_head_tail(q_ids, q_cap)
+            opts_fit = [_clip_head_tail(x, opt_cap) for x in option_ids]
+            packed = _assemble(tok, [], q_fit, opts_fit, max_length)
+
+        if packed is not None:
+            # If the shrunken question/options leave room, fill it with state.
+            base_len = len(packed[0])
+            state_budget = max(0, max_length - base_len)
+            was_truncated = was_truncated or len(state_ids) > state_budget
+            packed = _assemble(tok, _clip_head_tail(state_ids, state_budget), q_fit, opts_fit, max_length)
+
+    if packed is None:
+        return None
+    ids, option_positions, decide_position, option_spans = packed
+    return {
+        "input_ids": ids,
+        "option_positions": option_positions,
+        "option_spans": option_spans,
+        "decide_position": decide_position,
+        "target": question.get("target_distribution"),
+        "was_truncated": was_truncated,
+    }
+
+
+def collate_packed(tok, packed: list[dict]) -> dict[str, torch.Tensor]:
+    if not packed:
+        raise RuntimeError("No packable examples")
+    bsz = len(packed)
+    max_seq = max(len(x["input_ids"]) for x in packed)
+    max_opts = max(len(x["option_positions"]) for x in packed)
+    pad_id = tok.pad_token_id if tok.pad_token_id is not None else 0
+    input_ids = torch.full((bsz, max_seq), pad_id, dtype=torch.long)
+    attention_mask = torch.zeros((bsz, max_seq), dtype=torch.long)
+    option_positions = torch.zeros((bsz, max_opts), dtype=torch.long)
+    option_mask = torch.zeros((bsz, max_opts), dtype=torch.bool)
+    decide_positions = torch.zeros((bsz,), dtype=torch.long)
+    targets = torch.zeros((bsz, max_opts), dtype=torch.float32)
+    option_starts = torch.zeros((bsz, max_opts), dtype=torch.long)
+    option_ends = torch.zeros((bsz, max_opts), dtype=torch.long)
+    has_targets = all(x.get("target") is not None for x in packed)
+    for i, x in enumerate(packed):
+        n = len(x["input_ids"])
+        m = len(x["option_positions"])
+        input_ids[i, :n] = torch.tensor(x["input_ids"], dtype=torch.long)
+        attention_mask[i, :n] = 1
+        option_positions[i, :m] = torch.tensor(x["option_positions"], dtype=torch.long)
+        spans = x.get("option_spans") or [(int(v), int(v)+1) for v in x["option_positions"]]
+        option_starts[i, :m] = torch.tensor([a for a, _ in spans], dtype=torch.long)
+        option_ends[i, :m] = torch.tensor([b for _, b in spans], dtype=torch.long)
+        option_mask[i, :m] = True
+        decide_positions[i] = x["decide_position"]
+        if has_targets:
+            targets[i, :m] = torch.tensor(x["target"], dtype=torch.float32)
+    out = {
+        "input_ids": input_ids,
+        "attention_mask": attention_mask,
+        "option_positions": option_positions,
+        "option_starts": option_starts,
+        "option_ends": option_ends,
+        "option_mask": option_mask,
+        "decide_positions": decide_positions,
+    }
+    if has_targets:
+        out["targets"] = targets
+    return out
+
+
+def pack_shared_request(tok, state, questions: list[dict], max_length: int):
+    """Pack one common state prefix plus causal question suffixes.
+
+    Every question sees exactly the same serialized/clipped state. This is the
+    representation needed for safe KV-prefix reuse at inference time.
+    """
+    if max_length < 32 or not questions:
+        return None
+    bos = [tok.bos_token_id] if tok.bos_token_id is not None else []
+    state_marker = encode_piece(tok, "State:\n")
+    state_ids = encode_piece(tok, state_to_text(state))
+
+    suffixes = []
+    max_suffix_len = 0
+    for question in questions:
+        q_ids = encode_piece(tok, str(question["instruction"]))
+        option_ids = [encode_piece(tok, str(opt["text"])) for opt in question["options"]]
+        if not option_ids or any(not x for x in option_ids):
+            return None
+        ids = encode_piece(tok, "\n\nQuestion:\n") + q_ids + encode_piece(tok, "\n\nOptions:\n")
+        option_positions = []
+        for idx, opt_ids in enumerate(option_ids):
+            ids += encode_piece(tok, f"- [{idx}] ")
+            ids += opt_ids
+            option_positions.append(len(ids) - 1)
+            ids += encode_piece(tok, "\n")
+        ids += encode_piece(tok, "\nDecision:")
+        decide_position = len(ids) - 1
+        suffixes.append({
+            "input_ids": ids,
+            "option_positions": option_positions,
+            "decide_position": decide_position,
+        })
+        max_suffix_len = max(max_suffix_len, len(ids))
+
+    fixed_prefix_len = len(bos) + len(state_marker)
+    state_budget = max_length - fixed_prefix_len - max_suffix_len
+    if state_budget < 0:
+        return None
+    clipped = _clip_head_tail(state_ids, state_budget)
+    was_truncated = len(clipped) < len(state_ids)
+    prefix_ids = bos + state_marker + clipped
+    for s in suffixes:
+        s["was_truncated"] = was_truncated
+        if len(prefix_ids) + len(s["input_ids"]) > max_length:
+            return None
+    return {"prefix_ids": prefix_ids, "suffixes": suffixes, "was_truncated": was_truncated}

eval/harness/src/metrics.py

@@ -0,0 +1,223 @@
+from __future__ import annotations
+
+import math
+from collections import defaultdict
+
+import torch
+
+
+def ece(conf: list[float], correct: list[int], bins: int = 15) -> float | None:
+    if not conf:
+        return None
+    score = 0.0
+    total = len(conf)
+    for b in range(bins):
+        lo = b / bins
+        hi = (b + 1) / bins
+        ids = [i for i, x in enumerate(conf) if x >= lo and (x < hi or (b == bins - 1 and x <= hi))]
+        if not ids:
+            continue
+        acc = sum(correct[i] for i in ids) / len(ids)
+        cf = sum(conf[i] for i in ids) / len(ids)
+        score += len(ids) / total * abs(acc - cf)
+    return score
+
+
+def option_bucket(n: int) -> str:
+    if n <= 2:
+        return "2"
+    if n <= 5:
+        return "3-5"
+    if n <= 10:
+        return "6-10"
+    return ">10"
+
+
+def calibration_group(row: dict) -> str:
+    """Architecture-level calibration group, intentionally not dataset-specific."""
+    kind = str(row.get("kind", "choice"))
+    if kind == "choice":
+        return f"choice:{option_bucket(int(row.get('option_count', len(row.get('target', [])))))}"
+    return f"{kind}:*"
+
+
+def temperature_for_row(calibration, row: dict) -> float:
+    if isinstance(calibration, (int, float)):
+        return max(0.05, min(20.0, float(calibration)))
+    if not isinstance(calibration, dict):
+        return 1.0
+    if "temperature" in calibration and "groups" not in calibration and "global_temperature" not in calibration:
+        return max(0.05, min(20.0, float(calibration.get("temperature", 1.0))))
+    global_t = float(calibration.get("global_temperature", calibration.get("temperature", 1.0)))
+    group = calibration_group(row)
+    groups = calibration.get("groups") or {}
+    entry = groups.get(group)
+    if isinstance(entry, dict):
+        t = float(entry.get("temperature", global_t))
+    elif entry is not None:
+        t = float(entry)
+    else:
+        t = global_t
+    return max(0.05, min(20.0, t))
+
+
+def _score_subset(rows: list[dict], calibration=1.0) -> dict:
+    if not rows:
+        return {"n": 0, "accuracy": None, "nll": None, "brier": None, "ece": None, "mean_confidence": None}
+    conf: list[float] = []
+    correct: list[int] = []
+    nll: list[float] = []
+    brier: list[float] = []
+    for r in rows:
+        T = temperature_for_row(calibration, r)
+        logits = torch.tensor(r["logits"], dtype=torch.float32) / T
+        target = torch.tensor(r["target"], dtype=torch.float32)
+        probs = torch.softmax(logits, dim=-1)
+        pred = int(probs.argmax())
+        gold = int(target.argmax())
+        cf = float(probs[pred])
+        conf.append(cf)
+        correct.append(int(pred == gold))
+        nll.append(float(-(target * torch.log(probs.clamp_min(1e-9))).sum()))
+        brier.append(float(((probs - target) ** 2).sum()))
+    return {
+        "n": len(rows),
+        "accuracy": sum(correct) / len(correct),
+        "nll": sum(nll) / len(nll),
+        "brier": sum(brier) / len(brier),
+        "ece": ece(conf, correct),
+        "mean_confidence": sum(conf) / len(conf),
+    }
+
+
+def score_rows(rows: list[dict], calibration=1.0, grouped: bool = True) -> dict:
+    out = _score_subset(rows, calibration)
+    if isinstance(calibration, (int, float)):
+        out["temperature"] = float(calibration)
+    elif isinstance(calibration, dict):
+        out["calibration"] = calibration
+    if not grouped:
+        return out
+
+    group_specs = {
+        "by_dataset": lambda r: r.get("dataset", "unknown"),
+        "by_kind": lambda r: r.get("kind", "choice"),
+        "by_transform": lambda r: r.get("transform", "unknown"),
+        "by_option_bucket": lambda r: option_bucket(int(r.get("option_count", len(r.get("target", []))))),
+        "by_truncation": lambda r: "truncated" if r.get("truncated", False) else "not_truncated",
+        "by_dataset_truncation": lambda r: f"{r.get('dataset','unknown')}|{'truncated' if r.get('truncated',False) else 'not_truncated'}",
+        "by_kind_truncation": lambda r: f"{r.get('kind','choice')}|{'truncated' if r.get('truncated',False) else 'not_truncated'}",
+    }
+    for out_key, key_fn in group_specs.items():
+        groups: dict[str, list[dict]] = defaultdict(list)
+        for r in rows:
+            groups[str(key_fn(r))].append(r)
+        out[out_key] = {k: _score_subset(v, calibration) for k, v in sorted(groups.items())}
+
+    # Fixed-label NLI confusion is especially useful in v1.3 because the
+    # production options are shuffled while the auxiliary semantic labels are
+    # always entailment/neutral/contradiction.
+    labels = ["entailment", "neutral", "contradiction"]
+    matrix = {g: {p: 0 for p in labels} for g in labels}
+    n_nli = 0
+    for r in rows:
+        gold_idx = int(r.get("aux_nli_label", -1))
+        ids = list(r.get("option_ids") or [])
+        if gold_idx < 0 or gold_idx >= 3 or not ids:
+            continue
+        T = temperature_for_row(calibration, r)
+        probs = torch.softmax(torch.tensor(r["logits"], dtype=torch.float32) / T, dim=-1)
+        pred_i = int(probs.argmax())
+        pred_id = str(ids[pred_i]) if pred_i < len(ids) else ""
+        if pred_id not in labels:
+            continue
+        matrix[labels[gold_idx]][pred_id] += 1
+        n_nli += 1
+    if n_nli:
+        out["nli_confusion"] = {"n": n_nli, "labels": labels, "matrix": matrix}
+    return out
+
+
+def fit_temperature(rows: list[dict], max_iter: int = 50) -> float:
+    if not rows:
+        return 1.0
+    log_t = torch.tensor(0.0, requires_grad=True)
+    opt = torch.optim.LBFGS([log_t], lr=0.2, max_iter=max_iter, line_search_fn="strong_wolfe")
+
+    def closure():
+        opt.zero_grad()
+        T = log_t.exp().clamp(0.05, 20.0)
+        total = torch.tensor(0.0)
+        for r in rows:
+            logits = torch.tensor(r["logits"], dtype=torch.float32)
+            target = torch.tensor(r["target"], dtype=torch.float32)
+            total = total - (target * torch.log_softmax(logits / T, dim=-1)).sum()
+        loss = total / max(1, len(rows))
+        loss.backward()
+        return loss
+
+    opt.step(closure)
+    return float(log_t.exp().clamp(0.05, 20.0).detach())
+
+
+def fit_calibration_policy(
+    rows: list[dict],
+    min_group: int = 40,
+    prior_strength: float = 100.0,
+    max_iter: int = 50,
+) -> dict:
+    """Fit shrinkage temperatures by primitive/option-count.
+
+    Dataset-specific temperatures are intentionally avoided. Small groups shrink
+    in log-temperature space toward the global fit to reduce calibration overfit.
+    """
+    global_t = fit_temperature(rows, max_iter=max_iter)
+    grouped: dict[str, list[dict]] = defaultdict(list)
+    for row in rows:
+        grouped[calibration_group(row)].append(row)
+    groups: dict[str, dict] = {}
+    for key, subset in sorted(grouped.items()):
+        n = len(subset)
+        if n < min_group:
+            continue
+        raw_t = fit_temperature(subset, max_iter=max_iter)
+        w = n / (n + float(prior_strength))
+        shrunk = math.exp(w * math.log(raw_t) + (1.0 - w) * math.log(global_t))
+        groups[key] = {
+            "temperature": float(max(0.05, min(20.0, shrunk))),
+            "raw_temperature": float(raw_t),
+            "n": n,
+            "shrinkage_weight": w,
+        }
+    return {
+        "version": 2,
+        "type": "primitive_option_temperature",
+        "global_temperature": float(global_t),
+        "groups": groups,
+        "min_group": int(min_group),
+        "prior_strength": float(prior_strength),
+        "n": len(rows),
+    }
+
+
+def qualification_value(metrics: dict, metric: str) -> float:
+    """Return a lower-is-better scalar for checkpoint selection."""
+    if metric in {"calibrated_nll", "policy_nll"}:
+        return float(metrics["calibrated"]["nll"])
+    if metric in {"macro_dataset_calibrated_nll", "macro_dataset_policy_nll"}:
+        vals = [float(x["nll"]) for x in metrics["calibrated"].get("by_dataset", {}).values() if x.get("nll") is not None]
+        if not vals:
+            return float(metrics["calibrated"]["nll"])
+        return sum(vals) / len(vals)
+    if metric in {"macro_kind_calibrated_nll", "macro_kind_policy_nll"}:
+        vals = [float(x["nll"]) for x in metrics["calibrated"].get("by_kind", {}).values() if x.get("nll") is not None]
+        if not vals:
+            return float(metrics["calibrated"]["nll"])
+        return sum(vals) / len(vals)
+    if metric in {"calibrated_brier", "policy_brier"}:
+        return float(metrics["calibrated"]["brier"])
+    if metric == "raw_nll":
+        return float(metrics["raw"]["nll"])
+    if metric == "raw_brier":
+        return float(metrics["raw"]["brier"])
+    raise ValueError(f"Unknown selection metric: {metric}")

eval/multimodal_smoke/lines.tsv

@@ -0,0 +1,14 @@
+q_billing_1	billing	I was charged twice for my subscription this month.
+q_billing_2	billing	Can I get a refund for the invoice I paid yesterday?
+q_access_1	account access	I forgot my password and the reset email never arrives.
+q_access_2	account access	My account is locked after too many login attempts.
+q_tech_1	technical support	The app crashes every time I open the settings page.
+q_tech_2	technical support	The website shows an error when I upload a file.
+q_sales_1	sales	I would like a price quote for fifty team licenses.
+q_sales_2	sales	Do you offer a discount for annual enterprise plans?
+s_pos_1	positive	I absolutely love this phone, the camera is amazing.
+s_pos_2	positive	Great service, the delivery arrived early and everything works.
+s_pos_3	positive	This is the best purchase I have made all year.
+s_neg_1	negative	The battery died after two days and support never replied.
+s_neg_2	negative	Terrible experience, the product broke on the first use.
+s_neg_3	negative	I want my money back, this was a complete waste.

eval/multimodal_smoke/make_audio_windows.ps1

@@ -0,0 +1,17 @@
+# Speak every line of lines.tsv with the two built-in Windows voices (16 kHz, 16-bit, mono).
+# Usage: powershell -ExecutionPolicy Bypass -File make_audio_windows.ps1 -Out .\work\audio
+param([string]$Out = (Join-Path $PSScriptRoot "work\audio"))
+Add-Type -AssemblyName System.Speech
+New-Item -ItemType Directory -Force $Out | Out-Null
+$fmt = New-Object System.Speech.AudioFormat.SpeechAudioFormatInfo(16000, [System.Speech.AudioFormat.AudioBitsPerSample]::Sixteen, [System.Speech.AudioFormat.AudioChannel]::Mono)
+$s = New-Object System.Speech.Synthesis.SpeechSynthesizer
+$n = 0
+foreach ($voice in @('Microsoft David Desktop', 'Microsoft Zira Desktop')) {
+  $s.SelectVoice($voice); $tag = $voice.Split(' ')[1].ToLower()
+  foreach ($line in Get-Content -LiteralPath (Join-Path $PSScriptRoot 'lines.tsv')) {
+    $parts = $line.Split("`t")
+    $s.SetOutputToWaveFile((Join-Path $Out ($parts[0] + "-" + $tag + ".wav")), $fmt)
+    $s.Speak($parts[2]); $n++
+  }
+}
+$s.SetOutputToNull(); $s.Dispose(); "wrote $n clips to $Out"

eval/multimodal_smoke/results.json

@@ -0,0 +1,1044 @@
+{
+  "release": "OpenJEV E4B 1.0 (internal v1.6 R7)",
+  "load_mode": "nf4",
+  "device": "NVIDIA GeForce RTX 3060 12 GB",
+  "date": "2026-09-27",
+  "text_vs_multimodal_path_max_prob_diff": 0.0,
+  "peak_vram_gib": 10.73,
+  "suites": {
+    "image: shape (3-way)": {
+      "n": 12,
+      "correct": 12,
+      "accuracy": 1.0,
+      "median_ms": 4881
+    },
+    "image: color (4-way)": {
+      "n": 12,
+      "correct": 12,
+      "accuracy": 1.0,
+      "median_ms": 4738
+    },
+    "control, no image: shape": {
+      "n": 12,
+      "correct": 4,
+      "accuracy": 0.3333333333333333,
+      "median_ms": 191
+    },
+    "control, no image: color": {
+      "n": 12,
+      "correct": 3,
+      "accuracy": 0.25,
+      "median_ms": 202
+    },
+    "image: printed message -> queue (4-way)": {
+      "n": 8,
+      "correct": 8,
+      "accuracy": 1.0,
+      "median_ms": 5284
+    },
+    "control, no image: queue": {
+      "n": 8,
+      "correct": 2,
+      "accuracy": 0.25,
+      "median_ms": 189
+    },
+    "reference, same message as text: queue": {
+      "n": 8,
+      "correct": 7,
+      "accuracy": 0.875,
+      "median_ms": 196
+    },
+    "audio: spoken message -> queue (4-way)": {
+      "n": 16,
+      "correct": 16,
+      "accuracy": 1.0,
+      "median_ms": 2262
+    },
+    "control, no audio: queue": {
+      "n": 8,
+      "correct": 2,
+      "accuracy": 0.25,
+      "median_ms": 190
+    },
+    "audio: spoken review -> sentiment (2-way)": {
+      "n": 12,
+      "correct": 12,
+      "accuracy": 1.0,
+      "median_ms": 2424
+    },
+    "control, no audio: sentiment": {
+      "n": 6,
+      "correct": 3,
+      "accuracy": 0.5,
+      "median_ms": 190
+    },
+    "reference, same review as text: sentiment": {
+      "n": 6,
+      "correct": 6,
+      "accuracy": 1.0,
+      "median_ms": 192
+    }
+  },
+  "rows": [
+    {
+      "suite": "image: shape (3-way)",
+      "kind": "image",
+      "gold": "square",
+      "pred": "square",
+      "p_gold": 0.9991449117660522,
+      "ms": 4963
+    },
+    {
+      "suite": "image: color (4-way)",
+      "kind": "image",
+      "gold": "red",
+      "pred": "red",
+      "p_gold": 0.999085545539856,
+      "ms": 4712
+    },
+    {
+      "suite": "control, no image: shape",
+      "kind": "none",
+      "gold": "square",
+      "pred": "circle",
+      "p_gold": 0.3252781629562378,
+      "ms": 191
+    },
+    {
+      "suite": "control, no image: color",
+      "kind": "none",
+      "gold": "red",
+      "pred": "red",
+      "p_gold": 0.34349796175956726,
+      "ms": 223
+    },
+    {
+      "suite": "image: shape (3-way)",
+      "kind": "image",
+      "gold": "square",
+      "pred": "square",
+      "p_gold": 0.9993372559547424,
+      "ms": 4881
+    },
+    {
+      "suite": "image: color (4-way)",
+      "kind": "image",
+      "gold": "blue",
+      "pred": "blue",
+      "p_gold": 0.9998289346694946,
+      "ms": 4711
+    },
+    {
+      "suite": "control, no image: shape",
+      "kind": "none",
+      "gold": "square",
+      "pred": "circle",
+      "p_gold": 0.3252781629562378,
+      "ms": 207
+    },
+    {
+      "suite": "control, no image: color",
+      "kind": "none",
+      "gold": "blue",
+      "pred": "red",
+      "p_gold": 0.22613903880119324,
+      "ms": 191
+    },
+    {
+      "suite": "image: shape (3-way)",
+      "kind": "image",
+      "gold": "square",
+      "pred": "square",
+      "p_gold": 0.9991328120231628,
+      "ms": 4809
+    },
+    {
+      "suite": "image: color (4-way)",
+      "kind": "image",
+      "gold": "green",
+      "pred": "green",
+      "p_gold": 0.9999126195907593,
+      "ms": 4725
+    },
+    {
+      "suite": "control, no image: shape",
+      "kind": "none",
+      "gold": "square",
+      "pred": "circle",
+      "p_gold": 0.3252781629562378,
+      "ms": 186
+    },
+    {
+      "suite": "control, no image: color",
+      "kind": "none",
+      "gold": "green",
+      "pred": "red",
+      "p_gold": 0.22800041735172272,
+      "ms": 180
+    },
+    {
+      "suite": "image: shape (3-way)",
+      "kind": "image",
+      "gold": "square",
+      "pred": "square",
+      "p_gold": 0.999433696269989,
+      "ms": 4700
+    },
+    {
+      "suite": "image: color (4-way)",
+      "kind": "image",
+      "gold": "yellow",
+      "pred": "yellow",
+      "p_gold": 0.9988981485366821,
+      "ms": 4706
+    },
+    {
+      "suite": "control, no image: shape",
+      "kind": "none",
+      "gold": "square",
+      "pred": "circle",
+      "p_gold": 0.3252781629562378,
+      "ms": 181
+    },
+    {
+      "suite": "control, no image: color",
+      "kind": "none",
+      "gold": "yellow",
+      "pred": "red",
+      "p_gold": 0.20236249268054962,
+      "ms": 200
+    },
+    {
+      "suite": "image: shape (3-way)",
+      "kind": "image",
+      "gold": "circle",
+      "pred": "circle",
+      "p_gold": 0.9994489550590515,
+      "ms": 4697
+    },
+    {
+      "suite": "image: color (4-way)",
+      "kind": "image",
+      "gold": "red",
+      "pred": "red",
+      "p_gold": 0.998988687992096,
+      "ms": 4705
+    },
+    {
+      "suite": "control, no image: shape",
+      "kind": "none",
+      "gold": "circle",
+      "pred": "circle",
+      "p_gold": 0.451766699552536,
+      "ms": 189
+    },
+    {
+      "suite": "control, no image: color",
+      "kind": "none",
+      "gold": "red",
+      "pred": "red",
+      "p_gold": 0.34349796175956726,
+      "ms": 190
+    },
+    {
+      "suite": "image: shape (3-way)",
+      "kind": "image",
+      "gold": "circle",
+      "pred": "circle",
+      "p_gold": 0.9999669790267944,
+      "ms": 4698
+    },
+    {
+      "suite": "image: color (4-way)",
+      "kind": "image",
+      "gold": "blue",
+      "pred": "blue",
+      "p_gold": 0.9999053478240967,
+      "ms": 4738
+    },
+    {
+      "suite": "control, no image: shape",
+      "kind": "none",
+      "gold": "circle",
+      "pred": "circle",
+      "p_gold": 0.451766699552536,
+      "ms": 211
+    },
+    {
+      "suite": "control, no image: color",
+      "kind": "none",
+      "gold": "blue",
+      "pred": "red",
+      "p_gold": 0.22613903880119324,
+      "ms": 179
+    },
+    {
+      "suite": "image: shape (3-way)",
+      "kind": "image",
+      "gold": "circle",
+      "pred": "circle",
+      "p_gold": 0.9999372959136963,
+      "ms": 4692
+    },
+    {
+      "suite": "image: color (4-way)",
+      "kind": "image",
+      "gold": "green",
+      "pred": "green",
+      "p_gold": 0.9998636245727539,
+      "ms": 4783
+    },
+    {
+      "suite": "control, no image: shape",
+      "kind": "none",
+      "gold": "circle",
+      "pred": "circle",
+      "p_gold": 0.451766699552536,
+      "ms": 216
+    },
+    {
+      "suite": "control, no image: color",
+      "kind": "none",
+      "gold": "green",
+      "pred": "red",
+      "p_gold": 0.22800041735172272,
+      "ms": 202
+    },
+    {
+      "suite": "image: shape (3-way)",
+      "kind": "image",
+      "gold": "circle",
+      "pred": "circle",
+      "p_gold": 0.9997873902320862,
+      "ms": 4726
+    },
+    {
+      "suite": "image: color (4-way)",
+      "kind": "image",
+      "gold": "yellow",
+      "pred": "yellow",
+      "p_gold": 0.9992806315422058,
+      "ms": 4724
+    },
+    {
+      "suite": "control, no image: shape",
+      "kind": "none",
+      "gold": "circle",
+      "pred": "circle",
+      "p_gold": 0.451766699552536,
+      "ms": 185
+    },
+    {
+      "suite": "control, no image: color",
+      "kind": "none",
+      "gold": "yellow",
+      "pred": "red",
+      "p_gold": 0.20236249268054962,
+      "ms": 202
+    },
+    {
+      "suite": "image: shape (3-way)",
+      "kind": "image",
+      "gold": "triangle",
+      "pred": "triangle",
+      "p_gold": 0.9999802112579346,
+      "ms": 4963
+    },
+    {
+      "suite": "image: color (4-way)",
+      "kind": "image",
+      "gold": "red",
+      "pred": "red",
+      "p_gold": 0.9987249970436096,
+      "ms": 4991
+    },
+    {
+      "suite": "control, no image: shape",
+      "kind": "none",
+      "gold": "triangle",
+      "pred": "circle",
+      "p_gold": 0.22295516729354858,
+      "ms": 251
+    },
+    {
+      "suite": "control, no image: color",
+      "kind": "none",
+      "gold": "red",
+      "pred": "red",
+      "p_gold": 0.34349796175956726,
+      "ms": 285
+    },
+    {
+      "suite": "image: shape (3-way)",
+      "kind": "image",
+      "gold": "triangle",
+      "pred": "triangle",
+      "p_gold": 0.9999686479568481,
+      "ms": 6177
+    },
+    {
+      "suite": "image: color (4-way)",
+      "kind": "image",
+      "gold": "blue",
+      "pred": "blue",
+      "p_gold": 0.9998495578765869,
+      "ms": 5227
+    },
+    {
+      "suite": "control, no image: shape",
+      "kind": "none",
+      "gold": "triangle",
+      "pred": "circle",
+      "p_gold": 0.22295516729354858,
+      "ms": 205
+    },
+    {
+      "suite": "control, no image: color",
+      "kind": "none",
+      "gold": "blue",
+      "pred": "red",
+      "p_gold": 0.22613903880119324,
+      "ms": 220
+    },
+    {
+      "suite": "image: shape (3-way)",
+      "kind": "image",
+      "gold": "triangle",
+      "pred": "triangle",
+      "p_gold": 0.9999749660491943,
+      "ms": 5371
+    },
+    {
+      "suite": "image: color (4-way)",
+      "kind": "image",
+      "gold": "green",
+      "pred": "green",
+      "p_gold": 0.9999052286148071,
+      "ms": 4874
+    },
+    {
+      "suite": "control, no image: shape",
+      "kind": "none",
+      "gold": "triangle",
+      "pred": "circle",
+      "p_gold": 0.22295516729354858,
+      "ms": 190
+    },
+    {
+      "suite": "control, no image: color",
+      "kind": "none",
+      "gold": "green",
+      "pred": "red",
+      "p_gold": 0.22800041735172272,
+      "ms": 208
+    },
+    {
+      "suite": "image: shape (3-way)",
+      "kind": "image",
+      "gold": "triangle",
+      "pred": "triangle",
+      "p_gold": 0.9999715089797974,
+      "ms": 5469
+    },
+    {
+      "suite": "image: color (4-way)",
+      "kind": "image",
+      "gold": "yellow",
+      "pred": "yellow",
+      "p_gold": 0.9982457160949707,
+      "ms": 5095
+    },
+    {
+      "suite": "control, no image: shape",
+      "kind": "none",
+      "gold": "triangle",
+      "pred": "circle",
+      "p_gold": 0.22295516729354858,
+      "ms": 187
+    },
+    {
+      "suite": "control, no image: color",
+      "kind": "none",
+      "gold": "yellow",
+      "pred": "red",
+      "p_gold": 0.20236249268054962,
+      "ms": 185
+    },
+    {
+      "suite": "image: printed message -> queue (4-way)",
+      "kind": "image",
+      "gold": "billing",
+      "pred": "billing",
+      "p_gold": 0.9587600827217102,
+      "ms": 5605
+    },
+    {
+      "suite": "control, no image: queue",
+      "kind": "none",
+      "gold": "billing",
+      "pred": "technical support",
+      "p_gold": 0.02786904200911522,
+      "ms": 203
+    },
+    {
+      "suite": "reference, same message as text: queue",
+      "kind": "text",
+      "gold": "billing",
+      "pred": "billing",
+      "p_gold": 0.9986612796783447,
+      "ms": 196
+    },
+    {
+      "suite": "image: printed message -> queue (4-way)",
+      "kind": "image",
+      "gold": "billing",
+      "pred": "billing",
+      "p_gold": 0.9929062128067017,
+      "ms": 5024
+    },
+    {
+      "suite": "control, no image: queue",
+      "kind": "none",
+      "gold": "billing",
+      "pred": "technical support",
+      "p_gold": 0.02786904200911522,
+      "ms": 188
+    },
+    {
+      "suite": "reference, same message as text: queue",
+      "kind": "text",
+      "gold": "billing",
+      "pred": "billing",
+      "p_gold": 0.9986044764518738,
+      "ms": 181
+    },
+    {
+      "suite": "image: printed message -> queue (4-way)",
+      "kind": "image",
+      "gold": "account access",
+      "pred": "account access",
+      "p_gold": 0.7705231308937073,
+      "ms": 5284
+    },
+    {
+      "suite": "control, no image: queue",
+      "kind": "none",
+      "gold": "account access",
+      "pred": "technical support",
+      "p_gold": 0.15655532479286194,
+      "ms": 189
+    },
+    {
+      "suite": "reference, same message as text: queue",
+      "kind": "text",
+      "gold": "account access",
+      "pred": "account access",
+      "p_gold": 0.8988062143325806,
+      "ms": 202
+    },
+    {
+      "suite": "image: printed message -> queue (4-way)",
+      "kind": "image",
+      "gold": "account access",
+      "pred": "account access",
+      "p_gold": 0.6306372284889221,
+      "ms": 5310
+    },
+    {
+      "suite": "control, no image: queue",
+      "kind": "none",
+      "gold": "account access",
+      "pred": "technical support",
+      "p_gold": 0.15655532479286194,
+      "ms": 181
+    },
+    {
+      "suite": "reference, same message as text: queue",
+      "kind": "text",
+      "gold": "account access",
+      "pred": "account access",
+      "p_gold": 0.8048309087753296,
+      "ms": 192
+    },
+    {
+      "suite": "image: printed message -> queue (4-way)",
+      "kind": "image",
+      "gold": "technical support",
+      "pred": "technical support",
+      "p_gold": 0.9995842576026917,
+      "ms": 5288
+    },
+    {
+      "suite": "control, no image: queue",
+      "kind": "none",
+      "gold": "technical support",
+      "pred": "technical support",
+      "p_gold": 0.8107117414474487,
+      "ms": 202
+    },
+    {
+      "suite": "reference, same message as text: queue",
+      "kind": "text",
+      "gold": "technical support",
+      "pred": "technical support",
+      "p_gold": 0.9998304843902588,
+      "ms": 204
+    },
+    {
+      "suite": "image: printed message -> queue (4-way)",
+      "kind": "image",
+      "gold": "technical support",
+      "pred": "technical support",
+      "p_gold": 0.998561441898346,
+      "ms": 4919
+    },
+    {
+      "suite": "control, no image: queue",
+      "kind": "none",
+      "gold": "technical support",
+      "pred": "technical support",
+      "p_gold": 0.8107117414474487,
+      "ms": 209
+    },
+    {
+      "suite": "reference, same message as text: queue",
+      "kind": "text",
+      "gold": "technical support",
+      "pred": "technical support",
+      "p_gold": 0.999686598777771,
+      "ms": 206
+    },
+    {
+      "suite": "image: printed message -> queue (4-way)",
+      "kind": "image",
+      "gold": "sales",
+      "pred": "sales",
+      "p_gold": 0.9524871706962585,
+      "ms": 5105
+    },
+    {
+      "suite": "control, no image: queue",
+      "kind": "none",
+      "gold": "sales",
+      "pred": "technical support",
+      "p_gold": 0.004863882903009653,
+      "ms": 188
+    },
+    {
+      "suite": "reference, same message as text: queue",
+      "kind": "text",
+      "gold": "sales",
+      "pred": "sales",
+      "p_gold": 0.9405021667480469,
+      "ms": 193
+    },
+    {
+      "suite": "image: printed message -> queue (4-way)",
+      "kind": "image",
+      "gold": "sales",
+      "pred": "sales",
+      "p_gold": 0.8253994584083557,
+      "ms": 5032
+    },
+    {
+      "suite": "control, no image: queue",
+      "kind": "none",
+      "gold": "sales",
+      "pred": "technical support",
+      "p_gold": 0.004863882903009653,
+      "ms": 179
+    },
+    {
+      "suite": "reference, same message as text: queue",
+      "kind": "text",
+      "gold": "sales",
+      "pred": "billing",
+      "p_gold": 0.4147074818611145,
+      "ms": 192
+    },
+    {
+      "suite": "audio: spoken message -> queue (4-way)",
+      "kind": "audio",
+      "gold": "billing",
+      "pred": "billing",
+      "p_gold": 0.991688072681427,
+      "ms": 2480
+    },
+    {
+      "suite": "audio: spoken message -> queue (4-way)",
+      "kind": "audio",
+      "gold": "billing",
+      "pred": "billing",
+      "p_gold": 0.9903938174247742,
+      "ms": 2274
+    },
+    {
+      "suite": "control, no audio: queue",
+      "kind": "none",
+      "gold": "billing",
+      "pred": "technical support",
+      "p_gold": 0.005660666152834892,
+      "ms": 184
+    },
+    {
+      "suite": "audio: spoken message -> queue (4-way)",
+      "kind": "audio",
+      "gold": "billing",
+      "pred": "billing",
+      "p_gold": 0.9894062876701355,
+      "ms": 2247
+    },
+    {
+      "suite": "audio: spoken message -> queue (4-way)",
+      "kind": "audio",
+      "gold": "billing",
+      "pred": "billing",
+      "p_gold": 0.9892838597297668,
+      "ms": 2272
+    },
+    {
+      "suite": "control, no audio: queue",
+      "kind": "none",
+      "gold": "billing",
+      "pred": "technical support",
+      "p_gold": 0.005660666152834892,
+      "ms": 193
+    },
+    {
+      "suite": "audio: spoken message -> queue (4-way)",
+      "kind": "audio",
+      "gold": "account access",
+      "pred": "account access",
+      "p_gold": 0.898837149143219,
+      "ms": 2256
+    },
+    {
+      "suite": "audio: spoken message -> queue (4-way)",
+      "kind": "audio",
+      "gold": "account access",
+      "pred": "account access",
+      "p_gold": 0.8590515851974487,
+      "ms": 2290
+    },
+    {
+      "suite": "control, no audio: queue",
+      "kind": "none",
+      "gold": "account access",
+      "pred": "technical support",
+      "p_gold": 0.01911778189241886,
+      "ms": 183
+    },
+    {
+      "suite": "audio: spoken message -> queue (4-way)",
+      "kind": "audio",
+      "gold": "account access",
+      "pred": "account access",
+      "p_gold": 0.7258855104446411,
+      "ms": 2258
+    },
+    {
+      "suite": "audio: spoken message -> queue (4-way)",
+      "kind": "audio",
+      "gold": "account access",
+      "pred": "account access",
+      "p_gold": 0.6762276291847229,
+      "ms": 2268
+    },
+    {
+      "suite": "control, no audio: queue",
+      "kind": "none",
+      "gold": "account access",
+      "pred": "technical support",
+      "p_gold": 0.01911778189241886,
+      "ms": 189
+    },
+    {
+      "suite": "audio: spoken message -> queue (4-way)",
+      "kind": "audio",
+      "gold": "technical support",
+      "pred": "technical support",
+      "p_gold": 0.9997541308403015,
+      "ms": 2246
+    },
+    {
+      "suite": "audio: spoken message -> queue (4-way)",
+      "kind": "audio",
+      "gold": "technical support",
+      "pred": "technical support",
+      "p_gold": 0.9997640252113342,
+      "ms": 2263
+    },
+    {
+      "suite": "control, no audio: queue",
+      "kind": "none",
+      "gold": "technical support",
+      "pred": "technical support",
+      "p_gold": 0.955605149269104,
+      "ms": 201
+    },
+    {
+      "suite": "audio: spoken message -> queue (4-way)",
+      "kind": "audio",
+      "gold": "technical support",
+      "pred": "technical support",
+      "p_gold": 0.9992349147796631,
+      "ms": 2257
+    },
+    {
+      "suite": "audio: spoken message -> queue (4-way)",
+      "kind": "audio",
+      "gold": "technical support",
+      "pred": "technical support",
+      "p_gold": 0.9991409778594971,
+      "ms": 2267
+    },
+    {
+      "suite": "control, no audio: queue",
+      "kind": "none",
+      "gold": "technical support",
+      "pred": "technical support",
+      "p_gold": 0.955605149269104,
+      "ms": 190
+    },
+    {
+      "suite": "audio: spoken message -> queue (4-way)",
+      "kind": "audio",
+      "gold": "sales",
+      "pred": "sales",
+      "p_gold": 0.9358583092689514,
+      "ms": 2255
+    },
+    {
+      "suite": "audio: spoken message -> queue (4-way)",
+      "kind": "audio",
+      "gold": "sales",
+      "pred": "sales",
+      "p_gold": 0.9297321438789368,
+      "ms": 2247
+    },
+    {
+      "suite": "control, no audio: queue",
+      "kind": "none",
+      "gold": "sales",
+      "pred": "technical support",
+      "p_gold": 0.019616365432739258,
+      "ms": 216
+    },
+    {
+      "suite": "audio: spoken message -> queue (4-way)",
+      "kind": "audio",
+      "gold": "sales",
+      "pred": "sales",
+      "p_gold": 0.6899588108062744,
+      "ms": 2262
+    },
+    {
+      "suite": "audio: spoken message -> queue (4-way)",
+      "kind": "audio",
+      "gold": "sales",
+      "pred": "sales",
+      "p_gold": 0.7804462909698486,
+      "ms": 2259
+    },
+    {
+      "suite": "control, no audio: queue",
+      "kind": "none",
+      "gold": "sales",
+      "pred": "technical support",
+      "p_gold": 0.019616365432739258,
+      "ms": 175
+    },
+    {
+      "suite": "audio: spoken review -> sentiment (2-way)",
+      "kind": "audio",
+      "gold": "positive",
+      "pred": "positive",
+      "p_gold": 0.9998955726623535,
+      "ms": 2278
+    },
+    {
+      "suite": "audio: spoken review -> sentiment (2-way)",
+      "kind": "audio",
+      "gold": "positive",
+      "pred": "positive",
+      "p_gold": 0.9998762607574463,
+      "ms": 2751
+    },
+    {
+      "suite": "control, no audio: sentiment",
+      "kind": "none",
+      "gold": "positive",
+      "pred": "positive",
+      "p_gold": 0.8081900477409363,
+      "ms": 202
+    },
+    {
+      "suite": "reference, same review as text: sentiment",
+      "kind": "text",
+      "gold": "positive",
+      "pred": "positive",
+      "p_gold": 0.999956488609314,
+      "ms": 200
+    },
+    {
+      "suite": "audio: spoken review -> sentiment (2-way)",
+      "kind": "audio",
+      "gold": "positive",
+      "pred": "positive",
+      "p_gold": 0.9998339414596558,
+      "ms": 2356
+    },
+    {
+      "suite": "audio: spoken review -> sentiment (2-way)",
+      "kind": "audio",
+      "gold": "positive",
+      "pred": "positive",
+      "p_gold": 0.9998350143432617,
+      "ms": 2681
+    },
+    {
+      "suite": "control, no audio: sentiment",
+      "kind": "none",
+      "gold": "positive",
+      "pred": "positive",
+      "p_gold": 0.8081900477409363,
+      "ms": 190
+    },
+    {
+      "suite": "reference, same review as text: sentiment",
+      "kind": "text",
+      "gold": "positive",
+      "pred": "positive",
+      "p_gold": 0.9999436140060425,
+      "ms": 188
+    },
+    {
+      "suite": "audio: spoken review -> sentiment (2-way)",
+      "kind": "audio",
+      "gold": "positive",
+      "pred": "positive",
+      "p_gold": 0.9999054670333862,
+      "ms": 265
+    },
+    {
+      "suite": "audio: spoken review -> sentiment (2-way)",
+      "kind": "audio",
+      "gold": "positive",
+      "pred": "positive",
+      "p_gold": 0.9998829364776611,
+      "ms": 261
+    },
+    {
+      "suite": "control, no audio: sentiment",
+      "kind": "none",
+      "gold": "positive",
+      "pred": "positive",
+      "p_gold": 0.8081900477409363,
+      "ms": 200
+    },
+    {
+      "suite": "reference, same review as text: sentiment",
+      "kind": "text",
+      "gold": "positive",
+      "pred": "positive",
+      "p_gold": 0.9999107122421265,
+      "ms": 213
+    },
+    {
+      "suite": "audio: spoken review -> sentiment (2-way)",
+      "kind": "audio",
+      "gold": "negative",
+      "pred": "negative",
+      "p_gold": 0.9996554851531982,
+      "ms": 2513
+    },
+    {
+      "suite": "audio: spoken review -> sentiment (2-way)",
+      "kind": "audio",
+      "gold": "negative",
+      "pred": "negative",
+      "p_gold": 0.9996836185455322,
+      "ms": 2424
+    },
+    {
+      "suite": "control, no audio: sentiment",
+      "kind": "none",
+      "gold": "negative",
+      "pred": "positive",
+      "p_gold": 0.19180989265441895,
+      "ms": 188
+    },
+    {
+      "suite": "reference, same review as text: sentiment",
+      "kind": "text",
+      "gold": "negative",
+      "pred": "negative",
+      "p_gold": 0.9992406368255615,
+      "ms": 189
+    },
+    {
+      "suite": "audio: spoken review -> sentiment (2-way)",
+      "kind": "audio",
+      "gold": "negative",
+      "pred": "negative",
+      "p_gold": 0.9999104738235474,
+      "ms": 2465
+    },
+    {
+      "suite": "audio: spoken review -> sentiment (2-way)",
+      "kind": "audio",
+      "gold": "negative",
+      "pred": "negative",
+      "p_gold": 0.9998811483383179,
+      "ms": 2744
+    },
+    {
+      "suite": "control, no audio: sentiment",
+      "kind": "none",
+      "gold": "negative",
+      "pred": "positive",
+      "p_gold": 0.19180989265441895,
+      "ms": 188
+    },
+    {
+      "suite": "reference, same review as text: sentiment",
+      "kind": "text",
+      "gold": "negative",
+      "pred": "negative",
+      "p_gold": 0.999881386756897,
+      "ms": 192
+    },
+    {
+      "suite": "audio: spoken review -> sentiment (2-way)",
+      "kind": "audio",
+      "gold": "negative",
+      "pred": "negative",
+      "p_gold": 0.9994496703147888,
+      "ms": 2155
+    },
+    {
+      "suite": "audio: spoken review -> sentiment (2-way)",
+      "kind": "audio",
+      "gold": "negative",
+      "pred": "negative",
+      "p_gold": 0.9993879795074463,
+      "ms": 2120
+    },
+    {
+      "suite": "control, no audio: sentiment",
+      "kind": "none",
+      "gold": "negative",
+      "pred": "positive",
+      "p_gold": 0.19180989265441895,
+      "ms": 190
+    },
+    {
+      "suite": "reference, same review as text: sentiment",
+      "kind": "text",
+      "gold": "negative",
+      "pred": "negative",
+      "p_gold": 0.9989564418792725,
+      "ms": 184
+    }
+  ]
+}

eval/multimodal_smoke/run_smoke.py

@@ -0,0 +1,139 @@
+"""Smoke test for the experimental image/audio path (docs/MULTIMODAL_EXPERIMENTAL.md).
+
+Run from the release root after make_audio_windows.ps1 has written work/audio:
+    python -m eval.multimodal_smoke.run_smoke
+Images are generated here. Every question is also asked with the media removed,
+which should score at chance.
+"""
+
+import json
+import time
+from pathlib import Path
+
+import torch
+from PIL import Image, ImageDraw, ImageFont
+
+from openjev import OpenJEV
+from openjev.formatting import collate_packed, pack_question
+from openjev.modeling import _temperature
+
+HERE = Path(__file__).resolve().parent
+REPO = HERE.parents[1]
+WORK = HERE / "work"
+QUEUES = ["billing", "technical support", "account access", "sales"]
+COLORS = {"red": (220, 30, 30), "blue": (30, 70, 220), "green": (30, 160, 60), "yellow": (240, 200, 20)}
+SHAPES = ["square", "circle", "triangle"]
+TEXT_CASES = [
+    ("The battery died after two days and support never replied.", "What is the sentiment of this review?", ["positive", "negative", "neutral"]),
+    ("The central bank raised interest rates by half a point to curb inflation.", "What is the topic of this headline?", ["sports", "economy", "technology", "health"]),
+    ("I forgot my password and the reset email never arrives.", "Which support queue should handle this message?", QUEUES),
+    ("Premise: The store is closed on Sundays. Hypothesis: The store is open every day.", "Does the premise entail the hypothesis?", ["entailment", "neutral", "contradiction"]),
+]
+
+
+def make_images(lines):
+    out = WORK / "images"
+    out.mkdir(parents=True, exist_ok=True)
+    shapes = []
+    for shape in SHAPES:
+        for name, rgb in COLORS.items():
+            im = Image.new("RGB", (512, 512), "white")
+            d = ImageDraw.Draw(im)
+            if shape == "square":
+                d.rectangle((110, 110, 402, 402), fill=rgb)
+            elif shape == "circle":
+                d.ellipse((100, 100, 412, 412), fill=rgb)
+            else:
+                d.polygon([(256, 80), (80, 420), (432, 420)], fill=rgb)
+            path = out / f"{name}-{shape}.png"
+            im.save(path)
+            shapes.append((path, shape, name))
+    try:
+        font = ImageFont.truetype("arial.ttf", 34)
+    except OSError:
+        font = ImageFont.load_default()
+    messages = []
+    for rid, label, text in lines:
+        if not rid.startswith("q_"):
+            continue
+        im = Image.new("RGB", (768, 384), "white")
+        d = ImageDraw.Draw(im)
+        row, y = "", 60
+        for word in text.split():
+            if d.textlength(row + " " + word, font=font) > 680:
+                d.text((40, y), row.strip(), fill="black", font=font)
+                y, row = y + 52, ""
+            row += " " + word
+        d.text((40, y), row.strip(), fill="black", font=font)
+        path = out / f"{rid}.png"
+        im.save(path)
+        messages.append((path, label, text))
+    return shapes, messages
+
+
+def main():
+    lines = [l.split("\t") for l in (HERE / "lines.tsv").read_text(encoding="utf-8").splitlines() if l.strip()]
+    audio_dir = WORK / "audio"
+    if not any(audio_dir.glob("*.wav")):
+        raise SystemExit("Run make_audio_windows.ps1 first (writes eval/multimodal_smoke/work/audio).")
+    shapes, messages = make_images(lines)
+    model = OpenJEV.from_pretrained(REPO, device="cuda", load_mode="nf4", multimodal=True)
+    suites, rows = {}, []
+
+    def run(suite, gold, question, options, **kw):
+        t = time.perf_counter()
+        r = model.choice(instruction=question, options=options, **kw)
+        ms = 1000 * (time.perf_counter() - t)
+        s = suites.setdefault(suite, {"n": 0, "correct": 0, "ms": []})
+        s["n"] += 1
+        s["correct"] += r["selected_option"] == gold
+        s["ms"].append(ms)
+        rows.append({"suite": suite, "gold": gold, "pred": r["selected_option"],
+                     "p_gold": r["probabilities"][options.index(gold)], "ms": round(ms)})
+
+    # Text-only inputs must score identically through the text and multimodal forward paths.
+    max_diff = 0.0
+    for state, q, opts in TEXT_CASES:
+        a = model.choice(state=state, instruction=q, options=opts)["probabilities"]
+        packed = pack_question(model.tokenizer, state, {"instruction": q, "options": [{"text": o} for o in opts]}, 8192)
+        batch = {k: v.to("cuda") for k, v in collate_packed(model.tokenizer, [packed]).items()}
+        with torch.inference_mode():
+            logits = model._media_logits(batch, {})[0, :len(opts)].float()
+        b = torch.softmax(logits / _temperature(model.calibration), -1).cpu().tolist()
+        max_diff = max(max_diff, max(abs(x - y) for x, y in zip(a, b)))
+
+    for path, shape, color in shapes:
+        run("image: shape (3-way)", shape, "What shape is shown in the image?", SHAPES, state="", image=path)
+        run("image: color (4-way)", color, "What color is the shape in the image?", list(COLORS), state="", image=path)
+        run("control, no image: shape", shape, "What shape is shown in the image?", SHAPES, state="")
+        run("control, no image: color", color, "What color is the shape in the image?", list(COLORS), state="")
+    for path, label, text in messages:
+        q = "Which support queue should handle the message in the image?"
+        run("image: printed message -> queue (4-way)", label, q, QUEUES, state="", image=path)
+        run("control, no image: queue", label, q, QUEUES, state="")
+        run("reference, same message as text: queue", label, "Which support queue should handle this message?", QUEUES, state=text)
+    for rid, label, text in lines:
+        for voice in ("david", "zira"):
+            clip = audio_dir / f"{rid}-{voice}.wav"
+            if rid.startswith("q_"):
+                run("audio: spoken message -> queue (4-way)", label, "Which support queue should handle the spoken message?", QUEUES, state="", audio=clip)
+            else:
+                run("audio: spoken review -> sentiment (2-way)", label, "What is the sentiment of the spoken review?", ["positive", "negative"], state="", audio=clip)
+        if rid.startswith("q_"):
+            run("control, no audio: queue", label, "Which support queue should handle the spoken message?", QUEUES, state="")
+        else:
+            run("control, no audio: sentiment", label, "What is the sentiment of the spoken review?", ["positive", "negative"], state="")
+            run("reference, same review as text: sentiment", label, "What is the sentiment of this review?", ["positive", "negative"], state=text)
+
+    print(f"text vs multimodal path, max probability difference: {max_diff:.2e}")
+    for name, s in suites.items():
+        s["median_ms"] = round(sorted(s.pop("ms"))[s["n"] // 2])
+        s["accuracy"] = s["correct"] / s["n"]
+        print(f"{name:45s} {s['correct']:>2}/{s['n']:<2} {100 * s['accuracy']:5.1f}%  median {s['median_ms']} ms")
+    result = {"text_vs_multimodal_path_max_prob_diff": max_diff,
+              "peak_vram_gib": round(torch.cuda.max_memory_allocated() / 2**30, 2), "suites": suites, "rows": rows}
+    (WORK / "results.json").write_text(json.dumps(result, indent=2) + "\n", encoding="utf-8")
+
+
+if __name__ == "__main__":
+    main()

eval/openjev_1_0_public_benchmark.csv

@@ -0,0 +1,6 @@
+task,display,status,n,accuracy,nll,brier,ece,p50_ms,p95_ms,raw_e4b_nf4_accuracy,delta_vs_raw_pp
+arc_challenge,ARC-Challenge,measured,1172,0.878839590443686,0.3434354264088279,0.18004018282065612,0.018640702693919233,173.45179200128769,241.31601209974178,0.8737,0.5139590443685949
+winogrande,WinoGrande,measured,1267,0.664561957379637,0.8732282598352692,0.5079363220206212,0.1986676781891346,168.07529400102794,241.4132092009823,0.6014,6.316195737963692
+arc_easy,ARC-Easy,measured,2376,0.9562289562289562,0.15331558248276225,0.07269179989034444,0.029571857402955024,172.40582649901626,231.58336125015921,0.9558,0.04289562289562232
+hellaswag,HellaSwag,measured,10042,0.8977295359490142,0.2736215284007136,0.14571887516017584,0.012820075397336044,230.12878749796073,276.0439652007335,0.7651,13.262953594901417
+mmlu,MMLU,measured,14042,0.6441390115368181,0.8705675106095903,0.45396635565340543,0.03314363695835977,196.04511249963252,303.28189720094076,0.6619,-1.776098846318197

eval/openjev_1_0_public_benchmark.json

@@ -0,0 +1,74 @@
+{
+  "model_line": "openjev-e4b",
+  "release": "1.0",
+  "benchmark_artifact": "local NF4 frozen base + dense adapted layers + LoRA + decision head",
+  "source_summary": "eval/raw_run/summary.json",
+  "raw_control_summary": "eval/raw_e4b_nf4_control_summary.json",
+  "tasks": {
+    "arc_challenge": {
+      "status": "measured",
+      "display": "ARC-Challenge",
+      "n": 1172,
+      "accuracy": 0.878839590443686,
+      "nll": 0.3434354264088279,
+      "brier": 0.18004018282065612,
+      "ece": 0.018640702693919233,
+      "p50_ms": 173.45179200128769,
+      "p95_ms": 241.31601209974178,
+      "raw_e4b_nf4_accuracy": 0.8737,
+      "delta_vs_raw_pp": 0.5139590443685949
+    },
+    "winogrande": {
+      "status": "measured",
+      "display": "WinoGrande",
+      "n": 1267,
+      "accuracy": 0.664561957379637,
+      "nll": 0.8732282598352692,
+      "brier": 0.5079363220206212,
+      "ece": 0.1986676781891346,
+      "p50_ms": 168.07529400102794,
+      "p95_ms": 241.4132092009823,
+      "raw_e4b_nf4_accuracy": 0.6014,
+      "delta_vs_raw_pp": 6.316195737963692
+    },
+    "arc_easy": {
+      "status": "measured",
+      "display": "ARC-Easy",
+      "n": 2376,
+      "accuracy": 0.9562289562289562,
+      "nll": 0.15331558248276225,
+      "brier": 0.07269179989034444,
+      "ece": 0.029571857402955024,
+      "p50_ms": 172.40582649901626,
+      "p95_ms": 231.58336125015921,
+      "raw_e4b_nf4_accuracy": 0.9558,
+      "delta_vs_raw_pp": 0.04289562289562232
+    },
+    "hellaswag": {
+      "status": "measured",
+      "display": "HellaSwag",
+      "n": 10042,
+      "accuracy": 0.8977295359490142,
+      "nll": 0.2736215284007136,
+      "brier": 0.14571887516017584,
+      "ece": 0.012820075397336044,
+      "p50_ms": 230.12878749796073,
+      "p95_ms": 276.0439652007335,
+      "raw_e4b_nf4_accuracy": 0.7651,
+      "delta_vs_raw_pp": 13.262953594901417
+    },
+    "mmlu": {
+      "status": "measured",
+      "display": "MMLU",
+      "n": 14042,
+      "accuracy": 0.6441390115368181,
+      "nll": 0.8705675106095903,
+      "brier": 0.45396635565340543,
+      "ece": 0.03314363695835977,
+      "p50_ms": 196.04511249963252,
+      "p95_ms": 303.28189720094076,
+      "raw_e4b_nf4_accuracy": 0.6619,
+      "delta_vs_raw_pp": -1.776098846318197
+    }
+  }
+}

eval/raw_e4b_nf4_control_run/details.jsonl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9cca1e4e95851c986e0f34a5ee9fb1b795feab6471ed211ae145c176dd8a3e03
+size 11112822

eval/raw_e4b_nf4_control_run/summary.csv

@@ -0,0 +1,6 @@
+task,display,n,accuracy,nll,brier,ece,mean_confidence,latency_p50_ms,latency_p95_ms
+arc_challenge,ARC-Challenge,1172,0.8737201365187713,0.6831067400476731,0.21717192716282274,0.0939787680494793,0.9676989045682511,158.1707515015296,239.32236594882852
+winogrande,WinoGrande,1267,0.601420678768745,1.2788768116978686,0.6614413444986983,0.3052572014015491,0.9024290672675535,154.72152499933145,223.75241669906248
+arc_easy,ARC-Easy,2376,0.9558080808080808,0.22179265867232187,0.07866352502747752,0.03341212654329345,0.9868527784304538,153.7982515001204,216.1515892476018
+hellaswag,HellaSwag,10042,0.7650866361282613,1.1006902005839516,0.3895498661609883,0.16545367857507678,0.9305298846809178,259.9183570000605,484.0487622521322
+mmlu,MMLU,14042,0.6618715282723259,1.6791403682676391,0.5577819683925032,0.24181974065906128,0.9036912689313833,176.33997150187497,324.31165310117643

eval/raw_e4b_nf4_control_run/summary.json

@@ -0,0 +1,52 @@
+{
+  "arc_easy": {
+    "n": 2376,
+    "accuracy": 0.9558080808080808,
+    "nll": 0.22179265867232187,
+    "brier": 0.07866352502747752,
+    "ece": 0.03341212654329345,
+    "latency_p50_ms": 153.7982515001204,
+    "latency_p95_ms": 216.1515892476018,
+    "mean_confidence": 0.9868527784304538
+  },
+  "arc_challenge": {
+    "n": 1172,
+    "accuracy": 0.8737201365187713,
+    "nll": 0.6831067400476731,
+    "brier": 0.21717192716282274,
+    "ece": 0.0939787680494793,
+    "latency_p50_ms": 158.1707515015296,
+    "latency_p95_ms": 239.32236594882852,
+    "mean_confidence": 0.9676989045682511
+  },
+  "winogrande": {
+    "n": 1267,
+    "accuracy": 0.601420678768745,
+    "nll": 1.2788768116978686,
+    "brier": 0.6614413444986983,
+    "ece": 0.3052572014015491,
+    "latency_p50_ms": 154.72152499933145,
+    "latency_p95_ms": 223.75241669906248,
+    "mean_confidence": 0.9024290672675535
+  },
+  "hellaswag": {
+    "n": 10042,
+    "accuracy": 0.7650866361282613,
+    "nll": 1.1006902005839516,
+    "brier": 0.3895498661609883,
+    "ece": 0.16545367857507678,
+    "latency_p50_ms": 259.9183570000605,
+    "latency_p95_ms": 484.0487622521322,
+    "mean_confidence": 0.9305298846809178
+  },
+  "mmlu": {
+    "n": 14042,
+    "accuracy": 0.6618715282723259,
+    "nll": 1.6791403682676391,
+    "brier": 0.5577819683925032,
+    "ece": 0.24181974065906128,
+    "latency_p50_ms": 176.33997150187497,
+    "latency_p95_ms": 324.31165310117643,
+    "mean_confidence": 0.9036912689313833
+  }
+}

eval/raw_e4b_nf4_control_summary.json

@@ -0,0 +1,46 @@
+{
+  "model": "google/gemma-4-E4B-it",
+  "precision": "NF4",
+  "readout": "native LM-head answer-label scoring",
+  "complete": true,
+  "task_count": 5,
+  "source": "final benchmark output recorded 2026-09-26",
+  "methodological_note": "Quantization-matched base control for the local path. End-to-end system comparison versus OpenJEV because the readout architecture differs.",
+  "results": {
+    "arc_easy": {
+      "accuracy": 0.9558,
+      "nll": 0.2218,
+      "ece": 0.0334,
+      "precision": "NF4",
+      "readout": "native_lm_head_answer_label_scoring"
+    },
+    "arc_challenge": {
+      "accuracy": 0.8737,
+      "nll": 0.6831,
+      "ece": 0.094,
+      "precision": "NF4",
+      "readout": "native_lm_head_answer_label_scoring"
+    },
+    "winogrande": {
+      "accuracy": 0.6014,
+      "nll": 1.2789,
+      "ece": 0.3053,
+      "precision": "NF4",
+      "readout": "native_lm_head_answer_label_scoring"
+    },
+    "hellaswag": {
+      "accuracy": 0.7651,
+      "nll": 1.1007,
+      "ece": 0.1655,
+      "precision": "NF4",
+      "readout": "native_lm_head_answer_label_scoring"
+    },
+    "mmlu": {
+      "accuracy": 0.6619,
+      "nll": 1.6791,
+      "ece": 0.2418,
+      "precision": "NF4",
+      "readout": "native_lm_head_answer_label_scoring"
+    }
+  }
+}