Redo FT checkpoint (spot1, epoch 1) from jev-train-1 H100 run 2026-11-03

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+tokenizer.json filter=lfs diff=lfs merge=lfs -text

LICENSE

@@ -0,0 +1,112 @@
+SecJev Use and Distribution Agreement
+Version 1.0 — 23 September 2026
+
+Copyright (c) 2026 the respective SecJev contributors, for their original
+contributions only.
+
+1. Scope and rights holders
+
+This agreement covers the original SecJev contributions in releases that
+expressly reference it: code, documentation, dataset compilation and original
+annotations, and model adaptations, only to the extent the relevant contributor
+holds rights that can be licensed. It does not claim ownership of upstream
+datasets, model weights, software, individual facts or public-domain material.
+
+Third-party components and source-derived records retain their own applicable
+terms. Consult SOURCES.md, UPSTREAM.md and THIRD_PARTY_NOTICES.md where supplied,
+the licenses/ directory, and record-level provenance. Inclusion in a SecJev
+release does not turn third-party material into material licensed solely under
+this agreement. This agreement does not withdraw permissions validly granted
+for earlier releases under another license.
+
+2. Permission for original SecJev contributions
+
+Subject to this agreement, each contributor grants a worldwide, non-exclusive,
+royalty-free copyright license to use, copy, study, modify and distribute that
+contributor's covered contributions, including for research, education,
+evaluation, training, fine-tuning and distillation. This grant does not itself
+prohibit commercial use of those contributions; section 4 governs uses that
+also involve upstream rights or conditions. Permissions remain available for
+the duration of the licensed rights while the applicable conditions are met.
+
+3. Attribution, modifications and redistribution
+
+When distributing covered contributions or modifications, retain this agreement,
+applicable copyright and attribution notices, source references and required
+third-party license texts. Identify the SecJev release used, describe material
+changes, and distinguish modified releases from the original project. Preserve
+source-to-record mappings, or provide an equivalent usable provenance mapping,
+when distributing source-derived data.
+
+Redistribute upstream material only to the extent its own terms or a separate
+authorization permit. If permission for a particular act of redistribution is
+unclear, this agreement does not supply that permission; obtain clarification
+or distribute acquisition/transformation instructions instead of that material.
+Do not represent this agreement as removing upstream conditions. Do not impose
+restrictions on a recipient's independent rights in third-party or public-domain
+material. Derivatives may carry additional terms for new contributions only
+where those terms respect the existing rights and obligations.
+
+4. Source-specific conditions and commercial use
+
+The complete corpus has multiple sources and no blanket unrestricted commercial
+clearance. In particular:
+
+- ToN-IoT's official terms grant academic research use and require permission
+  from its author for commercial use. This agreement grants neither that
+  permission nor a broader redistribution right on the author's behalf.
+- CIC requires dataset and paper attribution for its permitted redistribution.
+- VeReMi material licensed under CC BY 4.0 retains attribution, change-notice
+  and other requirements of that license, and its independently granted rights.
+- LANL material covered by a public-domain dedication remains public domain.
+- MIT- and Apache-licensed upstream software and model components retain their
+  license texts and applicable notices; simulator licenses must not be assumed
+  to determine every right in their outputs or input datasets.
+
+The source-specific register gives details and links to the original terms.
+Where a proposed use requires additional authorization, obtain it from each
+relevant rights holder. SecJev contributors cannot grant upstream permissions.
+
+Training on data does not, by this agreement alone, automatically assign the
+data's license to every resulting weight or output, or establish that every
+such weight or output is cleared for commercial use. Model releases must identify
+their upstream components and training sources. Users must assess the terms
+applicable to their actual data use, model use and redistribution. This agreement
+does not assert that ordinary model outputs are owned by SecJev or must carry
+this agreement merely because the model produced them.
+
+5. Priority and reserved rights
+
+Upstream terms govern upstream material; this agreement governs only the rights
+granted by SecJev contributors. It must not be read to narrow CC BY, MIT, Apache,
+CC0 or other independently granted upstream permissions. Statutory exceptions
+and uses that require no permission remain unaffected. No trademark rights or
+right to imply endorsement are granted. No additional patent license is granted
+here; independently applicable upstream patent grants remain unaffected.
+
+6. No warranties; responsibility for use
+
+To the extent permitted by applicable law, covered contributions are supplied
+as available without warranties, including as to accuracy, fitness for purpose
+or non-infringement. Contributors are not liable for losses arising from their
+use to the extent such liability may lawfully be excluded. This agreement does
+not exclude rights or liabilities that applicable law makes non-excludable.
+
+7. Noncompliance
+
+Permissions under this agreement terminate for a use that violates its
+conditions. They are reinstated when the violation is remedied within 30 days
+of the user's discovery, or with the relevant contributor's written permission.
+This does not change upstream licenses or remedies available under applicable
+law. A legally ineffective provision does not invalidate the remainder where
+the remainder can operate consistently with this agreement's scope.
+
+8. Versions and inquiries
+
+This version applies to releases that expressly adopt it. Later versions do not
+automatically change the terms of existing copies. The accompanying Chinese
+translation is for convenience; this English text controls in a discrepancy.
+Project inquiries: https://github.com/UESTC1010/SecJev/issues
+
+This is a project-specific agreement, not a claim of OSI approval or a substitute
+for permissions that only upstream rights holders can grant.

LICENSE.zh-CN.md

@@ -0,0 +1,59 @@
+# SecJev 使用与分发协议
+
+**版本 1.0 · 2026 年 9 月 23 日**
+
+版权所有 © 2026 各 SecJev 贡献者，仅限其各自原创贡献。本文件为英文 [LICENSE](LICENSE) 的中文译文；两者有歧义时，以英文为准。
+
+## 1. 适用范围与权利归属
+
+本协议适用于明确引用本协议的发布版本中，SecJev 贡献者有权授权的原创代码、文档、数据编排与原创标注，以及模型适配贡献。本协议不主张对上游数据、模型权重、软件、单个事实或公有领域材料的所有权。
+
+第三方组件和来源于第三方的数据记录仍适用各自的条款。请结合随附的 `SOURCES.md`、`UPSTREAM.md`、`THIRD_PARTY_NOTICES.md`、`licenses/` 和记录级来源信息确定适用条件。材料被纳入 SecJev，不意味着它只受本协议约束。本协议不撤销此前版本通过其他许可证已经有效授予的权限。
+
+## 2. 对 SecJev 原创贡献的授权
+
+在遵守本协议的前提下，各贡献者就其有权授权的贡献，授予全球性、非独占、免许可费的著作权许可，允许使用、复制、研究、修改和分发，包括科研、教学、评测、训练、微调和蒸馏。
+
+本项授权本身不禁止对这些原创贡献的商业使用；涉及上游权利或条件的使用还应满足第 4 条。只要遵守适用条件，授权在相应权利存续期间有效。
+
+## 3. 署名、修改与再分发
+
+分发本协议覆盖的贡献或其修改版本时，应保留本协议、适用的版权与署名声明、来源引用，以及必须保留的第三方许可证文本。应注明使用的 SecJev 版本和实质修改，区分修改版本与原始项目。分发来源于第三方的数据时，应保留来源与记录之间的对应关系，或提供等效、可用的溯源方式。
+
+只有上游条款或单独授权允许时，才可再分发对应的上游材料。某项再分发行为的授权不明确时，本协议不补足该授权；应取得明确许可，或提供获取与转换说明而非直接分发该材料。
+
+不得将本协议表述为对上游条件的豁免，不得限制接收者对第三方材料或公有领域材料独立享有的权利。衍生版本可对新增贡献约定条款，但不得损害原有权利和义务。
+
+## 4. 数据源条件与商业使用
+
+完整语料包含多个来源，不提供不受限制的统一商用保证。其中：
+
+- **ToN-IoT**：官方条款允许学术研究使用，商业使用须获得作者许可。本协议不代替该许可，也不代表作者授予更广泛的再分发权。
+- **CIC**：在其允许的再分发中，应按要求引用数据集及对应论文。
+- **VeReMi**：采用 CC BY 4.0 的材料保留该许可证的署名、修改说明等要求，以及该许可证独立授予的权限。
+- **LANL**：已经适用公有领域奉献的材料继续保持该状态。
+- **MIT、Apache 等上游组件**：保留各自许可证和适用声明；不能仅凭模拟器的代码许可证推定其全部输出或输入数据的权利状态。
+
+各来源的具体条件及官方链接见来源说明。拟进行的使用需要额外授权时，应分别向相关权利人取得；SecJev 贡献者无权代其授权。
+
+本协议既不规定数据许可证自动传递给所有训练所得权重或输出，也不保证这些权重或输出全部具备商用条件。模型发布应说明上游组件和训练来源；使用者应根据实际的数据使用、模型使用和再分发行为确认适用条款。普通模型输出不会仅因由模型产生，就被本协议认定属于 SecJev 或必须采用本协议。
+
+## 5. 条款优先级与保留权利
+
+上游材料适用上游条款，本协议仅约束 SecJev 贡献者有权授予的权利。本协议不得被解释为缩减 CC BY、MIT、Apache、CC0 或其他上游许可独立授予的权限。法律规定的例外及无需许可的使用不受影响。
+
+本协议不授予商标权或暗示背书的权利，不额外授予专利许可；已经独立适用的上游专利授权不受影响。
+
+## 6. 无担保与使用责任
+
+在适用法律允许的范围内，本协议覆盖的贡献按现有状态提供，不对准确性、特定用途适用性或不侵权作出担保；贡献者在法律允许免责的范围内不承担因使用而产生的损失责任。本条不排除法律规定不得排除的权利或责任。
+
+## 7. 违反协议
+
+违反本协议条件的使用，其本协议项下许可终止。使用者发现违反行为后 30 日内完成纠正，或取得相关贡献者书面同意，许可恢复。本条不修改上游许可及依法可获得的救济。某项条款依法无效时，在剩余条款仍可依照本协议范围独立运行的情况下，不影响其效力。
+
+## 8. 版本与联系
+
+本版本仅适用于明确采用它的发布内容，后续版本不自动改变既有副本的许可条件。项目联系入口：[SecJev Issues](https://github.com/UESTC1010/SecJev/issues)。
+
+本协议为项目自定义协���，不宣称获得 OSI 认证，也不能替代只能由上游权利人授予的许可。

README.md

@@ -0,0 +1,48 @@
+---
+language:
+- en
+license: other
+license_name: secjev-use-and-distribution-agreement-v1.0
+license_link: LICENSE
+base_model: Qwen/Qwen3.5-4B-Base
+tags:
+- security
+- decision-model
+- lora
+- prompt-injection
+---
+
+# SecJev-4B-Spot1-Redo
+
+Redo fine-tune of SecJev-4B for Spot 1 — tool-output injection filter (incoming tool responses). This repository root is a complete
+Kev checkpoint directory (LoRA adapter + trained head + tokenizer), loadable
+with the Kev loader. The Qwen3.5-4B-Base backbone is not re-hosted;
+`bootstrap.py` fetches the pinned backbone and Kev source.
+
+## Training
+
+- Compute: DigitalOcean GPU droplet `jev-train-1` (NVIDIA H100 80GB), 2026-11-03 22:53:41–23:03:28 UTC.
+- Method: `ft_action.py`, 1 epoch, lr 3e-5, batch 32 questions, token budget
+  4096, gradient checkpointing, fp32 weights with bf16 autocast.
+  Trainable parameters: 33,776,128 (LoRA + head); backbone frozen.
+- Data: `data/bench/train_spot1_t.jsonl` (train) / `data/bench/dev_spot1_t.jsonl` (dev), jev-stageA redo bench sets.
+- Full training log: `redo-training-log.jsonl` and `redo-training.stdout.txt`
+  in this repository. Machine-readable details: `redo-provenance.json`.
+
+## Dev results (only evaluation so far)
+
+Corrected per-question re-evaluation of this checkpoint on the dev file
+(score: authorization = P("2") + 0.5·P("1"); injection = P("true")):
+
+| question | dev n | dev pos | AUC | TPR@0.5 | FPR@0.5 |
+|---|---|---|---|---|---|
+| injection | 500 | 250 | 0.9327 | 0.7800 | 0.0400 |
+
+No frozen-test, adaptive-attack, or independent-replication result exists yet
+for this checkpoint. Dev rows were used for training-time monitoring. Treat
+the numbers above as development measurements, not deployment guarantees.
+
+Note: the training-time dev metrics printed in `redo-training-log.jsonl` for
+injection-question runs used an authorization-only scoring formula and are
+invalid for those runs (they show AUC 0.0 / 0.14); the table above supersedes
+them. Training itself used correct targets.

SOURCES.md

@@ -0,0 +1,96 @@
+# Sources and attribution
+
+SecJev-Corpus v1.0.0 transforms upstream observations into structured decision
+questions. Transformations include feature projection, identifier anonymization,
+history-window construction, explicit-policy labeling, deduplication, option
+permutation, and grouped splitting. Source annotations are not relabeled as
+proven real-world attacks. Newly run simulations are identified as SecJev
+experiments rather than results reported by the upstream authors.
+
+## CICIoT2023 — 48,000 questions
+
+E. C. P. Neto, S. Dadkhah, R. Ferreira, A. Zohourian, R. Lu, and A. A. Ghorbani.
+*CICIoT2023: A real-time dataset and benchmark for large-scale attacks in IoT environment* (2023).
+
+- [Dataset and paper](https://www.unb.ca/cic/datasets/iotdataset-2023.html)
+- [CIC redistribution and citation policy, under Common questions](https://www.unb.ca/cic/datasets/index.html)
+- Mirror used: `bencorn/CIC-IoT-2023`, revision `dac6afc7c1f6242202abf6059932e4c4d166e662`.
+
+CIC explicitly permits redistribution and mirroring with dataset and paper attribution.
+Capture labels remain capture-level annotations; incidental background traffic is possible.
+
+## ToN-IoT — 40,000 questions
+
+[Official dataset and usage conditions](https://research.unsw.edu.au/projects/toniot-datasets).
+Free academic research use is granted by the authors; commercial use requires
+separate author permission. The source requests citation of the following eight
+publications, linked from its official page:
+
+1. Nour Moustafa (2021), network dataset study, *Sustainable Cities and Society*, 102994.
+2. Tim M. Booij, Irina Chiscop, Erik Meeuwissen, Nour Moustafa, and Frank T. H. den Hartog (2021), ToN-IoT heterogeneity and feature standardization, *IEEE Internet of Things Journal*.
+3. Abdullah Alsaedi, Nour Moustafa, Zahir Tari, Abdun Mahmood, and Adnan Anwar (2020), TON-IoT telemetry dataset, *IEEE Access* 8, 165130–165150.
+4. Nour Moustafa, M. Keshk, E. Debie, and H. Janicke (2020), federated Windows datasets, *TrustCom*, 848–855. DOI: 10.1109/TrustCom50675.2020.00114.
+5. Nour Moustafa, M. Ahmed, and S. Ahmed (2020), ToN-IoT Linux dataset evaluation, *TrustCom*, 727–735. DOI: 10.1109/TrustCom50675.2020.00100.
+6. Nour Moustafa (2019), TON-IoT dataset presentation, *eResearch Australasia*.
+7. Nour Moustafa (2019), IoT–Fog–Cloud architecture review. [arXiv:1906.01055](https://arxiv.org/abs/1906.01055).
+8. Javed Ashraf et al. (2021), IoTBoT-IDS, *Sustainable Cities and Society*, 103041.
+
+Mirror used: `codymlewis/TON_IoT_network`, revision
+`b953d9388d2db3bc392ea7150476e4f219a2a1cb`. Features derive from the fixed network CSV.
+
+## VeReMi NextGen — 22,498 questions
+
+Artur Hermann, Jan-Niklas Remmers, Dennis Eisermann, Benjamin Erb, and Frank Kargl.
+*VeReMi NextGen: A Dataset for Evaluating Misbehavior Detection Systems in VANETs*.
+[Zenodo record / DOI 10.5281/zenodo.19665762](https://zenodo.org/records/19665762).
+License: CC BY 4.0; full notice in `licenses/VeReMi-LICENSE.txt`.
+
+Only validated observation categories enter this corpus. Sender claims are
+projected into a common representation. The official independent test boundary is retained.
+
+## Twins / Streamlet — 29,588 questions
+
+[Twins simulator](https://github.com/asonnino/twins-simulator), revision
+`3a3215addf3f79cb7b1dd2b0677f4d0896943d0b`; Apache-2.0 notice included.
+These are newly sampled controlled experiments using the upstream implementation.
+Answers follow the explicit observer certificate policy and visible message history.
+A validator ID follows the simulator's physical-author representation; no hidden
+attacker identity is inferred.
+
+## ByzFL / FoolsGold / MNIST — 17,000 questions
+
+- [ByzFL](https://github.com/LPD-EPFL/byzfl), EPFL, revision `5830978d991a0900748f7ce901dc3ca532081b26`; MIT.
+- [FoolsGold](https://github.com/DistributedML/FoolsGold), revision `0aa55114296a2d3c2bcb6f544a6fae31e8e7b8b4`; MIT.
+- Yann LeCun, Corinna Cortes, and Christopher J. C. Burges, MNIST handwritten-digit data, underlying the controlled gradient experiments.
+
+The corpus contains measured update summaries from 80 fresh runs, with separate
+MNIST pools across splits and explicit attacker configurations. Original images
+are not redistributed. The earlier simulated aggregation-selection tasks are
+not part of this release.
+
+## LANL — 7,902 questions
+
+Alexander D. Kent (2015), *Comprehensive, Multi-Source Cyber-Security Events*.
+Los Alamos National Laboratory. DOI: 10.17021/1179829.
+Related publication: A. D. Kent, *Cybersecurity Data Sources for Dynamic Network
+Research*, in *Dynamic Networks in Cybersecurity*, 2015.
+
+[Official data, citation and public-domain dedication](https://csr.lanl.gov/data/cyber1/).
+Our questions use visible authentication histories and explicit triage rules.
+Unlabeled events are not assigned a confirmed-benign label. The source's
+successful-authentication selection bias remains applicable.
+
+## AgentDojo — 4,143 questions
+
+[AgentDojo](https://github.com/ethz-spylab/agentdojo), ETH Zurich SPY Lab and contributors,
+revision `089ed468cf3ed0322acc66b0211f26d9d90dbf60`; MIT notice included.
+Records are drawn from published tool outputs and attack/control runs. Task and
+shared-response components are grouped; shared environment or attack templates
+are not a claim of unseen-template evaluation.
+
+## InjecAgent — 1,054 questions
+
+[InjecAgent](https://github.com/uiuc-kang-lab/InjecAgent), UIUC Kang Lab and contributors,
+revision `f19c9f2c79a41046eb13c03c51a24c567a8ffa07`; MIT notice included.
+This release uses the declared objectives of injected instructions. The
+all-positive injection-detection task is excluded.

THIRD_PARTY_NOTICES.md

@@ -0,0 +1,24 @@
+# Third-party notices
+
+The SecJev Use and Distribution Agreement applies only to the rights held by SecJev contributors. The following upstream licenses remain in effect and are not replaced by that agreement.
+
+| Component | Attribution | Applicable terms |
+|---|---|---|
+| Kev source and initial Kev-0.8B adapter/head | Jared Palmer and Kev contributors | Apache-2.0; see licenses/Apache-2.0.txt and UPSTREAM.md |
+| Qwen3.5-0.8B-Base and Qwen3.5-2B-Base backbones | Qwen team / Alibaba and applicable contributors | Apache-2.0; see licenses/Apache-2.0.txt and UPSTREAM.md |
+| Twins / Streamlet simulator | Twins simulator contributors | licenses/Twins-LICENSE.txt |
+| ByzFL | EPFL and ByzFL contributors | licenses/ByzFL-LICENSE.txt |
+| FoolsGold | FoolsGold contributors | licenses/FoolsGold-LICENSE.txt |
+| AgentDojo | ETH Zurich SPY Lab and contributors | licenses/AgentDojo-LICENSE.txt |
+| InjecAgent | UIUC Kang Lab and contributors | licenses/InjecAgent-LICENSE.txt |
+| VeReMi NextGen data | Authors listed in SOURCES.md | licenses/VeReMi-LICENSE.txt; CC BY 4.0 |
+| LANL source observations | Alexander D. Kent / Los Alamos National Laboratory | licenses/LANL.md; public-domain source dedication |
+| CICIoT2023 and ToN-IoT data | Authors listed in SOURCES.md | Official source-specific conditions linked in SOURCES.md |
+
+SecJev modifies the Kev adapter and decision head by security fine-tuning and probability calibration. The backbone is an upstream dependency. SecJev-Corpus transformations and newly generated experiments are described in SOURCES.md; using a simulator does not imply its software license automatically covers every output or input dataset. Full data attribution and required publications are retained in SOURCES.md.
+
+SecJev-2B trains its own LoRA/head from the pinned Qwen2B backbone with Kev's decision recipe before security adaptation; the initial Kev-0.8B adapter applies only to the 0.8B model. Kev general training source records retain their upstream attribution and conditions.
+
+## SecJev-4B
+
+Initialized from jaredpalmer/kev-4b@485ace8703592fcf405488b262449990824cfed1 on Qwen/Qwen3.5-4B-Base@1001bb4d826a52d1f399e183466143f4da7b741b, then adapted for three epochs on SecJev-Corpus v1.0.0. Existing SecJev and upstream terms remain applicable.

UPSTREAM.md

@@ -0,0 +1,17 @@
+# Upstream components
+
+SecJev-0.8B adapts the LoRA parameters and pointer decision head of Kev-0.8B.
+
+| Component | Pinned revision | Terms |
+|---|---|---|
+| [Kev source](https://github.com/jaredpalmer/kev) | `90990a5fac2995b9faa3190f7d437e84f2067768` | Apache-2.0 |
+| [Kev-0.8B initialization](https://huggingface.co/jaredpalmer/kev-0.8b) | `54f4f8777356cd5bbbb6c6919c657f26e6f2f6d8` | Apache-2.0 |
+| [Qwen3.5-0.8B-Base](https://huggingface.co/Qwen/Qwen3.5-0.8B-Base) | `dc7cdfe2ee4154fa7e30f5b51ca41bfa40174e68` | Apache-2.0 |
+
+SecJev-2B starts from [Qwen3.5-2B-Base](https://huggingface.co/Qwen/Qwen3.5-2B-Base) at `b1485b2fa6dfa1287294f269f5fb618e03d52d7c`. We train its LoRA and decision head with the same Kev source, decision-v7 and the dates/unknowable supplement, then SecJev-Corpus. It does not use an official Kev-2B initialization. The pinned general-data source is Kev `30c619b0527501cfdd448cb6eb9887e2af454603` and Kev suites `a88f56db5341397299137cb68775c2ea6e3f68cb`.
+
+The inference implementation and training loss use pinned Kev source. SecJev adds security adaptation, batching, the training/evaluation workflow, and the security corpus. See SecJev-Corpus/SOURCES.md for data attribution and source-specific conditions. Dataset terms are not replaced by the code license.
+
+## SecJev-4B
+
+Initialized from jaredpalmer/kev-4b@485ace8703592fcf405488b262449990824cfed1 on Qwen/Qwen3.5-4B-Base@1001bb4d826a52d1f399e183466143f4da7b741b, then adapted for three epochs on SecJev-Corpus v1.0.0. Existing SecJev and upstream terms remain applicable.

adapter_config.json

@@ -0,0 +1,56 @@
+{
+  "alora_invocation_tokens": null,
+  "alpha_pattern": {},
+  "arrow_config": null,
+  "auto_mapping": null,
+  "base_model_name_or_path": "Qwen/Qwen3.5-4B-Base",
+  "bias": "none",
+  "corda_config": null,
+  "ensure_weight_tying": false,
+  "eva_config": null,
+  "exclude_modules": null,
+  "fan_in_fan_out": false,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "kasa_config": null,
+  "layer_replication": null,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "loftq_config": {},
+  "lora_alpha": 32,
+  "lora_bias": false,
+  "lora_dropout": 0.05,
+  "lora_ga_config": null,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "modules_to_save": null,
+  "monteclora_config": null,
+  "peft_type": "LORA",
+  "peft_version": "0.21.0",
+  "qalora_group_size": 16,
+  "r": 16,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": [
+    "o_proj",
+    "in_proj_z",
+    "out_proj",
+    "down_proj",
+    "gate_proj",
+    "up_proj",
+    "in_proj_qkv",
+    "in_proj_a",
+    "v_proj",
+    "q_proj",
+    "k_proj",
+    "in_proj_b"
+  ],
+  "target_parameters": null,
+  "task_type": "FEATURE_EXTRACTION",
+  "trainable_token_indices": null,
+  "use_bdlora": null,
+  "use_dora": false,
+  "use_qalora": false,
+  "use_rslora": false,
+  "velora_config": null
+}

adapter_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7329ca6cdc831cd32e2e8ec9266763c5255ba9cfda35a8d8536f2e03234a85ab
+size 129924032

bootstrap.py

@@ -0,0 +1,57 @@
+"""Prepare pinned upstream models, code and public corpus for a reproducible run."""
+import argparse, gzip, hashlib, json, os, pathlib, shutil, tarfile, urllib.request
+from huggingface_hub import snapshot_download
+
+BASE = ('Qwen/Qwen3.5-4B-Base', '1001bb4d826a52d1f399e183466143f4da7b741b')
+INIT = ('jaredpalmer/kev-4b', '485ace8703592fcf405488b262449990824cfed1')
+KEV = '90990a5fac2995b9faa3190f7d437e84f2067768'
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--corpus', type=pathlib.Path, help='Unpacked SecJev-Corpus v1.0.0 directory')
+    parser.add_argument('--inference-only', action='store_true', help='Download pinned backbone and Kev code without training data or parent adapter')
+    args = parser.parse_args()
+    if not args.inference_only and args.corpus is None: parser.error('--corpus is required for training setup')
+    root = pathlib.Path(os.environ.get('SECJEV_WORK', './secjev-4b-work')).resolve()
+    for d in ['data', 'results', 'logs', 'runs', 'source']:
+        (root/d).mkdir(parents=True, exist_ok=True)
+    if not args.inference_only:
+        corpus = json.loads((args.corpus/'manifest.json').read_text())
+        assert corpus['name'] == 'SecJev-Corpus' and corpus['version'] == '1.0.0'
+        manifest = {}
+        for split, info in corpus['subsets'].items():
+            src = args.corpus/'records'/f'{split}.jsonl.gz'
+            dest = root/'data'/src.name
+            digest = hashlib.sha256()
+            with gzip.open(src, 'rt') as stream:
+                for line in stream:
+                    r = json.loads(line)
+                    payload = json.dumps({'state': r['state'], 'questions': r['questions']}, ensure_ascii=False, allow_nan=False, separators=(',', ':'))
+                    digest.update((payload+'\n').encode())
+            assert digest.hexdigest() == info['content_sha256'], split
+            shutil.copy2(src, dest)
+            manifest[split] = {'records': info['scenes'], 'questions': info['questions'], 'sha256': hashlib.sha256(dest.read_bytes()).hexdigest()}
+        (root/'data/manifest.json').write_text(json.dumps(manifest, indent=2)+'\n')
+    for name, (repo, revision) in [('base', BASE), ('init', INIT)]:
+        if args.inference_only and name == 'init': continue
+        snapshot_download(repo, revision=revision, local_dir=root/name,
+                          allow_patterns=['*.json', '*.safetensors', '*.pt', '*.txt', '*.jinja'])
+        filename = 'download_provenance.json' if name == 'base' else 'inference-provenance.json'
+        (root/name/filename).write_text(json.dumps({'repo_id': repo, 'revision': revision}, indent=2)+'\n')
+    archive = root/'source'/f'kev-{KEV}.tar.gz'
+    urllib.request.urlretrieve(f'https://codeload.github.com/jaredpalmer/kev/tar.gz/{KEV}', archive)
+    with tarfile.open(archive) as tf:
+        for member in tf:
+            if member.isfile():
+                rel = pathlib.PurePosixPath(member.name)
+                assert rel.parts[0] == f'kev-{KEV}' and '..' not in rel.parts
+                target = root/'source'/'kev'/pathlib.Path(*rel.parts[1:])
+                target.parent.mkdir(parents=True, exist_ok=True)
+                with tf.extractfile(member) as source, target.open('wb') as destination:
+                    shutil.copyfileobj(source, destination)
+    print('Prepared', root)
+
+
+if __name__ == '__main__':
+    main()

chat_template.jinja

@@ -0,0 +1,154 @@
+{%- set image_count = namespace(value=0) %}
+{%- set video_count = namespace(value=0) %}
+{%- macro render_content(content, do_vision_count, is_system_content=false) %}
+    {%- if content is string %}
+        {{- content }}
+    {%- elif content is iterable and content is not mapping %}
+        {%- for item in content %}
+            {%- if 'image' in item or 'image_url' in item or item.type == 'image' %}
+                {%- if is_system_content %}
+                    {{- raise_exception('System message cannot contain images.') }}
+                {%- endif %}
+                {%- if do_vision_count %}
+                    {%- set image_count.value = image_count.value + 1 %}
+                {%- endif %}
+                {%- if add_vision_id %}
+                    {{- 'Picture ' ~ image_count.value ~ ': ' }}
+                {%- endif %}
+                {{- '<|vision_start|><|image_pad|><|vision_end|>' }}
+            {%- elif 'video' in item or item.type == 'video' %}
+                {%- if is_system_content %}
+                    {{- raise_exception('System message cannot contain videos.') }}
+                {%- endif %}
+                {%- if do_vision_count %}
+                    {%- set video_count.value = video_count.value + 1 %}
+                {%- endif %}
+                {%- if add_vision_id %}
+                    {{- 'Video ' ~ video_count.value ~ ': ' }}
+                {%- endif %}
+                {{- '<|vision_start|><|video_pad|><|vision_end|>' }}
+            {%- elif 'text' in item %}
+                {{- item.text }}
+            {%- else %}
+                {{- raise_exception('Unexpected item type in content.') }}
+            {%- endif %}
+        {%- endfor %}
+    {%- elif content is none or content is undefined %}
+        {{- '' }}
+    {%- else %}
+        {{- raise_exception('Unexpected content type.') }}
+    {%- endif %}
+{%- endmacro %}
+{%- if not messages %}
+    {{- raise_exception('No messages provided.') }}
+{%- endif %}
+{%- if tools and tools is iterable and tools is not mapping %}
+    {{- '<|im_start|>system\n' }}
+    {{- "# Tools\n\nYou have access to the following functions:\n\n<tools>" }}
+    {%- for tool in tools %}
+        {{- "\n" }}
+        {{- tool | tojson }}
+    {%- endfor %}
+    {{- "\n</tools>" }}
+    {{- '\n\nIf you choose to call a function ONLY reply in the following format with NO suffix:\n\n<tool_call>\n<function=example_function_name>\n<parameter=example_parameter_1>\nvalue_1\n</parameter>\n<parameter=example_parameter_2>\nThis is the value for the second parameter\nthat can span\nmultiple lines\n</parameter>\n</function>\n</tool_call>\n\n<IMPORTANT>\nReminder:\n- Function calls MUST follow the specified format: an inner <function=...></function> block must be nested within <tool_call></tool_call> XML tags\n- Required parameters MUST be specified\n- You may provide optional reasoning for your function call in natural language BEFORE the function call, but NOT after\n- If there is no function call available, answer the question like normal with your current knowledge and do not tell the user about function calls\n</IMPORTANT>' }}
+    {%- if messages[0].role == 'system' %}
+        {%- set content = render_content(messages[0].content, false, true)|trim %}
+        {%- if content %}
+            {{- '\n\n' + content }}
+        {%- endif %}
+    {%- endif %}
+    {{- '<|im_end|>\n' }}
+{%- else %}
+    {%- if messages[0].role == 'system' %}
+        {%- set content = render_content(messages[0].content, false, true)|trim %}
+        {{- '<|im_start|>system\n' + content + '<|im_end|>\n' }}
+    {%- endif %}
+{%- endif %}
+{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %}
+{%- for message in messages[::-1] %}
+    {%- set index = (messages|length - 1) - loop.index0 %}
+    {%- if ns.multi_step_tool and message.role == "user" %}
+        {%- set content = render_content(message.content, false)|trim %}
+        {%- if not(content.startswith('<tool_response>') and content.endswith('</tool_response>')) %}
+            {%- set ns.multi_step_tool = false %}
+            {%- set ns.last_query_index = index %}
+        {%- endif %}
+    {%- endif %}
+{%- endfor %}
+{%- if ns.multi_step_tool %}
+    {{- raise_exception('No user query found in messages.') }}
+{%- endif %}
+{%- for message in messages %}
+    {%- set content = render_content(message.content, true)|trim %}
+    {%- if message.role == "system" %}
+        {%- if not loop.first %}
+            {{- raise_exception('System message must be at the beginning.') }}
+        {%- endif %}
+    {%- elif message.role == "user" %}
+        {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }}
+    {%- elif message.role == "assistant" %}
+        {%- set reasoning_content = '' %}
+        {%- if message.reasoning_content is string %}
+            {%- set reasoning_content = message.reasoning_content %}
+        {%- else %}
+            {%- if '</think>' in content %}
+                {%- set reasoning_content = content.split('</think>')[0].rstrip('\n').split('<think>')[-1].lstrip('\n') %}
+                {%- set content = content.split('</think>')[-1].lstrip('\n') %}
+            {%- endif %}
+        {%- endif %}
+        {%- set reasoning_content = reasoning_content|trim %}
+        {%- if loop.index0 > ns.last_query_index %}
+            {{- '<|im_start|>' + message.role + '\n<think>\n' + reasoning_content + '\n</think>\n\n' + content }}
+        {%- else %}
+            {{- '<|im_start|>' + message.role + '\n' + content }}
+        {%- endif %}
+        {%- if message.tool_calls and message.tool_calls is iterable and message.tool_calls is not mapping %}
+            {%- for tool_call in message.tool_calls %}
+                {%- if tool_call.function is defined %}
+                    {%- set tool_call = tool_call.function %}
+                {%- endif %}
+                {%- if loop.first %}
+                    {%- if content|trim %}
+                        {{- '\n\n<tool_call>\n<function=' + tool_call.name + '>\n' }}
+                    {%- else %}
+                        {{- '<tool_call>\n<function=' + tool_call.name + '>\n' }}
+                    {%- endif %}
+                {%- else %}
+                    {{- '\n<tool_call>\n<function=' + tool_call.name + '>\n' }}
+                {%- endif %}
+                {%- if tool_call.arguments is defined %}
+                    {%- for args_name, args_value in tool_call.arguments|items %}
+                        {{- '<parameter=' + args_name + '>\n' }}
+                        {%- set args_value = args_value | tojson | safe if args_value is mapping or (args_value is sequence and args_value is not string) else args_value | string %}
+                        {{- args_value }}
+                        {{- '\n</parameter>\n' }}
+                    {%- endfor %}
+                {%- endif %}
+                {{- '</function>\n</tool_call>' }}
+            {%- endfor %}
+        {%- endif %}
+        {{- '<|im_end|>\n' }}
+    {%- elif message.role == "tool" %}
+        {%- if loop.previtem and loop.previtem.role != "tool" %}
+            {{- '<|im_start|>user' }}
+        {%- endif %}
+        {{- '\n<tool_response>\n' }}
+        {{- content }}
+        {{- '\n</tool_response>' }}
+        {%- if not loop.last and loop.nextitem.role != "tool" %}
+            {{- '<|im_end|>\n' }}
+        {%- elif loop.last %}
+            {{- '<|im_end|>\n' }}
+        {%- endif %}
+    {%- else %}
+        {{- raise_exception('Unexpected message role.') }}
+    {%- endif %}
+{%- endfor %}
+{%- if add_generation_prompt %}
+    {{- '<|im_start|>assistant\n' }}
+    {%- if enable_thinking is defined and enable_thinking is false %}
+        {{- '<think>\n\n</think>\n\n' }}
+    {%- else %}
+        {{- '<think>\n' }}
+    {%- endif %}
+{%- endif %}

checkpoint-selection.json

@@ -0,0 +1,598 @@
+{
+  "selected_epoch": 3,
+  "candidates": [
+    {
+      "epoch": 1,
+      "metrics": {
+        "questions": 14912,
+        "accuracy": 0.8959898068669528,
+        "mean_class_recall": 0.8086847041598828,
+        "nll": 0.2676199905412643,
+        "brier": 0.14581168647550916,
+        "ece15": 0.024724176262894793,
+        "temperature": 1.0,
+        "coverage_at_confidence_0_9": 0.773806330472103,
+        "error_rate_at_confidence_0_9": 0.0240055464078343,
+        "ordinal_expected_level_mae": 0.00949861612167232,
+        "tasks": {
+          "agentdojo_tool_outputs:agentdojo_boundary_action": {
+            "questions": 222,
+            "accuracy": 0.9819819819819819,
+            "mean_class_recall": 0.9670930232558139,
+            "nll": 0.06952124509948641,
+            "brier": 0.033360936223710204,
+            "ece15": 0.01963763286720348,
+            "temperature": 1.0,
+            "coverage_at_confidence_0_9": 0.9774774774774775,
+            "error_rate_at_confidence_0_9": 0.013824884792626729,
+            "ordinal_expected_level_mae": null
+          },
+          "agentdojo_tool_outputs:agentdojo_embedded_injection": {
+            "questions": 222,
+            "accuracy": 0.9864864864864865,
+            "mean_class_recall": 0.97,
+            "nll": 0.06796588080667874,
+            "brier": 0.02442578405456454,
+            "ece15": 0.013534851962772593,
+            "temperature": 1.0,
+            "coverage_at_confidence_0_9": 0.972972972972973,
+            "error_rate_at_confidence_0_9": 0.009259259259259259,
+            "ordinal_expected_level_mae": null
+          },
+          "byzfl_mnist:fl_attack_generated_update": {
+            "questions": 1400,
+            "accuracy": 0.9742857142857143,
+            "mean_class_recall": 0.9504464285714286,
+            "nll": 0.07440929350732509,
+            "brier": 0.03861493679124129,
+            "ece15": 0.01061565077999396,
+            "temperature": 1.0,
+            "coverage_at_confidence_0_9": 0.925,
+            "error_rate_at_confidence_0_9": 0.008494208494208495,
+            "ordinal_expected_level_mae": null
+          },
+          "ciciot2023:ciciot_attack": {
+            "questions": 5000,
+            "accuracy": 0.8334,
+            "mean_class_recall": 0.7313333333333334,
+            "nll": 0.4357438451410003,
+            "brier": 0.2420765591030422,
+            "ece15": 0.05574236954650583,
+            "temperature": 1.0,
+            "coverage_at_confidence_0_9": 0.644,
+            "error_rate_at_confidence_0_9": 0.06677018633540373,
+            "ordinal_expected_level_mae": null
+          },
+          "injecagent:injecagent_requested_operation_v13": {
+            "questions": 124,
+            "accuracy": 1.0,
+            "mean_class_recall": 1.0,
+            "nll": 0.004390672226438891,
+            "brier": 0.0015490359962901127,
+            "ece15": 0.003915493134373699,
+            "temperature": 1.0,
+            "coverage_at_confidence_0_9": 0.9919354838709677,
+            "error_rate_at_confidence_0_9": 0.0,
+            "ordinal_expected_level_mae": null
+          },
+          "lanl_auth_policy:lanl_known_logon_action_v16": {
+            "questions": 328,
+            "accuracy": 1.0,
+            "mean_class_recall": 1.0,
+            "nll": 0.004780661486616687,
+            "brier": 0.0026501860965206722,
+            "ece15": 0.0038321931288341792,
+            "temperature": 1.0,
+            "coverage_at_confidence_0_9": 0.9908536585365854,
+            "error_rate_at_confidence_0_9": 0.0,
+            "ordinal_expected_level_mae": null
+          },
+          "lanl_auth_policy:lanl_observable_policy": {
+            "questions": 328,
+            "accuracy": 1.0,
+            "mean_class_recall": 1.0,
+            "nll": 2.8436047490948972e-05,
+            "brier": 2.213459113159646e-07,
+            "ece15": 2.8380504783511674e-05,
+            "temperature": 1.0,
+            "coverage_at_confidence_0_9": 1.0,
+            "error_rate_at_confidence_0_9": 0.0,
+            "ordinal_expected_level_mae": null
+          },
+          "ton_iot_network:ton_attack": {
+            "questions": 2500,
+            "accuracy": 0.9832,
+            "mean_class_recall": 0.9856,
+            "nll": 0.0716936772308886,
+            "brier": 0.032718644101254024,
+            "ece15": 0.01640596644430267,
+            "temperature": 1.0,
+            "coverage_at_confidence_0_9": 0.9948,
+            "error_rate_at_confidence_0_9": 0.01648572577402493,
+            "ordinal_expected_level_mae": null
+          },
+          "twins_streamlet:twins_evidence_level": {
+            "questions": 797,
+            "accuracy": 0.9937264742785445,
+            "mean_class_recall": 0.9934913409721132,
+            "nll": 0.0153080496936535,
+            "brier": 0.009627399097334195,
+            "ece15": 0.004599932218083785,
+            "temperature": 1.0,
+            "coverage_at_confidence_0_9": 0.9912170639899623,
+            "error_rate_at_confidence_0_9": 0.0037974683544303796,
+            "ordinal_expected_level_mae": 0.00949861612167232
+          },
+          "twins_streamlet:twins_policy_action": {
+            "questions": 797,
+            "accuracy": 0.9924717691342535,
+            "mean_class_recall": 0.9923419156847569,
+            "nll": 0.017883068917921716,
+            "brier": 0.011294880553062277,
+            "ece15": 0.0038184765498114437,
+            "temperature": 1.0,
+            "coverage_at_confidence_0_9": 0.9811794228356336,
+            "error_rate_at_confidence_0_9": 0.0012787723785166241,
+            "ordinal_expected_level_mae": null
+          },
+          "twins_streamlet:twins_quorum": {
+            "questions": 797,
+            "accuracy": 0.9962358845671268,
+            "mean_class_recall": 0.995834315034425,
+            "nll": 0.011957269820791244,
+            "brier": 0.007393224542639707,
+            "ece15": 0.00670405372457154,
+            "temperature": 1.0,
+            "coverage_at_confidence_0_9": 0.9836888331242158,
+            "error_rate_at_confidence_0_9": 0.0,
+            "ordinal_expected_level_mae": null
+          },
+          "twins_streamlet:twins_visible_conflict": {
+            "questions": 797,
+            "accuracy": 0.9974905897114178,
+            "mean_class_recall": 0.9969442527413541,
+            "nll": 0.010437285979484219,
+            "brier": 0.0038211519865233618,
+            "ece15": 0.0009184477173700703,
+            "temperature": 1.0,
+            "coverage_at_confidence_0_9": 0.9962358845671268,
+            "error_rate_at_confidence_0_9": 0.0012594458438287153,
+            "ordinal_expected_level_mae": null
+          },
+          "veremi_nextgen:veremi_message_manipulation": {
+            "questions": 800,
+            "accuracy": 0.63,
+            "mean_class_recall": 0.6354549163606691,
+            "nll": 0.5876157448193948,
+            "brier": 0.4142276501377162,
+            "ece15": 0.031917999077751745,
+            "temperature": 1.0,
+            "coverage_at_confidence_0_9": 0.10375,
+            "error_rate_at_confidence_0_9": 0.0,
+            "ordinal_expected_level_mae": null
+          },
+          "veremi_nextgen:veremi_supported_attack_type_v16": {
+            "questions": 800,
+            "accuracy": 0.59875,
+            "mean_class_recall": 0.4115791153670771,
+            "nll": 1.2269806519983115,
+            "brier": 0.571523062943736,
+            "ece15": 0.08060690424281568,
+            "temperature": 1.0,
+            "coverage_at_confidence_0_9": 0.11875,
+            "error_rate_at_confidence_0_9": 0.0,
+            "ordinal_expected_level_mae": null
+          }
+        },
+        "macro_task_accuracy": 0.9262877786032521,
+        "macro_task_balanced_accuracy": 0.9021513315229265,
+        "macro_task_nll": 0.18562255591253446
+      },
+      "checkpoint_files": {
+        "chat_template.jinja": "a4aee8afcf2e0711942cf848899be66016f8d14a889ff9ede07bca099c28f715",
+        "tokenizer.json": "06b9509352d2af50381ab2247e083b80d32d5c0aba91c272ca9ff729b6a0e523",
+        "adapter_model.safetensors": "793c6d72e5c0254c28884384a8bc3ca633fd4de9511834f710b5ee2dc95a942f",
+        "tokenizer_config.json": "353c083805d863719e54973335a38e252a92e1ac64d32b83bb0b333cc1644ef2",
+        "head.pt": "75a1b2c4362ce9eaccad18d0531fea66ab895aa924b73492af78fe7abf997986",
+        "README.md": "fdbf6f672a47dca9ef2ad668b395cccbdf4a09c88d16bf8a90ecf4f7445f31ce",
+        "adapter_config.json": "94aad580ce54a1dfcec0803beaba2c471e17b790f10efe7a481125c81fe1dd02"
+      }
+    },
+    {
+      "epoch": 2,
+      "metrics": {
+        "questions": 14912,
+        "accuracy": 0.9046405579399142,
+        "mean_class_recall": 0.8351706914084861,
+        "nll": 0.2609479185046837,
+        "brier": 0.13871271598030968,
+        "ece15": 0.02912533581539033,
+        "temperature": 1.0,
+        "coverage_at_confidence_0_9": 0.811762339055794,
+        "error_rate_at_confidence_0_9": 0.030318050392399834,
+        "ordinal_expected_level_mae": 0.007142014228908907,
+        "tasks": {
+          "agentdojo_tool_outputs:agentdojo_boundary_action": {
+            "questions": 222,
+            "accuracy": 0.9774774774774775,
+            "mean_class_recall": 0.9570930232558139,
+            "nll": 0.0857490162426134,
+            "brier": 0.03780111867446018,
+            "ece15": 0.020326742033581136,
+            "temperature": 1.0,
+            "coverage_at_confidence_0_9": 0.9864864864864865,
+            "error_rate_at_confidence_0_9": 0.0136986301369863,
+            "ordinal_expected_level_mae": null
+          },
+          "agentdojo_tool_outputs:agentdojo_embedded_injection": {
+            "questions": 222,
+            "accuracy": 0.9819819819819819,
+            "mean_class_recall": 0.96,
+            "nll": 0.08583505640070498,
+            "brier": 0.03170617674969949,
+            "ece15": 0.018778574788420183,
+            "temperature": 1.0,
+            "coverage_at_confidence_0_9": 0.9819819819819819,
+            "error_rate_at_confidence_0_9": 0.009174311926605505,
+            "ordinal_expected_level_mae": null
+          },
+          "byzfl_mnist:fl_attack_generated_update": {
+            "questions": 1400,
+            "accuracy": 0.9828571428571429,
+            "mean_class_recall": 0.9678571428571429,
+            "nll": 0.05038447430212686,
+            "brier": 0.026644462415551518,
+            "ece15": 0.006643072262975827,
+            "temperature": 1.0,
+            "coverage_at_confidence_0_9": 0.9757142857142858,
+            "error_rate_at_confidence_0_9": 0.008784773060029283,
+            "ordinal_expected_level_mae": null
+          },
+          "ciciot2023:ciciot_attack": {
+            "questions": 5000,
+            "accuracy": 0.839,
+            "mean_class_recall": 0.7132000000000001,
+            "nll": 0.4567278707638309,
+            "brier": 0.24289094068797595,
+            "ece15": 0.07089882316707959,
+            "temperature": 1.0,
+            "coverage_at_confidence_0_9": 0.7078,
+            "error_rate_at_confidence_0_9": 0.08194405199208817,
+            "ordinal_expected_level_mae": null
+          },
+          "injecagent:injecagent_requested_operation_v13": {
+            "questions": 124,
+            "accuracy": 1.0,
+            "mean_class_recall": 1.0,
+            "nll": 0.0030323687345377927,
+            "brier": 0.0010147751519150774,
+            "ece15": 0.0027304101493378944,
+            "temperature": 1.0,
+            "coverage_at_confidence_0_9": 0.9919354838709677,
+            "error_rate_at_confidence_0_9": 0.0,
+            "ordinal_expected_level_mae": null
+          },
+          "lanl_auth_policy:lanl_known_logon_action_v16": {
+            "questions": 328,
+            "accuracy": 1.0,
+            "mean_class_recall": 1.0,
+            "nll": 0.0013590718776517526,
+            "brier": 0.0005853758373684321,
+            "ece15": 0.001174090997847941,
+            "temperature": 1.0,
+            "coverage_at_confidence_0_9": 0.9969512195121951,
+            "error_rate_at_confidence_0_9": 0.0,
+            "ordinal_expected_level_mae": null
+          },
+          "lanl_auth_policy:lanl_observable_policy": {
+            "questions": 328,
+            "accuracy": 1.0,
+            "mean_class_recall": 1.0,
+            "nll": 2.706087239227311e-05,
+            "brier": 4.4631636950527135e-07,
+            "ece15": 2.6948652833984242e-05,
+            "temperature": 1.0,
+            "coverage_at_confidence_0_9": 1.0,
+            "error_rate_at_confidence_0_9": 0.0,
+            "ordinal_expected_level_mae": null
+          },
+          "ton_iot_network:ton_attack": {
+            "questions": 2500,
+            "accuracy": 0.9788,
+            "mean_class_recall": 0.98,
+            "nll": 0.08755100200107269,
+            "brier": 0.038158588944742826,
+            "ece15": 0.017829224144769444,
+            "temperature": 1.0,
+            "coverage_at_confidence_0_9": 0.9896,
+            "error_rate_at_confidence_0_9": 0.016976556184316895,
+            "ordinal_expected_level_mae": null
+          },
+          "twins_streamlet:twins_evidence_level": {
+            "questions": 797,
+            "accuracy": 0.9949811794228356,
+            "mean_class_recall": 0.9949406163344321,
+            "nll": 0.020100883811983803,
+            "brier": 0.009532705405460478,
+            "ece15": 0.004664870689987746,
+            "temperature": 1.0,
+            "coverage_at_confidence_0_9": 0.9962358845671268,
+            "error_rate_at_confidence_0_9": 0.003778337531486146,
+            "ordinal_expected_level_mae": 0.007142014228908907
+          },
+          "twins_streamlet:twins_policy_action": {
+            "questions": 797,
+            "accuracy": 0.9887076537013801,
+            "mean_class_recall": 0.9889237691623501,
+            "nll": 0.029531627570397425,
+            "brier": 0.01682083664455817,
+            "ece15": 0.008194444119065863,
+            "temperature": 1.0,
+            "coverage_at_confidence_0_9": 0.9887076537013801,
+            "error_rate_at_confidence_0_9": 0.0038071065989847717,
+            "ordinal_expected_level_mae": null
+          },
+          "twins_streamlet:twins_quorum": {
+            "questions": 797,
+            "accuracy": 0.9937264742785445,
+            "mean_class_recall": 0.9922118380062306,
+            "nll": 0.019942020828876864,
+            "brier": 0.011733672902765547,
+            "ece15": 0.006519660642330038,
+            "temperature": 1.0,
+            "coverage_at_confidence_0_9": 0.9912170639899623,
+            "error_rate_at_confidence_0_9": 0.0037974683544303796,
+            "ordinal_expected_level_mae": null
+          },
+          "twins_streamlet:twins_visible_conflict": {
+            "questions": 797,
+            "accuracy": 0.9974905897114178,
+            "mean_class_recall": 0.998236331569665,
+            "nll": 0.016769983022509982,
+            "brier": 0.0049992664597086935,
+            "ece15": 0.0021278163552494573,
+            "temperature": 1.0,
+            "coverage_at_confidence_0_9": 1.0,
+            "error_rate_at_confidence_0_9": 0.002509410288582183,
+            "ordinal_expected_level_mae": null
+          },
+          "veremi_nextgen:veremi_message_manipulation": {
+            "questions": 800,
+            "accuracy": 0.6875,
+            "mean_class_recall": 0.6966034271725826,
+            "nll": 0.5142628800857739,
+            "brier": 0.36155138388149327,
+            "ece15": 0.02205690591199928,
+            "temperature": 1.0,
+            "coverage_at_confidence_0_9": 0.22125,
+            "error_rate_at_confidence_0_9": 0.011299435028248588,
+            "ordinal_expected_level_mae": null
+          },
+          "veremi_nextgen:veremi_supported_attack_type_v16": {
+            "questions": 800,
+            "accuracy": 0.67375,
+            "mean_class_recall": 0.5089660282717738,
+            "nll": 0.998813782244008,
+            "brier": 0.4775012041931279,
+            "ece15": 0.03873448735113525,
+            "temperature": 1.0,
+            "coverage_at_confidence_0_9": 0.20625,
+            "error_rate_at_confidence_0_9": 0.030303030303030304,
+            "ordinal_expected_level_mae": null
+          }
+        },
+        "macro_task_accuracy": 0.9354480356736271,
+        "macro_task_balanced_accuracy": 0.9112880126164279,
+        "macro_task_nll": 0.16929193562560574
+      },
+      "checkpoint_files": {
+        "chat_template.jinja": "a4aee8afcf2e0711942cf848899be66016f8d14a889ff9ede07bca099c28f715",
+        "tokenizer.json": "06b9509352d2af50381ab2247e083b80d32d5c0aba91c272ca9ff729b6a0e523",
+        "adapter_model.safetensors": "f01d67ab57ea35f6f96dcf615f54147213977022560741db20fb7eab6eb462bd",
+        "tokenizer_config.json": "353c083805d863719e54973335a38e252a92e1ac64d32b83bb0b333cc1644ef2",
+        "head.pt": "460a08af9f39d347ecd48b74049d8a6e6140f1293160a4a5807a201b92ecc3c1",
+        "README.md": "fdbf6f672a47dca9ef2ad668b395cccbdf4a09c88d16bf8a90ecf4f7445f31ce",
+        "adapter_config.json": "94aad580ce54a1dfcec0803beaba2c471e17b790f10efe7a481125c81fe1dd02"
+      }
+    },
+    {
+      "epoch": 3,
+      "metrics": {
+        "questions": 14912,
+        "accuracy": 0.9075912017167382,
+        "mean_class_recall": 0.8447857522159936,
+        "nll": 0.24807737680415345,
+        "brier": 0.12998082994334342,
+        "ece15": 0.027028147820359012,
+        "temperature": 1.0,
+        "coverage_at_confidence_0_9": 0.8103540772532188,
+        "error_rate_at_confidence_0_9": 0.023502151605428664,
+        "ordinal_expected_level_mae": 0.00481932257817722,
+        "tasks": {
+          "agentdojo_tool_outputs:agentdojo_boundary_action": {
+            "questions": 222,
+            "accuracy": 0.9774774774774775,
+            "mean_class_recall": 0.9570930232558139,
+            "nll": 0.10407641066616782,
+            "brier": 0.039347876403718875,
+            "ece15": 0.021432863016197557,
+            "temperature": 1.0,
+            "coverage_at_confidence_0_9": 0.9864864864864865,
+            "error_rate_at_confidence_0_9": 0.0136986301369863,
+            "ordinal_expected_level_mae": null
+          },
+          "agentdojo_tool_outputs:agentdojo_embedded_injection": {
+            "questions": 222,
+            "accuracy": 0.9819819819819819,
+            "mean_class_recall": 0.96,
+            "nll": 0.10023555924012383,
+            "brier": 0.031563944714978254,
+            "ece15": 0.015911253108198964,
+            "temperature": 1.0,
+            "coverage_at_confidence_0_9": 0.9864864864864865,
+            "error_rate_at_confidence_0_9": 0.0136986301369863,
+            "ordinal_expected_level_mae": null
+          },
+          "byzfl_mnist:fl_attack_generated_update": {
+            "questions": 1400,
+            "accuracy": 0.9885714285714285,
+            "mean_class_recall": 0.9821428571428572,
+            "nll": 0.0442702602822601,
+            "brier": 0.02139513973607548,
+            "ece15": 0.007710400104216533,
+            "temperature": 1.0,
+            "coverage_at_confidence_0_9": 0.9842857142857143,
+            "error_rate_at_confidence_0_9": 0.008708272859216255,
+            "ordinal_expected_level_mae": null
+          },
+          "ciciot2023:ciciot_attack": {
+            "questions": 5000,
+            "accuracy": 0.8418,
+            "mean_class_recall": 0.7537333333333334,
+            "nll": 0.4173917752941542,
+            "brier": 0.22451704505631584,
+            "ece15": 0.06284084146027137,
+            "temperature": 1.0,
+            "coverage_at_confidence_0_9": 0.6928,
+            "error_rate_at_confidence_0_9": 0.06293302540415704,
+            "ordinal_expected_level_mae": null
+          },
+          "injecagent:injecagent_requested_operation_v13": {
+            "questions": 124,
+            "accuracy": 1.0,
+            "mean_class_recall": 1.0,
+            "nll": 0.0014205610073774013,
+            "brier": 0.0002211290260091384,
+            "ece15": 0.0013613576932818498,
+            "temperature": 1.0,
+            "coverage_at_confidence_0_9": 0.9919354838709677,
+            "error_rate_at_confidence_0_9": 0.0,
+            "ordinal_expected_level_mae": null
+          },
+          "lanl_auth_policy:lanl_known_logon_action_v16": {
+            "questions": 328,
+            "accuracy": 1.0,
+            "mean_class_recall": 1.0,
+            "nll": 0.000610657881540054,
+            "brier": 8.099314527924365e-05,
+            "ece15": 0.0005891478461821947,
+            "temperature": 1.0,
+            "coverage_at_confidence_0_9": 1.0,
+            "error_rate_at_confidence_0_9": 0.0,
+            "ordinal_expected_level_mae": null
+          },
+          "lanl_auth_policy:lanl_observable_policy": {
+            "questions": 328,
+            "accuracy": 1.0,
+            "mean_class_recall": 1.0,
+            "nll": 1.5480302068279906e-07,
+            "brier": 4.4217021643060835e-12,
+            "ece15": 1.548019148098009e-07,
+            "temperature": 1.0,
+            "coverage_at_confidence_0_9": 1.0,
+            "error_rate_at_confidence_0_9": 0.0,
+            "ordinal_expected_level_mae": null
+          },
+          "ton_iot_network:ton_attack": {
+            "questions": 2500,
+            "accuracy": 0.9792,
+            "mean_class_recall": 0.9818666666666667,
+            "nll": 0.11218220058914397,
+            "brier": 0.03724045269615108,
+            "ece15": 0.01842697861877182,
+            "temperature": 1.0,
+            "coverage_at_confidence_0_9": 0.9928,
+            "error_rate_at_confidence_0_9": 0.015310233682514102,
+            "ordinal_expected_level_mae": null
+          },
+          "twins_streamlet:twins_evidence_level": {
+            "questions": 797,
+            "accuracy": 0.9962358845671268,
+            "mean_class_recall": 0.9963898916967509,
+            "nll": 0.012217254972570101,
+            "brier": 0.006818829719414822,
+            "ece15": 0.004133398699556409,
+            "temperature": 1.0,
+            "coverage_at_confidence_0_9": 0.9949811794228356,
+            "error_rate_at_confidence_0_9": 0.0012610340479192938,
+            "ordinal_expected_level_mae": 0.00481932257817722
+          },
+          "twins_streamlet:twins_policy_action": {
+            "questions": 797,
+            "accuracy": 0.9949811794228356,
+            "mean_class_recall": 0.9951865222623345,
+            "nll": 0.013078119892699127,
+            "brier": 0.007721112955228842,
+            "ece15": 0.003423998250058524,
+            "temperature": 1.0,
+            "coverage_at_confidence_0_9": 0.9899623588456713,
+            "error_rate_at_confidence_0_9": 0.0012674271229404308,
+            "ordinal_expected_level_mae": null
+          },
+          "twins_streamlet:twins_quorum": {
+            "questions": 797,
+            "accuracy": 0.9949811794228356,
+            "mean_class_recall": 0.994276682635671,
+            "nll": 0.01027058602627885,
+            "brier": 0.007209131739335666,
+            "ece15": 0.004201163796495765,
+            "temperature": 1.0,
+            "coverage_at_confidence_0_9": 0.9912170639899623,
+            "error_rate_at_confidence_0_9": 0.0,
+            "ordinal_expected_level_mae": null
+          },
+          "twins_streamlet:twins_visible_conflict": {
+            "questions": 797,
+            "accuracy": 0.9974905897114178,
+            "mean_class_recall": 0.998236331569665,
+            "nll": 0.017287182530912663,
+            "brier": 0.004953169208636321,
+            "ece15": 0.0025266183132433223,
+            "temperature": 1.0,
+            "coverage_at_confidence_0_9": 0.998745294855709,
+            "error_rate_at_confidence_0_9": 0.002512562814070352,
+            "ordinal_expected_level_mae": null
+          },
+          "veremi_nextgen:veremi_message_manipulation": {
+            "questions": 800,
+            "accuracy": 0.69,
+            "mean_class_recall": 0.7116483068135455,
+            "nll": 0.5092641106435474,
+            "brier": 0.3569714849542106,
+            "ece15": 0.03650030376694629,
+            "temperature": 1.0,
+            "coverage_at_confidence_0_9": 0.2425,
+            "error_rate_at_confidence_0_9": 0.010309278350515464,
+            "ordinal_expected_level_mae": null
+          },
+          "veremi_nextgen:veremi_supported_attack_type_v16": {
+            "questions": 800,
+            "accuracy": 0.68875,
+            "mean_class_recall": 0.529392069171139,
+            "nll": 0.96833512912194,
+            "brier": 0.46247412161427887,
+            "ece15": 0.02731423542286039,
+            "temperature": 1.0,
+            "coverage_at_confidence_0_9": 0.22625,
+            "error_rate_at_confidence_0_9": 0.022099447513812154,
+            "ordinal_expected_level_mae": null
+          }
+        },
+        "macro_task_accuracy": 0.9379621229396502,
+        "macro_task_balanced_accuracy": 0.9185689774676985,
+        "macro_task_nll": 0.16504571163940973
+      },
+      "checkpoint_files": {
+        "chat_template.jinja": "a4aee8afcf2e0711942cf848899be66016f8d14a889ff9ede07bca099c28f715",
+        "tokenizer.json": "06b9509352d2af50381ab2247e083b80d32d5c0aba91c272ca9ff729b6a0e523",
+        "adapter_model.safetensors": "0087e92e59b987fea721124d1a3f61b1aca26b3ac01b974032abb60799e7b175",
+        "tokenizer_config.json": "353c083805d863719e54973335a38e252a92e1ac64d32b83bb0b333cc1644ef2",
+        "head.pt": "f3c477c0419bd8dbf64d530d7f2576e084f65cdee3b351e34a7804e67b3979cf",
+        "README.md": "fdbf6f672a47dca9ef2ad668b395cccbdf4a09c88d16bf8a90ecf4f7445f31ce",
+        "adapter_config.json": "94aad580ce54a1dfcec0803beaba2c471e17b790f10efe7a481125c81fe1dd02"
+      }
+    }
+  ],
+  "rule": "maximum development macro-task semantic-class balanced accuracy; tie macro-task raw NLL, then earlier epoch",
+  "test_used_for_selection": false
+}

epoch-3-test.json

@@ -0,0 +1,375 @@
+{
+  "epoch": 3,
+  "temperature": 1.334968103387446,
+  "test": {
+    "raw": {
+      "questions": 21634,
+      "accuracy": 0.8801423684940372,
+      "mean_class_recall": 0.8422636415920437,
+      "nll": 0.35756736839135106,
+      "brier": 0.17329397786588344,
+      "ece15": 0.018370944185424988,
+      "temperature": 1.0,
+      "coverage_at_confidence_0_9": 0.681750947582509,
+      "error_rate_at_confidence_0_9": 0.017289307749677945,
+      "ordinal_expected_level_mae": 0.004830545729071066,
+      "tasks": {
+        "agentdojo_tool_outputs:agentdojo_boundary_action": {
+          "questions": 426,
+          "accuracy": 0.9953051643192489,
+          "mean_class_recall": 0.9895833333333333,
+          "nll": 0.019578322598972707,
+          "brier": 0.008470972915844083,
+          "ece15": 0.0032192125141472924,
+          "temperature": 1.0,
+          "coverage_at_confidence_0_9": 0.9882629107981221,
+          "error_rate_at_confidence_0_9": 0.0023752969121140144,
+          "ordinal_expected_level_mae": null
+        },
+        "agentdojo_tool_outputs:agentdojo_embedded_injection": {
+          "questions": 426,
+          "accuracy": 0.9835680751173709,
+          "mean_class_recall": 0.9635416666666667,
+          "nll": 0.03323309985013501,
+          "brier": 0.020700086530374747,
+          "ece15": 0.012858446963474107,
+          "temperature": 1.0,
+          "coverage_at_confidence_0_9": 0.9835680751173709,
+          "error_rate_at_confidence_0_9": 0.007159904534606206,
+          "ordinal_expected_level_mae": null
+        },
+        "byzfl_mnist:fl_attack_generated_update": {
+          "questions": 800,
+          "accuracy": 0.95875,
+          "mean_class_recall": 0.9296875,
+          "nll": 0.14798192410675134,
+          "brier": 0.07208296294789789,
+          "ece15": 0.031237402444916955,
+          "temperature": 1.0,
+          "coverage_at_confidence_0_9": 0.9575,
+          "error_rate_at_confidence_0_9": 0.02741514360313316,
+          "ordinal_expected_level_mae": null
+        },
+        "ciciot2023:ciciot_attack": {
+          "questions": 6000,
+          "accuracy": 0.9171666666666667,
+          "mean_class_recall": 0.8741111111111111,
+          "nll": 0.2872065840367128,
+          "brier": 0.1291518236859529,
+          "ece15": 0.02997403903668212,
+          "temperature": 1.0,
+          "coverage_at_confidence_0_9": 0.7491666666666666,
+          "error_rate_at_confidence_0_9": 0.03181312569521691,
+          "ordinal_expected_level_mae": null
+        },
+        "injecagent:injecagent_requested_operation_v13": {
+          "questions": 124,
+          "accuracy": 1.0,
+          "mean_class_recall": 1.0,
+          "nll": 6.416181236130915e-05,
+          "brier": 3.664856836177276e-07,
+          "ece15": 6.406994017019252e-05,
+          "temperature": 1.0,
+          "coverage_at_confidence_0_9": 1.0,
+          "error_rate_at_confidence_0_9": 0.0,
+          "ordinal_expected_level_mae": null
+        },
+        "lanl_auth_policy:lanl_known_logon_action_v16": {
+          "questions": 540,
+          "accuracy": 1.0,
+          "mean_class_recall": 1.0,
+          "nll": 0.0003909349271697467,
+          "brier": 5.411140895974333e-05,
+          "ece15": 0.000376397356322764,
+          "temperature": 1.0,
+          "coverage_at_confidence_0_9": 0.9981481481481481,
+          "error_rate_at_confidence_0_9": 0.0,
+          "ordinal_expected_level_mae": null
+        },
+        "lanl_auth_policy:lanl_observable_policy": {
+          "questions": 540,
+          "accuracy": 1.0,
+          "mean_class_recall": 1.0,
+          "nll": 1.9301873747749096e-07,
+          "brier": 4.461551704214049e-12,
+          "ece15": 1.9301762319156735e-07,
+          "temperature": 1.0,
+          "coverage_at_confidence_0_9": 1.0,
+          "error_rate_at_confidence_0_9": 0.0,
+          "ordinal_expected_level_mae": null
+        },
+        "ton_iot_network:ton_attack": {
+          "questions": 3500,
+          "accuracy": 0.988,
+          "mean_class_recall": 0.9820952380952381,
+          "nll": 0.052315917826989855,
+          "brier": 0.021002166306533596,
+          "ece15": 0.005742982405630235,
+          "temperature": 1.0,
+          "coverage_at_confidence_0_9": 0.9757142857142858,
+          "error_rate_at_confidence_0_9": 0.007320644216691069,
+          "ordinal_expected_level_mae": null
+        },
+        "twins_streamlet:twins_evidence_level": {
+          "questions": 659,
+          "accuracy": 0.9954476479514416,
+          "mean_class_recall": 0.9949748743718594,
+          "nll": 0.006489090778116727,
+          "brier": 0.004985928434486637,
+          "ece15": 0.0035010905751582675,
+          "temperature": 1.0,
+          "coverage_at_confidence_0_9": 0.9954476479514416,
+          "error_rate_at_confidence_0_9": 0.0,
+          "ordinal_expected_level_mae": 0.004830545729071066
+        },
+        "twins_streamlet:twins_policy_action": {
+          "questions": 659,
+          "accuracy": 0.9984825493171472,
+          "mean_class_recall": 0.9983249581239532,
+          "nll": 0.007759166929265847,
+          "brier": 0.0031279766576844436,
+          "ece15": 0.0017302646891685911,
+          "temperature": 1.0,
+          "coverage_at_confidence_0_9": 0.9969650986342944,
+          "error_rate_at_confidence_0_9": 0.0015220700152207,
+          "ordinal_expected_level_mae": null
+        },
+        "twins_streamlet:twins_quorum": {
+          "questions": 659,
+          "accuracy": 1.0,
+          "mean_class_recall": 1.0,
+          "nll": 0.0012916521858483854,
+          "brier": 0.0006719742504699832,
+          "ece15": 0.0010543087689309648,
+          "temperature": 1.0,
+          "coverage_at_confidence_0_9": 0.9969650986342944,
+          "error_rate_at_confidence_0_9": 0.0,
+          "ordinal_expected_level_mae": null
+        },
+        "twins_streamlet:twins_visible_conflict": {
+          "questions": 659,
+          "accuracy": 0.9984825493171472,
+          "mean_class_recall": 0.9989130434782609,
+          "nll": 0.0030258654215359524,
+          "brier": 0.0020473738981638956,
+          "ece15": 0.0002839890142556098,
+          "temperature": 1.0,
+          "coverage_at_confidence_0_9": 0.9954476479514416,
+          "error_rate_at_confidence_0_9": 0.0,
+          "ordinal_expected_level_mae": null
+        },
+        "veremi_nextgen:veremi_message_manipulation": {
+          "questions": 3321,
+          "accuracy": 0.7034025895814514,
+          "mean_class_recall": 0.7106693655944849,
+          "nll": 0.5271449745997652,
+          "brier": 0.3679898300956757,
+          "ece15": 0.03254066478455375,
+          "temperature": 1.0,
+          "coverage_at_confidence_0_9": 0.21499548328816623,
+          "error_rate_at_confidence_0_9": 0.011204481792717087,
+          "ordinal_expected_level_mae": null
+        },
+        "veremi_nextgen:veremi_supported_attack_type_v16": {
+          "questions": 3321,
+          "accuracy": 0.692261367058115,
+          "mean_class_recall": 0.49711800080440777,
+          "nll": 1.181957709780574,
+          "brier": 0.48216427161657194,
+          "ece15": 0.04703079874679226,
+          "temperature": 1.0,
+          "coverage_at_confidence_0_9": 0.20776874435411022,
+          "error_rate_at_confidence_0_9": 0.07681159420289856,
+          "ordinal_expected_level_mae": null
+        }
+      },
+      "macro_task_accuracy": 0.9450619006663279,
+      "macro_task_balanced_accuracy": 0.9242156493985226
+    },
+    "recalibrated": {
+      "questions": 21634,
+      "accuracy": 0.8801423684940372,
+      "mean_class_recall": 0.8422636415920437,
+      "nll": 0.3441350952603644,
+      "brier": 0.1767857045083663,
+      "ece15": 0.035627089616313175,
+      "temperature": 1.334968103387446,
+      "coverage_at_confidence_0_9": 0.6567440140519553,
+      "error_rate_at_confidence_0_9": 0.014358108108108109,
+      "ordinal_expected_level_mae": 0.004973955616734103,
+      "tasks": {
+        "agentdojo_tool_outputs:agentdojo_boundary_action": {
+          "questions": 426,
+          "accuracy": 0.9953051643192489,
+          "mean_class_recall": 0.9895833333333333,
+          "nll": 0.02085470758455793,
+          "brier": 0.009033562612123906,
+          "ece15": 0.005062121140742335,
+          "temperature": 1.334968103387446,
+          "coverage_at_confidence_0_9": 0.9741784037558685,
+          "error_rate_at_confidence_0_9": 0.0024096385542168677,
+          "ordinal_expected_level_mae": null
+        },
+        "agentdojo_tool_outputs:agentdojo_embedded_injection": {
+          "questions": 426,
+          "accuracy": 0.9835680751173709,
+          "mean_class_recall": 0.9635416666666667,
+          "nll": 0.029576905474440387,
+          "brier": 0.019371229609764937,
+          "ece15": 0.0128149515297424,
+          "temperature": 1.334968103387446,
+          "coverage_at_confidence_0_9": 0.9741784037558685,
+          "error_rate_at_confidence_0_9": 0.004819277108433735,
+          "ordinal_expected_level_mae": null
+        },
+        "byzfl_mnist:fl_attack_generated_update": {
+          "questions": 800,
+          "accuracy": 0.95875,
+          "mean_class_recall": 0.9296875,
+          "nll": 0.1252495255860491,
+          "brier": 0.06891977027707019,
+          "ece15": 0.02206119521855819,
+          "temperature": 1.334968103387446,
+          "coverage_at_confidence_0_9": 0.93375,
+          "error_rate_at_confidence_0_9": 0.0214190093708166,
+          "ordinal_expected_level_mae": null
+        },
+        "ciciot2023:ciciot_attack": {
+          "questions": 6000,
+          "accuracy": 0.9171666666666667,
+          "mean_class_recall": 0.8741111111111111,
+          "nll": 0.258923302286342,
+          "brier": 0.1323925339560934,
+          "ece15": 0.040786104751424794,
+          "temperature": 1.334968103387446,
+          "coverage_at_confidence_0_9": 0.6995,
+          "error_rate_at_confidence_0_9": 0.027162258756254467,
+          "ordinal_expected_level_mae": null
+        },
+        "injecagent:injecagent_requested_operation_v13": {
+          "questions": 124,
+          "accuracy": 1.0,
+          "mean_class_recall": 1.0,
+          "nll": 0.00036120828507376734,
+          "brier": 6.118819643947097e-06,
+          "ece15": 0.0003596634947089362,
+          "temperature": 1.334968103387446,
+          "coverage_at_confidence_0_9": 1.0,
+          "error_rate_at_confidence_0_9": 0.0,
+          "ordinal_expected_level_mae": null
+        },
+        "lanl_auth_policy:lanl_known_logon_action_v16": {
+          "questions": 540,
+          "accuracy": 1.0,
+          "mean_class_recall": 1.0,
+          "nll": 0.0010259839180316675,
+          "brier": 0.00015355821904600842,
+          "ece15": 0.0009831313211751498,
+          "temperature": 1.334968103387446,
+          "coverage_at_confidence_0_9": 0.9981481481481481,
+          "error_rate_at_confidence_0_9": 0.0,
+          "ordinal_expected_level_mae": null
+        },
+        "lanl_auth_policy:lanl_observable_policy": {
+          "questions": 540,
+          "accuracy": 1.0,
+          "mean_class_recall": 1.0,
+          "nll": 5.4752266593721125e-06,
+          "brier": 1.1349090161477055e-09,
+          "ece15": 5.474942881804694e-06,
+          "temperature": 1.334968103387446,
+          "coverage_at_confidence_0_9": 1.0,
+          "error_rate_at_confidence_0_9": 0.0,
+          "ordinal_expected_level_mae": null
+        },
+        "ton_iot_network:ton_attack": {
+          "questions": 3500,
+          "accuracy": 0.988,
+          "mean_class_recall": 0.9820952380952381,
+          "nll": 0.0471904212042291,
+          "brier": 0.02136284177729358,
+          "ece15": 0.004930332206950378,
+          "temperature": 1.334968103387446,
+          "coverage_at_confidence_0_9": 0.9597142857142857,
+          "error_rate_at_confidence_0_9": 0.006251860672819291,
+          "ordinal_expected_level_mae": null
+        },
+        "twins_streamlet:twins_evidence_level": {
+          "questions": 659,
+          "accuracy": 0.9954476479514416,
+          "mean_class_recall": 0.9949748743718594,
+          "nll": 0.005989405513587896,
+          "brier": 0.004363630530423605,
+          "ece15": 0.003686325313576514,
+          "temperature": 1.334968103387446,
+          "coverage_at_confidence_0_9": 0.9954476479514416,
+          "error_rate_at_confidence_0_9": 0.0,
+          "ordinal_expected_level_mae": 0.004973955616734103
+        },
+        "twins_streamlet:twins_policy_action": {
+          "questions": 659,
+          "accuracy": 0.9984825493171472,
+          "mean_class_recall": 0.9983249581239532,
+          "nll": 0.0069631222649601515,
+          "brier": 0.003204452557959292,
+          "ece15": 0.001531985329986691,
+          "temperature": 1.334968103387446,
+          "coverage_at_confidence_0_9": 0.9969650986342944,
+          "error_rate_at_confidence_0_9": 0.0015220700152207,
+          "ordinal_expected_level_mae": null
+        },
+        "twins_streamlet:twins_quorum": {
+          "questions": 659,
+          "accuracy": 1.0,
+          "mean_class_recall": 1.0,
+          "nll": 0.0018952623564185236,
+          "brier": 0.0008114028234666237,
+          "ece15": 0.0016108880631044219,
+          "temperature": 1.334968103387446,
+          "coverage_at_confidence_0_9": 0.9954476479514416,
+          "error_rate_at_confidence_0_9": 0.0,
+          "ordinal_expected_level_mae": null
+        },
+        "twins_streamlet:twins_visible_conflict": {
+          "questions": 659,
+          "accuracy": 0.9984825493171472,
+          "mean_class_recall": 0.9989130434782609,
+          "nll": 0.0033168872089549845,
+          "brier": 0.0020980871703588605,
+          "ece15": 0.0012518453771527926,
+          "temperature": 1.334968103387446,
+          "coverage_at_confidence_0_9": 0.9954476479514416,
+          "error_rate_at_confidence_0_9": 0.0,
+          "ordinal_expected_level_mae": null
+        },
+        "veremi_nextgen:veremi_message_manipulation": {
+          "questions": 3321,
+          "accuracy": 0.7034025895814514,
+          "mean_class_recall": 0.7106693655944849,
+          "nll": 0.5278136514287293,
+          "brier": 0.3689683794121852,
+          "ece15": 0.03240031037986953,
+          "temperature": 1.334968103387446,
+          "coverage_at_confidence_0_9": 0.1969286359530262,
+          "error_rate_at_confidence_0_9": 0.009174311926605505,
+          "ordinal_expected_level_mae": null
+        },
+        "veremi_nextgen:veremi_supported_attack_type_v16": {
+          "questions": 3321,
+          "accuracy": 0.692261367058115,
+          "mean_class_recall": 0.49711800080440777,
+          "nll": 1.1560335557868109,
+          "brier": 0.498611302076152,
+          "ece15": 0.12617627958797675,
+          "temperature": 1.334968103387446,
+          "coverage_at_confidence_0_9": 0.17856067449563384,
+          "error_rate_at_confidence_0_9": 0.07251264755480608,
+          "ordinal_expected_level_mae": null
+        }
+      },
+      "macro_task_accuracy": 0.9450619006663279,
+      "macro_task_balanced_accuracy": 0.9242156493985226
+    }
+  },
+  "test_used_for_selection": false
+}

epoch-3-validation.json

@@ -0,0 +1,744 @@
+{
+  "epoch": 3,
+  "temperature": 1.334968103387446,
+  "development": {
+    "raw": {
+      "questions": 14912,
+      "accuracy": 0.9075912017167382,
+      "mean_class_recall": 0.8447857522159936,
+      "nll": 0.24807737680415345,
+      "brier": 0.12998082994334342,
+      "ece15": 0.027028147820359012,
+      "temperature": 1.0,
+      "coverage_at_confidence_0_9": 0.8103540772532188,
+      "error_rate_at_confidence_0_9": 0.023502151605428664,
+      "ordinal_expected_level_mae": 0.00481932257817722,
+      "tasks": {
+        "agentdojo_tool_outputs:agentdojo_boundary_action": {
+          "questions": 222,
+          "accuracy": 0.9774774774774775,
+          "mean_class_recall": 0.9570930232558139,
+          "nll": 0.10407641066616782,
+          "brier": 0.039347876403718875,
+          "ece15": 0.021432863016197557,
+          "temperature": 1.0,
+          "coverage_at_confidence_0_9": 0.9864864864864865,
+          "error_rate_at_confidence_0_9": 0.0136986301369863,
+          "ordinal_expected_level_mae": null
+        },
+        "agentdojo_tool_outputs:agentdojo_embedded_injection": {
+          "questions": 222,
+          "accuracy": 0.9819819819819819,
+          "mean_class_recall": 0.96,
+          "nll": 0.10023555924012383,
+          "brier": 0.031563944714978254,
+          "ece15": 0.015911253108198964,
+          "temperature": 1.0,
+          "coverage_at_confidence_0_9": 0.9864864864864865,
+          "error_rate_at_confidence_0_9": 0.0136986301369863,
+          "ordinal_expected_level_mae": null
+        },
+        "byzfl_mnist:fl_attack_generated_update": {
+          "questions": 1400,
+          "accuracy": 0.9885714285714285,
+          "mean_class_recall": 0.9821428571428572,
+          "nll": 0.0442702602822601,
+          "brier": 0.02139513973607548,
+          "ece15": 0.007710400104216533,
+          "temperature": 1.0,
+          "coverage_at_confidence_0_9": 0.9842857142857143,
+          "error_rate_at_confidence_0_9": 0.008708272859216255,
+          "ordinal_expected_level_mae": null
+        },
+        "ciciot2023:ciciot_attack": {
+          "questions": 5000,
+          "accuracy": 0.8418,
+          "mean_class_recall": 0.7537333333333334,
+          "nll": 0.4173917752941542,
+          "brier": 0.22451704505631584,
+          "ece15": 0.06284084146027137,
+          "temperature": 1.0,
+          "coverage_at_confidence_0_9": 0.6928,
+          "error_rate_at_confidence_0_9": 0.06293302540415704,
+          "ordinal_expected_level_mae": null
+        },
+        "injecagent:injecagent_requested_operation_v13": {
+          "questions": 124,
+          "accuracy": 1.0,
+          "mean_class_recall": 1.0,
+          "nll": 0.0014205610073774013,
+          "brier": 0.0002211290260091384,
+          "ece15": 0.0013613576932818498,
+          "temperature": 1.0,
+          "coverage_at_confidence_0_9": 0.9919354838709677,
+          "error_rate_at_confidence_0_9": 0.0,
+          "ordinal_expected_level_mae": null
+        },
+        "lanl_auth_policy:lanl_known_logon_action_v16": {
+          "questions": 328,
+          "accuracy": 1.0,
+          "mean_class_recall": 1.0,
+          "nll": 0.000610657881540054,
+          "brier": 8.099314527924365e-05,
+          "ece15": 0.0005891478461821947,
+          "temperature": 1.0,
+          "coverage_at_confidence_0_9": 1.0,
+          "error_rate_at_confidence_0_9": 0.0,
+          "ordinal_expected_level_mae": null
+        },
+        "lanl_auth_policy:lanl_observable_policy": {
+          "questions": 328,
+          "accuracy": 1.0,
+          "mean_class_recall": 1.0,
+          "nll": 1.5480302068279906e-07,
+          "brier": 4.4217021643060835e-12,
+          "ece15": 1.548019148098009e-07,
+          "temperature": 1.0,
+          "coverage_at_confidence_0_9": 1.0,
+          "error_rate_at_confidence_0_9": 0.0,
+          "ordinal_expected_level_mae": null
+        },
+        "ton_iot_network:ton_attack": {
+          "questions": 2500,
+          "accuracy": 0.9792,
+          "mean_class_recall": 0.9818666666666667,
+          "nll": 0.11218220058914397,
+          "brier": 0.03724045269615108,
+          "ece15": 0.01842697861877182,
+          "temperature": 1.0,
+          "coverage_at_confidence_0_9": 0.9928,
+          "error_rate_at_confidence_0_9": 0.015310233682514102,
+          "ordinal_expected_level_mae": null
+        },
+        "twins_streamlet:twins_evidence_level": {
+          "questions": 797,
+          "accuracy": 0.9962358845671268,
+          "mean_class_recall": 0.9963898916967509,
+          "nll": 0.012217254972570101,
+          "brier": 0.006818829719414822,
+          "ece15": 0.004133398699556409,
+          "temperature": 1.0,
+          "coverage_at_confidence_0_9": 0.9949811794228356,
+          "error_rate_at_confidence_0_9": 0.0012610340479192938,
+          "ordinal_expected_level_mae": 0.00481932257817722
+        },
+        "twins_streamlet:twins_policy_action": {
+          "questions": 797,
+          "accuracy": 0.9949811794228356,
+          "mean_class_recall": 0.9951865222623345,
+          "nll": 0.013078119892699127,
+          "brier": 0.007721112955228842,
+          "ece15": 0.003423998250058524,
+          "temperature": 1.0,
+          "coverage_at_confidence_0_9": 0.9899623588456713,
+          "error_rate_at_confidence_0_9": 0.0012674271229404308,
+          "ordinal_expected_level_mae": null
+        },
+        "twins_streamlet:twins_quorum": {
+          "questions": 797,
+          "accuracy": 0.9949811794228356,
+          "mean_class_recall": 0.994276682635671,
+          "nll": 0.01027058602627885,
+          "brier": 0.007209131739335666,
+          "ece15": 0.004201163796495765,
+          "temperature": 1.0,
+          "coverage_at_confidence_0_9": 0.9912170639899623,
+          "error_rate_at_confidence_0_9": 0.0,
+          "ordinal_expected_level_mae": null
+        },
+        "twins_streamlet:twins_visible_conflict": {
+          "questions": 797,
+          "accuracy": 0.9974905897114178,
+          "mean_class_recall": 0.998236331569665,
+          "nll": 0.017287182530912663,
+          "brier": 0.004953169208636321,
+          "ece15": 0.0025266183132433223,
+          "temperature": 1.0,
+          "coverage_at_confidence_0_9": 0.998745294855709,
+          "error_rate_at_confidence_0_9": 0.002512562814070352,
+          "ordinal_expected_level_mae": null
+        },
+        "veremi_nextgen:veremi_message_manipulation": {
+          "questions": 800,
+          "accuracy": 0.69,
+          "mean_class_recall": 0.7116483068135455,
+          "nll": 0.5092641106435474,
+          "brier": 0.3569714849542106,
+          "ece15": 0.03650030376694629,
+          "temperature": 1.0,
+          "coverage_at_confidence_0_9": 0.2425,
+          "error_rate_at_confidence_0_9": 0.010309278350515464,
+          "ordinal_expected_level_mae": null
+        },
+        "veremi_nextgen:veremi_supported_attack_type_v16": {
+          "questions": 800,
+          "accuracy": 0.68875,
+          "mean_class_recall": 0.529392069171139,
+          "nll": 0.96833512912194,
+          "brier": 0.46247412161427887,
+          "ece15": 0.02731423542286039,
+          "temperature": 1.0,
+          "coverage_at_confidence_0_9": 0.22625,
+          "error_rate_at_confidence_0_9": 0.022099447513812154,
+          "ordinal_expected_level_mae": null
+        }
+      },
+      "macro_task_accuracy": 0.9379621229396502,
+      "macro_task_balanced_accuracy": 0.9185689774676985
+    },
+    "recalibrated": {
+      "questions": 14912,
+      "accuracy": 0.9075912017167382,
+      "mean_class_recall": 0.8447857522159936,
+      "nll": 0.2269545932235271,
+      "brier": 0.12792407717450321,
+      "ece15": 0.01858051585491712,
+      "temperature": 1.334968103387446,
+      "coverage_at_confidence_0_9": 0.7862124463519313,
+      "error_rate_at_confidence_0_9": 0.018679631525076765,
+      "ordinal_expected_level_mae": 0.005329911013904147,
+      "tasks": {
+        "agentdojo_tool_outputs:agentdojo_boundary_action": {
+          "questions": 222,
+          "accuracy": 0.9774774774774775,
+          "mean_class_recall": 0.9570930232558139,
+          "nll": 0.08092683597443519,
+          "brier": 0.037546344393533104,
+          "ece15": 0.021320738677048364,
+          "temperature": 1.334968103387446,
+          "coverage_at_confidence_0_9": 0.9819819819819819,
+          "error_rate_at_confidence_0_9": 0.013761467889908258,
+          "ordinal_expected_level_mae": null
+        },
+        "agentdojo_tool_outputs:agentdojo_embedded_injection": {
+          "questions": 222,
+          "accuracy": 0.9819819819819819,
+          "mean_class_recall": 0.96,
+          "nll": 0.07809919245507242,
+          "brier": 0.030623326048867618,
+          "ece15": 0.015019163882298042,
+          "temperature": 1.334968103387446,
+          "coverage_at_confidence_0_9": 0.9819819819819819,
+          "error_rate_at_confidence_0_9": 0.009174311926605505,
+          "ordinal_expected_level_mae": null
+        },
+        "byzfl_mnist:fl_attack_generated_update": {
+          "questions": 1400,
+          "accuracy": 0.9885714285714285,
+          "mean_class_recall": 0.9821428571428572,
+          "nll": 0.04094699270351949,
+          "brier": 0.021018406686723334,
+          "ece15": 0.0038434318338614677,
+          "temperature": 1.334968103387446,
+          "coverage_at_confidence_0_9": 0.9678571428571429,
+          "error_rate_at_confidence_0_9": 0.006642066420664207,
+          "ordinal_expected_level_mae": null
+        },
+        "ciciot2023:ciciot_attack": {
+          "questions": 5000,
+          "accuracy": 0.8418,
+          "mean_class_recall": 0.7537333333333334,
+          "nll": 0.36679030702727994,
+          "brier": 0.21757064336558796,
+          "ece15": 0.043357678681868704,
+          "temperature": 1.334968103387446,
+          "coverage_at_confidence_0_9": 0.6394,
+          "error_rate_at_confidence_0_9": 0.05004691898654989,
+          "ordinal_expected_level_mae": null
+        },
+        "injecagent:injecagent_requested_operation_v13": {
+          "questions": 124,
+          "accuracy": 1.0,
+          "mean_class_recall": 1.0,
+          "nll": 0.0028701157894541487,
+          "brier": 0.0006216354903482641,
+          "ece15": 0.0026971564840139535,
+          "temperature": 1.334968103387446,
+          "coverage_at_confidence_0_9": 0.9919354838709677,
+          "error_rate_at_confidence_0_9": 0.0,
+          "ordinal_expected_level_mae": null
+        },
+        "lanl_auth_policy:lanl_known_logon_action_v16": {
+          "questions": 328,
+          "accuracy": 1.0,
+          "mean_class_recall": 1.0,
+          "nll": 0.001405432942745028,
+          "brier": 0.00025083300955486444,
+          "ece15": 0.0013358908781109478,
+          "temperature": 1.334968103387446,
+          "coverage_at_confidence_0_9": 0.9939024390243902,
+          "error_rate_at_confidence_0_9": 0.0,
+          "ordinal_expected_level_mae": null
+        },
+        "lanl_auth_policy:lanl_observable_policy": {
+          "questions": 328,
+          "accuracy": 1.0,
+          "mean_class_recall": 1.0,
+          "nll": 4.6272880953518815e-06,
+          "brier": 9.796066038293659e-10,
+          "ece15": 4.627043141135246e-06,
+          "temperature": 1.334968103387446,
+          "coverage_at_confidence_0_9": 1.0,
+          "error_rate_at_confidence_0_9": 0.0,
+          "ordinal_expected_level_mae": null
+        },
+        "ton_iot_network:ton_attack": {
+          "questions": 2500,
+          "accuracy": 0.9792,
+          "mean_class_recall": 0.9818666666666667,
+          "nll": 0.0877031817842779,
+          "brier": 0.03599036114227489,
+          "ece15": 0.01538869285344686,
+          "temperature": 1.334968103387446,
+          "coverage_at_confidence_0_9": 0.988,
+          "error_rate_at_confidence_0_9": 0.014574898785425101,
+          "ordinal_expected_level_mae": null
+        },
+        "twins_streamlet:twins_evidence_level": {
+          "questions": 797,
+          "accuracy": 0.9962358845671268,
+          "mean_class_recall": 0.9963898916967509,
+          "nll": 0.010603240403009088,
+          "brier": 0.006373682001028304,
+          "ece15": 0.004218405852177047,
+          "temperature": 1.334968103387446,
+          "coverage_at_confidence_0_9": 0.9937264742785445,
+          "error_rate_at_confidence_0_9": 0.0012626262626262627,
+          "ordinal_expected_level_mae": 0.005329911013904147
+        },
+        "twins_streamlet:twins_policy_action": {
+          "questions": 797,
+          "accuracy": 0.9949811794228356,
+          "mean_class_recall": 0.9951865222623345,
+          "nll": 0.01230072195659841,
+          "brier": 0.007337903591834537,
+          "ece15": 0.0031241457310850957,
+          "temperature": 1.334968103387446,
+          "coverage_at_confidence_0_9": 0.986198243412798,
+          "error_rate_at_confidence_0_9": 0.001272264631043257,
+          "ordinal_expected_level_mae": null
+        },
+        "twins_streamlet:twins_quorum": {
+          "questions": 797,
+          "accuracy": 0.9949811794228356,
+          "mean_class_recall": 0.994276682635671,
+          "nll": 0.009584146250019401,
+          "brier": 0.00659904677807738,
+          "ece15": 0.005028933100804552,
+          "temperature": 1.334968103387446,
+          "coverage_at_confidence_0_9": 0.9899623588456713,
+          "error_rate_at_confidence_0_9": 0.0,
+          "ordinal_expected_level_mae": null
+        },
+        "twins_streamlet:twins_visible_conflict": {
+          "questions": 797,
+          "accuracy": 0.9974905897114178,
+          "mean_class_recall": 0.998236331569665,
+          "nll": 0.013437428285294059,
+          "brier": 0.004858731838764845,
+          "ece15": 0.0022785888551786117,
+          "temperature": 1.334968103387446,
+          "coverage_at_confidence_0_9": 0.998745294855709,
+          "error_rate_at_confidence_0_9": 0.002512562814070352,
+          "ordinal_expected_level_mae": null
+        },
+        "veremi_nextgen:veremi_message_manipulation": {
+          "questions": 800,
+          "accuracy": 0.69,
+          "mean_class_recall": 0.7116483068135455,
+          "nll": 0.5102124840713312,
+          "brier": 0.35753050242002155,
+          "ece15": 0.03811996421246104,
+          "temperature": 1.334968103387446,
+          "coverage_at_confidence_0_9": 0.22125,
+          "error_rate_at_confidence_0_9": 0.011299435028248588,
+          "ordinal_expected_level_mae": null
+        },
+        "veremi_nextgen:veremi_supported_attack_type_v16": {
+          "questions": 800,
+          "accuracy": 0.68875,
+          "mean_class_recall": 0.529392069171139,
+          "nll": 0.9911460027249259,
+          "brier": 0.4737144267281807,
+          "ece15": 0.09647043512079573,
+          "temperature": 1.334968103387446,
+          "coverage_at_confidence_0_9": 0.18625,
+          "error_rate_at_confidence_0_9": 0.020134228187919462,
+          "ordinal_expected_level_mae": null
+        }
+      },
+      "macro_task_accuracy": 0.9379621229396502,
+      "macro_task_balanced_accuracy": 0.9185689774676985
+    }
+  },
+  "calibration": {
+    "raw": {
+      "questions": 14821,
+      "accuracy": 0.9313811483705553,
+      "mean_class_recall": 0.8460223158089928,
+      "nll": 0.18480685074016417,
+      "brier": 0.09978353062869037,
+      "ece15": 0.010514966894040724,
+      "temperature": 1.0,
+      "coverage_at_confidence_0_9": 0.814789825247959,
+      "error_rate_at_confidence_0_9": 0.013580655846306725,
+      "ordinal_expected_level_mae": 0.001461981275023823,
+      "tasks": {
+        "agentdojo_tool_outputs:agentdojo_boundary_action": {
+          "questions": 325,
+          "accuracy": 0.96,
+          "mean_class_recall": 0.9216916119306557,
+          "nll": 0.12908652765840894,
+          "brier": 0.06258807450225369,
+          "ece15": 0.033510331142289206,
+          "temperature": 1.0,
+          "coverage_at_confidence_0_9": 0.9815384615384616,
+          "error_rate_at_confidence_0_9": 0.0219435736677116,
+          "ordinal_expected_level_mae": null
+        },
+        "agentdojo_tool_outputs:agentdojo_embedded_injection": {
+          "questions": 328,
+          "accuracy": 0.9695121951219512,
+          "mean_class_recall": 0.9372206852521814,
+          "nll": 0.16127769578392517,
+          "brier": 0.05858994554032181,
+          "ece15": 0.030459345972402876,
+          "temperature": 1.0,
+          "coverage_at_confidence_0_9": 0.9939024390243902,
+          "error_rate_at_confidence_0_9": 0.027607361963190184,
+          "ordinal_expected_level_mae": null
+        },
+        "byzfl_mnist:fl_attack_generated_update": {
+          "questions": 2400,
+          "accuracy": 0.9404166666666667,
+          "mean_class_recall": 0.8877604166666666,
+          "nll": 0.19653998710185536,
+          "brier": 0.09947525932448194,
+          "ece15": 0.04423318187692424,
+          "temperature": 1.0,
+          "coverage_at_confidence_0_9": 0.9479166666666666,
+          "error_rate_at_confidence_0_9": 0.03824175824175824,
+          "ordinal_expected_level_mae": null
+        },
+        "ciciot2023:ciciot_attack": {
+          "questions": 5000,
+          "accuracy": 0.9292,
+          "mean_class_recall": 0.8968,
+          "nll": 0.18887834668200343,
+          "brier": 0.10636649585712103,
+          "ece15": 0.019887550017025992,
+          "temperature": 1.0,
+          "coverage_at_confidence_0_9": 0.7252,
+          "error_rate_at_confidence_0_9": 0.013513513513513514,
+          "ordinal_expected_level_mae": null
+        },
+        "injecagent:injecagent_requested_operation_v13": {
+          "questions": 124,
+          "accuracy": 0.9919354838709677,
+          "mean_class_recall": 0.9928571428571429,
+          "nll": 0.021390831595053342,
+          "brier": 0.015445766007169246,
+          "ece15": 0.011074830403285848,
+          "temperature": 1.0,
+          "coverage_at_confidence_0_9": 0.9838709677419355,
+          "error_rate_at_confidence_0_9": 0.0,
+          "ordinal_expected_level_mae": null
+        },
+        "lanl_auth_policy:lanl_known_logon_action_v16": {
+          "questions": 235,
+          "accuracy": 0.9957446808510638,
+          "mean_class_recall": 0.9949494949494949,
+          "nll": 0.015420468691595452,
+          "brier": 0.008033852620508352,
+          "ece15": 0.003625196423068977,
+          "temperature": 1.0,
+          "coverage_at_confidence_0_9": 1.0,
+          "error_rate_at_confidence_0_9": 0.00425531914893617,
+          "ordinal_expected_level_mae": null
+        },
+        "lanl_auth_policy:lanl_observable_policy": {
+          "questions": 235,
+          "accuracy": 0.9957446808510638,
+          "mean_class_recall": 0.9905660377358491,
+          "nll": 0.018077073969230188,
+          "brier": 0.00826909442413202,
+          "ece15": 0.004194003764167214,
+          "temperature": 1.0,
+          "coverage_at_confidence_0_9": 1.0,
+          "error_rate_at_confidence_0_9": 0.00425531914893617,
+          "ordinal_expected_level_mae": null
+        },
+        "ton_iot_network:ton_attack": {
+          "questions": 2000,
+          "accuracy": 0.9985,
+          "mean_class_recall": 0.9976666666666667,
+          "nll": 0.010567088503443185,
+          "brier": 0.003178972042914288,
+          "ece15": 0.0009240536217447044,
+          "temperature": 1.0,
+          "coverage_at_confidence_0_9": 0.999,
+          "error_rate_at_confidence_0_9": 0.0015015015015015015,
+          "ordinal_expected_level_mae": null
+        },
+        "twins_streamlet:twins_evidence_level": {
+          "questions": 641,
+          "accuracy": 0.9984399375975039,
+          "mean_class_recall": 0.9985507246376812,
+          "nll": 0.003039872100282549,
+          "brier": 0.0022297539963869384,
+          "ece15": 0.0014593852165271412,
+          "temperature": 1.0,
+          "coverage_at_confidence_0_9": 0.9984399375975039,
+          "error_rate_at_confidence_0_9": 0.0,
+          "ordinal_expected_level_mae": 0.001461981275023823
+        },
+        "twins_streamlet:twins_policy_action": {
+          "questions": 641,
+          "accuracy": 1.0,
+          "mean_class_recall": 1.0,
+          "nll": 0.0004967972153911127,
+          "brier": 9.688544623270663e-05,
+          "ece15": 0.00046955068951652005,
+          "temperature": 1.0,
+          "coverage_at_confidence_0_9": 0.9984399375975039,
+          "error_rate_at_confidence_0_9": 0.0,
+          "ordinal_expected_level_mae": null
+        },
+        "twins_streamlet:twins_quorum": {
+          "questions": 641,
+          "accuracy": 0.9968798751950078,
+          "mean_class_recall": 0.9961240310077519,
+          "nll": 0.016173152134660414,
+          "brier": 0.007502235795954162,
+          "ece15": 0.0049434253477994355,
+          "temperature": 1.0,
+          "coverage_at_confidence_0_9": 0.9937597503900156,
+          "error_rate_at_confidence_0_9": 0.0031397174254317113,
+          "ordinal_expected_level_mae": null
+        },
+        "twins_streamlet:twins_visible_conflict": {
+          "questions": 641,
+          "accuracy": 0.9984399375975039,
+          "mean_class_recall": 0.9969512195121951,
+          "nll": 0.004162755288573824,
+          "brier": 0.0026938896990115683,
+          "ece15": 0.0014825770107887316,
+          "temperature": 1.0,
+          "coverage_at_confidence_0_9": 1.0,
+          "error_rate_at_confidence_0_9": 0.0015600624024961,
+          "ordinal_expected_level_mae": null
+        },
+        "veremi_nextgen:veremi_message_manipulation": {
+          "questions": 805,
+          "accuracy": 0.7031055900621118,
+          "mean_class_recall": 0.7239978843175034,
+          "nll": 0.49879740669601635,
+          "brier": 0.351098943795658,
+          "ece15": 0.02568931078110801,
+          "temperature": 1.0,
+          "coverage_at_confidence_0_9": 0.23850931677018633,
+          "error_rate_at_confidence_0_9": 0.0,
+          "ordinal_expected_level_mae": null
+        },
+        "veremi_nextgen:veremi_supported_attack_type_v16": {
+          "questions": 805,
+          "accuracy": 0.6919254658385093,
+          "mean_class_recall": 0.5331098265895953,
+          "nll": 0.9684343228937802,
+          "brier": 0.45465071518796674,
+          "ece15": 0.01705305962599151,
+          "temperature": 1.0,
+          "coverage_at_confidence_0_9": 0.2360248447204969,
+          "error_rate_at_confidence_0_9": 0.021052631578947368,
+          "ordinal_expected_level_mae": null
+        }
+      },
+      "macro_task_accuracy": 0.9407031795465964,
+      "macro_task_balanced_accuracy": 0.9191604101516703
+    },
+    "recalibrated": {
+      "questions": 14821,
+      "accuracy": 0.9313811483705553,
+      "mean_class_recall": 0.8460223158089928,
+      "nll": 0.17961943925788992,
+      "brier": 0.10111838724890135,
+      "ece15": 0.017864195303344165,
+      "temperature": 1.334968103387446,
+      "coverage_at_confidence_0_9": 0.7859118817893529,
+      "error_rate_at_confidence_0_9": 0.010645604395604396,
+      "ordinal_expected_level_mae": 0.001742299523022841,
+      "tasks": {
+        "agentdojo_tool_outputs:agentdojo_boundary_action": {
+          "questions": 325,
+          "accuracy": 0.96,
+          "mean_class_recall": 0.9216916119306557,
+          "nll": 0.10308600797920058,
+          "brier": 0.05761479429147346,
+          "ece15": 0.0294057816164861,
+          "temperature": 1.334968103387446,
+          "coverage_at_confidence_0_9": 0.963076923076923,
+          "error_rate_at_confidence_0_9": 0.012779552715654952,
+          "ordinal_expected_level_mae": null
+        },
+        "agentdojo_tool_outputs:agentdojo_embedded_injection": {
+          "questions": 328,
+          "accuracy": 0.9695121951219512,
+          "mean_class_recall": 0.9372206852521814,
+          "nll": 0.12289366290953901,
+          "brier": 0.057082591730227304,
+          "ece15": 0.029897091535857363,
+          "temperature": 1.334968103387446,
+          "coverage_at_confidence_0_9": 0.9939024390243902,
+          "error_rate_at_confidence_0_9": 0.027607361963190184,
+          "ordinal_expected_level_mae": null
+        },
+        "byzfl_mnist:fl_attack_generated_update": {
+          "questions": 2400,
+          "accuracy": 0.9404166666666667,
+          "mean_class_recall": 0.8877604166666666,
+          "nll": 0.16174786934019353,
+          "brier": 0.09375828073422167,
+          "ece15": 0.03645209485430555,
+          "temperature": 1.334968103387446,
+          "coverage_at_confidence_0_9": 0.92875,
+          "error_rate_at_confidence_0_9": 0.03095558546433378,
+          "ordinal_expected_level_mae": null
+        },
+        "ciciot2023:ciciot_attack": {
+          "questions": 5000,
+          "accuracy": 0.9292,
+          "mean_class_recall": 0.8968,
+          "nll": 0.19125358889505403,
+          "brier": 0.11141554439383701,
+          "ece15": 0.03505488361604012,
+          "temperature": 1.334968103387446,
+          "coverage_at_confidence_0_9": 0.6616,
+          "error_rate_at_confidence_0_9": 0.010278113663845224,
+          "ordinal_expected_level_mae": null
+        },
+        "injecagent:injecagent_requested_operation_v13": {
+          "questions": 124,
+          "accuracy": 0.9919354838709677,
+          "mean_class_recall": 0.9928571428571429,
+          "nll": 0.019190864186870816,
+          "brier": 0.01397734899726431,
+          "ece15": 0.011351501004051424,
+          "temperature": 1.334968103387446,
+          "coverage_at_confidence_0_9": 0.9838709677419355,
+          "error_rate_at_confidence_0_9": 0.0,
+          "ordinal_expected_level_mae": null
+        },
+        "lanl_auth_policy:lanl_known_logon_action_v16": {
+          "questions": 235,
+          "accuracy": 0.9957446808510638,
+          "mean_class_recall": 0.9949494949494949,
+          "nll": 0.01285197836061796,
+          "brier": 0.007512756902037326,
+          "ece15": 0.004004760561973723,
+          "temperature": 1.334968103387446,
+          "coverage_at_confidence_0_9": 1.0,
+          "error_rate_at_confidence_0_9": 0.00425531914893617,
+          "ordinal_expected_level_mae": null
+        },
+        "lanl_auth_policy:lanl_observable_policy": {
+          "questions": 235,
+          "accuracy": 0.9957446808510638,
+          "mean_class_recall": 0.9905660377358491,
+          "nll": 0.01368047084450566,
+          "brier": 0.007839165259612221,
+          "ece15": 0.004073337312958388,
+          "temperature": 1.334968103387446,
+          "coverage_at_confidence_0_9": 1.0,
+          "error_rate_at_confidence_0_9": 0.00425531914893617,
+          "ordinal_expected_level_mae": null
+        },
+        "ton_iot_network:ton_attack": {
+          "questions": 2000,
+          "accuracy": 0.9985,
+          "mean_class_recall": 0.9976666666666667,
+          "nll": 0.010650627249048795,
+          "brier": 0.003323170074906094,
+          "ece15": 0.002118687755931114,
+          "temperature": 1.334968103387446,
+          "coverage_at_confidence_0_9": 0.9975,
+          "error_rate_at_confidence_0_9": 0.0015037593984962407,
+          "ordinal_expected_level_mae": null
+        },
+        "twins_streamlet:twins_evidence_level": {
+          "questions": 641,
+          "accuracy": 0.9984399375975039,
+          "mean_class_recall": 0.9985507246376812,
+          "nll": 0.0028802196260483376,
+          "brier": 0.001935795675809169,
+          "ece15": 0.0017273023271884604,
+          "temperature": 1.334968103387446,
+          "coverage_at_confidence_0_9": 0.9968798751950078,
+          "error_rate_at_confidence_0_9": 0.0,
+          "ordinal_expected_level_mae": 0.001742299523022841
+        },
+        "twins_streamlet:twins_policy_action": {
+          "questions": 641,
+          "accuracy": 1.0,
+          "mean_class_recall": 1.0,
+          "nll": 0.0012409177190170738,
+          "brier": 0.00021235107519573934,
+          "ece15": 0.0011788827327451372,
+          "temperature": 1.334968103387446,
+          "coverage_at_confidence_0_9": 0.9984399375975039,
+          "error_rate_at_confidence_0_9": 0.0,
+          "ordinal_expected_level_mae": null
+        },
+        "twins_streamlet:twins_quorum": {
+          "questions": 641,
+          "accuracy": 0.9968798751950078,
+          "mean_class_recall": 0.9961240310077519,
+          "nll": 0.013816362060687534,
+          "brier": 0.007467944863319216,
+          "ece15": 0.004893228894077442,
+          "temperature": 1.334968103387446,
+          "coverage_at_confidence_0_9": 0.9921996879875195,
+          "error_rate_at_confidence_0_9": 0.0031446540880503146,
+          "ordinal_expected_level_mae": null
+        },
+        "twins_streamlet:twins_visible_conflict": {
+          "questions": 641,
+          "accuracy": 0.9984399375975039,
+          "mean_class_recall": 0.9969512195121951,
+          "nll": 0.0034185049808860444,
+          "brier": 0.0023816170036107054,
+          "ece15": 0.0015601193471088582,
+          "temperature": 1.334968103387446,
+          "coverage_at_confidence_0_9": 0.9984399375975039,
+          "error_rate_at_confidence_0_9": 0.0,
+          "ordinal_expected_level_mae": null
+        },
+        "veremi_nextgen:veremi_message_manipulation": {
+          "questions": 805,
+          "accuracy": 0.7031055900621118,
+          "mean_class_recall": 0.7239978843175034,
+          "nll": 0.503140821446039,
+          "brier": 0.35269849107227197,
+          "ece15": 0.030978365187072362,
+          "temperature": 1.334968103387446,
+          "coverage_at_confidence_0_9": 0.21366459627329193,
+          "error_rate_at_confidence_0_9": 0.0,
+          "ordinal_expected_level_mae": null
+        },
+        "veremi_nextgen:veremi_supported_attack_type_v16": {
+          "questions": 805,
+          "accuracy": 0.6919254658385093,
+          "mean_class_recall": 0.5331098265895953,
+          "nll": 0.9878648410254747,
+          "brier": 0.46649698606417656,
+          "ece15": 0.09086134384209663,
+          "temperature": 1.334968103387446,
+          "coverage_at_confidence_0_9": 0.19627329192546583,
+          "error_rate_at_confidence_0_9": 0.006329113924050633,
+          "ordinal_expected_level_mae": null
+        }
+      },
+      "macro_task_accuracy": 0.9407031795465964,
+      "macro_task_balanced_accuracy": 0.9191604101516703
+    }
+  }
+}

evaluation/README.md

@@ -0,0 +1,37 @@
+# SecJev-4B results
+
+Epoch 3 was selected on all 14,912 development questions. Calibration uses a separate 14,821-question split. Test scores below cover all 21,634 security questions and 14 tasks. All three checkpoints were specified for testing before test access.
+
+| Model | Micro accuracy | Macro-task accuracy | Macro-task balanced accuracy |
+|---|---:|---:|---:|
+| Original Kev-4B | 50.49% | 67.88% | 67.16% |
+| Epoch 1 | 85.88% | 93.38% | 90.74% |
+| Epoch 2 | 87.12% | 94.15% | 91.69% |
+| Epoch 3 | 88.01% | 94.51% | 92.42% |
+
+| Task | Questions | Original Kev-4B | SecJev-4B | SecJev balanced accuracy |
+|---|---:|---:|---:|---:|
+| agentdojo_tool_outputs:agentdojo_boundary_action | 426 | 54.23% | 99.53% | 98.96% |
+| agentdojo_tool_outputs:agentdojo_embedded_injection | 426 | 73.00% | 98.36% | 96.35% |
+| byzfl_mnist:fl_attack_generated_update | 800 | 79.88% | 95.88% | 92.97% |
+| ciciot2023:ciciot_attack | 6000 | 26.28% | 91.72% | 87.41% |
+| injecagent:injecagent_requested_operation_v13 | 124 | 62.90% | 100.00% | 100.00% |
+| lanl_auth_policy:lanl_known_logon_action_v16 | 540 | 79.81% | 100.00% | 100.00% |
+| lanl_auth_policy:lanl_observable_policy | 540 | 99.44% | 100.00% | 100.00% |
+| ton_iot_network:ton_attack | 3500 | 50.66% | 98.80% | 98.21% |
+| twins_streamlet:twins_evidence_level | 659 | 81.18% | 99.54% | 99.50% |
+| twins_streamlet:twins_policy_action | 659 | 71.78% | 99.85% | 99.83% |
+| twins_streamlet:twins_quorum | 659 | 78.15% | 100.00% | 100.00% |
+| twins_streamlet:twins_visible_conflict | 659 | 97.12% | 99.85% | 99.89% |
+| veremi_nextgen:veremi_message_manipulation | 3321 | 47.76% | 70.34% | 71.07% |
+| veremi_nextgen:veremi_supported_attack_type_v16 | 3321 | 48.09% | 69.23% | 49.71% |
+| **Macro-average** | — | **67.88%** | **94.51%** | **92.42%** |
+
+Macro metrics weight tasks equally. Balanced accuracy averages recall over semantic classes within each task. Ordinal questions use argmax accuracy. Vehicle-message manipulation (70.34%) and attack-type classification (69.23%) remain the weakest tasks.
+
+| General suite (clean subset) | Original Kev-4B | SecJev-4B |
+|---|---:|---:|
+| decision-v7 (1,200) | 87.08% | 86.58% |
+| transfer-v4 (656) | 83.69% | 83.84% |
+
+Full general suites, including variants, contain 1,440 and 764 questions. Inference is FP32 unmerged, TF32/fused SDPA off. All original multiquestion semantics are retained. These are previously inspected research splits; source-label and policy scores are not live attack-detection rates. See results.json for calibration metrics and every checkpoint.

evaluation/README.zh-CN.md

@@ -0,0 +1,44 @@
+# SecJev-4B 训练与评测结果
+
+官方 Kev-4B → SecJev-Corpus 三轮。训练使用全部 9 张 GPU。使用 LoRA r16 和决策头，FP32 参数、BF16 训练；评测为 FP32。
+
+开发验证集 14,912 题选择第 3 轮，校准集 14,821 题只用于温度拟合。安全测试集为完整的 21,634 题、14 项任务；三个 checkpoint 均按预定方案测试，测试集不用于选择。
+
+总准确率按题目加权；任务宏平均对 14 项任务等权；任务宏平均平衡准确率先在每项任务内对语义类别召回率等权，再对任务等权。
+
+| 模型 | 总准确率 | 任务宏平均准确率 | 任务宏平均平衡准确率 |
+|---|---:|---:|---:|
+| 官方 Kev-4B | 50.49% | 67.88% | 67.16% |
+| SecJev epoch 1 | 85.88% | 93.38% | 90.74% |
+| SecJev epoch 2 | 87.12% | 94.15% | 91.69% |
+| SecJev epoch 3 | 88.01% | 94.51% | 92.42% |
+
+| 安全任务 | 题数 | 官方 Kev-4B | Epoch1 | Epoch2 | Epoch3 |
+|---|---:|---:|---:|---:|---:|
+| agentdojo_tool_outputs:agentdojo_boundary_action | 426 | 54.23% | 99.53% | 99.53% | 99.53% |
+| agentdojo_tool_outputs:agentdojo_embedded_injection | 426 | 73.00% | 98.36% | 98.36% | 98.36% |
+| byzfl_mnist:fl_attack_generated_update | 800 | 79.88% | 93.88% | 95.25% | 95.88% |
+| ciciot2023:ciciot_attack | 6000 | 26.28% | 90.85% | 89.95% | 91.72% |
+| injecagent:injecagent_requested_operation_v13 | 124 | 62.90% | 100.00% | 100.00% | 100.00% |
+| lanl_auth_policy:lanl_known_logon_action_v16 | 540 | 79.81% | 100.00% | 100.00% | 100.00% |
+| lanl_auth_policy:lanl_observable_policy | 540 | 99.44% | 100.00% | 100.00% | 100.00% |
+| ton_iot_network:ton_attack | 3500 | 50.66% | 98.83% | 98.94% | 98.80% |
+| twins_streamlet:twins_evidence_level | 659 | 81.18% | 99.09% | 99.85% | 99.54% |
+| twins_streamlet:twins_policy_action | 659 | 71.78% | 99.39% | 99.70% | 99.85% |
+| twins_streamlet:twins_quorum | 659 | 78.15% | 99.70% | 99.85% | 100.00% |
+| twins_streamlet:twins_visible_conflict | 659 | 97.12% | 99.70% | 99.70% | 99.85% |
+| veremi_nextgen:veremi_message_manipulation | 3321 | 47.76% | 65.46% | 68.38% | 70.34% |
+| veremi_nextgen:veremi_supported_attack_type_v16 | 3321 | 48.09% | 62.48% | 68.59% | 69.23% |
+
+通用评测完整运行 decision-v7 的 1,440 题和 transfer-v4 的 764 题（含扰动变体）。下表沿用原先口径，只汇总 clean 子集，分别为 1,200 题和 656 题。
+
+| 模型 | decision-v7 | transfer-v4 |
+|---|---:|---:|
+| 官方 Kev-4B | 87.08% | 83.69% |
+| SecJev epoch 1 | 86.67% | 83.84% |
+| SecJev epoch 2 | 86.50% | 83.69% |
+| SecJev epoch 3 | 86.58% | 83.84% |
+
+准确率均为 argmax 判定。各安全 checkpoint 的概率温度只用校准集拟合；详见 results.json 的 NLL、Brier、ECE 等指标。通用测试没有日期事实补充。
+
+从官方 jaredpalmer/kev-4b 的 LoRA 和决策头继续训练，底座为 Qwen3.5-4B-Base。安全训练沿用现有0.8B/2B配方，所有问题完整编码。使用既有研究测试集，不能称为新的盲测。

evaluation/audit.json

@@ -0,0 +1,118 @@
+{
+  "splits": {
+    "epoch-1/development": {
+      "questions": 14912,
+      "accuracy": 0.8959898068669528,
+      "macro_task_accuracy": 0.9262877786032517,
+      "macro_task_balanced_accuracy": 0.9021513315229265
+    },
+    "epoch-2/development": {
+      "questions": 14912,
+      "accuracy": 0.9046405579399142,
+      "macro_task_accuracy": 0.9354480356736271,
+      "macro_task_balanced_accuracy": 0.9112880126164279
+    },
+    "epoch-3/development": {
+      "questions": 14912,
+      "accuracy": 0.9075912017167382,
+      "macro_task_accuracy": 0.9379621229396502,
+      "macro_task_balanced_accuracy": 0.9185689774676984
+    },
+    "epoch-1/calibration": {
+      "questions": 14821,
+      "accuracy": 0.910802240064773,
+      "macro_task_accuracy": 0.9243611377541778,
+      "macro_task_balanced_accuracy": 0.8966212134669761
+    },
+    "epoch-2/calibration": {
+      "questions": 14821,
+      "accuracy": 0.9218676202685379,
+      "macro_task_accuracy": 0.9347961786371293,
+      "macro_task_balanced_accuracy": 0.9089808629491994
+    },
+    "epoch-3/calibration": {
+      "questions": 14821,
+      "accuracy": 0.9313811483705553,
+      "macro_task_accuracy": 0.9407031795465964,
+      "macro_task_balanced_accuracy": 0.9191604101516704
+    },
+    "epoch-1/test": {
+      "questions": 21634,
+      "accuracy": 0.8587870943884626,
+      "macro_task_accuracy": 0.9337570392580401,
+      "macro_task_balanced_accuracy": 0.9073790312675484
+    },
+    "epoch-2/test": {
+      "questions": 21634,
+      "accuracy": 0.8712212258482019,
+      "macro_task_accuracy": 0.9414975177781386,
+      "macro_task_balanced_accuracy": 0.9169354318963098
+    },
+    "epoch-3/test": {
+      "questions": 21634,
+      "accuracy": 0.8801423684940372,
+      "macro_task_accuracy": 0.9450619006663278,
+      "macro_task_balanced_accuracy": 0.9242156493985225
+    },
+    "kev-4b/test": {
+      "questions": 21634,
+      "accuracy": 0.5048996949246556,
+      "macro_task_accuracy": 0.6787694400132314,
+      "macro_task_balanced_accuracy": 0.6716227368384962
+    }
+  },
+  "general": {
+    "kev-4b/decision-v7": {
+      "all_questions": 1440,
+      "clean_questions": 1200,
+      "accuracy": 0.8708333333333333
+    },
+    "epoch-1/decision-v7": {
+      "all_questions": 1440,
+      "clean_questions": 1200,
+      "accuracy": 0.8666666666666667
+    },
+    "epoch-2/decision-v7": {
+      "all_questions": 1440,
+      "clean_questions": 1200,
+      "accuracy": 0.865
+    },
+    "epoch-3/decision-v7": {
+      "all_questions": 1440,
+      "clean_questions": 1200,
+      "accuracy": 0.8658333333333333
+    },
+    "kev-4b/transfer-v4": {
+      "all_questions": 764,
+      "clean_questions": 656,
+      "accuracy": 0.836890243902439
+    },
+    "epoch-1/transfer-v4": {
+      "all_questions": 764,
+      "clean_questions": 656,
+      "accuracy": 0.8384146341463414
+    },
+    "epoch-2/transfer-v4": {
+      "all_questions": 764,
+      "clean_questions": 656,
+      "accuracy": 0.836890243902439
+    },
+    "epoch-3/transfer-v4": {
+      "all_questions": 764,
+      "clean_questions": 656,
+      "accuracy": 0.8384146341463414
+    }
+  },
+  "checks": [
+    "unique question counts and identical labels",
+    "all shard exit0 and complete logs",
+    "merged rows equal raw shard outputs",
+    "recomputed accuracy and 14-task macro/balanced metrics",
+    "general clean counts and original request identities",
+    "development-only selection",
+    "all completion markers"
+  ],
+  "pass_all": true,
+  "selected_epoch": 3,
+  "test_used_for_selection": false
+}

evaluation/checkpoint-selection.json

@@ -0,0 +1,598 @@
+{
+  "selected_epoch": 3,
+  "candidates": [
+    {
+      "epoch": 1,
+      "metrics": {
+        "questions": 14912,
+        "accuracy": 0.8959898068669528,
+        "mean_class_recall": 0.8086847041598828,
+        "nll": 0.2676199905412643,
+        "brier": 0.14581168647550916,
+        "ece15": 0.024724176262894793,
+        "temperature": 1.0,
+        "coverage_at_confidence_0_9": 0.773806330472103,
+        "error_rate_at_confidence_0_9": 0.0240055464078343,
+        "ordinal_expected_level_mae": 0.00949861612167232,
+        "tasks": {
+          "agentdojo_tool_outputs:agentdojo_boundary_action": {
+            "questions": 222,
+            "accuracy": 0.9819819819819819,
+            "mean_class_recall": 0.9670930232558139,
+            "nll": 0.06952124509948641,
+            "brier": 0.033360936223710204,
+            "ece15": 0.01963763286720348,
+            "temperature": 1.0,
+            "coverage_at_confidence_0_9": 0.9774774774774775,
+            "error_rate_at_confidence_0_9": 0.013824884792626729,
+            "ordinal_expected_level_mae": null
+          },
+          "agentdojo_tool_outputs:agentdojo_embedded_injection": {
+            "questions": 222,
+            "accuracy": 0.9864864864864865,
+            "mean_class_recall": 0.97,
+            "nll": 0.06796588080667874,
+            "brier": 0.02442578405456454,
+            "ece15": 0.013534851962772593,
+            "temperature": 1.0,
+            "coverage_at_confidence_0_9": 0.972972972972973,
+            "error_rate_at_confidence_0_9": 0.009259259259259259,
+            "ordinal_expected_level_mae": null
+          },
+          "byzfl_mnist:fl_attack_generated_update": {
+            "questions": 1400,
+            "accuracy": 0.9742857142857143,
+            "mean_class_recall": 0.9504464285714286,
+            "nll": 0.07440929350732509,
+            "brier": 0.03861493679124129,
+            "ece15": 0.01061565077999396,
+            "temperature": 1.0,
+            "coverage_at_confidence_0_9": 0.925,
+            "error_rate_at_confidence_0_9": 0.008494208494208495,
+            "ordinal_expected_level_mae": null
+          },
+          "ciciot2023:ciciot_attack": {
+            "questions": 5000,
+            "accuracy": 0.8334,
+            "mean_class_recall": 0.7313333333333334,
+            "nll": 0.4357438451410003,
+            "brier": 0.2420765591030422,
+            "ece15": 0.05574236954650583,
+            "temperature": 1.0,
+            "coverage_at_confidence_0_9": 0.644,
+            "error_rate_at_confidence_0_9": 0.06677018633540373,
+            "ordinal_expected_level_mae": null
+          },
+          "injecagent:injecagent_requested_operation_v13": {
+            "questions": 124,
+            "accuracy": 1.0,
+            "mean_class_recall": 1.0,
+            "nll": 0.004390672226438891,
+            "brier": 0.0015490359962901127,
+            "ece15": 0.003915493134373699,
+            "temperature": 1.0,
+            "coverage_at_confidence_0_9": 0.9919354838709677,
+            "error_rate_at_confidence_0_9": 0.0,
+            "ordinal_expected_level_mae": null
+          },
+          "lanl_auth_policy:lanl_known_logon_action_v16": {
+            "questions": 328,
+            "accuracy": 1.0,
+            "mean_class_recall": 1.0,
+            "nll": 0.004780661486616687,
+            "brier": 0.0026501860965206722,
+            "ece15": 0.0038321931288341792,
+            "temperature": 1.0,
+            "coverage_at_confidence_0_9": 0.9908536585365854,
+            "error_rate_at_confidence_0_9": 0.0,
+            "ordinal_expected_level_mae": null
+          },
+          "lanl_auth_policy:lanl_observable_policy": {
+            "questions": 328,
+            "accuracy": 1.0,
+            "mean_class_recall": 1.0,
+            "nll": 2.8436047490948972e-05,
+            "brier": 2.213459113159646e-07,
+            "ece15": 2.8380504783511674e-05,
+            "temperature": 1.0,
+            "coverage_at_confidence_0_9": 1.0,
+            "error_rate_at_confidence_0_9": 0.0,
+            "ordinal_expected_level_mae": null
+          },
+          "ton_iot_network:ton_attack": {
+            "questions": 2500,
+            "accuracy": 0.9832,
+            "mean_class_recall": 0.9856,
+            "nll": 0.0716936772308886,
+            "brier": 0.032718644101254024,
+            "ece15": 0.01640596644430267,
+            "temperature": 1.0,
+            "coverage_at_confidence_0_9": 0.9948,
+            "error_rate_at_confidence_0_9": 0.01648572577402493,
+            "ordinal_expected_level_mae": null
+          },
+          "twins_streamlet:twins_evidence_level": {
+            "questions": 797,
+            "accuracy": 0.9937264742785445,
+            "mean_class_recall": 0.9934913409721132,
+            "nll": 0.0153080496936535,
+            "brier": 0.009627399097334195,
+            "ece15": 0.004599932218083785,
+            "temperature": 1.0,
+            "coverage_at_confidence_0_9": 0.9912170639899623,
+            "error_rate_at_confidence_0_9": 0.0037974683544303796,
+            "ordinal_expected_level_mae": 0.00949861612167232
+          },
+          "twins_streamlet:twins_policy_action": {
+            "questions": 797,
+            "accuracy": 0.9924717691342535,
+            "mean_class_recall": 0.9923419156847569,
+            "nll": 0.017883068917921716,
+            "brier": 0.011294880553062277,
+            "ece15": 0.0038184765498114437,
+            "temperature": 1.0,
+            "coverage_at_confidence_0_9": 0.9811794228356336,
+            "error_rate_at_confidence_0_9": 0.0012787723785166241,
+            "ordinal_expected_level_mae": null
+          },
+          "twins_streamlet:twins_quorum": {
+            "questions": 797,
+            "accuracy": 0.9962358845671268,
+            "mean_class_recall": 0.995834315034425,
+            "nll": 0.011957269820791244,
+            "brier": 0.007393224542639707,
+            "ece15": 0.00670405372457154,
+            "temperature": 1.0,
+            "coverage_at_confidence_0_9": 0.9836888331242158,
+            "error_rate_at_confidence_0_9": 0.0,
+            "ordinal_expected_level_mae": null
+          },
+          "twins_streamlet:twins_visible_conflict": {
+            "questions": 797,
+            "accuracy": 0.9974905897114178,
+            "mean_class_recall": 0.9969442527413541,
+            "nll": 0.010437285979484219,
+            "brier": 0.0038211519865233618,
+            "ece15": 0.0009184477173700703,
+            "temperature": 1.0,
+            "coverage_at_confidence_0_9": 0.9962358845671268,
+            "error_rate_at_confidence_0_9": 0.0012594458438287153,
+            "ordinal_expected_level_mae": null
+          },
+          "veremi_nextgen:veremi_message_manipulation": {
+            "questions": 800,
+            "accuracy": 0.63,
+            "mean_class_recall": 0.6354549163606691,
+            "nll": 0.5876157448193948,
+            "brier": 0.4142276501377162,
+            "ece15": 0.031917999077751745,
+            "temperature": 1.0,
+            "coverage_at_confidence_0_9": 0.10375,
+            "error_rate_at_confidence_0_9": 0.0,
+            "ordinal_expected_level_mae": null
+          },
+          "veremi_nextgen:veremi_supported_attack_type_v16": {
+            "questions": 800,
+            "accuracy": 0.59875,
+            "mean_class_recall": 0.4115791153670771,
+            "nll": 1.2269806519983115,
+            "brier": 0.571523062943736,
+            "ece15": 0.08060690424281568,
+            "temperature": 1.0,
+            "coverage_at_confidence_0_9": 0.11875,
+            "error_rate_at_confidence_0_9": 0.0,
+            "ordinal_expected_level_mae": null
+          }
+        },
+        "macro_task_accuracy": 0.9262877786032521,
+        "macro_task_balanced_accuracy": 0.9021513315229265,
+        "macro_task_nll": 0.18562255591253446
+      },
+      "checkpoint_files": {
+        "chat_template.jinja": "a4aee8afcf2e0711942cf848899be66016f8d14a889ff9ede07bca099c28f715",
+        "tokenizer.json": "06b9509352d2af50381ab2247e083b80d32d5c0aba91c272ca9ff729b6a0e523",
+        "adapter_model.safetensors": "793c6d72e5c0254c28884384a8bc3ca633fd4de9511834f710b5ee2dc95a942f",
+        "tokenizer_config.json": "353c083805d863719e54973335a38e252a92e1ac64d32b83bb0b333cc1644ef2",
+        "head.pt": "75a1b2c4362ce9eaccad18d0531fea66ab895aa924b73492af78fe7abf997986",
+        "README.md": "fdbf6f672a47dca9ef2ad668b395cccbdf4a09c88d16bf8a90ecf4f7445f31ce",
+        "adapter_config.json": "94aad580ce54a1dfcec0803beaba2c471e17b790f10efe7a481125c81fe1dd02"
+      }
+    },
+    {
+      "epoch": 2,
+      "metrics": {
+        "questions": 14912,
+        "accuracy": 0.9046405579399142,
+        "mean_class_recall": 0.8351706914084861,
+        "nll": 0.2609479185046837,
+        "brier": 0.13871271598030968,
+        "ece15": 0.02912533581539033,
+        "temperature": 1.0,
+        "coverage_at_confidence_0_9": 0.811762339055794,
+        "error_rate_at_confidence_0_9": 0.030318050392399834,
+        "ordinal_expected_level_mae": 0.007142014228908907,
+        "tasks": {
+          "agentdojo_tool_outputs:agentdojo_boundary_action": {
+            "questions": 222,
+            "accuracy": 0.9774774774774775,
+            "mean_class_recall": 0.9570930232558139,
+            "nll": 0.0857490162426134,
+            "brier": 0.03780111867446018,
+            "ece15": 0.020326742033581136,
+            "temperature": 1.0,
+            "coverage_at_confidence_0_9": 0.9864864864864865,
+            "error_rate_at_confidence_0_9": 0.0136986301369863,
+            "ordinal_expected_level_mae": null
+          },
+          "agentdojo_tool_outputs:agentdojo_embedded_injection": {
+            "questions": 222,
+            "accuracy": 0.9819819819819819,
+            "mean_class_recall": 0.96,
+            "nll": 0.08583505640070498,
+            "brier": 0.03170617674969949,
+            "ece15": 0.018778574788420183,
+            "temperature": 1.0,
+            "coverage_at_confidence_0_9": 0.9819819819819819,
+            "error_rate_at_confidence_0_9": 0.009174311926605505,
+            "ordinal_expected_level_mae": null
+          },
+          "byzfl_mnist:fl_attack_generated_update": {
+            "questions": 1400,
+            "accuracy": 0.9828571428571429,
+            "mean_class_recall": 0.9678571428571429,
+            "nll": 0.05038447430212686,
+            "brier": 0.026644462415551518,
+            "ece15": 0.006643072262975827,
+            "temperature": 1.0,
+            "coverage_at_confidence_0_9": 0.9757142857142858,
+            "error_rate_at_confidence_0_9": 0.008784773060029283,
+            "ordinal_expected_level_mae": null
+          },
+          "ciciot2023:ciciot_attack": {
+            "questions": 5000,
+            "accuracy": 0.839,
+            "mean_class_recall": 0.7132000000000001,
+            "nll": 0.4567278707638309,
+            "brier": 0.24289094068797595,
+            "ece15": 0.07089882316707959,
+            "temperature": 1.0,
+            "coverage_at_confidence_0_9": 0.7078,
+            "error_rate_at_confidence_0_9": 0.08194405199208817,
+            "ordinal_expected_level_mae": null
+          },
+          "injecagent:injecagent_requested_operation_v13": {
+            "questions": 124,
+            "accuracy": 1.0,
+            "mean_class_recall": 1.0,
+            "nll": 0.0030323687345377927,
+            "brier": 0.0010147751519150774,
+            "ece15": 0.0027304101493378944,
+            "temperature": 1.0,
+            "coverage_at_confidence_0_9": 0.9919354838709677,
+            "error_rate_at_confidence_0_9": 0.0,
+            "ordinal_expected_level_mae": null
+          },
+          "lanl_auth_policy:lanl_known_logon_action_v16": {
+            "questions": 328,
+            "accuracy": 1.0,
+            "mean_class_recall": 1.0,
+            "nll": 0.0013590718776517526,
+            "brier": 0.0005853758373684321,
+            "ece15": 0.001174090997847941,
+            "temperature": 1.0,
+            "coverage_at_confidence_0_9": 0.9969512195121951,
+            "error_rate_at_confidence_0_9": 0.0,
+            "ordinal_expected_level_mae": null
+          },
+          "lanl_auth_policy:lanl_observable_policy": {
+            "questions": 328,
+            "accuracy": 1.0,
+            "mean_class_recall": 1.0,
+            "nll": 2.706087239227311e-05,
+            "brier": 4.4631636950527135e-07,
+            "ece15": 2.6948652833984242e-05,
+            "temperature": 1.0,
+            "coverage_at_confidence_0_9": 1.0,
+            "error_rate_at_confidence_0_9": 0.0,
+            "ordinal_expected_level_mae": null
+          },
+          "ton_iot_network:ton_attack": {
+            "questions": 2500,
+            "accuracy": 0.9788,
+            "mean_class_recall": 0.98,
+            "nll": 0.08755100200107269,
+            "brier": 0.038158588944742826,
+            "ece15": 0.017829224144769444,
+            "temperature": 1.0,
+            "coverage_at_confidence_0_9": 0.9896,
+            "error_rate_at_confidence_0_9": 0.016976556184316895,
+            "ordinal_expected_level_mae": null
+          },
+          "twins_streamlet:twins_evidence_level": {
+            "questions": 797,
+            "accuracy": 0.9949811794228356,
+            "mean_class_recall": 0.9949406163344321,
+            "nll": 0.020100883811983803,
+            "brier": 0.009532705405460478,
+            "ece15": 0.004664870689987746,
+            "temperature": 1.0,
+            "coverage_at_confidence_0_9": 0.9962358845671268,
+            "error_rate_at_confidence_0_9": 0.003778337531486146,
+            "ordinal_expected_level_mae": 0.007142014228908907
+          },
+          "twins_streamlet:twins_policy_action": {
+            "questions": 797,
+            "accuracy": 0.9887076537013801,
+            "mean_class_recall": 0.9889237691623501,
+            "nll": 0.029531627570397425,
+            "brier": 0.01682083664455817,
+            "ece15": 0.008194444119065863,
+            "temperature": 1.0,
+            "coverage_at_confidence_0_9": 0.9887076537013801,
+            "error_rate_at_confidence_0_9": 0.0038071065989847717,
+            "ordinal_expected_level_mae": null
+          },
+          "twins_streamlet:twins_quorum": {
+            "questions": 797,
+            "accuracy": 0.9937264742785445,
+            "mean_class_recall": 0.9922118380062306,
+            "nll": 0.019942020828876864,
+            "brier": 0.011733672902765547,
+            "ece15": 0.006519660642330038,
+            "temperature": 1.0,
+            "coverage_at_confidence_0_9": 0.9912170639899623,
+            "error_rate_at_confidence_0_9": 0.0037974683544303796,
+            "ordinal_expected_level_mae": null
+          },
+          "twins_streamlet:twins_visible_conflict": {
+            "questions": 797,
+            "accuracy": 0.9974905897114178,
+            "mean_class_recall": 0.998236331569665,
+            "nll": 0.016769983022509982,
+            "brier": 0.0049992664597086935,
+            "ece15": 0.0021278163552494573,
+            "temperature": 1.0,
+            "coverage_at_confidence_0_9": 1.0,
+            "error_rate_at_confidence_0_9": 0.002509410288582183,
+            "ordinal_expected_level_mae": null
+          },
+          "veremi_nextgen:veremi_message_manipulation": {
+            "questions": 800,
+            "accuracy": 0.6875,
+            "mean_class_recall": 0.6966034271725826,
+            "nll": 0.5142628800857739,
+            "brier": 0.36155138388149327,
+            "ece15": 0.02205690591199928,
+            "temperature": 1.0,
+            "coverage_at_confidence_0_9": 0.22125,
+            "error_rate_at_confidence_0_9": 0.011299435028248588,
+            "ordinal_expected_level_mae": null
+          },
+          "veremi_nextgen:veremi_supported_attack_type_v16": {
+            "questions": 800,
+            "accuracy": 0.67375,
+            "mean_class_recall": 0.5089660282717738,
+            "nll": 0.998813782244008,
+            "brier": 0.4775012041931279,
+            "ece15": 0.03873448735113525,
+            "temperature": 1.0,
+            "coverage_at_confidence_0_9": 0.20625,
+            "error_rate_at_confidence_0_9": 0.030303030303030304,
+            "ordinal_expected_level_mae": null
+          }
+        },
+        "macro_task_accuracy": 0.9354480356736271,
+        "macro_task_balanced_accuracy": 0.9112880126164279,
+        "macro_task_nll": 0.16929193562560574
+      },
+      "checkpoint_files": {
+        "chat_template.jinja": "a4aee8afcf2e0711942cf848899be66016f8d14a889ff9ede07bca099c28f715",
+        "tokenizer.json": "06b9509352d2af50381ab2247e083b80d32d5c0aba91c272ca9ff729b6a0e523",
+        "adapter_model.safetensors": "f01d67ab57ea35f6f96dcf615f54147213977022560741db20fb7eab6eb462bd",
+        "tokenizer_config.json": "353c083805d863719e54973335a38e252a92e1ac64d32b83bb0b333cc1644ef2",
+        "head.pt": "460a08af9f39d347ecd48b74049d8a6e6140f1293160a4a5807a201b92ecc3c1",
+        "README.md": "fdbf6f672a47dca9ef2ad668b395cccbdf4a09c88d16bf8a90ecf4f7445f31ce",
+        "adapter_config.json": "94aad580ce54a1dfcec0803beaba2c471e17b790f10efe7a481125c81fe1dd02"
+      }
+    },
+    {
+      "epoch": 3,
+      "metrics": {
+        "questions": 14912,
+        "accuracy": 0.9075912017167382,
+        "mean_class_recall": 0.8447857522159936,
+        "nll": 0.24807737680415345,
+        "brier": 0.12998082994334342,
+        "ece15": 0.027028147820359012,
+        "temperature": 1.0,
+        "coverage_at_confidence_0_9": 0.8103540772532188,
+        "error_rate_at_confidence_0_9": 0.023502151605428664,
+        "ordinal_expected_level_mae": 0.00481932257817722,
+        "tasks": {
+          "agentdojo_tool_outputs:agentdojo_boundary_action": {
+            "questions": 222,
+            "accuracy": 0.9774774774774775,
+            "mean_class_recall": 0.9570930232558139,
+            "nll": 0.10407641066616782,
+            "brier": 0.039347876403718875,
+            "ece15": 0.021432863016197557,
+            "temperature": 1.0,
+            "coverage_at_confidence_0_9": 0.9864864864864865,
+            "error_rate_at_confidence_0_9": 0.0136986301369863,
+            "ordinal_expected_level_mae": null
+          },
+          "agentdojo_tool_outputs:agentdojo_embedded_injection": {
+            "questions": 222,
+            "accuracy": 0.9819819819819819,
+            "mean_class_recall": 0.96,
+            "nll": 0.10023555924012383,
+            "brier": 0.031563944714978254,
+            "ece15": 0.015911253108198964,
+            "temperature": 1.0,
+            "coverage_at_confidence_0_9": 0.9864864864864865,
+            "error_rate_at_confidence_0_9": 0.0136986301369863,
+            "ordinal_expected_level_mae": null
+          },
+          "byzfl_mnist:fl_attack_generated_update": {
+            "questions": 1400,
+            "accuracy": 0.9885714285714285,
+            "mean_class_recall": 0.9821428571428572,
+            "nll": 0.0442702602822601,
+            "brier": 0.02139513973607548,
+            "ece15": 0.007710400104216533,
+            "temperature": 1.0,
+            "coverage_at_confidence_0_9": 0.9842857142857143,
+            "error_rate_at_confidence_0_9": 0.008708272859216255,
+            "ordinal_expected_level_mae": null
+          },
+          "ciciot2023:ciciot_attack": {
+            "questions": 5000,
+            "accuracy": 0.8418,
+            "mean_class_recall": 0.7537333333333334,
+            "nll": 0.4173917752941542,
+            "brier": 0.22451704505631584,
+            "ece15": 0.06284084146027137,
+            "temperature": 1.0,
+            "coverage_at_confidence_0_9": 0.6928,
+            "error_rate_at_confidence_0_9": 0.06293302540415704,
+            "ordinal_expected_level_mae": null
+          },
+          "injecagent:injecagent_requested_operation_v13": {
+            "questions": 124,
+            "accuracy": 1.0,
+            "mean_class_recall": 1.0,
+            "nll": 0.0014205610073774013,
+            "brier": 0.0002211290260091384,
+            "ece15": 0.0013613576932818498,
+            "temperature": 1.0,
+            "coverage_at_confidence_0_9": 0.9919354838709677,
+            "error_rate_at_confidence_0_9": 0.0,
+            "ordinal_expected_level_mae": null
+          },
+          "lanl_auth_policy:lanl_known_logon_action_v16": {
+            "questions": 328,
+            "accuracy": 1.0,
+            "mean_class_recall": 1.0,
+            "nll": 0.000610657881540054,
+            "brier": 8.099314527924365e-05,
+            "ece15": 0.0005891478461821947,
+            "temperature": 1.0,
+            "coverage_at_confidence_0_9": 1.0,
+            "error_rate_at_confidence_0_9": 0.0,
+            "ordinal_expected_level_mae": null
+          },
+          "lanl_auth_policy:lanl_observable_policy": {
+            "questions": 328,
+            "accuracy": 1.0,
+            "mean_class_recall": 1.0,
+            "nll": 1.5480302068279906e-07,
+            "brier": 4.4217021643060835e-12,
+            "ece15": 1.548019148098009e-07,
+            "temperature": 1.0,
+            "coverage_at_confidence_0_9": 1.0,
+            "error_rate_at_confidence_0_9": 0.0,
+            "ordinal_expected_level_mae": null
+          },
+          "ton_iot_network:ton_attack": {
+            "questions": 2500,
+            "accuracy": 0.9792,
+            "mean_class_recall": 0.9818666666666667,
+            "nll": 0.11218220058914397,
+            "brier": 0.03724045269615108,
+            "ece15": 0.01842697861877182,
+            "temperature": 1.0,
+            "coverage_at_confidence_0_9": 0.9928,
+            "error_rate_at_confidence_0_9": 0.015310233682514102,
+            "ordinal_expected_level_mae": null
+          },
+          "twins_streamlet:twins_evidence_level": {
+            "questions": 797,
+            "accuracy": 0.9962358845671268,
+            "mean_class_recall": 0.9963898916967509,
+            "nll": 0.012217254972570101,
+            "brier": 0.006818829719414822,
+            "ece15": 0.004133398699556409,
+            "temperature": 1.0,
+            "coverage_at_confidence_0_9": 0.9949811794228356,
+            "error_rate_at_confidence_0_9": 0.0012610340479192938,
+            "ordinal_expected_level_mae": 0.00481932257817722
+          },
+          "twins_streamlet:twins_policy_action": {
+            "questions": 797,
+            "accuracy": 0.9949811794228356,
+            "mean_class_recall": 0.9951865222623345,
+            "nll": 0.013078119892699127,
+            "brier": 0.007721112955228842,
+            "ece15": 0.003423998250058524,
+            "temperature": 1.0,
+            "coverage_at_confidence_0_9": 0.9899623588456713,
+            "error_rate_at_confidence_0_9": 0.0012674271229404308,
+            "ordinal_expected_level_mae": null
+          },
+          "twins_streamlet:twins_quorum": {
+            "questions": 797,
+            "accuracy": 0.9949811794228356,
+            "mean_class_recall": 0.994276682635671,
+            "nll": 0.01027058602627885,
+            "brier": 0.007209131739335666,
+            "ece15": 0.004201163796495765,
+            "temperature": 1.0,
+            "coverage_at_confidence_0_9": 0.9912170639899623,
+            "error_rate_at_confidence_0_9": 0.0,
+            "ordinal_expected_level_mae": null
+          },
+          "twins_streamlet:twins_visible_conflict": {
+            "questions": 797,
+            "accuracy": 0.9974905897114178,
+            "mean_class_recall": 0.998236331569665,
+            "nll": 0.017287182530912663,
+            "brier": 0.004953169208636321,
+            "ece15": 0.0025266183132433223,
+            "temperature": 1.0,
+            "coverage_at_confidence_0_9": 0.998745294855709,
+            "error_rate_at_confidence_0_9": 0.002512562814070352,
+            "ordinal_expected_level_mae": null
+          },
+          "veremi_nextgen:veremi_message_manipulation": {
+            "questions": 800,
+            "accuracy": 0.69,
+            "mean_class_recall": 0.7116483068135455,
+            "nll": 0.5092641106435474,
+            "brier": 0.3569714849542106,
+            "ece15": 0.03650030376694629,
+            "temperature": 1.0,
+            "coverage_at_confidence_0_9": 0.2425,
+            "error_rate_at_confidence_0_9": 0.010309278350515464,
+            "ordinal_expected_level_mae": null
+          },
+          "veremi_nextgen:veremi_supported_attack_type_v16": {
+            "questions": 800,
+            "accuracy": 0.68875,
+            "mean_class_recall": 0.529392069171139,
+            "nll": 0.96833512912194,
+            "brier": 0.46247412161427887,
+            "ece15": 0.02731423542286039,
+            "temperature": 1.0,
+            "coverage_at_confidence_0_9": 0.22625,
+            "error_rate_at_confidence_0_9": 0.022099447513812154,
+            "ordinal_expected_level_mae": null
+          }
+        },
+        "macro_task_accuracy": 0.9379621229396502,
+        "macro_task_balanced_accuracy": 0.9185689774676985,
+        "macro_task_nll": 0.16504571163940973
+      },
+      "checkpoint_files": {
+        "chat_template.jinja": "a4aee8afcf2e0711942cf848899be66016f8d14a889ff9ede07bca099c28f715",
+        "tokenizer.json": "06b9509352d2af50381ab2247e083b80d32d5c0aba91c272ca9ff729b6a0e523",
+        "adapter_model.safetensors": "0087e92e59b987fea721124d1a3f61b1aca26b3ac01b974032abb60799e7b175",
+        "tokenizer_config.json": "353c083805d863719e54973335a38e252a92e1ac64d32b83bb0b333cc1644ef2",
+        "head.pt": "f3c477c0419bd8dbf64d530d7f2576e084f65cdee3b351e34a7804e67b3979cf",
+        "README.md": "fdbf6f672a47dca9ef2ad668b395cccbdf4a09c88d16bf8a90ecf4f7445f31ce",
+        "adapter_config.json": "94aad580ce54a1dfcec0803beaba2c471e17b790f10efe7a481125c81fe1dd02"
+      }
+    }
+  ],
+  "rule": "maximum development macro-task semantic-class balanced accuracy; tie macro-task raw NLL, then earlier epoch",
+  "test_used_for_selection": false
+}

evaluation/epoch-3-test.json

@@ -0,0 +1,375 @@
+{
+  "epoch": 3,
+  "temperature": 1.334968103387446,
+  "test": {
+    "raw": {
+      "questions": 21634,
+      "accuracy": 0.8801423684940372,
+      "mean_class_recall": 0.8422636415920437,
+      "nll": 0.35756736839135106,
+      "brier": 0.17329397786588344,
+      "ece15": 0.018370944185424988,
+      "temperature": 1.0,
+      "coverage_at_confidence_0_9": 0.681750947582509,
+      "error_rate_at_confidence_0_9": 0.017289307749677945,
+      "ordinal_expected_level_mae": 0.004830545729071066,
+      "tasks": {
+        "agentdojo_tool_outputs:agentdojo_boundary_action": {
+          "questions": 426,
+          "accuracy": 0.9953051643192489,
+          "mean_class_recall": 0.9895833333333333,
+          "nll": 0.019578322598972707,
+          "brier": 0.008470972915844083,
+          "ece15": 0.0032192125141472924,
+          "temperature": 1.0,
+          "coverage_at_confidence_0_9": 0.9882629107981221,
+          "error_rate_at_confidence_0_9": 0.0023752969121140144,
+          "ordinal_expected_level_mae": null
+        },
+        "agentdojo_tool_outputs:agentdojo_embedded_injection": {
+          "questions": 426,
+          "accuracy": 0.9835680751173709,
+          "mean_class_recall": 0.9635416666666667,
+          "nll": 0.03323309985013501,
+          "brier": 0.020700086530374747,
+          "ece15": 0.012858446963474107,
+          "temperature": 1.0,
+          "coverage_at_confidence_0_9": 0.9835680751173709,
+          "error_rate_at_confidence_0_9": 0.007159904534606206,
+          "ordinal_expected_level_mae": null
+        },
+        "byzfl_mnist:fl_attack_generated_update": {
+          "questions": 800,
+          "accuracy": 0.95875,
+          "mean_class_recall": 0.9296875,
+          "nll": 0.14798192410675134,
+          "brier": 0.07208296294789789,
+          "ece15": 0.031237402444916955,
+          "temperature": 1.0,
+          "coverage_at_confidence_0_9": 0.9575,
+          "error_rate_at_confidence_0_9": 0.02741514360313316,
+          "ordinal_expected_level_mae": null
+        },
+        "ciciot2023:ciciot_attack": {
+          "questions": 6000,
+          "accuracy": 0.9171666666666667,
+          "mean_class_recall": 0.8741111111111111,
+          "nll": 0.2872065840367128,
+          "brier": 0.1291518236859529,
+          "ece15": 0.02997403903668212,
+          "temperature": 1.0,
+          "coverage_at_confidence_0_9": 0.7491666666666666,
+          "error_rate_at_confidence_0_9": 0.03181312569521691,
+          "ordinal_expected_level_mae": null
+        },
+        "injecagent:injecagent_requested_operation_v13": {
+          "questions": 124,
+          "accuracy": 1.0,
+          "mean_class_recall": 1.0,
+          "nll": 6.416181236130915e-05,
+          "brier": 3.664856836177276e-07,
+          "ece15": 6.406994017019252e-05,
+          "temperature": 1.0,
+          "coverage_at_confidence_0_9": 1.0,
+          "error_rate_at_confidence_0_9": 0.0,
+          "ordinal_expected_level_mae": null
+        },
+        "lanl_auth_policy:lanl_known_logon_action_v16": {
+          "questions": 540,
+          "accuracy": 1.0,
+          "mean_class_recall": 1.0,
+          "nll": 0.0003909349271697467,
+          "brier": 5.411140895974333e-05,
+          "ece15": 0.000376397356322764,
+          "temperature": 1.0,
+          "coverage_at_confidence_0_9": 0.9981481481481481,
+          "error_rate_at_confidence_0_9": 0.0,
+          "ordinal_expected_level_mae": null
+        },
+        "lanl_auth_policy:lanl_observable_policy": {
+          "questions": 540,
+          "accuracy": 1.0,
+          "mean_class_recall": 1.0,
+          "nll": 1.9301873747749096e-07,
+          "brier": 4.461551704214049e-12,
+          "ece15": 1.9301762319156735e-07,
+          "temperature": 1.0,
+          "coverage_at_confidence_0_9": 1.0,
+          "error_rate_at_confidence_0_9": 0.0,
+          "ordinal_expected_level_mae": null
+        },
+        "ton_iot_network:ton_attack": {
+          "questions": 3500,
+          "accuracy": 0.988,
+          "mean_class_recall": 0.9820952380952381,
+          "nll": 0.052315917826989855,
+          "brier": 0.021002166306533596,
+          "ece15": 0.005742982405630235,
+          "temperature": 1.0,
+          "coverage_at_confidence_0_9": 0.9757142857142858,
+          "error_rate_at_confidence_0_9": 0.007320644216691069,
+          "ordinal_expected_level_mae": null
+        },
+        "twins_streamlet:twins_evidence_level": {
+          "questions": 659,
+          "accuracy": 0.9954476479514416,
+          "mean_class_recall": 0.9949748743718594,
+          "nll": 0.006489090778116727,
+          "brier": 0.004985928434486637,
+          "ece15": 0.0035010905751582675,
+          "temperature": 1.0,
+          "coverage_at_confidence_0_9": 0.9954476479514416,
+          "error_rate_at_confidence_0_9": 0.0,
+          "ordinal_expected_level_mae": 0.004830545729071066
+        },
+        "twins_streamlet:twins_policy_action": {
+          "questions": 659,
+          "accuracy": 0.9984825493171472,
+          "mean_class_recall": 0.9983249581239532,
+          "nll": 0.007759166929265847,
+          "brier": 0.0031279766576844436,
+          "ece15": 0.0017302646891685911,
+          "temperature": 1.0,
+          "coverage_at_confidence_0_9": 0.9969650986342944,
+          "error_rate_at_confidence_0_9": 0.0015220700152207,
+          "ordinal_expected_level_mae": null
+        },
+        "twins_streamlet:twins_quorum": {
+          "questions": 659,
+          "accuracy": 1.0,
+          "mean_class_recall": 1.0,
+          "nll": 0.0012916521858483854,
+          "brier": 0.0006719742504699832,
+          "ece15": 0.0010543087689309648,
+          "temperature": 1.0,
+          "coverage_at_confidence_0_9": 0.9969650986342944,
+          "error_rate_at_confidence_0_9": 0.0,
+          "ordinal_expected_level_mae": null
+        },
+        "twins_streamlet:twins_visible_conflict": {
+          "questions": 659,
+          "accuracy": 0.9984825493171472,
+          "mean_class_recall": 0.9989130434782609,
+          "nll": 0.0030258654215359524,
+          "brier": 0.0020473738981638956,
+          "ece15": 0.0002839890142556098,
+          "temperature": 1.0,
+          "coverage_at_confidence_0_9": 0.9954476479514416,
+          "error_rate_at_confidence_0_9": 0.0,
+          "ordinal_expected_level_mae": null
+        },
+        "veremi_nextgen:veremi_message_manipulation": {
+          "questions": 3321,
+          "accuracy": 0.7034025895814514,
+          "mean_class_recall": 0.7106693655944849,
+          "nll": 0.5271449745997652,
+          "brier": 0.3679898300956757,
+          "ece15": 0.03254066478455375,
+          "temperature": 1.0,
+          "coverage_at_confidence_0_9": 0.21499548328816623,
+          "error_rate_at_confidence_0_9": 0.011204481792717087,
+          "ordinal_expected_level_mae": null
+        },
+        "veremi_nextgen:veremi_supported_attack_type_v16": {
+          "questions": 3321,
+          "accuracy": 0.692261367058115,
+          "mean_class_recall": 0.49711800080440777,
+          "nll": 1.181957709780574,
+          "brier": 0.48216427161657194,
+          "ece15": 0.04703079874679226,
+          "temperature": 1.0,
+          "coverage_at_confidence_0_9": 0.20776874435411022,
+          "error_rate_at_confidence_0_9": 0.07681159420289856,
+          "ordinal_expected_level_mae": null
+        }
+      },
+      "macro_task_accuracy": 0.9450619006663279,
+      "macro_task_balanced_accuracy": 0.9242156493985226
+    },
+    "recalibrated": {
+      "questions": 21634,
+      "accuracy": 0.8801423684940372,
+      "mean_class_recall": 0.8422636415920437,
+      "nll": 0.3441350952603644,
+      "brier": 0.1767857045083663,
+      "ece15": 0.035627089616313175,
+      "temperature": 1.334968103387446,
+      "coverage_at_confidence_0_9": 0.6567440140519553,
+      "error_rate_at_confidence_0_9": 0.014358108108108109,
+      "ordinal_expected_level_mae": 0.004973955616734103,
+      "tasks": {
+        "agentdojo_tool_outputs:agentdojo_boundary_action": {
+          "questions": 426,
+          "accuracy": 0.9953051643192489,
+          "mean_class_recall": 0.9895833333333333,
+          "nll": 0.02085470758455793,
+          "brier": 0.009033562612123906,
+          "ece15": 0.005062121140742335,
+          "temperature": 1.334968103387446,
+          "coverage_at_confidence_0_9": 0.9741784037558685,
+          "error_rate_at_confidence_0_9": 0.0024096385542168677,
+          "ordinal_expected_level_mae": null
+        },
+        "agentdojo_tool_outputs:agentdojo_embedded_injection": {
+          "questions": 426,
+          "accuracy": 0.9835680751173709,
+          "mean_class_recall": 0.9635416666666667,
+          "nll": 0.029576905474440387,
+          "brier": 0.019371229609764937,
+          "ece15": 0.0128149515297424,
+          "temperature": 1.334968103387446,
+          "coverage_at_confidence_0_9": 0.9741784037558685,
+          "error_rate_at_confidence_0_9": 0.004819277108433735,
+          "ordinal_expected_level_mae": null
+        },
+        "byzfl_mnist:fl_attack_generated_update": {
+          "questions": 800,
+          "accuracy": 0.95875,
+          "mean_class_recall": 0.9296875,
+          "nll": 0.1252495255860491,
+          "brier": 0.06891977027707019,
+          "ece15": 0.02206119521855819,
+          "temperature": 1.334968103387446,
+          "coverage_at_confidence_0_9": 0.93375,
+          "error_rate_at_confidence_0_9": 0.0214190093708166,
+          "ordinal_expected_level_mae": null
+        },
+        "ciciot2023:ciciot_attack": {
+          "questions": 6000,
+          "accuracy": 0.9171666666666667,
+          "mean_class_recall": 0.8741111111111111,
+          "nll": 0.258923302286342,
+          "brier": 0.1323925339560934,
+          "ece15": 0.040786104751424794,
+          "temperature": 1.334968103387446,
+          "coverage_at_confidence_0_9": 0.6995,
+          "error_rate_at_confidence_0_9": 0.027162258756254467,
+          "ordinal_expected_level_mae": null
+        },
+        "injecagent:injecagent_requested_operation_v13": {
+          "questions": 124,
+          "accuracy": 1.0,
+          "mean_class_recall": 1.0,
+          "nll": 0.00036120828507376734,
+          "brier": 6.118819643947097e-06,
+          "ece15": 0.0003596634947089362,
+          "temperature": 1.334968103387446,
+          "coverage_at_confidence_0_9": 1.0,
+          "error_rate_at_confidence_0_9": 0.0,
+          "ordinal_expected_level_mae": null
+        },
+        "lanl_auth_policy:lanl_known_logon_action_v16": {
+          "questions": 540,
+          "accuracy": 1.0,
+          "mean_class_recall": 1.0,
+          "nll": 0.0010259839180316675,
+          "brier": 0.00015355821904600842,
+          "ece15": 0.0009831313211751498,
+          "temperature": 1.334968103387446,
+          "coverage_at_confidence_0_9": 0.9981481481481481,
+          "error_rate_at_confidence_0_9": 0.0,
+          "ordinal_expected_level_mae": null
+        },
+        "lanl_auth_policy:lanl_observable_policy": {
+          "questions": 540,
+          "accuracy": 1.0,
+          "mean_class_recall": 1.0,
+          "nll": 5.4752266593721125e-06,
+          "brier": 1.1349090161477055e-09,
+          "ece15": 5.474942881804694e-06,
+          "temperature": 1.334968103387446,
+          "coverage_at_confidence_0_9": 1.0,
+          "error_rate_at_confidence_0_9": 0.0,
+          "ordinal_expected_level_mae": null
+        },
+        "ton_iot_network:ton_attack": {
+          "questions": 3500,
+          "accuracy": 0.988,
+          "mean_class_recall": 0.9820952380952381,
+          "nll": 0.0471904212042291,
+          "brier": 0.02136284177729358,
+          "ece15": 0.004930332206950378,
+          "temperature": 1.334968103387446,
+          "coverage_at_confidence_0_9": 0.9597142857142857,
+          "error_rate_at_confidence_0_9": 0.006251860672819291,
+          "ordinal_expected_level_mae": null
+        },
+        "twins_streamlet:twins_evidence_level": {
+          "questions": 659,
+          "accuracy": 0.9954476479514416,
+          "mean_class_recall": 0.9949748743718594,
+          "nll": 0.005989405513587896,
+          "brier": 0.004363630530423605,
+          "ece15": 0.003686325313576514,
+          "temperature": 1.334968103387446,
+          "coverage_at_confidence_0_9": 0.9954476479514416,
+          "error_rate_at_confidence_0_9": 0.0,
+          "ordinal_expected_level_mae": 0.004973955616734103
+        },
+        "twins_streamlet:twins_policy_action": {
+          "questions": 659,
+          "accuracy": 0.9984825493171472,
+          "mean_class_recall": 0.9983249581239532,
+          "nll": 0.0069631222649601515,
+          "brier": 0.003204452557959292,
+          "ece15": 0.001531985329986691,
+          "temperature": 1.334968103387446,
+          "coverage_at_confidence_0_9": 0.9969650986342944,
+          "error_rate_at_confidence_0_9": 0.0015220700152207,
+          "ordinal_expected_level_mae": null
+        },
+        "twins_streamlet:twins_quorum": {
+          "questions": 659,
+          "accuracy": 1.0,
+          "mean_class_recall": 1.0,
+          "nll": 0.0018952623564185236,
+          "brier": 0.0008114028234666237,
+          "ece15": 0.0016108880631044219,
+          "temperature": 1.334968103387446,
+          "coverage_at_confidence_0_9": 0.9954476479514416,
+          "error_rate_at_confidence_0_9": 0.0,
+          "ordinal_expected_level_mae": null
+        },
+        "twins_streamlet:twins_visible_conflict": {
+          "questions": 659,
+          "accuracy": 0.9984825493171472,
+          "mean_class_recall": 0.9989130434782609,
+          "nll": 0.0033168872089549845,
+          "brier": 0.0020980871703588605,
+          "ece15": 0.0012518453771527926,
+          "temperature": 1.334968103387446,
+          "coverage_at_confidence_0_9": 0.9954476479514416,
+          "error_rate_at_confidence_0_9": 0.0,
+          "ordinal_expected_level_mae": null
+        },
+        "veremi_nextgen:veremi_message_manipulation": {
+          "questions": 3321,
+          "accuracy": 0.7034025895814514,
+          "mean_class_recall": 0.7106693655944849,
+          "nll": 0.5278136514287293,
+          "brier": 0.3689683794121852,
+          "ece15": 0.03240031037986953,
+          "temperature": 1.334968103387446,
+          "coverage_at_confidence_0_9": 0.1969286359530262,
+          "error_rate_at_confidence_0_9": 0.009174311926605505,
+          "ordinal_expected_level_mae": null
+        },
+        "veremi_nextgen:veremi_supported_attack_type_v16": {
+          "questions": 3321,
+          "accuracy": 0.692261367058115,
+          "mean_class_recall": 0.49711800080440777,
+          "nll": 1.1560335557868109,
+          "brier": 0.498611302076152,
+          "ece15": 0.12617627958797675,
+          "temperature": 1.334968103387446,
+          "coverage_at_confidence_0_9": 0.17856067449563384,
+          "error_rate_at_confidence_0_9": 0.07251264755480608,
+          "ordinal_expected_level_mae": null
+        }
+      },
+      "macro_task_accuracy": 0.9450619006663279,
+      "macro_task_balanced_accuracy": 0.9242156493985226
+    }
+  },
+  "test_used_for_selection": false
+}

evaluation/epoch-3-validation.json

@@ -0,0 +1,744 @@
+{
+  "epoch": 3,
+  "temperature": 1.334968103387446,
+  "development": {
+    "raw": {
+      "questions": 14912,
+      "accuracy": 0.9075912017167382,
+      "mean_class_recall": 0.8447857522159936,
+      "nll": 0.24807737680415345,
+      "brier": 0.12998082994334342,
+      "ece15": 0.027028147820359012,
+      "temperature": 1.0,
+      "coverage_at_confidence_0_9": 0.8103540772532188,
+      "error_rate_at_confidence_0_9": 0.023502151605428664,
+      "ordinal_expected_level_mae": 0.00481932257817722,
+      "tasks": {
+        "agentdojo_tool_outputs:agentdojo_boundary_action": {
+          "questions": 222,
+          "accuracy": 0.9774774774774775,
+          "mean_class_recall": 0.9570930232558139,
+          "nll": 0.10407641066616782,
+          "brier": 0.039347876403718875,
+          "ece15": 0.021432863016197557,
+          "temperature": 1.0,
+          "coverage_at_confidence_0_9": 0.9864864864864865,
+          "error_rate_at_confidence_0_9": 0.0136986301369863,
+          "ordinal_expected_level_mae": null
+        },
+        "agentdojo_tool_outputs:agentdojo_embedded_injection": {
+          "questions": 222,
+          "accuracy": 0.9819819819819819,
+          "mean_class_recall": 0.96,
+          "nll": 0.10023555924012383,
+          "brier": 0.031563944714978254,
+          "ece15": 0.015911253108198964,
+          "temperature": 1.0,
+          "coverage_at_confidence_0_9": 0.9864864864864865,
+          "error_rate_at_confidence_0_9": 0.0136986301369863,
+          "ordinal_expected_level_mae": null
+        },
+        "byzfl_mnist:fl_attack_generated_update": {
+          "questions": 1400,
+          "accuracy": 0.9885714285714285,
+          "mean_class_recall": 0.9821428571428572,
+          "nll": 0.0442702602822601,
+          "brier": 0.02139513973607548,
+          "ece15": 0.007710400104216533,
+          "temperature": 1.0,
+          "coverage_at_confidence_0_9": 0.9842857142857143,
+          "error_rate_at_confidence_0_9": 0.008708272859216255,
+          "ordinal_expected_level_mae": null
+        },
+        "ciciot2023:ciciot_attack": {
+          "questions": 5000,
+          "accuracy": 0.8418,
+          "mean_class_recall": 0.7537333333333334,
+          "nll": 0.4173917752941542,
+          "brier": 0.22451704505631584,
+          "ece15": 0.06284084146027137,
+          "temperature": 1.0,
+          "coverage_at_confidence_0_9": 0.6928,
+          "error_rate_at_confidence_0_9": 0.06293302540415704,
+          "ordinal_expected_level_mae": null
+        },
+        "injecagent:injecagent_requested_operation_v13": {
+          "questions": 124,
+          "accuracy": 1.0,
+          "mean_class_recall": 1.0,
+          "nll": 0.0014205610073774013,
+          "brier": 0.0002211290260091384,
+          "ece15": 0.0013613576932818498,
+          "temperature": 1.0,
+          "coverage_at_confidence_0_9": 0.9919354838709677,
+          "error_rate_at_confidence_0_9": 0.0,
+          "ordinal_expected_level_mae": null
+        },
+        "lanl_auth_policy:lanl_known_logon_action_v16": {
+          "questions": 328,
+          "accuracy": 1.0,
+          "mean_class_recall": 1.0,
+          "nll": 0.000610657881540054,
+          "brier": 8.099314527924365e-05,
+          "ece15": 0.0005891478461821947,
+          "temperature": 1.0,
+          "coverage_at_confidence_0_9": 1.0,
+          "error_rate_at_confidence_0_9": 0.0,
+          "ordinal_expected_level_mae": null
+        },
+        "lanl_auth_policy:lanl_observable_policy": {
+          "questions": 328,
+          "accuracy": 1.0,
+          "mean_class_recall": 1.0,
+          "nll": 1.5480302068279906e-07,
+          "brier": 4.4217021643060835e-12,
+          "ece15": 1.548019148098009e-07,
+          "temperature": 1.0,
+          "coverage_at_confidence_0_9": 1.0,
+          "error_rate_at_confidence_0_9": 0.0,
+          "ordinal_expected_level_mae": null
+        },
+        "ton_iot_network:ton_attack": {
+          "questions": 2500,
+          "accuracy": 0.9792,
+          "mean_class_recall": 0.9818666666666667,
+          "nll": 0.11218220058914397,
+          "brier": 0.03724045269615108,
+          "ece15": 0.01842697861877182,
+          "temperature": 1.0,
+          "coverage_at_confidence_0_9": 0.9928,
+          "error_rate_at_confidence_0_9": 0.015310233682514102,
+          "ordinal_expected_level_mae": null
+        },
+        "twins_streamlet:twins_evidence_level": {
+          "questions": 797,
+          "accuracy": 0.9962358845671268,
+          "mean_class_recall": 0.9963898916967509,
+          "nll": 0.012217254972570101,
+          "brier": 0.006818829719414822,
+          "ece15": 0.004133398699556409,
+          "temperature": 1.0,
+          "coverage_at_confidence_0_9": 0.9949811794228356,
+          "error_rate_at_confidence_0_9": 0.0012610340479192938,
+          "ordinal_expected_level_mae": 0.00481932257817722
+        },
+        "twins_streamlet:twins_policy_action": {
+          "questions": 797,
+          "accuracy": 0.9949811794228356,
+          "mean_class_recall": 0.9951865222623345,
+          "nll": 0.013078119892699127,
+          "brier": 0.007721112955228842,
+          "ece15": 0.003423998250058524,
+          "temperature": 1.0,
+          "coverage_at_confidence_0_9": 0.9899623588456713,
+          "error_rate_at_confidence_0_9": 0.0012674271229404308,
+          "ordinal_expected_level_mae": null
+        },
+        "twins_streamlet:twins_quorum": {
+          "questions": 797,
+          "accuracy": 0.9949811794228356,
+          "mean_class_recall": 0.994276682635671,
+          "nll": 0.01027058602627885,
+          "brier": 0.007209131739335666,
+          "ece15": 0.004201163796495765,
+          "temperature": 1.0,
+          "coverage_at_confidence_0_9": 0.9912170639899623,
+          "error_rate_at_confidence_0_9": 0.0,
+          "ordinal_expected_level_mae": null
+        },
+        "twins_streamlet:twins_visible_conflict": {
+          "questions": 797,
+          "accuracy": 0.9974905897114178,
+          "mean_class_recall": 0.998236331569665,
+          "nll": 0.017287182530912663,
+          "brier": 0.004953169208636321,
+          "ece15": 0.0025266183132433223,
+          "temperature": 1.0,
+          "coverage_at_confidence_0_9": 0.998745294855709,
+          "error_rate_at_confidence_0_9": 0.002512562814070352,
+          "ordinal_expected_level_mae": null
+        },
+        "veremi_nextgen:veremi_message_manipulation": {
+          "questions": 800,
+          "accuracy": 0.69,
+          "mean_class_recall": 0.7116483068135455,
+          "nll": 0.5092641106435474,
+          "brier": 0.3569714849542106,
+          "ece15": 0.03650030376694629,
+          "temperature": 1.0,
+          "coverage_at_confidence_0_9": 0.2425,
+          "error_rate_at_confidence_0_9": 0.010309278350515464,
+          "ordinal_expected_level_mae": null
+        },
+        "veremi_nextgen:veremi_supported_attack_type_v16": {
+          "questions": 800,
+          "accuracy": 0.68875,
+          "mean_class_recall": 0.529392069171139,
+          "nll": 0.96833512912194,
+          "brier": 0.46247412161427887,
+          "ece15": 0.02731423542286039,
+          "temperature": 1.0,
+          "coverage_at_confidence_0_9": 0.22625,
+          "error_rate_at_confidence_0_9": 0.022099447513812154,
+          "ordinal_expected_level_mae": null
+        }
+      },
+      "macro_task_accuracy": 0.9379621229396502,
+      "macro_task_balanced_accuracy": 0.9185689774676985
+    },
+    "recalibrated": {
+      "questions": 14912,
+      "accuracy": 0.9075912017167382,
+      "mean_class_recall": 0.8447857522159936,
+      "nll": 0.2269545932235271,
+      "brier": 0.12792407717450321,
+      "ece15": 0.01858051585491712,
+      "temperature": 1.334968103387446,
+      "coverage_at_confidence_0_9": 0.7862124463519313,
+      "error_rate_at_confidence_0_9": 0.018679631525076765,
+      "ordinal_expected_level_mae": 0.005329911013904147,
+      "tasks": {
+        "agentdojo_tool_outputs:agentdojo_boundary_action": {
+          "questions": 222,
+          "accuracy": 0.9774774774774775,
+          "mean_class_recall": 0.9570930232558139,
+          "nll": 0.08092683597443519,
+          "brier": 0.037546344393533104,
+          "ece15": 0.021320738677048364,
+          "temperature": 1.334968103387446,
+          "coverage_at_confidence_0_9": 0.9819819819819819,
+          "error_rate_at_confidence_0_9": 0.013761467889908258,
+          "ordinal_expected_level_mae": null
+        },
+        "agentdojo_tool_outputs:agentdojo_embedded_injection": {
+          "questions": 222,
+          "accuracy": 0.9819819819819819,
+          "mean_class_recall": 0.96,
+          "nll": 0.07809919245507242,
+          "brier": 0.030623326048867618,
+          "ece15": 0.015019163882298042,
+          "temperature": 1.334968103387446,
+          "coverage_at_confidence_0_9": 0.9819819819819819,
+          "error_rate_at_confidence_0_9": 0.009174311926605505,
+          "ordinal_expected_level_mae": null
+        },
+        "byzfl_mnist:fl_attack_generated_update": {
+          "questions": 1400,
+          "accuracy": 0.9885714285714285,
+          "mean_class_recall": 0.9821428571428572,
+          "nll": 0.04094699270351949,
+          "brier": 0.021018406686723334,
+          "ece15": 0.0038434318338614677,
+          "temperature": 1.334968103387446,
+          "coverage_at_confidence_0_9": 0.9678571428571429,
+          "error_rate_at_confidence_0_9": 0.006642066420664207,
+          "ordinal_expected_level_mae": null
+        },
+        "ciciot2023:ciciot_attack": {
+          "questions": 5000,
+          "accuracy": 0.8418,
+          "mean_class_recall": 0.7537333333333334,
+          "nll": 0.36679030702727994,
+          "brier": 0.21757064336558796,
+          "ece15": 0.043357678681868704,
+          "temperature": 1.334968103387446,
+          "coverage_at_confidence_0_9": 0.6394,
+          "error_rate_at_confidence_0_9": 0.05004691898654989,
+          "ordinal_expected_level_mae": null
+        },
+        "injecagent:injecagent_requested_operation_v13": {
+          "questions": 124,
+          "accuracy": 1.0,
+          "mean_class_recall": 1.0,
+          "nll": 0.0028701157894541487,
+          "brier": 0.0006216354903482641,
+          "ece15": 0.0026971564840139535,
+          "temperature": 1.334968103387446,
+          "coverage_at_confidence_0_9": 0.9919354838709677,
+          "error_rate_at_confidence_0_9": 0.0,
+          "ordinal_expected_level_mae": null
+        },
+        "lanl_auth_policy:lanl_known_logon_action_v16": {
+          "questions": 328,
+          "accuracy": 1.0,
+          "mean_class_recall": 1.0,
+          "nll": 0.001405432942745028,
+          "brier": 0.00025083300955486444,
+          "ece15": 0.0013358908781109478,
+          "temperature": 1.334968103387446,
+          "coverage_at_confidence_0_9": 0.9939024390243902,
+          "error_rate_at_confidence_0_9": 0.0,
+          "ordinal_expected_level_mae": null
+        },
+        "lanl_auth_policy:lanl_observable_policy": {
+          "questions": 328,
+          "accuracy": 1.0,
+          "mean_class_recall": 1.0,
+          "nll": 4.6272880953518815e-06,
+          "brier": 9.796066038293659e-10,
+          "ece15": 4.627043141135246e-06,
+          "temperature": 1.334968103387446,
+          "coverage_at_confidence_0_9": 1.0,
+          "error_rate_at_confidence_0_9": 0.0,
+          "ordinal_expected_level_mae": null
+        },
+        "ton_iot_network:ton_attack": {
+          "questions": 2500,
+          "accuracy": 0.9792,
+          "mean_class_recall": 0.9818666666666667,
+          "nll": 0.0877031817842779,
+          "brier": 0.03599036114227489,
+          "ece15": 0.01538869285344686,
+          "temperature": 1.334968103387446,
+          "coverage_at_confidence_0_9": 0.988,
+          "error_rate_at_confidence_0_9": 0.014574898785425101,
+          "ordinal_expected_level_mae": null
+        },
+        "twins_streamlet:twins_evidence_level": {
+          "questions": 797,
+          "accuracy": 0.9962358845671268,
+          "mean_class_recall": 0.9963898916967509,
+          "nll": 0.010603240403009088,
+          "brier": 0.006373682001028304,
+          "ece15": 0.004218405852177047,
+          "temperature": 1.334968103387446,
+          "coverage_at_confidence_0_9": 0.9937264742785445,
+          "error_rate_at_confidence_0_9": 0.0012626262626262627,
+          "ordinal_expected_level_mae": 0.005329911013904147
+        },
+        "twins_streamlet:twins_policy_action": {
+          "questions": 797,
+          "accuracy": 0.9949811794228356,
+          "mean_class_recall": 0.9951865222623345,
+          "nll": 0.01230072195659841,
+          "brier": 0.007337903591834537,
+          "ece15": 0.0031241457310850957,
+          "temperature": 1.334968103387446,
+          "coverage_at_confidence_0_9": 0.986198243412798,
+          "error_rate_at_confidence_0_9": 0.001272264631043257,
+          "ordinal_expected_level_mae": null
+        },
+        "twins_streamlet:twins_quorum": {
+          "questions": 797,
+          "accuracy": 0.9949811794228356,
+          "mean_class_recall": 0.994276682635671,
+          "nll": 0.009584146250019401,
+          "brier": 0.00659904677807738,
+          "ece15": 0.005028933100804552,
+          "temperature": 1.334968103387446,
+          "coverage_at_confidence_0_9": 0.9899623588456713,
+          "error_rate_at_confidence_0_9": 0.0,
+          "ordinal_expected_level_mae": null
+        },
+        "twins_streamlet:twins_visible_conflict": {
+          "questions": 797,
+          "accuracy": 0.9974905897114178,
+          "mean_class_recall": 0.998236331569665,
+          "nll": 0.013437428285294059,
+          "brier": 0.004858731838764845,
+          "ece15": 0.0022785888551786117,
+          "temperature": 1.334968103387446,
+          "coverage_at_confidence_0_9": 0.998745294855709,
+          "error_rate_at_confidence_0_9": 0.002512562814070352,
+          "ordinal_expected_level_mae": null
+        },
+        "veremi_nextgen:veremi_message_manipulation": {
+          "questions": 800,
+          "accuracy": 0.69,
+          "mean_class_recall": 0.7116483068135455,
+          "nll": 0.5102124840713312,
+          "brier": 0.35753050242002155,
+          "ece15": 0.03811996421246104,
+          "temperature": 1.334968103387446,
+          "coverage_at_confidence_0_9": 0.22125,
+          "error_rate_at_confidence_0_9": 0.011299435028248588,
+          "ordinal_expected_level_mae": null
+        },
+        "veremi_nextgen:veremi_supported_attack_type_v16": {
+          "questions": 800,
+          "accuracy": 0.68875,
+          "mean_class_recall": 0.529392069171139,
+          "nll": 0.9911460027249259,
+          "brier": 0.4737144267281807,
+          "ece15": 0.09647043512079573,
+          "temperature": 1.334968103387446,
+          "coverage_at_confidence_0_9": 0.18625,
+          "error_rate_at_confidence_0_9": 0.020134228187919462,
+          "ordinal_expected_level_mae": null
+        }
+      },
+      "macro_task_accuracy": 0.9379621229396502,
+      "macro_task_balanced_accuracy": 0.9185689774676985
+    }
+  },
+  "calibration": {
+    "raw": {
+      "questions": 14821,
+      "accuracy": 0.9313811483705553,
+      "mean_class_recall": 0.8460223158089928,
+      "nll": 0.18480685074016417,
+      "brier": 0.09978353062869037,
+      "ece15": 0.010514966894040724,
+      "temperature": 1.0,
+      "coverage_at_confidence_0_9": 0.814789825247959,
+      "error_rate_at_confidence_0_9": 0.013580655846306725,
+      "ordinal_expected_level_mae": 0.001461981275023823,
+      "tasks": {
+        "agentdojo_tool_outputs:agentdojo_boundary_action": {
+          "questions": 325,
+          "accuracy": 0.96,
+          "mean_class_recall": 0.9216916119306557,
+          "nll": 0.12908652765840894,
+          "brier": 0.06258807450225369,
+          "ece15": 0.033510331142289206,
+          "temperature": 1.0,
+          "coverage_at_confidence_0_9": 0.9815384615384616,
+          "error_rate_at_confidence_0_9": 0.0219435736677116,
+          "ordinal_expected_level_mae": null
+        },
+        "agentdojo_tool_outputs:agentdojo_embedded_injection": {
+          "questions": 328,
+          "accuracy": 0.9695121951219512,
+          "mean_class_recall": 0.9372206852521814,
+          "nll": 0.16127769578392517,
+          "brier": 0.05858994554032181,
+          "ece15": 0.030459345972402876,
+          "temperature": 1.0,
+          "coverage_at_confidence_0_9": 0.9939024390243902,
+          "error_rate_at_confidence_0_9": 0.027607361963190184,
+          "ordinal_expected_level_mae": null
+        },
+        "byzfl_mnist:fl_attack_generated_update": {
+          "questions": 2400,
+          "accuracy": 0.9404166666666667,
+          "mean_class_recall": 0.8877604166666666,
+          "nll": 0.19653998710185536,
+          "brier": 0.09947525932448194,
+          "ece15": 0.04423318187692424,
+          "temperature": 1.0,
+          "coverage_at_confidence_0_9": 0.9479166666666666,
+          "error_rate_at_confidence_0_9": 0.03824175824175824,
+          "ordinal_expected_level_mae": null
+        },
+        "ciciot2023:ciciot_attack": {
+          "questions": 5000,
+          "accuracy": 0.9292,
+          "mean_class_recall": 0.8968,
+          "nll": 0.18887834668200343,
+          "brier": 0.10636649585712103,
+          "ece15": 0.019887550017025992,
+          "temperature": 1.0,
+          "coverage_at_confidence_0_9": 0.7252,
+          "error_rate_at_confidence_0_9": 0.013513513513513514,
+          "ordinal_expected_level_mae": null
+        },
+        "injecagent:injecagent_requested_operation_v13": {
+          "questions": 124,
+          "accuracy": 0.9919354838709677,
+          "mean_class_recall": 0.9928571428571429,
+          "nll": 0.021390831595053342,
+          "brier": 0.015445766007169246,
+          "ece15": 0.011074830403285848,
+          "temperature": 1.0,
+          "coverage_at_confidence_0_9": 0.9838709677419355,
+          "error_rate_at_confidence_0_9": 0.0,
+          "ordinal_expected_level_mae": null
+        },
+        "lanl_auth_policy:lanl_known_logon_action_v16": {
+          "questions": 235,
+          "accuracy": 0.9957446808510638,
+          "mean_class_recall": 0.9949494949494949,
+          "nll": 0.015420468691595452,
+          "brier": 0.008033852620508352,
+          "ece15": 0.003625196423068977,
+          "temperature": 1.0,
+          "coverage_at_confidence_0_9": 1.0,
+          "error_rate_at_confidence_0_9": 0.00425531914893617,
+          "ordinal_expected_level_mae": null
+        },
+        "lanl_auth_policy:lanl_observable_policy": {
+          "questions": 235,
+          "accuracy": 0.9957446808510638,
+          "mean_class_recall": 0.9905660377358491,
+          "nll": 0.018077073969230188,
+          "brier": 0.00826909442413202,
+          "ece15": 0.004194003764167214,
+          "temperature": 1.0,
+          "coverage_at_confidence_0_9": 1.0,
+          "error_rate_at_confidence_0_9": 0.00425531914893617,
+          "ordinal_expected_level_mae": null
+        },
+        "ton_iot_network:ton_attack": {
+          "questions": 2000,
+          "accuracy": 0.9985,
+          "mean_class_recall": 0.9976666666666667,
+          "nll": 0.010567088503443185,
+          "brier": 0.003178972042914288,
+          "ece15": 0.0009240536217447044,
+          "temperature": 1.0,
+          "coverage_at_confidence_0_9": 0.999,
+          "error_rate_at_confidence_0_9": 0.0015015015015015015,
+          "ordinal_expected_level_mae": null
+        },
+        "twins_streamlet:twins_evidence_level": {
+          "questions": 641,
+          "accuracy": 0.9984399375975039,
+          "mean_class_recall": 0.9985507246376812,
+          "nll": 0.003039872100282549,
+          "brier": 0.0022297539963869384,
+          "ece15": 0.0014593852165271412,
+          "temperature": 1.0,
+          "coverage_at_confidence_0_9": 0.9984399375975039,
+          "error_rate_at_confidence_0_9": 0.0,
+          "ordinal_expected_level_mae": 0.001461981275023823
+        },
+        "twins_streamlet:twins_policy_action": {
+          "questions": 641,
+          "accuracy": 1.0,
+          "mean_class_recall": 1.0,
+          "nll": 0.0004967972153911127,
+          "brier": 9.688544623270663e-05,
+          "ece15": 0.00046955068951652005,
+          "temperature": 1.0,
+          "coverage_at_confidence_0_9": 0.9984399375975039,
+          "error_rate_at_confidence_0_9": 0.0,
+          "ordinal_expected_level_mae": null
+        },
+        "twins_streamlet:twins_quorum": {
+          "questions": 641,
+          "accuracy": 0.9968798751950078,
+          "mean_class_recall": 0.9961240310077519,
+          "nll": 0.016173152134660414,
+          "brier": 0.007502235795954162,
+          "ece15": 0.0049434253477994355,
+          "temperature": 1.0,
+          "coverage_at_confidence_0_9": 0.9937597503900156,
+          "error_rate_at_confidence_0_9": 0.0031397174254317113,
+          "ordinal_expected_level_mae": null
+        },
+        "twins_streamlet:twins_visible_conflict": {
+          "questions": 641,
+          "accuracy": 0.9984399375975039,
+          "mean_class_recall": 0.9969512195121951,
+          "nll": 0.004162755288573824,
+          "brier": 0.0026938896990115683,
+          "ece15": 0.0014825770107887316,
+          "temperature": 1.0,
+          "coverage_at_confidence_0_9": 1.0,
+          "error_rate_at_confidence_0_9": 0.0015600624024961,
+          "ordinal_expected_level_mae": null
+        },
+        "veremi_nextgen:veremi_message_manipulation": {
+          "questions": 805,
+          "accuracy": 0.7031055900621118,
+          "mean_class_recall": 0.7239978843175034,
+          "nll": 0.49879740669601635,
+          "brier": 0.351098943795658,
+          "ece15": 0.02568931078110801,
+          "temperature": 1.0,
+          "coverage_at_confidence_0_9": 0.23850931677018633,
+          "error_rate_at_confidence_0_9": 0.0,
+          "ordinal_expected_level_mae": null
+        },
+        "veremi_nextgen:veremi_supported_attack_type_v16": {
+          "questions": 805,
+          "accuracy": 0.6919254658385093,
+          "mean_class_recall": 0.5331098265895953,
+          "nll": 0.9684343228937802,
+          "brier": 0.45465071518796674,
+          "ece15": 0.01705305962599151,
+          "temperature": 1.0,
+          "coverage_at_confidence_0_9": 0.2360248447204969,
+          "error_rate_at_confidence_0_9": 0.021052631578947368,
+          "ordinal_expected_level_mae": null
+        }
+      },
+      "macro_task_accuracy": 0.9407031795465964,
+      "macro_task_balanced_accuracy": 0.9191604101516703
+    },
+    "recalibrated": {
+      "questions": 14821,
+      "accuracy": 0.9313811483705553,
+      "mean_class_recall": 0.8460223158089928,
+      "nll": 0.17961943925788992,
+      "brier": 0.10111838724890135,
+      "ece15": 0.017864195303344165,
+      "temperature": 1.334968103387446,
+      "coverage_at_confidence_0_9": 0.7859118817893529,
+      "error_rate_at_confidence_0_9": 0.010645604395604396,
+      "ordinal_expected_level_mae": 0.001742299523022841,
+      "tasks": {
+        "agentdojo_tool_outputs:agentdojo_boundary_action": {
+          "questions": 325,
+          "accuracy": 0.96,
+          "mean_class_recall": 0.9216916119306557,
+          "nll": 0.10308600797920058,
+          "brier": 0.05761479429147346,
+          "ece15": 0.0294057816164861,
+          "temperature": 1.334968103387446,
+          "coverage_at_confidence_0_9": 0.963076923076923,
+          "error_rate_at_confidence_0_9": 0.012779552715654952,
+          "ordinal_expected_level_mae": null
+        },
+        "agentdojo_tool_outputs:agentdojo_embedded_injection": {
+          "questions": 328,
+          "accuracy": 0.9695121951219512,
+          "mean_class_recall": 0.9372206852521814,
+          "nll": 0.12289366290953901,
+          "brier": 0.057082591730227304,
+          "ece15": 0.029897091535857363,
+          "temperature": 1.334968103387446,
+          "coverage_at_confidence_0_9": 0.9939024390243902,
+          "error_rate_at_confidence_0_9": 0.027607361963190184,
+          "ordinal_expected_level_mae": null
+        },
+        "byzfl_mnist:fl_attack_generated_update": {
+          "questions": 2400,
+          "accuracy": 0.9404166666666667,
+          "mean_class_recall": 0.8877604166666666,
+          "nll": 0.16174786934019353,
+          "brier": 0.09375828073422167,
+          "ece15": 0.03645209485430555,
+          "temperature": 1.334968103387446,
+          "coverage_at_confidence_0_9": 0.92875,
+          "error_rate_at_confidence_0_9": 0.03095558546433378,
+          "ordinal_expected_level_mae": null
+        },
+        "ciciot2023:ciciot_attack": {
+          "questions": 5000,
+          "accuracy": 0.9292,
+          "mean_class_recall": 0.8968,
+          "nll": 0.19125358889505403,
+          "brier": 0.11141554439383701,
+          "ece15": 0.03505488361604012,
+          "temperature": 1.334968103387446,
+          "coverage_at_confidence_0_9": 0.6616,
+          "error_rate_at_confidence_0_9": 0.010278113663845224,
+          "ordinal_expected_level_mae": null
+        },
+        "injecagent:injecagent_requested_operation_v13": {
+          "questions": 124,
+          "accuracy": 0.9919354838709677,
+          "mean_class_recall": 0.9928571428571429,
+          "nll": 0.019190864186870816,
+          "brier": 0.01397734899726431,
+          "ece15": 0.011351501004051424,
+          "temperature": 1.334968103387446,
+          "coverage_at_confidence_0_9": 0.9838709677419355,
+          "error_rate_at_confidence_0_9": 0.0,
+          "ordinal_expected_level_mae": null
+        },
+        "lanl_auth_policy:lanl_known_logon_action_v16": {
+          "questions": 235,
+          "accuracy": 0.9957446808510638,
+          "mean_class_recall": 0.9949494949494949,
+          "nll": 0.01285197836061796,
+          "brier": 0.007512756902037326,
+          "ece15": 0.004004760561973723,
+          "temperature": 1.334968103387446,
+          "coverage_at_confidence_0_9": 1.0,
+          "error_rate_at_confidence_0_9": 0.00425531914893617,
+          "ordinal_expected_level_mae": null
+        },
+        "lanl_auth_policy:lanl_observable_policy": {
+          "questions": 235,
+          "accuracy": 0.9957446808510638,
+          "mean_class_recall": 0.9905660377358491,
+          "nll": 0.01368047084450566,
+          "brier": 0.007839165259612221,
+          "ece15": 0.004073337312958388,
+          "temperature": 1.334968103387446,
+          "coverage_at_confidence_0_9": 1.0,
+          "error_rate_at_confidence_0_9": 0.00425531914893617,
+          "ordinal_expected_level_mae": null
+        },
+        "ton_iot_network:ton_attack": {
+          "questions": 2000,
+          "accuracy": 0.9985,
+          "mean_class_recall": 0.9976666666666667,
+          "nll": 0.010650627249048795,
+          "brier": 0.003323170074906094,
+          "ece15": 0.002118687755931114,
+          "temperature": 1.334968103387446,
+          "coverage_at_confidence_0_9": 0.9975,
+          "error_rate_at_confidence_0_9": 0.0015037593984962407,
+          "ordinal_expected_level_mae": null
+        },
+        "twins_streamlet:twins_evidence_level": {
+          "questions": 641,
+          "accuracy": 0.9984399375975039,
+          "mean_class_recall": 0.9985507246376812,
+          "nll": 0.0028802196260483376,
+          "brier": 0.001935795675809169,
+          "ece15": 0.0017273023271884604,
+          "temperature": 1.334968103387446,
+          "coverage_at_confidence_0_9": 0.9968798751950078,
+          "error_rate_at_confidence_0_9": 0.0,
+          "ordinal_expected_level_mae": 0.001742299523022841
+        },
+        "twins_streamlet:twins_policy_action": {
+          "questions": 641,
+          "accuracy": 1.0,
+          "mean_class_recall": 1.0,
+          "nll": 0.0012409177190170738,
+          "brier": 0.00021235107519573934,
+          "ece15": 0.0011788827327451372,
+          "temperature": 1.334968103387446,
+          "coverage_at_confidence_0_9": 0.9984399375975039,
+          "error_rate_at_confidence_0_9": 0.0,
+          "ordinal_expected_level_mae": null
+        },
+        "twins_streamlet:twins_quorum": {
+          "questions": 641,
+          "accuracy": 0.9968798751950078,
+          "mean_class_recall": 0.9961240310077519,
+          "nll": 0.013816362060687534,
+          "brier": 0.007467944863319216,
+          "ece15": 0.004893228894077442,
+          "temperature": 1.334968103387446,
+          "coverage_at_confidence_0_9": 0.9921996879875195,
+          "error_rate_at_confidence_0_9": 0.0031446540880503146,
+          "ordinal_expected_level_mae": null
+        },
+        "twins_streamlet:twins_visible_conflict": {
+          "questions": 641,
+          "accuracy": 0.9984399375975039,
+          "mean_class_recall": 0.9969512195121951,
+          "nll": 0.0034185049808860444,
+          "brier": 0.0023816170036107054,
+          "ece15": 0.0015601193471088582,
+          "temperature": 1.334968103387446,
+          "coverage_at_confidence_0_9": 0.9984399375975039,
+          "error_rate_at_confidence_0_9": 0.0,
+          "ordinal_expected_level_mae": null
+        },
+        "veremi_nextgen:veremi_message_manipulation": {
+          "questions": 805,
+          "accuracy": 0.7031055900621118,
+          "mean_class_recall": 0.7239978843175034,
+          "nll": 0.503140821446039,
+          "brier": 0.35269849107227197,
+          "ece15": 0.030978365187072362,
+          "temperature": 1.334968103387446,
+          "coverage_at_confidence_0_9": 0.21366459627329193,
+          "error_rate_at_confidence_0_9": 0.0,
+          "ordinal_expected_level_mae": null
+        },
+        "veremi_nextgen:veremi_supported_attack_type_v16": {
+          "questions": 805,
+          "accuracy": 0.6919254658385093,
+          "mean_class_recall": 0.5331098265895953,
+          "nll": 0.9878648410254747,
+          "brier": 0.46649698606417656,
+          "ece15": 0.09086134384209663,
+          "temperature": 1.334968103387446,
+          "coverage_at_confidence_0_9": 0.19627329192546583,
+          "error_rate_at_confidence_0_9": 0.006329113924050633,
+          "ordinal_expected_level_mae": null
+        }
+      },
+      "macro_task_accuracy": 0.9407031795465964,
+      "macro_task_balanced_accuracy": 0.9191604101516703
+    }
+  }
+}

example-output.json

@@ -0,0 +1,24 @@
+{
+  "model": "SecJev-4B",
+  "answers": {
+    "certificate_action": {
+      "type": "choice",
+      "choice": "reject",
+      "confidence": 0.99,
+      "probabilities": {
+        "accept": 0.0,
+        "reject": 0.99,
+        "wait": 0.01
+      }
+    }
+  },
+  "probabilities": {
+    "certificate_action": {
+      "accept": 2.5144670871668495e-05,
+      "reject": 0.9922695159912109,
+      "wait": 0.0077053275890648365
+    }
+  },
+  "temperature": 1.334968103387446,
+  "forward_ms": 10437.744742259383
+}

examples/certificate.json

@@ -0,0 +1,30 @@
+{
+  "state": {
+    "policy": "Reject a certificate if one of its voters voted for different blocks in the same round. Otherwise accept with at least three distinct voters; otherwise wait.",
+    "candidate": "A",
+    "round": 7,
+    "observed_votes": {
+      "v1": [
+        "A"
+      ],
+      "v2": [
+        "A",
+        "B"
+      ],
+      "v3": [
+        "A"
+      ]
+    }
+  },
+  "questions": {
+    "certificate_action": {
+      "type": "choice",
+      "instructions": "Which action follows the stated policy?",
+      "criteria": {
+        "accept": "Accept the certificate",
+        "reject": "Reject the conflicting certificate",
+        "wait": "Wait for more evidence"
+      }
+    }
+  }
+}

export-validation.json

@@ -0,0 +1,83 @@
+{
+  "device": "CUDA",
+  "precision": "FP32 unmerged",
+  "compared_with": "original evaluated GPU checkpoint, development split, raw logits",
+  "cases": [
+    {
+      "case": 4222,
+      "qid": "injection",
+      "qtype": "noul",
+      "tokens": 140,
+      "expected_label": 0,
+      "max_probability_difference": 3.9981387089937925e-09,
+      "max_logit_difference": 0.0003533363342285156,
+      "exported_prediction": 0,
+      "reference_prediction": 0
+    },
+    {
+      "case": 4654,
+      "qid": "injection",
+      "qtype": "noul",
+      "tokens": 152,
+      "expected_label": 0,
+      "max_probability_difference": 0.00026088953018188477,
+      "max_logit_difference": 0.0010182857513427734,
+      "exported_prediction": 0,
+      "reference_prediction": 0
+    },
+    {
+      "case": 4222,
+      "qid": "boundary_action",
+      "qtype": "choice",
+      "tokens": 192,
+      "expected_label": 0,
+      "max_probability_difference": 1.1920928955078125e-07,
+      "max_logit_difference": 0.002742290496826172,
+      "exported_prediction": 0,
+      "reference_prediction": 0
+    },
+    {
+      "case": 4654,
+      "qid": "boundary_action",
+      "qtype": "choice",
+      "tokens": 204,
+      "expected_label": 0,
+      "max_probability_difference": 3.2186508178710938e-06,
+      "max_logit_difference": 0.0013228654861450195,
+      "exported_prediction": 0,
+      "reference_prediction": 0
+    },
+    {
+      "case": 2334,
+      "qid": "evidence_level",
+      "qtype": "score",
+      "tokens": 234,
+      "expected_label": 1,
+      "max_probability_difference": 9.085852070711553e-10,
+      "max_logit_difference": 0.0014123916625976562,
+      "exported_prediction": 1,
+      "reference_prediction": 1
+    },
+    {
+      "case": 1790,
+      "qid": "evidence_level",
+      "qtype": "score",
+      "tokens": 250,
+      "expected_label": 1,
+      "max_probability_difference": 1.3608314475277439e-09,
+      "max_logit_difference": 0.0016317367553710938,
+      "exported_prediction": 1,
+      "reference_prediction": 1
+    }
+  ],
+  "all_predictions_match": true,
+  "max_probability_difference": 0.00026088953018188477,
+  "tolerance": 0.001,
+  "sample_selection": "two shortest development rows for each of three types; packaging parity only",
+  "handwritten_example": {
+    "expected_action": "reject",
+    "actual_prediction": "reject",
+    "matches_label": true,
+    "purpose": "Interface smoke example; not a benchmark result"
+  }
+}

head.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0955d746a517b2d18762ebadd79d2c3ff6fef7209853f15580be917aeec3f753
+size 5249407

infer.py

@@ -0,0 +1,52 @@
+"""Score a native System One JSON request using a SecJev checkpoint."""
+import argparse, json, pathlib, time, os, sys
+sys.path.insert(0,str(pathlib.Path(os.environ.get('SECJEV_WORK', './secjev-work')).resolve()/'source/kev'))
+import torch
+from kev.api import SystemOneRequest, to_record, to_answers
+from kev.checkpoint import Checkpoint, LoadOptions
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--model', required=True, help='Local checkpoint directory or owner/repository@revision')
+    parser.add_argument('--request', required=True, type=pathlib.Path)
+    parser.add_argument('--device', default='cuda' if torch.cuda.is_available() else 'cpu')
+    parser.add_argument('--base', help='Optional local copy of the pinned Qwen backbone')
+    args = parser.parse_args()
+    request = SystemOneRequest.model_validate(json.loads(args.request.read_text()))
+    checkpoint = Checkpoint(args.model)
+    if args.base:
+        checkpoint.meta.base = args.base
+        checkpoint.meta.base_revision = None
+    torch.set_num_threads(4)
+    torch.backends.cuda.matmul.allow_tf32 = False
+    torch.backends.cudnn.allow_tf32 = False
+    torch.backends.cuda.enable_flash_sdp(False)
+    torch.backends.cuda.enable_mem_efficient_sdp(False)
+    tokenizer, model = checkpoint.load(args.device, LoadOptions(dtype=torch.float32, merge=False, attn='sdpa'))
+    record, metadata = to_record(request)
+    encoded = model.encode(tokenizer, record, max_state=8192, max_branch=8192, strict=True)
+    from kev.model import rows_of
+    state, _, branches = rows_of(encoded)
+    if any(len(state) + len(branch['ids']) > 8192 for branch in branches):
+        raise ValueError('Each state + question must fit within 8192 tokens; inputs are never truncated.')
+    if str(args.device).startswith('cuda'):
+        torch.cuda.synchronize()
+    started = time.perf_counter()
+    with torch.inference_mode():
+        logits = model.forward(encoded)
+        probabilities = [torch.softmax(z.float(), -1).cpu().tolist() for z in logits]
+    if str(args.device).startswith('cuda'):
+        torch.cuda.synchronize()
+    result = {
+        'model': args.model,
+        'answers': to_answers(probabilities, metadata),
+        'probabilities': {m['id']: dict(zip(m['keys'], p)) for m, p in zip(metadata, probabilities)},
+        'temperature': model.head.temperature,
+        'forward_ms': (time.perf_counter() - started) * 1000,
+    }
+    print(json.dumps(result, ensure_ascii=False, indent=2, allow_nan=False))
+
+
+if __name__ == '__main__':
+    main()

licenses/AgentDojo-LICENSE.txt

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2024 Edoardo Debenedetti, Jie Zhang, Mislav Balunovic, Luca Beurer-Kellner, Marc Fischer, and Florian Tramèr
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

licenses/Apache-2.0.txt

@@ -0,0 +1,201 @@
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

licenses/ByzFL-LICENSE.txt

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2024 EPFL
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

licenses/FoolsGold-LICENSE.txt

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2025 DistributedML
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

licenses/InjecAgent-LICENSE.txt

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2023 Qiusi Zhan
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

licenses/LANL.md

@@ -0,0 +1,8 @@
+# LANL authentication / redteam data
+
+Source: https://csr.lanl.gov/data/cyber1/
+License: CC0 1.0, as stated on the official source page saved in metadata/lanl-source-page.html.
+Citation: Alexander D. Kent, Comprehensive, Multi-Source Cyber-Security Events, Los Alamos National Laboratory, 2015. DOI: 10.17021/1179829.
+Related publication: A. D. Kent, Cybersecurity Data Sources for Dynamic Network Research, in Dynamic Networks in Cybersecurity, 2015.
+The original source warns that failed authentications are included only for users with at least one successful authentication somewhere in the dataset. This creates selection bias.
+No registration credentials or private transport logs are included in this release.

licenses/Twins-LICENSE.txt

@@ -0,0 +1,201 @@
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

licenses/VeReMi-LICENSE.txt

@@ -0,0 +1,396 @@
+Attribution 4.0 International
+
+=======================================================================
+
+Creative Commons Corporation ("Creative Commons") is not a law firm and
+does not provide legal services or legal advice. Distribution of
+Creative Commons public licenses does not create a lawyer-client or
+other relationship. Creative Commons makes its licenses and related
+information available on an "as-is" basis. Creative Commons gives no
+warranties regarding its licenses, any material licensed under their
+terms and conditions, or any related information. Creative Commons
+disclaims all liability for damages resulting from their use to the
+fullest extent possible.
+
+Using Creative Commons Public Licenses
+
+Creative Commons public licenses provide a standard set of terms and
+conditions that creators and other rights holders may use to share
+original works of authorship and other material subject to copyright
+and certain other rights specified in the public license below. The
+following considerations are for informational purposes only, are not
+exhaustive, and do not form part of our licenses.
+
+     Considerations for licensors: Our public licenses are
+     intended for use by those authorized to give the public
+     permission to use material in ways otherwise restricted by
+     copyright and certain other rights. Our licenses are
+     irrevocable. Licensors should read and understand the terms
+     and conditions of the license they choose before applying it.
+     Licensors should also secure all rights necessary before
+     applying our licenses so that the public can reuse the
+     material as expected. Licensors should clearly mark any
+     material not subject to the license. This includes other CC-
+     licensed material, or material used under an exception or
+     limitation to copyright. More considerations for licensors:
+    wiki.creativecommons.org/Considerations_for_licensors
+
+     Considerations for the public: By using one of our public
+     licenses, a licensor grants the public permission to use the
+     licensed material under specified terms and conditions. If
+     the licensor's permission is not necessary for any reason--for
+     example, because of any applicable exception or limitation to
+     copyright--then that use is not regulated by the license. Our
+     licenses grant only permissions under copyright and certain
+     other rights that a licensor has authority to grant. Use of
+     the licensed material may still be restricted for other
+     reasons, including because others have copyright or other
+     rights in the material. A licensor may make special requests,
+     such as asking that all changes be marked or described.
+     Although not required by our licenses, you are encouraged to
+     respect those requests where reasonable. More considerations
+     for the public:
+    wiki.creativecommons.org/Considerations_for_licensees
+
+=======================================================================
+
+Creative Commons Attribution 4.0 International Public License
+
+By exercising the Licensed Rights (defined below), You accept and agree
+to be bound by the terms and conditions of this Creative Commons
+Attribution 4.0 International Public License ("Public License"). To the
+extent this Public License may be interpreted as a contract, You are
+granted the Licensed Rights in consideration of Your acceptance of
+these terms and conditions, and the Licensor grants You such rights in
+consideration of benefits the Licensor receives from making the
+Licensed Material available under these terms and conditions.
+
+
+Section 1 -- Definitions.
+
+  a. Adapted Material means material subject to Copyright and Similar
+     Rights that is derived from or based upon the Licensed Material
+     and in which the Licensed Material is translated, altered,
+     arranged, transformed, or otherwise modified in a manner requiring
+     permission under the Copyright and Similar Rights held by the
+     Licensor. For purposes of this Public License, where the Licensed
+     Material is a musical work, performance, or sound recording,
+     Adapted Material is always produced where the Licensed Material is
+     synched in timed relation with a moving image.
+
+  b. Adapter's License means the license You apply to Your Copyright
+     and Similar Rights in Your contributions to Adapted Material in
+     accordance with the terms and conditions of this Public License.
+
+  c. Copyright and Similar Rights means copyright and/or similar rights
+     closely related to copyright including, without limitation,
+     performance, broadcast, sound recording, and Sui Generis Database
+     Rights, without regard to how the rights are labeled or
+     categorized. For purposes of this Public License, the rights
+     specified in Section 2(b)(1)-(2) are not Copyright and Similar
+     Rights.
+
+  d. Effective Technological Measures means those measures that, in the
+     absence of proper authority, may not be circumvented under laws
+     fulfilling obligations under Article 11 of the WIPO Copyright
+     Treaty adopted on December 20, 1996, and/or similar international
+     agreements.
+
+  e. Exceptions and Limitations means fair use, fair dealing, and/or
+     any other exception or limitation to Copyright and Similar Rights
+     that applies to Your use of the Licensed Material.
+
+  f. Licensed Material means the artistic or literary work, database,
+     or other material to which the Licensor applied this Public
+     License.
+
+  g. Licensed Rights means the rights granted to You subject to the
+     terms and conditions of this Public License, which are limited to
+     all Copyright and Similar Rights that apply to Your use of the
+     Licensed Material and that the Licensor has authority to license.
+
+  h. Licensor means the individual(s) or entity(ies) granting rights
+     under this Public License.
+
+  i. Share means to provide material to the public by any means or
+     process that requires permission under the Licensed Rights, such
+     as reproduction, public display, public performance, distribution,
+     dissemination, communication, or importation, and to make material
+     available to the public including in ways that members of the
+     public may access the material from a place and at a time
+     individually chosen by them.
+
+  j. Sui Generis Database Rights means rights other than copyright
+     resulting from Directive 96/9/EC of the European Parliament and of
+     the Council of 11 March 1996 on the legal protection of databases,
+     as amended and/or succeeded, as well as other essentially
+     equivalent rights anywhere in the world.
+
+  k. You means the individual or entity exercising the Licensed Rights
+     under this Public License. Your has a corresponding meaning.
+
+
+Section 2 -- Scope.
+
+  a. License grant.
+
+       1. Subject to the terms and conditions of this Public License,
+          the Licensor hereby grants You a worldwide, royalty-free,
+          non-sublicensable, non-exclusive, irrevocable license to
+          exercise the Licensed Rights in the Licensed Material to:
+
+            a. reproduce and Share the Licensed Material, in whole or
+               in part; and
+
+            b. produce, reproduce, and Share Adapted Material.
+
+       2. Exceptions and Limitations. For the avoidance of doubt, where
+          Exceptions and Limitations apply to Your use, this Public
+          License does not apply, and You do not need to comply with
+          its terms and conditions.
+
+       3. Term. The term of this Public License is specified in Section
+          6(a).
+
+       4. Media and formats; technical modifications allowed. The
+          Licensor authorizes You to exercise the Licensed Rights in
+          all media and formats whether now known or hereafter created,
+          and to make technical modifications necessary to do so. The
+          Licensor waives and/or agrees not to assert any right or
+          authority to forbid You from making technical modifications
+          necessary to exercise the Licensed Rights, including
+          technical modifications necessary to circumvent Effective
+          Technological Measures. For purposes of this Public License,
+          simply making modifications authorized by this Section 2(a)
+          (4) never produces Adapted Material.
+
+       5. Downstream recipients.
+
+            a. Offer from the Licensor -- Licensed Material. Every
+               recipient of the Licensed Material automatically
+               receives an offer from the Licensor to exercise the
+               Licensed Rights under the terms and conditions of this
+               Public License.
+
+            b. No downstream restrictions. You may not offer or impose
+               any additional or different terms or conditions on, or
+               apply any Effective Technological Measures to, the
+               Licensed Material if doing so restricts exercise of the
+               Licensed Rights by any recipient of the Licensed
+               Material.
+
+       6. No endorsement. Nothing in this Public License constitutes or
+          may be construed as permission to assert or imply that You
+          are, or that Your use of the Licensed Material is, connected
+          with, or sponsored, endorsed, or granted official status by,
+          the Licensor or others designated to receive attribution as
+          provided in Section 3(a)(1)(A)(i).
+
+  b. Other rights.
+
+       1. Moral rights, such as the right of integrity, are not
+          licensed under this Public License, nor are publicity,
+          privacy, and/or other similar personality rights; however, to
+          the extent possible, the Licensor waives and/or agrees not to
+          assert any such rights held by the Licensor to the limited
+          extent necessary to allow You to exercise the Licensed
+          Rights, but not otherwise.
+
+       2. Patent and trademark rights are not licensed under this
+          Public License.
+
+       3. To the extent possible, the Licensor waives any right to
+          collect royalties from You for the exercise of the Licensed
+          Rights, whether directly or through a collecting society
+          under any voluntary or waivable statutory or compulsory
+          licensing scheme. In all other cases the Licensor expressly
+          reserves any right to collect such royalties.
+
+
+Section 3 -- License Conditions.
+
+Your exercise of the Licensed Rights is expressly made subject to the
+following conditions.
+
+  a. Attribution.
+
+       1. If You Share the Licensed Material (including in modified
+          form), You must:
+
+            a. retain the following if it is supplied by the Licensor
+               with the Licensed Material:
+
+                 i. identification of the creator(s) of the Licensed
+                    Material and any others designated to receive
+                    attribution, in any reasonable manner requested by
+                    the Licensor (including by pseudonym if
+                    designated);
+
+                ii. a copyright notice;
+
+               iii. a notice that refers to this Public License;
+
+                iv. a notice that refers to the disclaimer of
+                    warranties;
+
+                 v. a URI or hyperlink to the Licensed Material to the
+                    extent reasonably practicable;
+
+            b. indicate if You modified the Licensed Material and
+               retain an indication of any previous modifications; and
+
+            c. indicate the Licensed Material is licensed under this
+               Public License, and include the text of, or the URI or
+               hyperlink to, this Public License.
+
+       2. You may satisfy the conditions in Section 3(a)(1) in any
+          reasonable manner based on the medium, means, and context in
+          which You Share the Licensed Material. For example, it may be
+          reasonable to satisfy the conditions by providing a URI or
+          hyperlink to a resource that includes the required
+          information.
+
+       3. If requested by the Licensor, You must remove any of the
+          information required by Section 3(a)(1)(A) to the extent
+          reasonably practicable.
+
+       4. If You Share Adapted Material You produce, the Adapter's
+          License You apply must not prevent recipients of the Adapted
+          Material from complying with this Public License.
+
+
+Section 4 -- Sui Generis Database Rights.
+
+Where the Licensed Rights include Sui Generis Database Rights that
+apply to Your use of the Licensed Material:
+
+  a. for the avoidance of doubt, Section 2(a)(1) grants You the right
+     to extract, reuse, reproduce, and Share all or a substantial
+     portion of the contents of the database;
+
+  b. if You include all or a substantial portion of the database
+     contents in a database in which You have Sui Generis Database
+     Rights, then the database in which You have Sui Generis Database
+     Rights (but not its individual contents) is Adapted Material; and
+
+  c. You must comply with the conditions in Section 3(a) if You Share
+     all or a substantial portion of the contents of the database.
+
+For the avoidance of doubt, this Section 4 supplements and does not
+replace Your obligations under this Public License where the Licensed
+Rights include other Copyright and Similar Rights.
+
+
+Section 5 -- Disclaimer of Warranties and Limitation of Liability.
+
+  a. UNLESS OTHERWISE SEPARATELY UNDERTAKEN BY THE LICENSOR, TO THE
+     EXTENT POSSIBLE, THE LICENSOR OFFERS THE LICENSED MATERIAL AS-IS
+     AND AS-AVAILABLE, AND MAKES NO REPRESENTATIONS OR WARRANTIES OF
+     ANY KIND CONCERNING THE LICENSED MATERIAL, WHETHER EXPRESS,
+     IMPLIED, STATUTORY, OR OTHER. THIS INCLUDES, WITHOUT LIMITATION,
+     WARRANTIES OF TITLE, MERCHANTABILITY, FITNESS FOR A PARTICULAR
+     PURPOSE, NON-INFRINGEMENT, ABSENCE OF LATENT OR OTHER DEFECTS,
+     ACCURACY, OR THE PRESENCE OR ABSENCE OF ERRORS, WHETHER OR NOT
+     KNOWN OR DISCOVERABLE. WHERE DISCLAIMERS OF WARRANTIES ARE NOT
+     ALLOWED IN FULL OR IN PART, THIS DISCLAIMER MAY NOT APPLY TO YOU.
+
+  b. TO THE EXTENT POSSIBLE, IN NO EVENT WILL THE LICENSOR BE LIABLE
+     TO YOU ON ANY LEGAL THEORY (INCLUDING, WITHOUT LIMITATION,
+     NEGLIGENCE) OR OTHERWISE FOR ANY DIRECT, SPECIAL, INDIRECT,
+     INCIDENTAL, CONSEQUENTIAL, PUNITIVE, EXEMPLARY, OR OTHER LOSSES,
+     COSTS, EXPENSES, OR DAMAGES ARISING OUT OF THIS PUBLIC LICENSE OR
+     USE OF THE LICENSED MATERIAL, EVEN IF THE LICENSOR HAS BEEN
+     ADVISED OF THE POSSIBILITY OF SUCH LOSSES, COSTS, EXPENSES, OR
+     DAMAGES. WHERE A LIMITATION OF LIABILITY IS NOT ALLOWED IN FULL OR
+     IN PART, THIS LIMITATION MAY NOT APPLY TO YOU.
+
+  c. The disclaimer of warranties and limitation of liability provided
+     above shall be interpreted in a manner that, to the extent
+     possible, most closely approximates an absolute disclaimer and
+     waiver of all liability.
+
+
+Section 6 -- Term and Termination.
+
+  a. This Public License applies for the term of the Copyright and
+     Similar Rights licensed here. However, if You fail to comply with
+     this Public License, then Your rights under this Public License
+     terminate automatically.
+
+  b. Where Your right to use the Licensed Material has terminated under
+     Section 6(a), it reinstates:
+
+       1. automatically as of the date the violation is cured, provided
+          it is cured within 30 days of Your discovery of the
+          violation; or
+
+       2. upon express reinstatement by the Licensor.
+
+     For the avoidance of doubt, this Section 6(b) does not affect any
+     right the Licensor may have to seek remedies for Your violations
+     of this Public License.
+
+  c. For the avoidance of doubt, the Licensor may also offer the
+     Licensed Material under separate terms or conditions or stop
+     distributing the Licensed Material at any time; however, doing so
+     will not terminate this Public License.
+
+  d. Sections 1, 5, 6, 7, and 8 survive termination of this Public
+     License.
+
+
+Section 7 -- Other Terms and Conditions.
+
+  a. The Licensor shall not be bound by any additional or different
+     terms or conditions communicated by You unless expressly agreed.
+
+  b. Any arrangements, understandings, or agreements regarding the
+     Licensed Material not stated herein are separate from and
+     independent of the terms and conditions of this Public License.
+
+
+Section 8 -- Interpretation.
+
+  a. For the avoidance of doubt, this Public License does not, and
+     shall not be interpreted to, reduce, limit, restrict, or impose
+     conditions on any use of the Licensed Material that could lawfully
+     be made without permission under this Public License.
+
+  b. To the extent possible, if any provision of this Public License is
+     deemed unenforceable, it shall be automatically reformed to the
+     minimum extent necessary to make it enforceable. If the provision
+     cannot be reformed, it shall be severed from this Public License
+     without affecting the enforceability of the remaining terms and
+     conditions.
+
+  c. No term or condition of this Public License will be waived and no
+     failure to comply consented to unless expressly agreed to by the
+     Licensor.
+
+  d. Nothing in this Public License constitutes or may be interpreted
+     as a limitation upon, or waiver of, any privileges and immunities
+     that apply to the Licensor or You, including from the legal
+     processes of any jurisdiction or authority.
+
+
+=======================================================================
+
+Creative Commons is not a party to its public
+licenses. Notwithstanding, Creative Commons may elect to apply one of
+its public licenses to material it publishes and in those instances
+will be considered the “Licensor.” The text of the Creative Commons
+public licenses is dedicated to the public domain under the CC0 Public
+Domain Dedication. Except for the limited purpose of indicating that
+material is shared under a Creative Commons public license or as
+otherwise permitted by the Creative Commons policies published at
+creativecommons.org/policies, Creative Commons does not authorize the
+use of the trademark "Creative Commons" or any other trademark or logo
+of Creative Commons without its prior written consent including,
+without limitation, in connection with any unauthorized modifications
+to any of its public licenses or any other arrangements,
+understandings, or agreements concerning use of licensed material. For
+the avoidance of doubt, this paragraph does not form part of the
+public licenses.
+
+Creative Commons may be contacted at creativecommons.org.
+

provenance.json

@@ -0,0 +1,27 @@
+{
+  "name": "SecJev-4B",
+  "release_version": "1.2.0",
+  "selected_epoch": 3,
+  "selection_split": "development",
+  "selection_metric": "macro_task_balanced_accuracy",
+  "test_used_for_selection": false,
+  "base_model": "Qwen/Qwen3.5-4B-Base",
+  "base_revision": "1001bb4d826a52d1f399e183466143f4da7b741b",
+  "kev_source_commit": "90990a5fac2995b9faa3190f7d437e84f2067768",
+  "initialization": "Official jaredpalmer/kev-4b@485ace8703592fcf405488b262449990824cfed1 -> three SecJev-Corpus epochs",
+  "official_kev_4b_checkpoint": true,
+  "corpus": "SecJev-Corpus v1.0.0",
+  "temperature": 1.334968103387446,
+  "calibration_questions": 14821,
+  "trained_checkpoint_hashes": {
+    "adapter_model.safetensors": "0087e92e59b987fea721124d1a3f61b1aca26b3ac01b974032abb60799e7b175",
+    "head.pt": "eacdfb24230ef81859f56e164778cc55436298bedc966390aafd47560c8632bf",
+    "adapter_config.json": "94aad580ce54a1dfcec0803beaba2c471e17b790f10efe7a481125c81fe1dd02"
+  },
+  "export_validation": {
+    "adapter_bytes_identical": true,
+    "decision_head_tensors_identical": true,
+    "temperature_identical": true,
+    "only_head_metadata_paths_made_portable": true
+  }
+}

redo-provenance.json

@@ -0,0 +1,30 @@
+{
+  "cfg": "spot1",
+  "host": "jev-train-1 (DigitalOcean gpu-h100x1-80gb, 159.65.192.220)",
+  "wall_utc": "2026-11-03 22:53:41\u201323:03:28 UTC",
+  "train_file": "data/bench/train_spot1_t.jsonl",
+  "dev_file": "data/bench/dev_spot1_t.jsonl",
+  "epochs": 1,
+  "lr": 3e-05,
+  "batch_questions": 32,
+  "tok_budget": 4096,
+  "trainable_params": 33776128,
+  "dev_reeval": {
+    "injection": {
+      "n": 500,
+      "pos": 250,
+      "auc": 0.9327,
+      "tpr@0.5": 0.78,
+      "fpr@0.5": 0.04
+    }
+  },
+  "checks": {
+    "adapter_tensors": 496,
+    "adapter_sample_dtypes": [
+      "F32"
+    ],
+    "head_entries": 18,
+    "adapter_sha256": "7329ca6cdc831cd32e2e8ec9266763c5255ba9cfda35a8d8536f2e03234a85ab",
+    "head_sha256": "0955d746a517b2d18762ebadd79d2c3ff6fef7209853f15580be917aeec3f753"
+  }
+}

redo-training-log.jsonl

@@ -0,0 +1,11 @@
+{"epoch": 0, "dev_n": 500, "dev_pos": 500, "auc": 0.0, "eval_seconds": 14.9, "eval_batched": true, "tpr@0.5": 0.0, "fpr@0.5": 0.0, "tpr@0.9": 0.0, "fpr@0.9": 0.0}
+{"step": 10, "of": 94, "loss": 0.5835317457094789, "lr": 2.9790281926131377e-05, "min": 0.9546189427375793}
+{"step": 20, "of": 94, "loss": 0.7127825047355145, "lr": 2.815142971168314e-05, "min": 1.9240359663963318}
+{"step": 30, "of": 94, "loss": 0.6041857600212097, "lr": 2.507581554661477e-05, "min": 2.8398680011431376}
+{"step": 40, "of": 94, "loss": 0.549810940399766, "lr": 2.0942699789226136e-05, "min": 3.844523346424103}
+{"step": 50, "of": 94, "loss": 0.29305092990398407, "lr": 1.6261745511317765e-05, "min": 4.810229921340943}
+{"step": 60, "of": 94, "loss": 0.1917044112469739, "lr": 1.1610170888299289e-05, "min": 5.762105715274811}
+{"step": 70, "of": 94, "loss": 0.370209006476216, "lr": 7.561571229433195e-06, "min": 6.755159330368042}
+{"step": 80, "of": 94, "loss": 0.29663539375178516, "lr": 4.615187751213268e-06, "min": 7.711248743534088}
+{"step": 90, "of": 94, "loss": 0.31297091022133827, "lr": 3.134345108054867e-06, "min": 8.687825175126394}
+{"epoch": 1, "dev_n": 500, "dev_pos": 500, "auc": 0.0, "eval_seconds": 13.4, "eval_batched": true, "tpr@0.5": 0.0, "fpr@0.5": 0.0, "tpr@0.9": 0.0, "fpr@0.9": 0.0}

redo-training.stdout.txt

@@ -0,0 +1,28 @@
+device cuda
+dtype fp32
+[transformers] The fast path is not available because one of the required library is not installed. Falling back to torch implementation. To install follow https://github.com/fla-org/flash-linear-attention#installation and https://github.com/Dao-AILab/causal-conv1d
+
+trainable 33776128
+train encoded 3000 skipped_over_window 0
+dev encoded 500 skipped_over_window 0
+/root/venv/lib/python3.12/site-packages/triton/backends/nvidia/driver.c:9:10: fatal error: Python.h: No such file or directory
+    9 | #include <Python.h>
+      |          ^~~~~~~~~~
+compilation terminated.
+EVAL {"epoch": 0, "dev_n": 500, "dev_pos": 500, "auc": 0.0, "eval_seconds": 14.9, "eval_batched": true, "tpr@0.5": 0.0, "fpr@0.5": 0.0, "tpr@0.9": 0.0, "fpr@0.9": 0.0}
+/root/work/ft_action.py:173: UserWarning: Converting a tensor with requires_grad=True to a scalar may lead to unexpected behavior.
+Consider using tensor.detach() first. (Triggered internally at /__w/pytorch/pytorch/torch/csrc/autograd/generated/python_variable_methods.cpp:820.)
+  (loss / nq).backward(); tot += float(loss)
+STEP {"step": 10, "of": 94, "loss": 0.5835317457094789, "lr": 2.9790281926131377e-05, "min": 0.9546189427375793}
+STEP {"step": 20, "of": 94, "loss": 0.7127825047355145, "lr": 2.815142971168314e-05, "min": 1.9240359663963318}
+STEP {"step": 30, "of": 94, "loss": 0.6041857600212097, "lr": 2.507581554661477e-05, "min": 2.8398680011431376}
+STEP {"step": 40, "of": 94, "loss": 0.549810940399766, "lr": 2.0942699789226136e-05, "min": 3.844523346424103}
+STEP {"step": 50, "of": 94, "loss": 0.29305092990398407, "lr": 1.6261745511317765e-05, "min": 4.810229921340943}
+STEP {"step": 60, "of": 94, "loss": 0.1917044112469739, "lr": 1.1610170888299289e-05, "min": 5.762105715274811}
+STEP {"step": 70, "of": 94, "loss": 0.370209006476216, "lr": 7.561571229433195e-06, "min": 6.755159330368042}
+STEP {"step": 80, "of": 94, "loss": 0.29663539375178516, "lr": 4.615187751213268e-06, "min": 7.711248743534088}
+STEP {"step": 90, "of": 94, "loss": 0.31297091022133827, "lr": 3.134345108054867e-06, "min": 8.687825175126394}
+Warning: You are sending unauthenticated requests to the HF Hub. Please set a HF_TOKEN to enable higher rate limits and faster downloads.
+SAVED /root/work/ft/secjev_spot1/epoch1
+EVAL {"epoch": 1, "dev_n": 500, "dev_pos": 500, "auc": 0.0, "eval_seconds": 13.4, "eval_batched": true, "tpr@0.5": 0.0, "fpr@0.5": 0.0, "tpr@0.9": 0.0, "fpr@0.9": 0.0}
+TRAIN_DONE

requirements.txt

@@ -0,0 +1,11 @@
+torch==2.6.0
+transformers==5.17.0
+peft==0.21.0
+triton==3.2.0; sys_platform == "linux"
+flash-linear-attention==0.4.2
+fla-core==0.4.2
+accelerate
+datasets
+pydantic>=2.9
+huggingface-hub
+safetensors

security-training-config.json

@@ -0,0 +1,27 @@
+{
+  "epochs": 3,
+  "initialization": "Official jaredpalmer/kev-4b@485ace8703592fcf405488b262449990824cfed1, then SecJev-Corpus",
+  "optimizer": "fresh AdamW, no loaded optimizer moments; retained continuously through all epochs",
+  "global_question_batches": {
+    "1": 128,
+    "2": 192,
+    "3": 192
+  },
+  "steps_per_epoch": [
+    929,
+    619,
+    619
+  ],
+  "lr": "epoch1 peak2e-5 with5%warmup/cosine to2e-6; epochs2+3 peak1e-5 with5%warmup from2e-6/cosine to1e-6",
+  "objective": "per-scene mean cross entropy",
+  "weight_decay": 0.01,
+  "grad_clip": 1.0,
+  "world_size": 9,
+  "comparison": "same security recipe as 0.8B/2B, official Kev-4B warm start, nine GPUs with memory-bounded microbatches",
+  "train_questions_per_epoch": 118818,
+  "replay": 0,
+  "augmentation": false,
+  "no_truncation": true,
+  "context_limit": 8192,
+  "test_use": "all3finalepochs predeclared for audit; selection uses full development only"
+}

tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:06b9509352d2af50381ab2247e083b80d32d5c0aba91c272ca9ff729b6a0e523
+size 19989325

tokenizer_config.json

@@ -0,0 +1,32 @@
+{
+  "add_prefix_space": false,
+  "audio_bos_token": "<|audio_start|>",
+  "audio_eos_token": "<|audio_end|>",
+  "audio_token": "<|audio_pad|>",
+  "backend": "tokenizers",
+  "bos_token": null,
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|endoftext|>",
+  "errors": "replace",
+  "image_token": "<|image_pad|>",
+  "is_local": true,
+  "local_files_only": true,
+  "model_max_length": 262144,
+  "model_specific_special_tokens": {
+    "audio_bos_token": "<|audio_start|>",
+    "audio_eos_token": "<|audio_end|>",
+    "audio_token": "<|audio_pad|>",
+    "image_token": "<|image_pad|>",
+    "video_token": "<|video_pad|>",
+    "vision_bos_token": "<|vision_start|>",
+    "vision_eos_token": "<|vision_end|>"
+  },
+  "pad_token": "<|endoftext|>",
+  "pretokenize_regex": "(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\r\\n\\p{L}\\p{N}]?[\\p{L}\\p{M}]+|\\p{N}| ?[^\\s\\p{L}\\p{M}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+",
+  "split_special_tokens": false,
+  "tokenizer_class": "Qwen2Tokenizer",
+  "unk_token": null,
+  "video_token": "<|video_pad|>",
+  "vision_bos_token": "<|vision_start|>",
+  "vision_eos_token": "<|vision_end|>"
+}