Independent re-verification 2026-09-13: fixed-harness fp16 evals, all merges 11/11 plan-valid, matching 2026-08-28 MEASURED receipt
Browse files
eval_verified_2026-09-13.json
ADDED
|
@@ -0,0 +1,45 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"label": "MEASURED-REVERIFICATION",
|
| 3 |
+
"repo": "SZLHOLDINGS/KHIPU-R2",
|
| 4 |
+
"verifiedAt": "2026-09-13T23:31:00Z",
|
| 5 |
+
"verifiedBy": "owner host (Windows, CPU-only torch)",
|
| 6 |
+
"harness": "szl_khipu_eval.py - fixed 2026-09-13",
|
| 7 |
+
"harnessFix": "generate_plan called tok.decode on a 2-D batch tensor and indexed the result as a string; it returned prompt characters (always 'y') instead of model output. Fixed to tok.batch_decode(...)[0] with prompt-length stripping. All prior fp16 eval scores produced by the buggy harness are void.",
|
| 8 |
+
"method": "transformers AutoModelForCausalLM fp16 on CPU, temperature 0 (do_sample=False), max_new_tokens 1024, held-out curriculum sha256-verified from SZLHOLDINGS/SZL-Khipu-1.5B/repro/",
|
| 9 |
+
"runs": [
|
| 10 |
+
{
|
| 11 |
+
"model": "merged/KHIPU-R2-fixed - fp16 merge of this adapter onto Qwen/Qwen2.5-1.5B-Instruct (rebuilt 2026-09-13)",
|
| 12 |
+
"evaluatedAt": "2026-09-13T23:31:55Z",
|
| 13 |
+
"planValid": "11/11",
|
| 14 |
+
"grounding": "5/5",
|
| 15 |
+
"abstain": "3/6",
|
| 16 |
+
"hallucinatedCitations": 0
|
| 17 |
+
},
|
| 18 |
+
{
|
| 19 |
+
"model": "merged/KHIPU-R2-fixed rebuilt again 19:56 local (third confirmation)",
|
| 20 |
+
"evaluatedAt": "2026-09-13T23:57:26Z",
|
| 21 |
+
"planValid": "11/11",
|
| 22 |
+
"grounding": "5/5",
|
| 23 |
+
"abstain": "3/6",
|
| 24 |
+
"hallucinatedCitations": 0
|
| 25 |
+
},
|
| 26 |
+
{
|
| 27 |
+
"model": "merged/KHIPU-R2 - published fp32 merge, weights unchanged",
|
| 28 |
+
"evaluatedAt": "2026-09-13T23:41:59Z",
|
| 29 |
+
"planValid": "11/11",
|
| 30 |
+
"grounding": "5/5",
|
| 31 |
+
"abstain": "3/6",
|
| 32 |
+
"hallucinatedCitations": 0
|
| 33 |
+
}
|
| 34 |
+
],
|
| 35 |
+
"agreement": "All runs reproduce the 2026-08-28 hub eval_measured.json receipt exactly (plan-valid 11/11, grounding 5/5, abstain 3/6, hallucinated citations 0).",
|
| 36 |
+
"notes": [
|
| 37 |
+
"GPU UNAVAILABLE - CPU-only evaluation",
|
| 38 |
+
"This is verification evidence, not model promotion; publication and autonomy gates remain closed",
|
| 39 |
+
"adapter sha256 e44d53f29f2d443598e06d6c0441557fd3a5010888c7aa97b56ec3c0e050d349 (per card)"
|
| 40 |
+
],
|
| 41 |
+
"localArtifacts": [
|
| 42 |
+
"reports/khipu_fixed.json",
|
| 43 |
+
"reports/khipu_published_rescored.json"
|
| 44 |
+
]
|
| 45 |
+
}
|