| { |
| "status": "source_pytorch_reference_not_full_onnx_rerun", |
| "dataset": "eval/heldout supported phone/address records", |
| "records_total": 2000, |
| "routing": "learned_router", |
| "source_model": "epoch-100 last.pt; heldout was excluded from training and intermediate checkpointing, then used post hoc to compare e79 best.pt with e100 last.pt; e100 last.pt was selected for higher heldout answer exact; the source checkpoint container is not distributed in this package", |
| "onnx_fp32_agreement_note": "The source PyTorch reference and the final ONNX FP32 rerun agree exactly on all nine reported accuracy metrics across the overall, address, and phone subsets.", |
| "summary": { |
| "address": { |
| "n": 1032, |
| "answer_exact": 0.9496124031007752, |
| "recomputed_n": 1032, |
| "recomputed_answer_exact": 0.9486434108527132, |
| "target_exact": 0.015503875968992248 |
| }, |
| "overall": { |
| "n": 2000, |
| "answer_exact": 0.9705, |
| "recomputed_n": 2000, |
| "recomputed_answer_exact": 0.97, |
| "target_exact": 0.453 |
| }, |
| "phone_number": { |
| "n": 968, |
| "answer_exact": 0.9927685950413223, |
| "recomputed_n": 968, |
| "recomputed_answer_exact": 0.9927685950413223, |
| "target_exact": 0.9194214876033058 |
| } |
| } |
| } |
|
|
