File size: 1,266 Bytes
ac68cef | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 | {
"status": "source_pytorch_reference_not_full_onnx_rerun",
"dataset": "eval/heldout supported phone/address records",
"records_total": 2000,
"routing": "learned_router",
"source_model": "epoch-100 last.pt; heldout was excluded from training and intermediate checkpointing, then used post hoc to compare e79 best.pt with e100 last.pt; e100 last.pt was selected for higher heldout answer exact; the source checkpoint container is not distributed in this package",
"onnx_fp32_agreement_note": "The source PyTorch reference and the final ONNX FP32 rerun agree exactly on all nine reported accuracy metrics across the overall, address, and phone subsets.",
"summary": {
"address": {
"n": 1032,
"answer_exact": 0.9496124031007752,
"recomputed_n": 1032,
"recomputed_answer_exact": 0.9486434108527132,
"target_exact": 0.015503875968992248
},
"overall": {
"n": 2000,
"answer_exact": 0.9705,
"recomputed_n": 2000,
"recomputed_answer_exact": 0.97,
"target_exact": 0.453
},
"phone_number": {
"n": 968,
"answer_exact": 0.9927685950413223,
"recomputed_n": 968,
"recomputed_answer_exact": 0.9927685950413223,
"target_exact": 0.9194214876033058
}
}
}
|