| { |
| "name": "TinyReceiptVQA-StructuredQA-22M-BPE1536-DirectNoValue-LoRA-Router-e100", |
| "format": "tiny_receipt_vqa_split_onnx_v1", |
| "library": "onnxruntime", |
| "opset": 18, |
| "architecture": { |
| "vocab_size": 1536, |
| "d_model": 320, |
| "heads": 8, |
| "enc_layers": 6, |
| "dec_layers": 4, |
| "ff_mult": 4, |
| "dropout": 0.1, |
| "max_q_len": 192, |
| "max_out_len": 192, |
| "img_tokens": 210, |
| "vision_stride": 32, |
| "use_adapters": true, |
| "use_router": true, |
| "adapter_bottleneck": 64, |
| "adapter_families": 8, |
| "lora_r": 8, |
| "lora_alpha": 16.0, |
| "lora_dropout": 0.05, |
| "lora_targets": "transformer" |
| }, |
| "source": { |
| "selection": "epoch-100 last.pt; heldout was excluded from training and intermediate checkpointing, then used post hoc to compare e79 best.pt with e100 last.pt; e100 last.pt was selected for higher heldout answer exact", |
| "target_schema": "direct_answer_no_value_v1", |
| "training_precision": "BF16 autocast; saved source tensors are FP32", |
| "tokenizer": { |
| "type": "byte_fallback_bpe", |
| "version": 1, |
| "vocab_size": 1536, |
| "normalization": "NFC", |
| "tokenizer_hash": "5801826ac9092bdc984210af805d2ff20f0398b80adaf5963ef13b04771a8584" |
| }, |
| "heldout_usage_note": "Heldout was excluded from training and intermediate checkpointing. After training it was used to compare e79 best.pt and e100 last.pt and to select e100 last.pt, so it is a development test rather than a final blind benchmark.", |
| "pytorch_or_safetensors_checkpoint_distributed": false |
| }, |
| "artifacts": { |
| "encoder": { |
| "path": "encoder_model.onnx", |
| "bytes": 58500824, |
| "sha256": "0c6469ef75eeb266c3dbc77c414d658f98f704264b72bdc7dfeed9cf37217db7" |
| }, |
| "decoder": { |
| "path": "decoder_model.onnx", |
| "bytes": 29924405, |
| "sha256": "3fff897777fa5e9805c01a6779812f4adf33e1294f4e2003dd823c04a3300b05" |
| }, |
| "encoder_int8_w8a8": { |
| "path": "encoder_model_int8.onnx", |
| "bytes": 17808595, |
| "sha256": "924a60cfa8172456131aedb2e672af2ad1de7e4fa975bf14f4837a49e09cec94", |
| "quantize_linear_nodes": 120, |
| "dequantize_linear_nodes": 167 |
| }, |
| "decoder_int8_w8a8": { |
| "path": "decoder_model_int8.onnx", |
| "bytes": 10917677, |
| "sha256": "24dbf7411719fea928f1f249db9b981eedf4b3af6751bec0970f242ca3f99d0a", |
| "quantize_linear_nodes": 109, |
| "dequantize_linear_nodes": 146 |
| } |
| }, |
| "evaluation": { |
| "onnx_parity": { |
| "status": "passed", |
| "scope": "graph-level numerical parity on dynamic shapes", |
| "exporter_validation": { |
| "split_vs_model_max_abs_difference": 4.0531158447265625e-06, |
| "onnx_checker": "passed", |
| "onnxruntime": "1.23.2", |
| "provider": "CPUExecutionProvider", |
| "validation_shapes": { |
| "image": [ |
| 1, |
| 1, |
| 320, |
| 672 |
| ], |
| "question_ids": [ |
| 1, |
| 11 |
| ], |
| "decoder_input_ids": [ |
| 1, |
| 3 |
| ] |
| }, |
| "explicit_family_ids_validated": [ |
| 0, |
| 1, |
| 2, |
| 3, |
| 4, |
| 5, |
| 6, |
| 7 |
| ], |
| "explicit_family_encoder_memory_max_abs_difference": 1.996755599975586e-05, |
| "explicit_family_decoder_logits_max_abs_difference": 1.049041748046875e-05, |
| "encoder_memory_max_abs_difference": 1.9311904907226562e-05, |
| "encoder_mask_max_abs_difference": 0.0, |
| "router_logits_max_abs_difference": 1.1444091796875e-05, |
| "selected_family_ids_max_abs_difference": 0.0, |
| "decoder_logits_max_abs_difference": 6.198883056640625e-06 |
| }, |
| "lora_merge": { |
| "enabled": true, |
| "layers_merged": 20, |
| "module_names": [ |
| "encoder.layers.0.linear1", |
| "encoder.layers.0.linear2", |
| "encoder.layers.1.linear1", |
| "encoder.layers.1.linear2", |
| "encoder.layers.2.linear1", |
| "encoder.layers.2.linear2", |
| "encoder.layers.3.linear1", |
| "encoder.layers.3.linear2", |
| "encoder.layers.4.linear1", |
| "encoder.layers.4.linear2", |
| "encoder.layers.5.linear1", |
| "encoder.layers.5.linear2", |
| "decoder.layers.0.linear1", |
| "decoder.layers.0.linear2", |
| "decoder.layers.1.linear1", |
| "decoder.layers.1.linear2", |
| "decoder.layers.2.linear1", |
| "decoder.layers.2.linear2", |
| "decoder.layers.3.linear1", |
| "decoder.layers.3.linear2" |
| ], |
| "max_abs_difference": 5.245208740234375e-06 |
| } |
| }, |
| "int8_w8a8": { |
| "format": "tiny_receipt_vqa_onnx_w8a8_u8s8_qdq_v1", |
| "onnx_checker": "passed", |
| "runtime_validation": { |
| "onnxruntime": "passed", |
| "provider": "CPUExecutionProvider", |
| "family_ids": [ |
| -1, |
| 0, |
| 1, |
| 2, |
| 3, |
| 4, |
| 5, |
| 6, |
| 7 |
| ], |
| "dynamic_prefix_lengths": [ |
| 1, |
| 10, |
| 20 |
| ], |
| "onnxruntime_version": "1.23.2" |
| }, |
| "heldout_evaluation": { |
| "status": "passed", |
| "dataset": "eval/heldout supported phone/address records", |
| "records_total": 2000, |
| "routing": "learned_router", |
| "evaluation_mode": "final_split_onnx_files", |
| "runtime": { |
| "library": "onnxruntime", |
| "version": "1.23.2", |
| "provider": "CUDAExecutionProvider" |
| }, |
| "batch_size": 24, |
| "artifacts": { |
| "fp32": { |
| "encoder": { |
| "filename": "encoder_model.onnx", |
| "bytes": 58500824, |
| "sha256": "0c6469ef75eeb266c3dbc77c414d658f98f704264b72bdc7dfeed9cf37217db7" |
| }, |
| "decoder": { |
| "filename": "decoder_model.onnx", |
| "bytes": 29924405, |
| "sha256": "3fff897777fa5e9805c01a6779812f4adf33e1294f4e2003dd823c04a3300b05" |
| } |
| }, |
| "int8": { |
| "encoder": { |
| "filename": "encoder_model_int8.onnx", |
| "bytes": 17808595, |
| "sha256": "924a60cfa8172456131aedb2e672af2ad1de7e4fa975bf14f4837a49e09cec94" |
| }, |
| "decoder": { |
| "filename": "decoder_model_int8.onnx", |
| "bytes": 10917677, |
| "sha256": "24dbf7411719fea928f1f249db9b981eedf4b3af6751bec0970f242ca3f99d0a" |
| } |
| } |
| }, |
| "summary": { |
| "fp32": { |
| "address": { |
| "n": 1032, |
| "answer_exact": 0.9496124031007752, |
| "recomputed_n": 1032, |
| "recomputed_answer_exact": 0.9486434108527132, |
| "target_exact": 0.015503875968992248 |
| }, |
| "overall": { |
| "n": 2000, |
| "answer_exact": 0.9705, |
| "recomputed_n": 2000, |
| "recomputed_answer_exact": 0.97, |
| "target_exact": 0.453 |
| }, |
| "phone_number": { |
| "n": 968, |
| "answer_exact": 0.9927685950413223, |
| "recomputed_n": 968, |
| "recomputed_answer_exact": 0.9927685950413223, |
| "target_exact": 0.9194214876033058 |
| } |
| }, |
| "int8": { |
| "address": { |
| "n": 1032, |
| "answer_exact": 0.9437984496124031, |
| "recomputed_n": 1032, |
| "recomputed_answer_exact": 0.9457364341085271, |
| "target_exact": 0.01065891472868217 |
| }, |
| "overall": { |
| "n": 2000, |
| "answer_exact": 0.968, |
| "recomputed_n": 2000, |
| "recomputed_answer_exact": 0.969, |
| "target_exact": 0.4505 |
| }, |
| "phone_number": { |
| "n": 968, |
| "answer_exact": 0.993801652892562, |
| "recomputed_n": 968, |
| "recomputed_answer_exact": 0.993801652892562, |
| "target_exact": 0.9194214876033058 |
| } |
| } |
| }, |
| "comparison": { |
| "answer_agreement": 0.9925, |
| "changed_predictions": 388, |
| "n": 2000, |
| "prediction_exact_agreement": 0.806, |
| "selected_family_agreement": 1.0 |
| }, |
| "timing": { |
| "fp32": { |
| "encoder_seconds": 4.9500676593743265, |
| "decoder_seconds": 20.797384655103087, |
| "total_seconds": 25.747452314477414, |
| "decoder_calls": 2953, |
| "providers": [ |
| "CUDAExecutionProvider", |
| "CPUExecutionProvider" |
| ], |
| "encoder": "encoder_model.onnx", |
| "decoder": "decoder_model.onnx" |
| }, |
| "int8": { |
| "encoder_seconds": 5.913507376797497, |
| "decoder_seconds": 26.702236368786544, |
| "total_seconds": 32.61574374558404, |
| "decoder_calls": 2944, |
| "providers": [ |
| "CUDAExecutionProvider", |
| "CPUExecutionProvider" |
| ], |
| "encoder": "encoder_model_int8.onnx", |
| "decoder": "decoder_model_int8.onnx" |
| } |
| }, |
| "timing_note": "Static W8A8 speed depends on QDQ fusion and the selected execution provider." |
| } |
| }, |
| "source_heldout_reference": { |
| "status": "source_pytorch_reference_not_full_onnx_rerun", |
| "dataset": "eval/heldout supported phone/address records", |
| "records_total": 2000, |
| "routing": "learned_router", |
| "source_model": "epoch-100 last.pt; heldout was excluded from training and intermediate checkpointing, then used post hoc to compare e79 best.pt with e100 last.pt; e100 last.pt was selected for higher heldout answer exact; the source checkpoint container is not distributed in this package", |
| "onnx_fp32_agreement_note": "The source PyTorch reference and the final ONNX FP32 rerun agree exactly on all nine reported accuracy metrics across the overall, address, and phone subsets.", |
| "summary": { |
| "address": { |
| "n": 1032, |
| "answer_exact": 0.9496124031007752, |
| "recomputed_n": 1032, |
| "recomputed_answer_exact": 0.9486434108527132, |
| "target_exact": 0.015503875968992248 |
| }, |
| "overall": { |
| "n": 2000, |
| "answer_exact": 0.9705, |
| "recomputed_n": 2000, |
| "recomputed_answer_exact": 0.97, |
| "target_exact": 0.453 |
| }, |
| "phone_number": { |
| "n": 968, |
| "answer_exact": 0.9927685950413223, |
| "recomputed_n": 968, |
| "recomputed_answer_exact": 0.9927685950413223, |
| "target_exact": 0.9194214876033058 |
| } |
| } |
| } |
| }, |
| "license": "Apache-2.0" |
| } |
|
|
