| { |
| "onnx_artifact": { |
| "format": "tiny_receipt_vqa_split_onnx_v1", |
| "opset": 18, |
| "validation": { |
| "status": "passed", |
| "scope": "graph-level numerical parity on dynamic shapes", |
| "exporter_validation": { |
| "split_vs_model_max_abs_difference": 4.0531158447265625e-06, |
| "onnx_checker": "passed", |
| "onnxruntime": "1.23.2", |
| "provider": "CPUExecutionProvider", |
| "validation_shapes": { |
| "image": [ |
| 1, |
| 1, |
| 320, |
| 672 |
| ], |
| "question_ids": [ |
| 1, |
| 11 |
| ], |
| "decoder_input_ids": [ |
| 1, |
| 3 |
| ] |
| }, |
| "explicit_family_ids_validated": [ |
| 0, |
| 1, |
| 2, |
| 3, |
| 4, |
| 5, |
| 6, |
| 7 |
| ], |
| "explicit_family_encoder_memory_max_abs_difference": 1.996755599975586e-05, |
| "explicit_family_decoder_logits_max_abs_difference": 1.049041748046875e-05, |
| "encoder_memory_max_abs_difference": 1.9311904907226562e-05, |
| "encoder_mask_max_abs_difference": 0.0, |
| "router_logits_max_abs_difference": 1.1444091796875e-05, |
| "selected_family_ids_max_abs_difference": 0.0, |
| "decoder_logits_max_abs_difference": 6.198883056640625e-06 |
| }, |
| "lora_merge": { |
| "enabled": true, |
| "layers_merged": 20, |
| "module_names": [ |
| "encoder.layers.0.linear1", |
| "encoder.layers.0.linear2", |
| "encoder.layers.1.linear1", |
| "encoder.layers.1.linear2", |
| "encoder.layers.2.linear1", |
| "encoder.layers.2.linear2", |
| "encoder.layers.3.linear1", |
| "encoder.layers.3.linear2", |
| "encoder.layers.4.linear1", |
| "encoder.layers.4.linear2", |
| "encoder.layers.5.linear1", |
| "encoder.layers.5.linear2", |
| "decoder.layers.0.linear1", |
| "decoder.layers.0.linear2", |
| "decoder.layers.1.linear1", |
| "decoder.layers.1.linear2", |
| "decoder.layers.2.linear1", |
| "decoder.layers.2.linear2", |
| "decoder.layers.3.linear1", |
| "decoder.layers.3.linear2" |
| ], |
| "max_abs_difference": 5.245208740234375e-06 |
| } |
| }, |
| "variants": { |
| "fp32": { |
| "encoder": "encoder_model.onnx", |
| "decoder": "decoder_model.onnx", |
| "weight_dtype": "float32", |
| "activation_dtype": "float32", |
| "compute_dtype": "float32", |
| "encoder_sha256": "0c6469ef75eeb266c3dbc77c414d658f98f704264b72bdc7dfeed9cf37217db7", |
| "encoder_bytes": 58500824, |
| "decoder_sha256": "3fff897777fa5e9805c01a6779812f4adf33e1294f4e2003dd823c04a3300b05", |
| "decoder_bytes": 29924405 |
| }, |
| "int8_w8a8": { |
| "format": "tiny_receipt_vqa_onnx_w8a8_u8s8_qdq_v1", |
| "encoder": "encoder_model_int8.onnx", |
| "decoder": "decoder_model_int8.onnx", |
| "mode": "static_w8a8_qdq", |
| "quant_format": "QDQ", |
| "scheme": "U8S8", |
| "weight_dtype": "int8", |
| "weight_granularity": "per_output_channel", |
| "activation_dtype": "uint8", |
| "activation_granularity": "per_tensor", |
| "accumulation_dtype": "int32", |
| "operators": [ |
| "Conv", |
| "MatMul", |
| "Gemm" |
| ], |
| "encoder_sha256": "924a60cfa8172456131aedb2e672af2ad1de7e4fa975bf14f4837a49e09cec94", |
| "encoder_bytes": 17808595, |
| "decoder_sha256": "24dbf7411719fea928f1f249db9b981eedf4b3af6751bec0970f242ca3f99d0a", |
| "decoder_bytes": 10917677, |
| "calibration": { |
| "source": "synthetic_training_split", |
| "heldout_used": false, |
| "records": 256, |
| "decoder_feeds": 768, |
| "receipt_limit": 1024, |
| "prefixes_per_record": 3, |
| "family_counts": { |
| "address": 43, |
| "item_lookup": 42, |
| "item_math": 42, |
| "item_row": 43, |
| "phone": 43, |
| "store": 43 |
| }, |
| "seed": 71 |
| }, |
| "validation": { |
| "onnx_checker": "passed", |
| "runtime": { |
| "onnxruntime": "passed", |
| "provider": "CPUExecutionProvider", |
| "family_ids": [ |
| -1, |
| 0, |
| 1, |
| 2, |
| 3, |
| 4, |
| 5, |
| 6, |
| 7 |
| ], |
| "dynamic_prefix_lengths": [ |
| 1, |
| 10, |
| 20 |
| ], |
| "onnxruntime_version": "1.23.2" |
| }, |
| "optimized_graph": { |
| "encoder": { |
| "nodes": { |
| "Add": 69, |
| "ArgMax": 1, |
| "Cast": 1, |
| "Clip": 2, |
| "Concat": 24, |
| "DequantizeLinear": 67, |
| "Div": 9, |
| "Equal": 1, |
| "Erf": 8, |
| "Expand": 3, |
| "Gather": 5, |
| "GreaterOrEqual": 1, |
| "InstanceNormalization": 13, |
| "LayerNormalization": 12, |
| "Mul": 44, |
| "Not": 1, |
| "QGemm": 8, |
| "QLinearConv": 13, |
| "QLinearMatMul": 32, |
| "QuantizeLinear": 61, |
| "QuickGelu": 13, |
| "ReduceSum": 2, |
| "Reshape": 100, |
| "Shape": 22, |
| "Slice": 19, |
| "Softmax": 6, |
| "Split": 6, |
| "Squeeze": 26, |
| "Transpose": 77, |
| "Unsqueeze": 9, |
| "Where": 2 |
| }, |
| "initializers": { |
| "BOOL": 1, |
| "FLOAT": 240, |
| "INT32": 8, |
| "INT64": 42, |
| "INT8": 78, |
| "UINT8": 120 |
| } |
| }, |
| "decoder": { |
| "nodes": { |
| "Add": 51, |
| "Clip": 1, |
| "Concat": 31, |
| "DequantizeLinear": 40, |
| "Div": 5, |
| "Equal": 1, |
| "Erf": 5, |
| "Expand": 6, |
| "Gather": 5, |
| "LayerNormalization": 13, |
| "Mul": 28, |
| "QGemm": 8, |
| "QLinearMatMul": 39, |
| "QuantizeLinear": 54, |
| "Reshape": 97, |
| "Shape": 14, |
| "Slice": 25, |
| "Softmax": 8, |
| "Split": 8, |
| "Squeeze": 30, |
| "Transpose": 67, |
| "Trilu": 1, |
| "Unsqueeze": 11, |
| "Where": 3 |
| }, |
| "initializers": { |
| "BOOL": 1, |
| "FLOAT": 186, |
| "INT32": 8, |
| "INT64": 44, |
| "INT8": 58, |
| "UINT8": 109 |
| } |
| } |
| }, |
| "heldout": { |
| "status": "passed", |
| "dataset": "eval/heldout supported phone/address records", |
| "records_total": 2000, |
| "routing": "learned_router", |
| "evaluation_mode": "final_split_onnx_files", |
| "runtime": { |
| "library": "onnxruntime", |
| "version": "1.23.2", |
| "provider": "CUDAExecutionProvider" |
| }, |
| "batch_size": 24, |
| "artifacts": { |
| "fp32": { |
| "encoder": { |
| "filename": "encoder_model.onnx", |
| "bytes": 58500824, |
| "sha256": "0c6469ef75eeb266c3dbc77c414d658f98f704264b72bdc7dfeed9cf37217db7" |
| }, |
| "decoder": { |
| "filename": "decoder_model.onnx", |
| "bytes": 29924405, |
| "sha256": "3fff897777fa5e9805c01a6779812f4adf33e1294f4e2003dd823c04a3300b05" |
| } |
| }, |
| "int8": { |
| "encoder": { |
| "filename": "encoder_model_int8.onnx", |
| "bytes": 17808595, |
| "sha256": "924a60cfa8172456131aedb2e672af2ad1de7e4fa975bf14f4837a49e09cec94" |
| }, |
| "decoder": { |
| "filename": "decoder_model_int8.onnx", |
| "bytes": 10917677, |
| "sha256": "24dbf7411719fea928f1f249db9b981eedf4b3af6751bec0970f242ca3f99d0a" |
| } |
| } |
| }, |
| "summary": { |
| "fp32": { |
| "address": { |
| "n": 1032, |
| "answer_exact": 0.9496124031007752, |
| "recomputed_n": 1032, |
| "recomputed_answer_exact": 0.9486434108527132, |
| "target_exact": 0.015503875968992248 |
| }, |
| "overall": { |
| "n": 2000, |
| "answer_exact": 0.9705, |
| "recomputed_n": 2000, |
| "recomputed_answer_exact": 0.97, |
| "target_exact": 0.453 |
| }, |
| "phone_number": { |
| "n": 968, |
| "answer_exact": 0.9927685950413223, |
| "recomputed_n": 968, |
| "recomputed_answer_exact": 0.9927685950413223, |
| "target_exact": 0.9194214876033058 |
| } |
| }, |
| "int8": { |
| "address": { |
| "n": 1032, |
| "answer_exact": 0.9437984496124031, |
| "recomputed_n": 1032, |
| "recomputed_answer_exact": 0.9457364341085271, |
| "target_exact": 0.01065891472868217 |
| }, |
| "overall": { |
| "n": 2000, |
| "answer_exact": 0.968, |
| "recomputed_n": 2000, |
| "recomputed_answer_exact": 0.969, |
| "target_exact": 0.4505 |
| }, |
| "phone_number": { |
| "n": 968, |
| "answer_exact": 0.993801652892562, |
| "recomputed_n": 968, |
| "recomputed_answer_exact": 0.993801652892562, |
| "target_exact": 0.9194214876033058 |
| } |
| } |
| }, |
| "comparison": { |
| "answer_agreement": 0.9925, |
| "changed_predictions": 388, |
| "n": 2000, |
| "prediction_exact_agreement": 0.806, |
| "selected_family_agreement": 1.0 |
| }, |
| "timing": { |
| "fp32": { |
| "encoder_seconds": 4.9500676593743265, |
| "decoder_seconds": 20.797384655103087, |
| "total_seconds": 25.747452314477414, |
| "decoder_calls": 2953, |
| "providers": [ |
| "CUDAExecutionProvider", |
| "CPUExecutionProvider" |
| ], |
| "encoder": "encoder_model.onnx", |
| "decoder": "decoder_model.onnx" |
| }, |
| "int8": { |
| "encoder_seconds": 5.913507376797497, |
| "decoder_seconds": 26.702236368786544, |
| "total_seconds": 32.61574374558404, |
| "decoder_calls": 2944, |
| "providers": [ |
| "CUDAExecutionProvider", |
| "CPUExecutionProvider" |
| ], |
| "encoder": "encoder_model_int8.onnx", |
| "decoder": "decoder_model_int8.onnx" |
| } |
| }, |
| "timing_note": "Static W8A8 speed depends on QDQ fusion and the selected execution provider." |
| } |
| } |
| } |
| } |
| }, |
| "heldout_metrics": { |
| "status": "source_pytorch_reference_not_full_onnx_rerun", |
| "dataset": "eval/heldout supported phone/address records", |
| "records_total": 2000, |
| "routing": "learned_router", |
| "source_model": "epoch-100 last.pt; heldout was excluded from training and intermediate checkpointing, then used post hoc to compare e79 best.pt with e100 last.pt; e100 last.pt was selected for higher heldout answer exact; the source checkpoint container is not distributed in this package", |
| "onnx_fp32_agreement_note": "The source PyTorch reference and the final ONNX FP32 rerun agree exactly on all nine reported accuracy metrics across the overall, address, and phone subsets.", |
| "summary": { |
| "address": { |
| "n": 1032, |
| "answer_exact": 0.9496124031007752, |
| "recomputed_n": 1032, |
| "recomputed_answer_exact": 0.9486434108527132, |
| "target_exact": 0.015503875968992248 |
| }, |
| "overall": { |
| "n": 2000, |
| "answer_exact": 0.9705, |
| "recomputed_n": 2000, |
| "recomputed_answer_exact": 0.97, |
| "target_exact": 0.453 |
| }, |
| "phone_number": { |
| "n": 968, |
| "answer_exact": 0.9927685950413223, |
| "recomputed_n": 968, |
| "recomputed_answer_exact": 0.9927685950413223, |
| "target_exact": 0.9194214876033058 |
| } |
| } |
| }, |
| "int8_w8a8_heldout": { |
| "status": "passed", |
| "dataset": "eval/heldout supported phone/address records", |
| "records_total": 2000, |
| "routing": "learned_router", |
| "evaluation_mode": "final_split_onnx_files", |
| "runtime": { |
| "library": "onnxruntime", |
| "version": "1.23.2", |
| "provider": "CUDAExecutionProvider" |
| }, |
| "batch_size": 24, |
| "artifacts": { |
| "fp32": { |
| "encoder": { |
| "filename": "encoder_model.onnx", |
| "bytes": 58500824, |
| "sha256": "0c6469ef75eeb266c3dbc77c414d658f98f704264b72bdc7dfeed9cf37217db7" |
| }, |
| "decoder": { |
| "filename": "decoder_model.onnx", |
| "bytes": 29924405, |
| "sha256": "3fff897777fa5e9805c01a6779812f4adf33e1294f4e2003dd823c04a3300b05" |
| } |
| }, |
| "int8": { |
| "encoder": { |
| "filename": "encoder_model_int8.onnx", |
| "bytes": 17808595, |
| "sha256": "924a60cfa8172456131aedb2e672af2ad1de7e4fa975bf14f4837a49e09cec94" |
| }, |
| "decoder": { |
| "filename": "decoder_model_int8.onnx", |
| "bytes": 10917677, |
| "sha256": "24dbf7411719fea928f1f249db9b981eedf4b3af6751bec0970f242ca3f99d0a" |
| } |
| } |
| }, |
| "summary": { |
| "fp32": { |
| "address": { |
| "n": 1032, |
| "answer_exact": 0.9496124031007752, |
| "recomputed_n": 1032, |
| "recomputed_answer_exact": 0.9486434108527132, |
| "target_exact": 0.015503875968992248 |
| }, |
| "overall": { |
| "n": 2000, |
| "answer_exact": 0.9705, |
| "recomputed_n": 2000, |
| "recomputed_answer_exact": 0.97, |
| "target_exact": 0.453 |
| }, |
| "phone_number": { |
| "n": 968, |
| "answer_exact": 0.9927685950413223, |
| "recomputed_n": 968, |
| "recomputed_answer_exact": 0.9927685950413223, |
| "target_exact": 0.9194214876033058 |
| } |
| }, |
| "int8": { |
| "address": { |
| "n": 1032, |
| "answer_exact": 0.9437984496124031, |
| "recomputed_n": 1032, |
| "recomputed_answer_exact": 0.9457364341085271, |
| "target_exact": 0.01065891472868217 |
| }, |
| "overall": { |
| "n": 2000, |
| "answer_exact": 0.968, |
| "recomputed_n": 2000, |
| "recomputed_answer_exact": 0.969, |
| "target_exact": 0.4505 |
| }, |
| "phone_number": { |
| "n": 968, |
| "answer_exact": 0.993801652892562, |
| "recomputed_n": 968, |
| "recomputed_answer_exact": 0.993801652892562, |
| "target_exact": 0.9194214876033058 |
| } |
| } |
| }, |
| "comparison": { |
| "answer_agreement": 0.9925, |
| "changed_predictions": 388, |
| "n": 2000, |
| "prediction_exact_agreement": 0.806, |
| "selected_family_agreement": 1.0 |
| }, |
| "timing": { |
| "fp32": { |
| "encoder_seconds": 4.9500676593743265, |
| "decoder_seconds": 20.797384655103087, |
| "total_seconds": 25.747452314477414, |
| "decoder_calls": 2953, |
| "providers": [ |
| "CUDAExecutionProvider", |
| "CPUExecutionProvider" |
| ], |
| "encoder": "encoder_model.onnx", |
| "decoder": "decoder_model.onnx" |
| }, |
| "int8": { |
| "encoder_seconds": 5.913507376797497, |
| "decoder_seconds": 26.702236368786544, |
| "total_seconds": 32.61574374558404, |
| "decoder_calls": 2944, |
| "providers": [ |
| "CUDAExecutionProvider", |
| "CPUExecutionProvider" |
| ], |
| "encoder": "encoder_model_int8.onnx", |
| "decoder": "decoder_model_int8.onnx" |
| } |
| }, |
| "timing_note": "Static W8A8 speed depends on QDQ fusion and the selected execution provider." |
| }, |
| "metric_note": "heldout_metrics is the selected source-model reference; int8_w8a8_heldout, when present, is a direct run of the final split FP32 and INT8 ONNX files." |
| } |
|
|
