mars-gend / serving.json
ahmedyradwan02's picture
ONNX: batch axis, attribution outputs, float16 file, usual ranges, serving facts
3dd6619 verified
Raw History Blame Contribute Delete
1.02 kB
{
"onnxruntime": "onnxruntime-gpu 1.32.0",
"measured_on": "NVIDIA A100-SXM4-80GB, CUDA execution provider, arena_extend_strategy kSameAsRequested, cudnn_conv_algo_search HEURISTIC; peak is process total in GiB including the CUDA context; time is the median of 5 runs after 2 warm-up runs",
"onnx_opset": 17,
"batch_axis": "dynamic",
"n_frames": 64,
"encoder_chunk": 16,
"outputs": [
"fake_prob",
"logits",
"frame_probs"
],
"files": {
"model.onnx": {
"precision": "float32",
"peak_vram_gb": {
"batch1": 2.82,
"batch2": 3.96
},
"seconds_per_clip": {
"batch1": 0.59,
"batch2": 0.57
}
},
"model.fp16.onnx": {
"precision": "float16",
"peak_vram_gb": {
"batch1": 2.25,
"batch2": 3.43
},
"seconds_per_clip": {
"batch1": 0.47,
"batch2": 0.45
}
}
},
"pytorch_bf16_reference": {
"precision": "bfloat16 autocast",
"peak_vram_gb": {
"batch1": 3.72,
"batch2": 3.88
},
"seconds_per_clip": {
"batch1": 0.22,
"batch2": 0.22
}
}
}