File size: 1,448 Bytes
faeb8fa 114a678 faeb8fa | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 | {
"batch_e2e": {
"fp16_baseline": {
"e2e_s": 7.66,
"decode_ms_per_tok": 44.1,
"tokens": "161/161"
},
"int8": {
"e2e_s": 6.24,
"decode_ms_per_tok": 37.0,
"tokens": "161/161"
},
"int4": {
"e2e_s": 5.44,
"decode_ms_per_tok": 32.1,
"tokens": "161/161"
},
"int4_qkv": {
"e2e_s": 5.26,
"decode_ms_per_tok": 30.9,
"tokens": "161/161"
},
"int4_qkv_sdpa": {
"e2e_s": 4.74,
"decode_ms_per_tok": 28.1,
"tokens": "161/161"
},
"int4_qkv_sdpa_batched_first_chunk": {
"e2e_s": 4.45,
"decode_ms_per_tok": 26.6,
"tokens": "161/161"
},
"fp16_qkv_sdpa": {
"e2e_s": 7.93,
"decode_ms_per_tok": 45.3,
"tokens": "161/161"
},
"int8_qkv_sdpa": {
"e2e_s": 6.11,
"decode_ms_per_tok": 35.6,
"tokens": "161/161"
}
},
"streaming": {
"serialized_ttft_s": 1.53,
"batched_ttft_s": 0.148,
"chunk_decode_ms": 59.6,
"tokens": "128/128",
"passes": 5
},
"protocol": {
"fixture": "12.02s synthetic say audio, 16kHz mono float32",
"warmup": "1 untimed pass + 3 timed passes (batch), 1 warmup + 5 timed passes (streaming)",
"hardware": "Apple M4 Max, 64GB unified memory",
"os": "macOS 27.0 (Darwin 27.0.0)",
"runtime": "Core AI 1.0.0b2 (beta 4)",
"backend": "GPU (xctrace: 23,470 GPU Compute intervals, 0 ANE intervals)"
}
} |