| { | |
| "environment": { | |
| "model": "Qwen/Qwen2.5-1.5B-Instruct", | |
| "revision": "989aa7980e4cf806f80c7fef2b1adb7bc71aa306", | |
| "torch": "2.11.0", | |
| "transformers": "4.57.6", | |
| "gpu": null, | |
| "platform": "macOS-26.6.2-arm64-arm-64bit", | |
| "dtype": "torch.float32", | |
| "attention": "sdpa" | |
| }, | |
| "arguments": { | |
| "model": "Qwen/Qwen2.5-1.5B-Instruct", | |
| "presets": [ | |
| "fintech_fraud" | |
| ], | |
| "fields": [ | |
| 4, | |
| 16, | |
| 28 | |
| ], | |
| "steps": 1, | |
| "repeats": 8, | |
| "warmup": 2, | |
| "output": "mac_cpu_initial.json" | |
| }, | |
| "cases": [ | |
| { | |
| "preset": "fintech_fraud", | |
| "fields": 4, | |
| "prefix_tokens": 291, | |
| "suffix_tokens": 28, | |
| "batch_padded_tokens": 32, | |
| "correctness": { | |
| "max_abs_logit_error": 5.14984130859375e-05, | |
| "mean_abs_logit_error": 6.680697879346553e-06, | |
| "full_vocab_argmax_agreement": 1.0, | |
| "candidate_decisions_equal": true, | |
| "candidate_collision_fields": [] | |
| }, | |
| "decode": { | |
| "batch": { | |
| "median_ms": 125.02806199336192, | |
| "min_ms": 119.85650000860915, | |
| "p90_ms": 156.35541699884925, | |
| "samples_ms": [ | |
| 126.25195799046196, | |
| 123.17129199800547, | |
| 124.2745829949854, | |
| 125.78154099173844, | |
| 121.04620899481233, | |
| 119.85650000860915, | |
| 156.35541699884925, | |
| 128.15308300196193 | |
| ] | |
| }, | |
| "tree": { | |
| "median_ms": 110.84147949441103, | |
| "min_ms": 109.36420800862834, | |
| "p90_ms": 120.09329099964816, | |
| "samples_ms": [ | |
| 110.04775000037625, | |
| 120.09329099964816, | |
| 113.18729200866073, | |
| 111.36766699200962, | |
| 109.36420800862834, | |
| 110.31529199681245, | |
| 109.927166995476, | |
| 112.77408400201239 | |
| ] | |
| } | |
| }, | |
| "end_to_end": { | |
| "batch": { | |
| "median_ms": 494.8375209933147, | |
| "min_ms": 490.0297499989392, | |
| "p90_ms": 505.98341700970195, | |
| "samples_ms": [ | |
| 497.0009999960894, | |
| 494.0542499971343, | |
| 492.7669589960715, | |
| 490.0297499989392, | |
| 495.62079198949505, | |
| 493.570708000334, | |
| 505.98341700970195, | |
| 503.2984999998007 | |
| ] | |
| }, | |
| "tree": { | |
| "median_ms": 483.60870849865023, | |
| "min_ms": 473.5579160042107, | |
| "p90_ms": 495.45220800791867, | |
| "samples_ms": [ | |
| 490.6235829985235, | |
| 486.5667080011917, | |
| 482.09995799697936, | |
| 479.543959008879, | |
| 477.3575409926707, | |
| 473.5579160042107, | |
| 485.1174590003211, | |
| 495.45220800791867 | |
| ] | |
| } | |
| }, | |
| "memory": {}, | |
| "decode_speedup": 1.1279898334419671, | |
| "end_to_end_speedup": 1.0232187971335835, | |
| "decode_speedup_ci95": [ | |
| 1.0921950117174755, | |
| 1.1558847908819312 | |
| ], | |
| "end_to_end_speedup_ci95": [ | |
| 1.01418859675665, | |
| 1.0374776060151394 | |
| ] | |
| }, | |
| { | |
| "preset": "fintech_fraud", | |
| "fields": 16, | |
| "prefix_tokens": 468, | |
| "suffix_tokens": 127, | |
| "batch_padded_tokens": 160, | |
| "correctness": { | |
| "max_abs_logit_error": 7.2479248046875e-05, | |
| "mean_abs_logit_error": 6.526509423565585e-06, | |
| "full_vocab_argmax_agreement": 1.0, | |
| "candidate_decisions_equal": true, | |
| "candidate_collision_fields": [] | |
| }, | |
| "decode": { | |
| "batch": { | |
| "median_ms": 306.6326870030025, | |
| "min_ms": 296.44145800557453, | |
| "p90_ms": 321.2097089999588, | |
| "samples_ms": [ | |
| 321.2097089999588, | |
| 306.82954100484494, | |
| 304.35929099621717, | |
| 308.5727500001667, | |
| 304.67591701017227, | |
| 309.20870800036937, | |
| 306.43583300116006, | |
| 296.44145800557453 | |
| ] | |
| }, | |
| "tree": { | |
| "median_ms": 208.24712500325404, | |
| "min_ms": 199.32016600796487, | |
| "p90_ms": 217.84774999832734, | |
| "samples_ms": [ | |
| 217.84774999832734, | |
| 209.30799999041483, | |
| 208.65054200112354, | |
| 207.7542079932755, | |
| 209.12945800228044, | |
| 207.84370800538454, | |
| 204.79975000489503, | |
| 199.32016600796487 | |
| ] | |
| } | |
| }, | |
| "end_to_end": { | |
| "batch": { | |
| "median_ms": 855.2939375003916, | |
| "min_ms": 834.634166007163, | |
| "p90_ms": 864.323791989591, | |
| "samples_ms": [ | |
| 844.4657090003602, | |
| 852.9589170066174, | |
| 864.323791989591, | |
| 857.6289579941658, | |
| 864.1029999998864, | |
| 834.634166007163, | |
| 839.8262090049684, | |
| 858.5307920002379 | |
| ] | |
| }, | |
| "tree": { | |
| "median_ms": 741.478771007678, | |
| "min_ms": 727.3588340030983, | |
| "p90_ms": 753.5777500015683, | |
| "samples_ms": [ | |
| 744.3955000053393, | |
| 738.5620420100167, | |
| 753.5777500015683, | |
| 747.171166993212, | |
| 737.5786250049714, | |
| 747.073749997071, | |
| 727.3588340030983, | |
| 730.0422090047505 | |
| ] | |
| } | |
| }, | |
| "memory": {}, | |
| "decode_speedup": 1.4724461958272466, | |
| "end_to_end_speedup": 1.1534975388951967, | |
| "decode_speedup_ci95": [ | |
| 1.45854716361573, | |
| 1.4876981890274918 | |
| ], | |
| "end_to_end_speedup_ci95": [ | |
| 1.1292837033030394, | |
| 1.169980246545309 | |
| ] | |
| }, | |
| { | |
| "preset": "fintech_fraud", | |
| "fields": 28, | |
| "prefix_tokens": 646, | |
| "suffix_tokens": 227, | |
| "batch_padded_tokens": 308, | |
| "correctness": { | |
| "max_abs_logit_error": 6.818771362304688e-05, | |
| "mean_abs_logit_error": 6.96640609021415e-06, | |
| "full_vocab_argmax_agreement": 1.0, | |
| "candidate_decisions_equal": true, | |
| "candidate_collision_fields": [ | |
| "counterparty_jurisdiction_risk" | |
| ] | |
| }, | |
| "decode": { | |
| "batch": { | |
| "median_ms": 670.6976869973005, | |
| "min_ms": 648.5337920021266, | |
| "p90_ms": 702.7558749978198, | |
| "samples_ms": [ | |
| 658.6372909951024, | |
| 681.5660840074997, | |
| 648.5337920021266, | |
| 666.1693329951959, | |
| 686.8903329886962, | |
| 702.7558749978198, | |
| 661.6408749978291, | |
| 675.2260409994051 | |
| ] | |
| }, | |
| "tree": { | |
| "median_ms": 373.8734794969787, | |
| "min_ms": 365.2136249875184, | |
| "p90_ms": 383.5883330029901, | |
| "samples_ms": [ | |
| 381.38704099401366, | |
| 372.52491699473467, | |
| 365.2136249875184, | |
| 375.59429099201225, | |
| 383.5883330029901, | |
| 372.16383300255984, | |
| 365.7445830031065, | |
| 375.2220419992227 | |
| ] | |
| } | |
| }, | |
| "end_to_end": { | |
| "batch": { | |
| "median_ms": 1403.671021500486, | |
| "min_ms": 1317.3234580026474, | |
| "p90_ms": 1727.2021660028258, | |
| "samples_ms": [ | |
| 1353.7045839912025, | |
| 1389.389709001989, | |
| 1519.3337499949848, | |
| 1381.5615420026006, | |
| 1417.9523339989828, | |
| 1441.085959013435, | |
| 1727.2021660028258, | |
| 1317.3234580026474 | |
| ] | |
| }, | |
| "tree": { | |
| "median_ms": 1085.385687496455, | |
| "min_ms": 1019.9310419993708, | |
| "p90_ms": 1443.0702919926262, | |
| "samples_ms": [ | |
| 1094.387666991679, | |
| 1076.3837080012308, | |
| 1019.9310419993708, | |
| 1143.5619999974733, | |
| 1027.4991250043968, | |
| 1146.7937080014963, | |
| 1443.0702919926262, | |
| 1031.1697909928625 | |
| ] | |
| } | |
| }, | |
| "memory": {}, | |
| "decode_speedup": 1.7939161876356637, | |
| "end_to_end_speedup": 1.2932462973030225, | |
| "decode_speedup_ci95": [ | |
| 1.7520756879898927, | |
| 1.8376219823398265 | |
| ], | |
| "end_to_end_speedup_ci95": [ | |
| 1.21154395258443, | |
| 1.4115168584410118 | |
| ] | |
| } | |
| ] | |
| } | |