| { | |
| "environment": { | |
| "model": "Qwen/Qwen2.5-1.5B-Instruct", | |
| "revision": "989aa7980e4cf806f80c7fef2b1adb7bc71aa306", | |
| "torch": "2.11.0", | |
| "transformers": "4.57.6", | |
| "gpu": "mps", | |
| "platform": "macOS-26.6.2-arm64-arm-64bit", | |
| "dtype": "torch.float16", | |
| "attention": "sdpa" | |
| }, | |
| "arguments": { | |
| "model": "Qwen/Qwen2.5-1.5B-Instruct", | |
| "presets": [ | |
| "code_security" | |
| ], | |
| "fields": [ | |
| 4, | |
| 16, | |
| 28 | |
| ], | |
| "steps": 1, | |
| "repeats": 8, | |
| "warmup": 2, | |
| "output": "mac_mps_code_security.json", | |
| "device": "mps", | |
| "dtype": "float16" | |
| }, | |
| "cases": [ | |
| { | |
| "preset": "code_security", | |
| "fields": 4, | |
| "prefix_tokens": 302, | |
| "suffix_tokens": 31, | |
| "batch_padded_tokens": 40, | |
| "correctness": { | |
| "max_abs_logit_error": 0.0361328125, | |
| "mean_abs_logit_error": 0.004900083411484957, | |
| "full_vocab_argmax_agreement": 1.0, | |
| "candidate_decisions_equal": false, | |
| "candidate_disagreements": [ | |
| { | |
| "field": "primary_cwe", | |
| "batch_choice": 2, | |
| "tree_choice": 1, | |
| "top_two_margin": 0.015625, | |
| "distinct_candidate_tokens": 4, | |
| "candidates": 4 | |
| } | |
| ], | |
| "candidate_collision_fields": [ | |
| "secondary_cwe" | |
| ] | |
| }, | |
| "decode": { | |
| "batch": { | |
| "median_ms": 50.879604496003594, | |
| "min_ms": 50.278832990443334, | |
| "p90_ms": 51.27383299986832, | |
| "samples_ms": [ | |
| 51.26725000445731, | |
| 51.01349999313243, | |
| 51.27383299986832, | |
| 50.79987499630079, | |
| 50.41279099532403, | |
| 50.61795900110155, | |
| 50.278832990443334, | |
| 50.9593339957064 | |
| ] | |
| }, | |
| "tree": { | |
| "median_ms": 34.881520994531456, | |
| "min_ms": 34.403541998472065, | |
| "p90_ms": 35.53900000406429, | |
| "samples_ms": [ | |
| 34.994041998288594, | |
| 35.53900000406429, | |
| 34.6182079956634, | |
| 35.06995800125878, | |
| 34.84349999052938, | |
| 34.91954199853353, | |
| 34.403541998472065, | |
| 34.67412501049694 | |
| ] | |
| } | |
| }, | |
| "end_to_end": { | |
| "batch": { | |
| "median_ms": 195.58491700445302, | |
| "min_ms": 195.04941700142808, | |
| "p90_ms": 197.6404999877559, | |
| "samples_ms": [ | |
| 195.11904100363608, | |
| 195.04941700142808, | |
| 195.87837500148453, | |
| 197.6404999877559, | |
| 196.34754200524185, | |
| 195.12154199765064, | |
| 195.2914590074215, | |
| 196.0291249997681 | |
| ] | |
| }, | |
| "tree": { | |
| "median_ms": 181.7374789970927, | |
| "min_ms": 181.1812919913791, | |
| "p90_ms": 183.7198329885723, | |
| "samples_ms": [ | |
| 181.40674999449402, | |
| 181.48845799441915, | |
| 181.1812919913791, | |
| 182.22454200440552, | |
| 182.78208299307153, | |
| 183.7198329885723, | |
| 181.2051250017248, | |
| 181.98649999976624 | |
| ] | |
| } | |
| }, | |
| "memory": { | |
| "batch": { | |
| "driver_high_water_mib": 3293.015625, | |
| "extra_driver_mib": 120.0 | |
| }, | |
| "tree": { | |
| "driver_high_water_mib": 3205.015625, | |
| "extra_driver_mib": 32.0 | |
| } | |
| }, | |
| "decode_speedup": 1.4586406511338836, | |
| "end_to_end_speedup": 1.0761947292532976, | |
| "decode_speedup_ci95": [ | |
| 1.4468348762043552, | |
| 1.4746103089743758 | |
| ], | |
| "end_to_end_speedup_ci95": [ | |
| 1.0696052462274368, | |
| 1.0803748498502816 | |
| ] | |
| }, | |
| { | |
| "preset": "code_security", | |
| "fields": 16, | |
| "prefix_tokens": 486, | |
| "suffix_tokens": 128, | |
| "batch_padded_tokens": 160, | |
| "correctness": { | |
| "max_abs_logit_error": 0.046875, | |
| "mean_abs_logit_error": 0.004968002438545227, | |
| "full_vocab_argmax_agreement": 1.0, | |
| "candidate_decisions_equal": true, | |
| "candidate_disagreements": [], | |
| "candidate_collision_fields": [ | |
| "secondary_cwe" | |
| ] | |
| }, | |
| "decode": { | |
| "batch": { | |
| "median_ms": 168.52770849800436, | |
| "min_ms": 168.16216701408848, | |
| "p90_ms": 168.7455420033075, | |
| "samples_ms": [ | |
| 168.42808299406897, | |
| 168.45774999819696, | |
| 168.16216701408848, | |
| 168.558541990933, | |
| 168.56449999613687, | |
| 168.53641699708533, | |
| 168.5189999989234, | |
| 168.7455420033075 | |
| ] | |
| }, | |
| "tree": { | |
| "median_ms": 83.24410449858988, | |
| "min_ms": 82.93445799790788, | |
| "p90_ms": 83.74645799631253, | |
| "samples_ms": [ | |
| 83.00320799753536, | |
| 83.34545799880289, | |
| 83.18249999138061, | |
| 82.93445799790788, | |
| 83.18066700303461, | |
| 83.30570900579914, | |
| 83.74645799631253, | |
| 83.31108299898915 | |
| ] | |
| } | |
| }, | |
| "end_to_end": { | |
| "batch": { | |
| "median_ms": 409.24462499970105, | |
| "min_ms": 408.52445900964085, | |
| "p90_ms": 409.74658299819566, | |
| "samples_ms": [ | |
| 409.74658299819566, | |
| 409.39200000138953, | |
| 409.33083300478756, | |
| 409.505249990616, | |
| 408.52445900964085, | |
| 408.82958300062455, | |
| 408.87237500282936, | |
| 409.1584169946145 | |
| ] | |
| }, | |
| "tree": { | |
| "median_ms": 324.1305210030987, | |
| "min_ms": 323.2846250029979, | |
| "p90_ms": 324.7729589929804, | |
| "samples_ms": [ | |
| 323.2846250029979, | |
| 323.65375000517815, | |
| 324.7729589929804, | |
| 324.6173330117017, | |
| 324.67654198990203, | |
| 323.4534579969477, | |
| 324.06320799782407, | |
| 324.1978340083733 | |
| ] | |
| } | |
| }, | |
| "memory": { | |
| "batch": { | |
| "driver_high_water_mib": 5618.421875, | |
| "extra_driver_mib": 2352.0 | |
| }, | |
| "tree": { | |
| "driver_high_water_mib": 3290.421875, | |
| "extra_driver_mib": 24.0 | |
| } | |
| }, | |
| "decode_speedup": 2.024500227530938, | |
| "end_to_end_speedup": 1.2625920685691572, | |
| "decode_speedup_ci95": [ | |
| 2.02161582125275, | |
| 2.029262774061603 | |
| ], | |
| "end_to_end_speedup_ci95": [ | |
| 1.25996231663521, | |
| 1.2652043168146683 | |
| ] | |
| }, | |
| { | |
| "preset": "code_security", | |
| "fields": 28, | |
| "prefix_tokens": 650, | |
| "suffix_tokens": 220, | |
| "batch_padded_tokens": 280, | |
| "correctness": { | |
| "max_abs_logit_error": 0.046875, | |
| "mean_abs_logit_error": 0.00478665204718709, | |
| "full_vocab_argmax_agreement": 1.0, | |
| "candidate_decisions_equal": true, | |
| "candidate_disagreements": [], | |
| "candidate_collision_fields": [ | |
| "secondary_cwe" | |
| ] | |
| }, | |
| "decode": { | |
| "batch": { | |
| "median_ms": 331.3711874943692, | |
| "min_ms": 330.6104579969542, | |
| "p90_ms": 333.8664999901084, | |
| "samples_ms": [ | |
| 330.6104579969542, | |
| 332.9330410051625, | |
| 333.8664999901084, | |
| 331.30533299117815, | |
| 331.4370419975603, | |
| 330.94691699079704, | |
| 331.45566600433085, | |
| 331.0064999968745 | |
| ] | |
| }, | |
| "tree": { | |
| "median_ms": 140.7133124957909, | |
| "min_ms": 140.09108299796935, | |
| "p90_ms": 147.16175000648946, | |
| "samples_ms": [ | |
| 141.0023330099648, | |
| 141.5980000019772, | |
| 140.92637499561533, | |
| 147.16175000648946, | |
| 140.50024999596644, | |
| 140.16799999808427, | |
| 140.39466599933803, | |
| 140.09108299796935 | |
| ] | |
| } | |
| }, | |
| "end_to_end": { | |
| "batch": { | |
| "median_ms": 661.668708504294, | |
| "min_ms": 660.7298750022892, | |
| "p90_ms": 665.3499589883722, | |
| "samples_ms": [ | |
| 662.3491670034127, | |
| 660.8333750045858, | |
| 662.0492920046672, | |
| 660.7298750022892, | |
| 661.2881250039209, | |
| 660.8159159950446, | |
| 662.5105409912067, | |
| 665.3499589883722 | |
| ] | |
| }, | |
| "tree": { | |
| "median_ms": 470.8869994938141, | |
| "min_ms": 469.48833399801515, | |
| "p90_ms": 473.8746670045657, | |
| "samples_ms": [ | |
| 471.05629098950885, | |
| 469.48833399801515, | |
| 473.8746670045657, | |
| 470.57508399302606, | |
| 470.7177079981193, | |
| 469.7416659910232, | |
| 472.0834579929942, | |
| 473.6256249889266 | |
| ] | |
| } | |
| }, | |
| "memory": { | |
| "batch": { | |
| "driver_high_water_mib": 5302.984375, | |
| "extra_driver_mib": 2022.0 | |
| }, | |
| "tree": { | |
| "driver_high_water_mib": 3270.984375, | |
| "extra_driver_mib": 32.0 | |
| } | |
| }, | |
| "decode_speedup": 2.354938432028465, | |
| "end_to_end_speedup": 1.405153909994464, | |
| "decode_speedup_ci95": [ | |
| 2.344691738046735, | |
| 2.3618243304159923 | |
| ], | |
| "end_to_end_speedup_ci95": [ | |
| 1.4001119454561137, | |
| 1.4067645342912223 | |
| ] | |
| } | |
| ] | |
| } | |