{
  "environment": {
    "model": "Qwen/Qwen2.5-1.5B-Instruct",
    "revision": "989aa7980e4cf806f80c7fef2b1adb7bc71aa306",
    "torch": "2.11.0",
    "transformers": "4.57.6",
    "gpu": "mps",
    "platform": "macOS-26.6.2-arm64-arm-64bit",
    "dtype": "torch.float16",
    "attention": "sdpa"
  },
  "arguments": {
    "model": "Qwen/Qwen2.5-1.5B-Instruct",
    "presets": [
      "code_security"
    ],
    "fields": [
      4,
      16,
      28
    ],
    "steps": 1,
    "repeats": 8,
    "warmup": 2,
    "output": "mac_mps_code_security.json",
    "device": "mps",
    "dtype": "float16"
  },
  "cases": [
    {
      "preset": "code_security",
      "fields": 4,
      "prefix_tokens": 302,
      "suffix_tokens": 31,
      "batch_padded_tokens": 40,
      "correctness": {
        "max_abs_logit_error": 0.0361328125,
        "mean_abs_logit_error": 0.004900083411484957,
        "full_vocab_argmax_agreement": 1.0,
        "candidate_decisions_equal": false,
        "candidate_disagreements": [
          {
            "field": "primary_cwe",
            "batch_choice": 2,
            "tree_choice": 1,
            "top_two_margin": 0.015625,
            "distinct_candidate_tokens": 4,
            "candidates": 4
          }
        ],
        "candidate_collision_fields": [
          "secondary_cwe"
        ]
      },
      "decode": {
        "batch": {
          "median_ms": 50.879604496003594,
          "min_ms": 50.278832990443334,
          "p90_ms": 51.27383299986832,
          "samples_ms": [
            51.26725000445731,
            51.01349999313243,
            51.27383299986832,
            50.79987499630079,
            50.41279099532403,
            50.61795900110155,
            50.278832990443334,
            50.9593339957064
          ]
        },
        "tree": {
          "median_ms": 34.881520994531456,
          "min_ms": 34.403541998472065,
          "p90_ms": 35.53900000406429,
          "samples_ms": [
            34.994041998288594,
            35.53900000406429,
            34.6182079956634,
            35.06995800125878,
            34.84349999052938,
            34.91954199853353,
            34.403541998472065,
            34.67412501049694
          ]
        }
      },
      "end_to_end": {
        "batch": {
          "median_ms": 195.58491700445302,
          "min_ms": 195.04941700142808,
          "p90_ms": 197.6404999877559,
          "samples_ms": [
            195.11904100363608,
            195.04941700142808,
            195.87837500148453,
            197.6404999877559,
            196.34754200524185,
            195.12154199765064,
            195.2914590074215,
            196.0291249997681
          ]
        },
        "tree": {
          "median_ms": 181.7374789970927,
          "min_ms": 181.1812919913791,
          "p90_ms": 183.7198329885723,
          "samples_ms": [
            181.40674999449402,
            181.48845799441915,
            181.1812919913791,
            182.22454200440552,
            182.78208299307153,
            183.7198329885723,
            181.2051250017248,
            181.98649999976624
          ]
        }
      },
      "memory": {
        "batch": {
          "driver_high_water_mib": 3293.015625,
          "extra_driver_mib": 120.0
        },
        "tree": {
          "driver_high_water_mib": 3205.015625,
          "extra_driver_mib": 32.0
        }
      },
      "decode_speedup": 1.4586406511338836,
      "end_to_end_speedup": 1.0761947292532976,
      "decode_speedup_ci95": [
        1.4468348762043552,
        1.4746103089743758
      ],
      "end_to_end_speedup_ci95": [
        1.0696052462274368,
        1.0803748498502816
      ]
    },
    {
      "preset": "code_security",
      "fields": 16,
      "prefix_tokens": 486,
      "suffix_tokens": 128,
      "batch_padded_tokens": 160,
      "correctness": {
        "max_abs_logit_error": 0.046875,
        "mean_abs_logit_error": 0.004968002438545227,
        "full_vocab_argmax_agreement": 1.0,
        "candidate_decisions_equal": true,
        "candidate_disagreements": [],
        "candidate_collision_fields": [
          "secondary_cwe"
        ]
      },
      "decode": {
        "batch": {
          "median_ms": 168.52770849800436,
          "min_ms": 168.16216701408848,
          "p90_ms": 168.7455420033075,
          "samples_ms": [
            168.42808299406897,
            168.45774999819696,
            168.16216701408848,
            168.558541990933,
            168.56449999613687,
            168.53641699708533,
            168.5189999989234,
            168.7455420033075
          ]
        },
        "tree": {
          "median_ms": 83.24410449858988,
          "min_ms": 82.93445799790788,
          "p90_ms": 83.74645799631253,
          "samples_ms": [
            83.00320799753536,
            83.34545799880289,
            83.18249999138061,
            82.93445799790788,
            83.18066700303461,
            83.30570900579914,
            83.74645799631253,
            83.31108299898915
          ]
        }
      },
      "end_to_end": {
        "batch": {
          "median_ms": 409.24462499970105,
          "min_ms": 408.52445900964085,
          "p90_ms": 409.74658299819566,
          "samples_ms": [
            409.74658299819566,
            409.39200000138953,
            409.33083300478756,
            409.505249990616,
            408.52445900964085,
            408.82958300062455,
            408.87237500282936,
            409.1584169946145
          ]
        },
        "tree": {
          "median_ms": 324.1305210030987,
          "min_ms": 323.2846250029979,
          "p90_ms": 324.7729589929804,
          "samples_ms": [
            323.2846250029979,
            323.65375000517815,
            324.7729589929804,
            324.6173330117017,
            324.67654198990203,
            323.4534579969477,
            324.06320799782407,
            324.1978340083733
          ]
        }
      },
      "memory": {
        "batch": {
          "driver_high_water_mib": 5618.421875,
          "extra_driver_mib": 2352.0
        },
        "tree": {
          "driver_high_water_mib": 3290.421875,
          "extra_driver_mib": 24.0
        }
      },
      "decode_speedup": 2.024500227530938,
      "end_to_end_speedup": 1.2625920685691572,
      "decode_speedup_ci95": [
        2.02161582125275,
        2.029262774061603
      ],
      "end_to_end_speedup_ci95": [
        1.25996231663521,
        1.2652043168146683
      ]
    },
    {
      "preset": "code_security",
      "fields": 28,
      "prefix_tokens": 650,
      "suffix_tokens": 220,
      "batch_padded_tokens": 280,
      "correctness": {
        "max_abs_logit_error": 0.046875,
        "mean_abs_logit_error": 0.00478665204718709,
        "full_vocab_argmax_agreement": 1.0,
        "candidate_decisions_equal": true,
        "candidate_disagreements": [],
        "candidate_collision_fields": [
          "secondary_cwe"
        ]
      },
      "decode": {
        "batch": {
          "median_ms": 331.3711874943692,
          "min_ms": 330.6104579969542,
          "p90_ms": 333.8664999901084,
          "samples_ms": [
            330.6104579969542,
            332.9330410051625,
            333.8664999901084,
            331.30533299117815,
            331.4370419975603,
            330.94691699079704,
            331.45566600433085,
            331.0064999968745
          ]
        },
        "tree": {
          "median_ms": 140.7133124957909,
          "min_ms": 140.09108299796935,
          "p90_ms": 147.16175000648946,
          "samples_ms": [
            141.0023330099648,
            141.5980000019772,
            140.92637499561533,
            147.16175000648946,
            140.50024999596644,
            140.16799999808427,
            140.39466599933803,
            140.09108299796935
          ]
        }
      },
      "end_to_end": {
        "batch": {
          "median_ms": 661.668708504294,
          "min_ms": 660.7298750022892,
          "p90_ms": 665.3499589883722,
          "samples_ms": [
            662.3491670034127,
            660.8333750045858,
            662.0492920046672,
            660.7298750022892,
            661.2881250039209,
            660.8159159950446,
            662.5105409912067,
            665.3499589883722
          ]
        },
        "tree": {
          "median_ms": 470.8869994938141,
          "min_ms": 469.48833399801515,
          "p90_ms": 473.8746670045657,
          "samples_ms": [
            471.05629098950885,
            469.48833399801515,
            473.8746670045657,
            470.57508399302606,
            470.7177079981193,
            469.7416659910232,
            472.0834579929942,
            473.6256249889266
          ]
        }
      },
      "memory": {
        "batch": {
          "driver_high_water_mib": 5302.984375,
          "extra_driver_mib": 2022.0
        },
        "tree": {
          "driver_high_water_mib": 3270.984375,
          "extra_driver_mib": 32.0
        }
      },
      "decode_speedup": 2.354938432028465,
      "end_to_end_speedup": 1.405153909994464,
      "decode_speedup_ci95": [
        2.344691738046735,
        2.3618243304159923
      ],
      "end_to_end_speedup_ci95": [
        1.4001119454561137,
        1.4067645342912223
      ]
    }
  ]
}