{
  "n_tasks": 27,
  "single": {
    "deepseek-v4-flash": {
      "pass": 0.7407407407407407,
      "score": 0.7194407407407408,
      "cost": 0.025244660000000002,
      "latency_p50": 32.60991859436035
    },
    "gpt-5.4-nano": {
      "pass": 0.7777777777777778,
      "score": 0.7777111111111111,
      "cost": 0.030138250000000002,
      "latency_p50": 6.396074295043945
    },
    "gpt-5.4-mini": {
      "pass": 0.8518518518518519,
      "score": 0.8517851851851852,
      "cost": 0.42622575,
      "latency_p50": 27.368282556533813
    },
    "gemini-3.5-flash": {
      "pass": 0.4444444444444444,
      "score": 0.4444444444444444,
      "cost": 0.8562555000000001,
      "latency_p50": 17.11278009414673
    },
    "gpt-5.5": {
      "pass": 0.7777777777777778,
      "score": 0.7777111111111111,
      "cost": 2.547895,
      "latency_p50": 27.903853178024292
    }
  },
  "router": {
    "0.4": {
      "pass": 0.7407407407407407,
      "score": 0.7194407407407408,
      "cost": 0.02524466
    },
    "0.5": {
      "pass": 0.7777777777777778,
      "score": 0.7564777777777778,
      "cost": 0.025137299999999998
    },
    "0.6": {
      "pass": 0.7407407407407407,
      "score": 0.7406740740740742,
      "cost": 0.03005728
    },
    "0.7": {
      "pass": 0.7777777777777778,
      "score": 0.7777111111111111,
      "cost": 0.14011191
    },
    "0.8": {
      "pass": 0.7777777777777778,
      "score": 0.7777111111111111,
      "cost": 0.25106635
    },
    "0.9": {
      "pass": 0.7777777777777778,
      "score": 0.7777111111111111,
      "cost": 0.25106635
    }
  },
  "cascade": {
    "price_order": {
      "pass": 1.0,
      "score": 0.9787,
      "cost": 0.11071721000000001
    },
    "router_start_0.5": {
      "pass": 1.0,
      "score": 0.9787,
      "cost": 0.10954415
    },
    "router_start_0.6": {
      "pass": 0.9629629629629629,
      "score": 0.9628962962962964,
      "cost": 0.67018278
    },
    "router_start_0.7": {
      "pass": 0.9259259259259259,
      "score": 0.9258592592592593,
      "cost": 0.93017776
    }
  },
  "k_ablation": {
    "1": {
      "pass": 0.7777777777777778,
      "score": 0.7777111111111111,
      "cost": 0.03041265
    },
    "2": {
      "pass": 0.7407407407407407,
      "score": 0.7406740740740742,
      "cost": 0.2955786
    },
    "4": {
      "pass": 0.7037037037037037,
      "score": 0.7036370370370371,
      "cost": 0.06224015000000001
    },
    "8": {
      "pass": 0.7407407407407407,
      "score": 0.7406740740740742,
      "cost": 0.03005728
    },
    "12": {
      "pass": 0.7407407407407407,
      "score": 0.7194407407407408,
      "cost": 0.024380539999999996
    },
    "16": {
      "pass": 0.7407407407407407,
      "score": 0.7194407407407408,
      "cost": 0.02590911
    }
  },
  "backend": {
    "pybed": {
      "pass": 0.7407407407407407,
      "score": 0.7406740740740742,
      "cost": 0.03005728
    },
    "hash": {
      "pass": 0.8518518518518519,
      "score": 0.8305518518518519,
      "cost": 0.02901353
    }
  },
  "oracle": {
    "pass": 1.0,
    "score": 1.0,
    "cost": 0.09019056000000003
  }
}