| { |
| "n_tasks": 27, |
| "single": { |
| "deepseek-v4-flash": { |
| "pass": 0.7407407407407407, |
| "score": 0.7194407407407408, |
| "cost": 0.025244660000000002, |
| "latency_p50": 32.60991859436035 |
| }, |
| "gpt-5.4-nano": { |
| "pass": 0.7777777777777778, |
| "score": 0.7777111111111111, |
| "cost": 0.030138250000000002, |
| "latency_p50": 6.396074295043945 |
| }, |
| "gpt-5.4-mini": { |
| "pass": 0.8518518518518519, |
| "score": 0.8517851851851852, |
| "cost": 0.42622575, |
| "latency_p50": 27.368282556533813 |
| }, |
| "gemini-3.5-flash": { |
| "pass": 0.4444444444444444, |
| "score": 0.4444444444444444, |
| "cost": 0.8562555000000001, |
| "latency_p50": 17.11278009414673 |
| }, |
| "gpt-5.5": { |
| "pass": 0.7777777777777778, |
| "score": 0.7777111111111111, |
| "cost": 2.547895, |
| "latency_p50": 27.903853178024292 |
| } |
| }, |
| "router": { |
| "0.4": { |
| "pass": 0.7407407407407407, |
| "score": 0.7194407407407408, |
| "cost": 0.02524466 |
| }, |
| "0.5": { |
| "pass": 0.7777777777777778, |
| "score": 0.7564777777777778, |
| "cost": 0.025137299999999998 |
| }, |
| "0.6": { |
| "pass": 0.7407407407407407, |
| "score": 0.7406740740740742, |
| "cost": 0.03005728 |
| }, |
| "0.7": { |
| "pass": 0.7777777777777778, |
| "score": 0.7777111111111111, |
| "cost": 0.14011191 |
| }, |
| "0.8": { |
| "pass": 0.7777777777777778, |
| "score": 0.7777111111111111, |
| "cost": 0.25106635 |
| }, |
| "0.9": { |
| "pass": 0.7777777777777778, |
| "score": 0.7777111111111111, |
| "cost": 0.25106635 |
| } |
| }, |
| "cascade": { |
| "price_order": { |
| "pass": 1.0, |
| "score": 0.9787, |
| "cost": 0.11071721000000001 |
| }, |
| "router_start_0.5": { |
| "pass": 1.0, |
| "score": 0.9787, |
| "cost": 0.10954415 |
| }, |
| "router_start_0.6": { |
| "pass": 0.9629629629629629, |
| "score": 0.9628962962962964, |
| "cost": 0.67018278 |
| }, |
| "router_start_0.7": { |
| "pass": 0.9259259259259259, |
| "score": 0.9258592592592593, |
| "cost": 0.93017776 |
| } |
| }, |
| "k_ablation": { |
| "1": { |
| "pass": 0.7777777777777778, |
| "score": 0.7777111111111111, |
| "cost": 0.03041265 |
| }, |
| "2": { |
| "pass": 0.7407407407407407, |
| "score": 0.7406740740740742, |
| "cost": 0.2955786 |
| }, |
| "4": { |
| "pass": 0.7037037037037037, |
| "score": 0.7036370370370371, |
| "cost": 0.06224015000000001 |
| }, |
| "8": { |
| "pass": 0.7407407407407407, |
| "score": 0.7406740740740742, |
| "cost": 0.03005728 |
| }, |
| "12": { |
| "pass": 0.7407407407407407, |
| "score": 0.7194407407407408, |
| "cost": 0.024380539999999996 |
| }, |
| "16": { |
| "pass": 0.7407407407407407, |
| "score": 0.7194407407407408, |
| "cost": 0.02590911 |
| } |
| }, |
| "backend": { |
| "pybed": { |
| "pass": 0.7407407407407407, |
| "score": 0.7406740740740742, |
| "cost": 0.03005728 |
| }, |
| "hash": { |
| "pass": 0.8518518518518519, |
| "score": 0.8305518518518519, |
| "cost": 0.02901353 |
| } |
| }, |
| "oracle": { |
| "pass": 1.0, |
| "score": 1.0, |
| "cost": 0.09019056000000003 |
| } |
| } |
