Buckets:
| { | |
| "run": "dock-eval50", | |
| "episodes": [ | |
| { | |
| "model": "anthropic:claude-sonnet-5-5", | |
| "task_id": "dock-24B-w06x1-busy-0", | |
| "split": "eval", | |
| "difficulty": "busy", | |
| "quay": "24B", | |
| "ships": 17, | |
| "reward": 0.944468, | |
| "submitted": true, | |
| "feasible": true, | |
| "cost": 92, | |
| "optimal_cost": 87, | |
| "naive_cost": 658, | |
| "turns": 2, | |
| "checks": 1, | |
| "seconds": 124.5, | |
| "input_tokens": 25105, | |
| "output_tokens": 17392, | |
| "end_reason": "submitted", | |
| "errors": 0 | |
| }, | |
| { | |
| "model": "anthropic:claude-sonnet-5-5", | |
| "task_id": "dock-24B-w06x1-standard-0", | |
| "split": "eval", | |
| "difficulty": "standard", | |
| "quay": "24B", | |
| "ships": 16, | |
| "reward": 1.0, | |
| "submitted": true, | |
| "feasible": true, | |
| "cost": 114, | |
| "optimal_cost": 114, | |
| "naive_cost": 311, | |
| "turns": 2, | |
| "checks": 1, | |
| "seconds": 45.7, | |
| "input_tokens": 13381, | |
| "output_tokens": 5796, | |
| "end_reason": "submitted", | |
| "errors": 0 | |
| }, | |
| { | |
| "model": "anthropic:claude-sonnet-5-5", | |
| "task_id": "dock-24B-w06x2-busy-0", | |
| "split": "eval", | |
| "difficulty": "busy", | |
| "quay": "24B", | |
| "ships": 31, | |
| "reward": 0.736256, | |
| "submitted": true, | |
| "feasible": true, | |
| "cost": 642, | |
| "optimal_cost": 616, | |
| "naive_cost": 1181, | |
| "turns": 2, | |
| "checks": 1, | |
| "seconds": 92.3, | |
| "input_tokens": 23387, | |
| "output_tokens": 13309, | |
| "end_reason": "submitted", | |
| "errors": 0 | |
| }, | |
| { | |
| "model": "anthropic:claude-sonnet-5-5", | |
| "task_id": "dock-24B-w06x2-extreme-0", | |
| "split": "eval", | |
| "difficulty": "extreme", | |
| "quay": "24B", | |
| "ships": 32, | |
| "reward": 1.0, | |
| "submitted": true, | |
| "feasible": true, | |
| "cost": 231, | |
| "optimal_cost": 231, | |
| "naive_cost": 1090, | |
| "turns": 2, | |
| "checks": 1, | |
| "seconds": 151.7, | |
| "input_tokens": 31156, | |
| "output_tokens": 20971, | |
| "end_reason": "submitted", | |
| "errors": 0 | |
| }, | |
| { | |
| "model": "anthropic:claude-sonnet-5-5", | |
| "task_id": "dock-24B-w06x2-storm-0", | |
| "split": "eval", | |
| "difficulty": "storm", | |
| "quay": "24B", | |
| "ships": 32, | |
| "reward": 0.878427, | |
| "submitted": true, | |
| "feasible": true, | |
| "cost": 169, | |
| "optimal_cost": 154, | |
| "naive_cost": 1259, | |
| "turns": 2, | |
| "checks": 1, | |
| "seconds": 214.6, | |
| "input_tokens": 40248, | |
| "output_tokens": 30004, | |
| "end_reason": "submitted", | |
| "errors": 0 | |
| }, | |
| { | |
| "model": "anthropic:claude-sonnet-5-5", | |
| "task_id": "dock-24B-w07x1-busy-0", | |
| "split": "eval", | |
| "difficulty": "busy", | |
| "quay": "24B", | |
| "ships": 17, | |
| "reward": 0.989333, | |
| "submitted": true, | |
| "feasible": true, | |
| "cost": 227, | |
| "optimal_cost": 226, | |
| "naive_cost": 641, | |
| "turns": 2, | |
| "checks": 1, | |
| "seconds": 97.5, | |
| "input_tokens": 21509, | |
| "output_tokens": 13170, | |
| "end_reason": "submitted", | |
| "errors": 0 | |
| }, | |
| { | |
| "model": "anthropic:claude-sonnet-5-5", | |
| "task_id": "dock-24B-w15x2-extreme-0", | |
| "split": "eval", | |
| "difficulty": "extreme", | |
| "quay": "24B", | |
| "ships": 24, | |
| "reward": 0.790861, | |
| "submitted": true, | |
| "feasible": true, | |
| "cost": 1054, | |
| "optimal_cost": 1009, | |
| "naive_cost": 3964, | |
| "turns": 2, | |
| "checks": 1, | |
| "seconds": 208.7, | |
| "input_tokens": 36001, | |
| "output_tokens": 27441, | |
| "end_reason": "submitted", | |
| "errors": 0 | |
| }, | |
| { | |
| "model": "anthropic:claude-sonnet-5-5", | |
| "task_id": "dock-24B-w16x1-busy-0", | |
| "split": "eval", | |
| "difficulty": "busy", | |
| "quay": "24B", | |
| "ships": 16, | |
| "reward": 0.964841, | |
| "submitted": true, | |
| "feasible": true, | |
| "cost": 1365, | |
| "optimal_cost": 1349, | |
| "naive_cost": 4069, | |
| "turns": 2, | |
| "checks": 1, | |
| "seconds": 195.4, | |
| "input_tokens": 35065, | |
| "output_tokens": 26627, | |
| "end_reason": "submitted", | |
| "errors": 0 | |
| }, | |
| { | |
| "model": "anthropic:claude-sonnet-5-5", | |
| "task_id": "dock-24B-w16x2-extreme-0", | |
| "split": "eval", | |
| "difficulty": "extreme", | |
| "quay": "24B", | |
| "ships": 28, | |
| "reward": 0.693042, | |
| "submitted": true, | |
| "feasible": true, | |
| "cost": 519, | |
| "optimal_cost": 466, | |
| "naive_cost": 1485, | |
| "turns": 3, | |
| "checks": 1, | |
| "seconds": 278.4, | |
| "input_tokens": 84660, | |
| "output_tokens": 36689, | |
| "end_reason": "submitted", | |
| "errors": 0 | |
| }, | |
| { | |
| "model": "anthropic:claude-sonnet-5-5", | |
| "task_id": "dock-24B-w16x2-storm-0", | |
| "split": "eval", | |
| "difficulty": "storm", | |
| "quay": "24B", | |
| "ships": 27, | |
| "reward": 0.886995, | |
| "submitted": true, | |
| "feasible": true, | |
| "cost": 216, | |
| "optimal_cost": 201, | |
| "naive_cost": 1569, | |
| "turns": 2, | |
| "checks": 1, | |
| "seconds": 159.6, | |
| "input_tokens": 31040, | |
| "output_tokens": 20875, | |
| "end_reason": "submitted", | |
| "errors": 0 | |
| }, | |
| { | |
| "model": "anthropic:claude-sonnet-5-5", | |
| "task_id": "dock-24B-w16x2-storm-1", | |
| "split": "eval", | |
| "difficulty": "storm", | |
| "quay": "24B", | |
| "ships": 26, | |
| "reward": 0.784635, | |
| "submitted": true, | |
| "feasible": true, | |
| "cost": 1248, | |
| "optimal_cost": 1187, | |
| "naive_cost": 2403, | |
| "turns": 3, | |
| "checks": 2, | |
| "seconds": 214.4, | |
| "input_tokens": 71918, | |
| "output_tokens": 29208, | |
| "end_reason": "submitted", | |
| "errors": 0 | |
| }, | |
| { | |
| "model": "anthropic:claude-sonnet-5-5", | |
| "task_id": "dock-24B-w35x1-busy-0", | |
| "split": "eval", | |
| "difficulty": "busy", | |
| "quay": "24B", | |
| "ships": 14, | |
| "reward": 0.845158, | |
| "submitted": true, | |
| "feasible": true, | |
| "cost": 1260, | |
| "optimal_cost": 1223, | |
| "naive_cost": 2892, | |
| "turns": 2, | |
| "checks": 1, | |
| "seconds": 171.5, | |
| "input_tokens": 30784, | |
| "output_tokens": 22655, | |
| "end_reason": "submitted", | |
| "errors": 0 | |
| }, | |
| { | |
| "model": "anthropic:claude-sonnet-5-5", | |
| "task_id": "dock-24B-w35x2-busy-0", | |
| "split": "eval", | |
| "difficulty": "busy", | |
| "quay": "24B", | |
| "ships": 24, | |
| "reward": 0.93526, | |
| "submitted": true, | |
| "feasible": true, | |
| "cost": 172, | |
| "optimal_cost": 162, | |
| "naive_cost": 1443, | |
| "turns": 3, | |
| "checks": 2, | |
| "seconds": 221.6, | |
| "input_tokens": 73025, | |
| "output_tokens": 29701, | |
| "end_reason": "submitted", | |
| "errors": 0 | |
| }, | |
| { | |
| "model": "anthropic:claude-sonnet-5-5", | |
| "task_id": "dock-24B-w35x2-extreme-0", | |
| "split": "eval", | |
| "difficulty": "extreme", | |
| "quay": "24B", | |
| "ships": 24, | |
| "reward": 1.0, | |
| "submitted": true, | |
| "feasible": true, | |
| "cost": 1792, | |
| "optimal_cost": 1792, | |
| "naive_cost": 4682, | |
| "turns": 3, | |
| "checks": 2, | |
| "seconds": 93.3, | |
| "input_tokens": 38820, | |
| "output_tokens": 11351, | |
| "end_reason": "submitted", | |
| "errors": 0 | |
| }, | |
| { | |
| "model": "anthropic:claude-sonnet-5-5", | |
| "task_id": "dock-24B-w35x2-extreme-1", | |
| "split": "eval", | |
| "difficulty": "extreme", | |
| "quay": "24B", | |
| "ships": 25, | |
| "reward": 1.0, | |
| "submitted": true, | |
| "feasible": true, | |
| "cost": 283, | |
| "optimal_cost": 283, | |
| "naive_cost": 1356, | |
| "turns": 3, | |
| "checks": 2, | |
| "seconds": 147.3, | |
| "input_tokens": 57923, | |
| "output_tokens": 20992, | |
| "end_reason": "submitted", | |
| "errors": 0 | |
| }, | |
| { | |
| "model": "anthropic:claude-sonnet-5-5", | |
| "task_id": "dock-24B-w35x2-storm-0", | |
| "split": "eval", | |
| "difficulty": "storm", | |
| "quay": "24B", | |
| "ships": 24, | |
| "reward": 0.918257, | |
| "submitted": true, | |
| "feasible": true, | |
| "cost": 2627, | |
| "optimal_cost": 2591, | |
| "naive_cost": 4835, | |
| "turns": 2, | |
| "checks": 1, | |
| "seconds": 171.5, | |
| "input_tokens": 32368, | |
| "output_tokens": 22860, | |
| "end_reason": "submitted", | |
| "errors": 0 | |
| }, | |
| { | |
| "model": "anthropic:claude-sonnet-5-5", | |
| "task_id": "dock-24B-w35x2-storm-1", | |
| "split": "eval", | |
| "difficulty": "storm", | |
| "quay": "24B", | |
| "ships": 24, | |
| "reward": 0.551842, | |
| "submitted": true, | |
| "feasible": true, | |
| "cost": 381, | |
| "optimal_cost": 312, | |
| "naive_cost": 2701, | |
| "turns": 2, | |
| "checks": 1, | |
| "seconds": 120.1, | |
| "input_tokens": 26228, | |
| "output_tokens": 16503, | |
| "end_reason": "submitted", | |
| "errors": 0 | |
| }, | |
| { | |
| "model": "anthropic:claude-sonnet-5-5", | |
| "task_id": "dock-24B-w35x3-extreme-0", | |
| "split": "eval", | |
| "difficulty": "extreme", | |
| "quay": "24B", | |
| "ships": 39, | |
| "reward": 0.954815, | |
| "submitted": true, | |
| "feasible": true, | |
| "cost": 1116, | |
| "optimal_cost": 1106, | |
| "naive_cost": 4303, | |
| "turns": 2, | |
| "checks": 1, | |
| "seconds": 233.1, | |
| "input_tokens": 44764, | |
| "output_tokens": 32151, | |
| "end_reason": "submitted", | |
| "errors": 0 | |
| }, | |
| { | |
| "model": "anthropic:claude-sonnet-5-5", | |
| "task_id": "dock-24B-w35x3-extreme-1", | |
| "split": "eval", | |
| "difficulty": "extreme", | |
| "quay": "24B", | |
| "ships": 38, | |
| "reward": 0.765633, | |
| "submitted": true, | |
| "feasible": true, | |
| "cost": 943, | |
| "optimal_cost": 865, | |
| "naive_cost": 2611, | |
| "turns": 3, | |
| "checks": 1, | |
| "seconds": 287.9, | |
| "input_tokens": 86172, | |
| "output_tokens": 36844, | |
| "end_reason": "submitted", | |
| "errors": 0 | |
| }, | |
| { | |
| "model": "anthropic:claude-sonnet-5-5", | |
| "task_id": "dock-24B-w36x2-extreme-0", | |
| "split": "eval", | |
| "difficulty": "extreme", | |
| "quay": "24B", | |
| "ships": 24, | |
| "reward": 0.934007, | |
| "submitted": true, | |
| "feasible": true, | |
| "cost": 661, | |
| "optimal_cost": 648, | |
| "naive_cost": 1736, | |
| "turns": 2, | |
| "checks": 1, | |
| "seconds": 202.5, | |
| "input_tokens": 35495, | |
| "output_tokens": 25206, | |
| "end_reason": "submitted", | |
| "errors": 0 | |
| }, | |
| { | |
| "model": "anthropic:claude-sonnet-5-5", | |
| "task_id": "dock-24B-w36x2-storm-0", | |
| "split": "eval", | |
| "difficulty": "storm", | |
| "quay": "24B", | |
| "ships": 23, | |
| "reward": 0.875598, | |
| "submitted": true, | |
| "feasible": true, | |
| "cost": 774, | |
| "optimal_cost": 756, | |
| "naive_cost": 1931, | |
| "turns": 2, | |
| "checks": 1, | |
| "seconds": 118.7, | |
| "input_tokens": 25508, | |
| "output_tokens": 15989, | |
| "end_reason": "submitted", | |
| "errors": 0 | |
| }, | |
| { | |
| "model": "anthropic:claude-sonnet-5-5", | |
| "task_id": "dock-24B-w36x2-storm-1", | |
| "split": "eval", | |
| "difficulty": "storm", | |
| "quay": "24B", | |
| "ships": 23, | |
| "reward": 1.0, | |
| "submitted": true, | |
| "feasible": true, | |
| "cost": 244, | |
| "optimal_cost": 244, | |
| "naive_cost": 549, | |
| "turns": 2, | |
| "checks": 1, | |
| "seconds": 59.0, | |
| "input_tokens": 17526, | |
| "output_tokens": 8401, | |
| "end_reason": "submitted", | |
| "errors": 0 | |
| }, | |
| { | |
| "model": "anthropic:claude-sonnet-5-5", | |
| "task_id": "dock-24B-w37x1-standard-0", | |
| "split": "eval", | |
| "difficulty": "standard", | |
| "quay": "24B", | |
| "ships": 15, | |
| "reward": 1.0, | |
| "submitted": true, | |
| "feasible": true, | |
| "cost": 229, | |
| "optimal_cost": 229, | |
| "naive_cost": 734, | |
| "turns": 2, | |
| "checks": 1, | |
| "seconds": 98.8, | |
| "input_tokens": 20589, | |
| "output_tokens": 13115, | |
| "end_reason": "submitted", | |
| "errors": 0 | |
| }, | |
| { | |
| "model": "anthropic:claude-sonnet-5-5", | |
| "task_id": "dock-36A-w05x1-busy-0", | |
| "split": "eval", | |
| "difficulty": "busy", | |
| "quay": "36A", | |
| "ships": 27, | |
| "reward": 1.0, | |
| "submitted": true, | |
| "feasible": true, | |
| "cost": 404, | |
| "optimal_cost": 404, | |
| "naive_cost": 791, | |
| "turns": 3, | |
| "checks": 1, | |
| "seconds": 96.6, | |
| "input_tokens": 31552, | |
| "output_tokens": 13065, | |
| "end_reason": "submitted", | |
| "errors": 0 | |
| }, | |
| { | |
| "model": "anthropic:claude-sonnet-5-5", | |
| "task_id": "dock-36A-w05x2-busy-0", | |
| "split": "eval", | |
| "difficulty": "busy", | |
| "quay": "36A", | |
| "ships": 51, | |
| "reward": 0.41216, | |
| "submitted": true, | |
| "feasible": true, | |
| "cost": 350, | |
| "optimal_cost": 277, | |
| "naive_cost": 1831, | |
| "turns": 4, | |
| "checks": 2, | |
| "seconds": 133.7, | |
| "input_tokens": 78745, | |
| "output_tokens": 19781, | |
| "end_reason": "submitted", | |
| "errors": 0 | |
| }, | |
| { | |
| "model": "anthropic:claude-sonnet-5-5", | |
| "task_id": "dock-36A-w05x2-extreme-0", | |
| "split": "eval", | |
| "difficulty": "extreme", | |
| "quay": "36A", | |
| "ships": 58, | |
| "reward": 0.337334, | |
| "submitted": true, | |
| "feasible": true, | |
| "cost": 1883, | |
| "optimal_cost": 1594, | |
| "naive_cost": 4448, | |
| "turns": 4, | |
| "checks": 2, | |
| "seconds": 505.7, | |
| "input_tokens": 199138, | |
| "output_tokens": 68108, | |
| "end_reason": "submitted", | |
| "errors": 0 | |
| }, | |
| { | |
| "model": "anthropic:claude-sonnet-5-5", | |
| "task_id": "dock-36A-w05x2-storm-0", | |
| "split": "eval", | |
| "difficulty": "storm", | |
| "quay": "36A", | |
| "ships": 59, | |
| "reward": 0.0, | |
| "submitted": false, | |
| "feasible": false, | |
| "cost": null, | |
| "optimal_cost": 290, | |
| "naive_cost": 4214, | |
| "turns": 2, | |
| "checks": 0, | |
| "seconds": 464.8, | |
| "input_tokens": 45528, | |
| "output_tokens": 64000, | |
| "end_reason": "no_tool_call", | |
| "errors": 0 | |
| }, | |
| { | |
| "model": "anthropic:claude-sonnet-5-5", | |
| "task_id": "dock-36A-w06x1-busy-0", | |
| "split": "eval", | |
| "difficulty": "busy", | |
| "quay": "36A", | |
| "ships": 25, | |
| "reward": 0.801182, | |
| "submitted": true, | |
| "feasible": true, | |
| "cost": 60, | |
| "optimal_cost": 40, | |
| "naive_cost": 824, | |
| "turns": 2, | |
| "checks": 1, | |
| "seconds": 190.7, | |
| "input_tokens": 34672, | |
| "output_tokens": 25429, | |
| "end_reason": "submitted", | |
| "errors": 0 | |
| }, | |
| { | |
| "model": "anthropic:claude-sonnet-5-5", | |
| "task_id": "dock-36A-w06x1-standard-0", | |
| "split": "eval", | |
| "difficulty": "standard", | |
| "quay": "36A", | |
| "ships": 24, | |
| "reward": 0.986087, | |
| "submitted": true, | |
| "feasible": true, | |
| "cost": 15, | |
| "optimal_cost": 14, | |
| "naive_cost": 591, | |
| "turns": 2, | |
| "checks": 1, | |
| "seconds": 76.8, | |
| "input_tokens": 18306, | |
| "output_tokens": 9887, | |
| "end_reason": "submitted", | |
| "errors": 0 | |
| }, | |
| { | |
| "model": "anthropic:claude-sonnet-5-5", | |
| "task_id": "dock-36A-w10x1-standard-0", | |
| "split": "eval", | |
| "difficulty": "standard", | |
| "quay": "36A", | |
| "ships": 32, | |
| "reward": 0.933374, | |
| "submitted": true, | |
| "feasible": true, | |
| "cost": 44, | |
| "optimal_cost": 39, | |
| "naive_cost": 606, | |
| "turns": 3, | |
| "checks": 2, | |
| "seconds": 71.4, | |
| "input_tokens": 29654, | |
| "output_tokens": 10641, | |
| "end_reason": "submitted", | |
| "errors": 0 | |
| }, | |
| { | |
| "model": "anthropic:claude-sonnet-5-5", | |
| "task_id": "dock-36A-w15x1-busy-0", | |
| "split": "eval", | |
| "difficulty": "busy", | |
| "quay": "36A", | |
| "ships": 29, | |
| "reward": 0.290887, | |
| "submitted": true, | |
| "feasible": true, | |
| "cost": 327, | |
| "optimal_cost": 153, | |
| "naive_cost": 1963, | |
| "turns": 3, | |
| "checks": 1, | |
| "seconds": 331.9, | |
| "input_tokens": 90215, | |
| "output_tokens": 45388, | |
| "end_reason": "submitted", | |
| "errors": 0 | |
| }, | |
| { | |
| "model": "anthropic:claude-sonnet-5-5", | |
| "task_id": "dock-36A-w15x1-standard-0", | |
| "split": "eval", | |
| "difficulty": "standard", | |
| "quay": "36A", | |
| "ships": 30, | |
| "reward": 0.67479, | |
| "submitted": true, | |
| "feasible": true, | |
| "cost": 149, | |
| "optimal_cost": 119, | |
| "naive_cost": 2613, | |
| "turns": 2, | |
| "checks": 1, | |
| "seconds": 192.6, | |
| "input_tokens": 37272, | |
| "output_tokens": 27717, | |
| "end_reason": "submitted", | |
| "errors": 0 | |
| }, | |
| { | |
| "model": "anthropic:claude-sonnet-5-5", | |
| "task_id": "dock-36A-w15x2-busy-0", | |
| "split": "eval", | |
| "difficulty": "busy", | |
| "quay": "36A", | |
| "ships": 55, | |
| "reward": 0.581884, | |
| "submitted": true, | |
| "feasible": true, | |
| "cost": 412, | |
| "optimal_cost": 368, | |
| "naive_cost": 3372, | |
| "turns": 3, | |
| "checks": 2, | |
| "seconds": 207.6, | |
| "input_tokens": 81972, | |
| "output_tokens": 30460, | |
| "end_reason": "submitted", | |
| "errors": 0 | |
| }, | |
| { | |
| "model": "anthropic:claude-sonnet-5-5", | |
| "task_id": "dock-36A-w15x2-storm-0", | |
| "split": "eval", | |
| "difficulty": "storm", | |
| "quay": "36A", | |
| "ships": 56, | |
| "reward": 0.378824, | |
| "submitted": true, | |
| "feasible": true, | |
| "cost": 578, | |
| "optimal_cost": 369, | |
| "naive_cost": 6062, | |
| "turns": 8, | |
| "checks": 6, | |
| "seconds": 632.4, | |
| "input_tokens": 583378, | |
| "output_tokens": 88084, | |
| "end_reason": "submitted", | |
| "errors": 0 | |
| }, | |
| { | |
| "model": "anthropic:claude-sonnet-5-5", | |
| "task_id": "dock-36A-w16x1-standard-0", | |
| "split": "eval", | |
| "difficulty": "standard", | |
| "quay": "36A", | |
| "ships": 27, | |
| "reward": 0.872296, | |
| "submitted": true, | |
| "feasible": true, | |
| "cost": 233, | |
| "optimal_cost": 223, | |
| "naive_cost": 1083, | |
| "turns": 3, | |
| "checks": 1, | |
| "seconds": 158.6, | |
| "input_tokens": 39615, | |
| "output_tokens": 21452, | |
| "end_reason": "submitted", | |
| "errors": 0 | |
| }, | |
| { | |
| "model": "anthropic:claude-sonnet-5-5", | |
| "task_id": "dock-36A-w16x2-busy-0", | |
| "split": "eval", | |
| "difficulty": "busy", | |
| "quay": "36A", | |
| "ships": 55, | |
| "reward": 0.754432, | |
| "submitted": true, | |
| "feasible": true, | |
| "cost": 425, | |
| "optimal_cost": 403, | |
| "naive_cost": 2046, | |
| "turns": 4, | |
| "checks": 2, | |
| "seconds": 334.6, | |
| "input_tokens": 149456, | |
| "output_tokens": 44955, | |
| "end_reason": "submitted", | |
| "errors": 0 | |
| }, | |
| { | |
| "model": "anthropic:claude-sonnet-5-5", | |
| "task_id": "dock-36A-w16x2-storm-0", | |
| "split": "eval", | |
| "difficulty": "storm", | |
| "quay": "36A", | |
| "ships": 57, | |
| "reward": 0.448112, | |
| "submitted": true, | |
| "feasible": true, | |
| "cost": 204, | |
| "optimal_cost": 108, | |
| "naive_cost": 6938, | |
| "turns": 5, | |
| "checks": 3, | |
| "seconds": 482.2, | |
| "input_tokens": 241256, | |
| "output_tokens": 65722, | |
| "end_reason": "submitted", | |
| "errors": 0 | |
| }, | |
| { | |
| "model": "anthropic:claude-sonnet-5-5", | |
| "task_id": "dock-36A-w16x2-storm-1", | |
| "split": "eval", | |
| "difficulty": "storm", | |
| "quay": "36A", | |
| "ships": 55, | |
| "reward": 0.454503, | |
| "submitted": true, | |
| "feasible": true, | |
| "cost": 761, | |
| "optimal_cost": 627, | |
| "naive_cost": 6819, | |
| "turns": 4, | |
| "checks": 2, | |
| "seconds": 298.0, | |
| "input_tokens": 144957, | |
| "output_tokens": 42191, | |
| "end_reason": "submitted", | |
| "errors": 0 | |
| }, | |
| { | |
| "model": "anthropic:claude-sonnet-5-5", | |
| "task_id": "dock-36A-w17x1-busy-0", | |
| "split": "eval", | |
| "difficulty": "busy", | |
| "quay": "36A", | |
| "ships": 28, | |
| "reward": 0.707789, | |
| "submitted": true, | |
| "feasible": true, | |
| "cost": 177, | |
| "optimal_cost": 152, | |
| "naive_cost": 1521, | |
| "turns": 3, | |
| "checks": 2, | |
| "seconds": 144.7, | |
| "input_tokens": 53445, | |
| "output_tokens": 19702, | |
| "end_reason": "submitted", | |
| "errors": 0 | |
| }, | |
| { | |
| "model": "anthropic:claude-sonnet-5-5", | |
| "task_id": "dock-36A-w17x1-standard-0", | |
| "split": "eval", | |
| "difficulty": "standard", | |
| "quay": "36A", | |
| "ships": 29, | |
| "reward": 0.936036, | |
| "submitted": true, | |
| "feasible": true, | |
| "cost": 105, | |
| "optimal_cost": 100, | |
| "naive_cost": 1794, | |
| "turns": 2, | |
| "checks": 1, | |
| "seconds": 193.0, | |
| "input_tokens": 36292, | |
| "output_tokens": 26745, | |
| "end_reason": "submitted", | |
| "errors": 0 | |
| }, | |
| { | |
| "model": "anthropic:claude-sonnet-5-5", | |
| "task_id": "dock-36A-w35x1-standard-0", | |
| "split": "eval", | |
| "difficulty": "standard", | |
| "quay": "36A", | |
| "ships": 19, | |
| "reward": 1.0, | |
| "submitted": true, | |
| "feasible": true, | |
| "cost": 10, | |
| "optimal_cost": 10, | |
| "naive_cost": 274, | |
| "turns": 2, | |
| "checks": 1, | |
| "seconds": 72.3, | |
| "input_tokens": 18006, | |
| "output_tokens": 9848, | |
| "end_reason": "submitted", | |
| "errors": 0 | |
| }, | |
| { | |
| "model": "anthropic:claude-sonnet-5-5", | |
| "task_id": "dock-36A-w35x2-extreme-0", | |
| "split": "eval", | |
| "difficulty": "extreme", | |
| "quay": "36A", | |
| "ships": 39, | |
| "reward": 0.922394, | |
| "submitted": true, | |
| "feasible": true, | |
| "cost": 764, | |
| "optimal_cost": 754, | |
| "naive_cost": 3430, | |
| "turns": 4, | |
| "checks": 2, | |
| "seconds": 411.7, | |
| "input_tokens": 168320, | |
| "output_tokens": 55468, | |
| "end_reason": "submitted", | |
| "errors": 0 | |
| }, | |
| { | |
| "model": "anthropic:claude-sonnet-5-5", | |
| "task_id": "dock-36A-w35x2-extreme-1", | |
| "split": "eval", | |
| "difficulty": "extreme", | |
| "quay": "36A", | |
| "ships": 35, | |
| "reward": 0.226932, | |
| "submitted": true, | |
| "feasible": true, | |
| "cost": 1510, | |
| "optimal_cost": 1315, | |
| "naive_cost": 3041, | |
| "turns": 3, | |
| "checks": 2, | |
| "seconds": 250.7, | |
| "input_tokens": 86549, | |
| "output_tokens": 34646, | |
| "end_reason": "submitted", | |
| "errors": 0 | |
| }, | |
| { | |
| "model": "anthropic:claude-sonnet-5-5", | |
| "task_id": "dock-36A-w35x2-storm-0", | |
| "split": "eval", | |
| "difficulty": "storm", | |
| "quay": "36A", | |
| "ships": 33, | |
| "reward": 0.600896, | |
| "submitted": true, | |
| "feasible": true, | |
| "cost": 140, | |
| "optimal_cost": 102, | |
| "naive_cost": 670, | |
| "turns": 3, | |
| "checks": 2, | |
| "seconds": 162.7, | |
| "input_tokens": 63296, | |
| "output_tokens": 23322, | |
| "end_reason": "submitted", | |
| "errors": 0 | |
| }, | |
| { | |
| "model": "anthropic:claude-sonnet-5-5", | |
| "task_id": "dock-36A-w35x3-extreme-0", | |
| "split": "eval", | |
| "difficulty": "extreme", | |
| "quay": "36A", | |
| "ships": 59, | |
| "reward": 0.915625, | |
| "submitted": true, | |
| "feasible": true, | |
| "cost": 971, | |
| "optimal_cost": 953, | |
| "naive_cost": 5403, | |
| "turns": 5, | |
| "checks": 3, | |
| "seconds": 466.5, | |
| "input_tokens": 266479, | |
| "output_tokens": 66032, | |
| "end_reason": "submitted", | |
| "errors": 0 | |
| }, | |
| { | |
| "model": "anthropic:claude-sonnet-5-5", | |
| "task_id": "dock-36A-w36x2-busy-0", | |
| "split": "eval", | |
| "difficulty": "busy", | |
| "quay": "36A", | |
| "ships": 37, | |
| "reward": 0.998792, | |
| "submitted": true, | |
| "feasible": true, | |
| "cost": 3144, | |
| "optimal_cost": 3143, | |
| "naive_cost": 9374, | |
| "turns": 3, | |
| "checks": 2, | |
| "seconds": 248.5, | |
| "input_tokens": 78144, | |
| "output_tokens": 33256, | |
| "end_reason": "submitted", | |
| "errors": 0 | |
| }, | |
| { | |
| "model": "anthropic:claude-sonnet-5-5", | |
| "task_id": "dock-36A-w36x2-extreme-0", | |
| "split": "eval", | |
| "difficulty": "extreme", | |
| "quay": "36A", | |
| "ships": 42, | |
| "reward": 0.516779, | |
| "submitted": true, | |
| "feasible": true, | |
| "cost": 427, | |
| "optimal_cost": 320, | |
| "naive_cost": 4753, | |
| "turns": 4, | |
| "checks": 2, | |
| "seconds": 501.7, | |
| "input_tokens": 184365, | |
| "output_tokens": 65160, | |
| "end_reason": "submitted", | |
| "errors": 0 | |
| }, | |
| { | |
| "model": "anthropic:claude-sonnet-5-5", | |
| "task_id": "dock-36A-w36x2-storm-0", | |
| "split": "eval", | |
| "difficulty": "storm", | |
| "quay": "36A", | |
| "ships": 37, | |
| "reward": 1.0, | |
| "submitted": true, | |
| "feasible": true, | |
| "cost": 28, | |
| "optimal_cost": 28, | |
| "naive_cost": 864, | |
| "turns": 3, | |
| "checks": 2, | |
| "seconds": 109.5, | |
| "input_tokens": 47499, | |
| "output_tokens": 14778, | |
| "end_reason": "submitted", | |
| "errors": 0 | |
| }, | |
| { | |
| "model": "anthropic:claude-sonnet-5-5", | |
| "task_id": "dock-36A-w37x1-busy-0", | |
| "split": "eval", | |
| "difficulty": "busy", | |
| "quay": "36A", | |
| "ships": 23, | |
| "reward": 1.0, | |
| "submitted": true, | |
| "feasible": true, | |
| "cost": 36, | |
| "optimal_cost": 36, | |
| "naive_cost": 598, | |
| "turns": 2, | |
| "checks": 1, | |
| "seconds": 37.5, | |
| "input_tokens": 14146, | |
| "output_tokens": 5168, | |
| "end_reason": "submitted", | |
| "errors": 0 | |
| }, | |
| { | |
| "model": "anthropic:claude-sonnet-5-5", | |
| "task_id": "dock-36A-w37x1-standard-0", | |
| "split": "eval", | |
| "difficulty": "standard", | |
| "quay": "36A", | |
| "ships": 22, | |
| "reward": 0.88409, | |
| "submitted": true, | |
| "feasible": true, | |
| "cost": 24, | |
| "optimal_cost": 15, | |
| "naive_cost": 429, | |
| "turns": 2, | |
| "checks": 1, | |
| "seconds": 41.5, | |
| "input_tokens": 14210, | |
| "output_tokens": 5775, | |
| "end_reason": "submitted", | |
| "errors": 0 | |
| }, | |
| { | |
| "model": "hf:Qwen/Qwen3.8-2.4T-A95B:together", | |
| "task_id": "dock-24B-w06x1-busy-0", | |
| "split": "eval", | |
| "difficulty": "busy", | |
| "quay": "24B", | |
| "ships": 17, | |
| "reward": 0.0, | |
| "submitted": false, | |
| "feasible": false, | |
| "cost": null, | |
| "optimal_cost": 87, | |
| "naive_cost": 658, | |
| "turns": 2, | |
| "checks": 0, | |
| "seconds": 642.2, | |
| "input_tokens": 10282, | |
| "output_tokens": 64000, | |
| "end_reason": "no_tool_call", | |
| "errors": 0 | |
| }, | |
| { | |
| "model": "hf:Qwen/Qwen3.8-2.4T-A95B:together", | |
| "task_id": "dock-24B-w06x1-standard-0", | |
| "split": "eval", | |
| "difficulty": "standard", | |
| "quay": "24B", | |
| "ships": 16, | |
| "reward": 1.0, | |
| "submitted": true, | |
| "feasible": true, | |
| "cost": 114, | |
| "optimal_cost": 114, | |
| "naive_cost": 311, | |
| "turns": 5, | |
| "checks": 3, | |
| "seconds": 282.8, | |
| "input_tokens": 28295, | |
| "output_tokens": 39263, | |
| "end_reason": "submitted", | |
| "errors": 0 | |
| }, | |
| { | |
| "model": "hf:Qwen/Qwen3.8-2.4T-A95B:together", | |
| "task_id": "dock-24B-w06x2-busy-0", | |
| "split": "eval", | |
| "difficulty": "busy", | |
| "quay": "24B", | |
| "ships": 31, | |
| "reward": 0.885923, | |
| "submitted": true, | |
| "feasible": true, | |
| "cost": 626, | |
| "optimal_cost": 616, | |
| "naive_cost": 1181, | |
| "turns": 11, | |
| "checks": 8, | |
| "seconds": 726.1, | |
| "input_tokens": 168888, | |
| "output_tokens": 93914, | |
| "end_reason": "submitted", | |
| "errors": 0 | |
| }, | |
| { | |
| "model": "hf:Qwen/Qwen3.8-2.4T-A95B:together", | |
| "task_id": "dock-24B-w06x2-extreme-0", | |
| "split": "eval", | |
| "difficulty": "extreme", | |
| "quay": "24B", | |
| "ships": 32, | |
| "reward": 0.18125, | |
| "submitted": true, | |
| "feasible": false, | |
| "cost": null, | |
| "optimal_cost": 231, | |
| "naive_cost": 1090, | |
| "turns": 11, | |
| "checks": 8, | |
| "seconds": 1430.8, | |
| "input_tokens": 202794, | |
| "output_tokens": 106832, | |
| "end_reason": "submitted", | |
| "errors": 0 | |
| }, | |
| { | |
| "model": "hf:Qwen/Qwen3.8-2.4T-A95B:together", | |
| "task_id": "dock-24B-w06x2-storm-0", | |
| "split": "eval", | |
| "difficulty": "storm", | |
| "quay": "24B", | |
| "ships": 32, | |
| "reward": 0.842156, | |
| "submitted": true, | |
| "feasible": true, | |
| "cost": 174, | |
| "optimal_cost": 154, | |
| "naive_cost": 1259, | |
| "turns": 11, | |
| "checks": 7, | |
| "seconds": 863.1, | |
| "input_tokens": 210000, | |
| "output_tokens": 116577, | |
| "end_reason": "submitted", | |
| "errors": 0 | |
| }, | |
| { | |
| "model": "hf:Qwen/Qwen3.8-2.4T-A95B:together", | |
| "task_id": "dock-24B-w07x1-busy-0", | |
| "split": "eval", | |
| "difficulty": "busy", | |
| "quay": "24B", | |
| "ships": 17, | |
| "reward": 0.938097, | |
| "submitted": true, | |
| "feasible": true, | |
| "cost": 232, | |
| "optimal_cost": 226, | |
| "naive_cost": 641, | |
| "turns": 5, | |
| "checks": 3, | |
| "seconds": 556.0, | |
| "input_tokens": 38786, | |
| "output_tokens": 71104, | |
| "end_reason": "submitted", | |
| "errors": 0 | |
| }, | |
| { | |
| "model": "hf:Qwen/Qwen3.8-2.4T-A95B:together", | |
| "task_id": "dock-24B-w15x2-extreme-0", | |
| "split": "eval", | |
| "difficulty": "extreme", | |
| "quay": "24B", | |
| "ships": 24, | |
| "reward": 0.0, | |
| "submitted": false, | |
| "feasible": false, | |
| "cost": null, | |
| "optimal_cost": 1009, | |
| "naive_cost": 3964, | |
| "turns": 2, | |
| "checks": 0, | |
| "seconds": 425.2, | |
| "input_tokens": 11310, | |
| "output_tokens": 64000, | |
| "end_reason": "no_tool_call", | |
| "errors": 0 | |
| }, | |
| { | |
| "model": "hf:Qwen/Qwen3.8-2.4T-A95B:together", | |
| "task_id": "dock-24B-w16x1-busy-0", | |
| "split": "eval", | |
| "difficulty": "busy", | |
| "quay": "24B", | |
| "ships": 16, | |
| "reward": 0.727887, | |
| "submitted": true, | |
| "feasible": true, | |
| "cost": 1497, | |
| "optimal_cost": 1349, | |
| "naive_cost": 4069, | |
| "turns": 5, | |
| "checks": 2, | |
| "seconds": 1129.6, | |
| "input_tokens": 34733, | |
| "output_tokens": 112169, | |
| "end_reason": "submitted", | |
| "errors": 0 | |
| }, | |
| { | |
| "model": "hf:Qwen/Qwen3.8-2.4T-A95B:together", | |
| "task_id": "dock-24B-w16x2-extreme-0", | |
| "split": "eval", | |
| "difficulty": "extreme", | |
| "quay": "24B", | |
| "ships": 28, | |
| "reward": 0.301564, | |
| "submitted": true, | |
| "feasible": true, | |
| "cost": 692, | |
| "optimal_cost": 466, | |
| "naive_cost": 1485, | |
| "turns": 11, | |
| "checks": 7, | |
| "seconds": 1005.8, | |
| "input_tokens": 189755, | |
| "output_tokens": 164814, | |
| "end_reason": "submitted", | |
| "errors": 0 | |
| }, | |
| { | |
| "model": "hf:Qwen/Qwen3.8-2.4T-A95B:together", | |
| "task_id": "dock-24B-w16x2-storm-0", | |
| "split": "eval", | |
| "difficulty": "storm", | |
| "quay": "24B", | |
| "ships": 27, | |
| "reward": 0.177778, | |
| "submitted": true, | |
| "feasible": false, | |
| "cost": null, | |
| "optimal_cost": 201, | |
| "naive_cost": 1569, | |
| "turns": 10, | |
| "checks": 6, | |
| "seconds": 854.1, | |
| "input_tokens": 152545, | |
| "output_tokens": 115849, | |
| "end_reason": "submitted", | |
| "errors": 0 | |
| }, | |
| { | |
| "model": "hf:Qwen/Qwen3.8-2.4T-A95B:together", | |
| "task_id": "dock-24B-w16x2-storm-1", | |
| "split": "eval", | |
| "difficulty": "storm", | |
| "quay": "24B", | |
| "ships": 26, | |
| "reward": 0.504305, | |
| "submitted": true, | |
| "feasible": true, | |
| "cost": 1375, | |
| "optimal_cost": 1187, | |
| "naive_cost": 2403, | |
| "turns": 11, | |
| "checks": 8, | |
| "seconds": 674.1, | |
| "input_tokens": 180848, | |
| "output_tokens": 121438, | |
| "end_reason": "submitted", | |
| "errors": 0 | |
| }, | |
| { | |
| "model": "hf:Qwen/Qwen3.8-2.4T-A95B:together", | |
| "task_id": "dock-24B-w35x1-busy-0", | |
| "split": "eval", | |
| "difficulty": "busy", | |
| "quay": "24B", | |
| "ships": 14, | |
| "reward": 0.0, | |
| "submitted": false, | |
| "feasible": false, | |
| "cost": null, | |
| "optimal_cost": 1223, | |
| "naive_cost": 2892, | |
| "turns": 2, | |
| "checks": 0, | |
| "seconds": 645.9, | |
| "input_tokens": 9330, | |
| "output_tokens": 64000, | |
| "end_reason": "no_tool_call", | |
| "errors": 0 | |
| }, | |
| { | |
| "model": "hf:Qwen/Qwen3.8-2.4T-A95B:together", | |
| "task_id": "dock-24B-w35x2-busy-0", | |
| "split": "eval", | |
| "difficulty": "busy", | |
| "quay": "24B", | |
| "ships": 24, | |
| "reward": 0.083333, | |
| "submitted": true, | |
| "feasible": false, | |
| "cost": null, | |
| "optimal_cost": 162, | |
| "naive_cost": 1443, | |
| "turns": 4, | |
| "checks": 2, | |
| "seconds": 888.1, | |
| "input_tokens": 33441, | |
| "output_tokens": 77851, | |
| "end_reason": "submitted", | |
| "errors": 0 | |
| }, | |
| { | |
| "model": "hf:Qwen/Qwen3.8-2.4T-A95B:together", | |
| "task_id": "dock-24B-w35x2-extreme-0", | |
| "split": "eval", | |
| "difficulty": "extreme", | |
| "quay": "24B", | |
| "ships": 24, | |
| "reward": 1.0, | |
| "submitted": true, | |
| "feasible": true, | |
| "cost": 1792, | |
| "optimal_cost": 1792, | |
| "naive_cost": 4682, | |
| "turns": 3, | |
| "checks": 1, | |
| "seconds": 304.1, | |
| "input_tokens": 18963, | |
| "output_tokens": 47799, | |
| "end_reason": "submitted", | |
| "errors": 0 | |
| }, | |
| { | |
| "model": "hf:Qwen/Qwen3.8-2.4T-A95B:together", | |
| "task_id": "dock-24B-w35x2-extreme-1", | |
| "split": "eval", | |
| "difficulty": "extreme", | |
| "quay": "24B", | |
| "ships": 25, | |
| "reward": 0.801182, | |
| "submitted": true, | |
| "feasible": true, | |
| "cost": 308, | |
| "optimal_cost": 283, | |
| "naive_cost": 1356, | |
| "turns": 7, | |
| "checks": 5, | |
| "seconds": 465.4, | |
| "input_tokens": 81331, | |
| "output_tokens": 77172, | |
| "end_reason": "submitted", | |
| "errors": 0 | |
| }, | |
| { | |
| "model": "hf:Qwen/Qwen3.8-2.4T-A95B:together", | |
| "task_id": "dock-24B-w35x2-storm-0", | |
| "split": "eval", | |
| "difficulty": "storm", | |
| "quay": "24B", | |
| "ships": 24, | |
| "reward": 0.554334, | |
| "submitted": true, | |
| "feasible": true, | |
| "cost": 2863, | |
| "optimal_cost": 2591, | |
| "naive_cost": 4835, | |
| "turns": 11, | |
| "checks": 9, | |
| "seconds": 1661.5, | |
| "input_tokens": 191886, | |
| "output_tokens": 140759, | |
| "end_reason": "submitted", | |
| "errors": 0 | |
| }, | |
| { | |
| "model": "hf:Qwen/Qwen3.8-2.4T-A95B:together", | |
| "task_id": "dock-24B-w35x2-storm-1", | |
| "split": "eval", | |
| "difficulty": "storm", | |
| "quay": "24B", | |
| "ships": 24, | |
| "reward": 0.759738, | |
| "submitted": true, | |
| "feasible": true, | |
| "cost": 342, | |
| "optimal_cost": 312, | |
| "naive_cost": 2701, | |
| "turns": 4, | |
| "checks": 1, | |
| "seconds": 271.9, | |
| "input_tokens": 31559, | |
| "output_tokens": 41916, | |
| "end_reason": "submitted", | |
| "errors": 0 | |
| }, | |
| { | |
| "model": "hf:Qwen/Qwen3.8-2.4T-A95B:together", | |
| "task_id": "dock-24B-w35x3-extreme-0", | |
| "split": "eval", | |
| "difficulty": "extreme", | |
| "quay": "24B", | |
| "ships": 39, | |
| "reward": 0.174359, | |
| "submitted": true, | |
| "feasible": false, | |
| "cost": null, | |
| "optimal_cost": 1106, | |
| "naive_cost": 4303, | |
| "turns": 11, | |
| "checks": 8, | |
| "seconds": 1092.4, | |
| "input_tokens": 243474, | |
| "output_tokens": 177915, | |
| "end_reason": "submitted", | |
| "errors": 0 | |
| }, | |
| { | |
| "model": "hf:Qwen/Qwen3.8-2.4T-A95B:together", | |
| "task_id": "dock-24B-w35x3-extreme-1", | |
| "split": "eval", | |
| "difficulty": "extreme", | |
| "quay": "24B", | |
| "ships": 38, | |
| "reward": 0.0, | |
| "submitted": false, | |
| "feasible": false, | |
| "cost": null, | |
| "optimal_cost": 865, | |
| "naive_cost": 2611, | |
| "turns": 3, | |
| "checks": 0, | |
| "seconds": 381.1, | |
| "input_tokens": 24305, | |
| "output_tokens": 64108, | |
| "end_reason": "no_tool_call", | |
| "errors": 0 | |
| }, | |
| { | |
| "model": "hf:Qwen/Qwen3.8-2.4T-A95B:together", | |
| "task_id": "dock-24B-w36x2-extreme-0", | |
| "split": "eval", | |
| "difficulty": "extreme", | |
| "quay": "24B", | |
| "ships": 24, | |
| "reward": 0.15, | |
| "submitted": true, | |
| "feasible": false, | |
| "cost": null, | |
| "optimal_cost": 648, | |
| "naive_cost": 1736, | |
| "turns": 11, | |
| "checks": 9, | |
| "seconds": 553.4, | |
| "input_tokens": 174791, | |
| "output_tokens": 85561, | |
| "end_reason": "submitted", | |
| "errors": 0 | |
| }, | |
| { | |
| "model": "hf:Qwen/Qwen3.8-2.4T-A95B:together", | |
| "task_id": "dock-24B-w36x2-storm-0", | |
| "split": "eval", | |
| "difficulty": "storm", | |
| "quay": "24B", | |
| "ships": 23, | |
| "reward": 0.173913, | |
| "submitted": true, | |
| "feasible": false, | |
| "cost": null, | |
| "optimal_cost": 756, | |
| "naive_cost": 1931, | |
| "turns": 8, | |
| "checks": 6, | |
| "seconds": 667.6, | |
| "input_tokens": 96397, | |
| "output_tokens": 93335, | |
| "end_reason": "submitted", | |
| "errors": 0 | |
| }, | |
| { | |
| "model": "hf:Qwen/Qwen3.8-2.4T-A95B:together", | |
| "task_id": "dock-24B-w36x2-storm-1", | |
| "split": "eval", | |
| "difficulty": "storm", | |
| "quay": "24B", | |
| "ships": 23, | |
| "reward": 0.0, | |
| "submitted": true, | |
| "feasible": false, | |
| "cost": null, | |
| "optimal_cost": 244, | |
| "naive_cost": 549, | |
| "turns": 3, | |
| "checks": 2, | |
| "seconds": 292.5, | |
| "input_tokens": 15727, | |
| "output_tokens": 44032, | |
| "end_reason": "submitted", | |
| "errors": 0 | |
| }, | |
| { | |
| "model": "hf:Qwen/Qwen3.8-2.4T-A95B:together", | |
| "task_id": "dock-24B-w37x1-standard-0", | |
| "split": "eval", | |
| "difficulty": "standard", | |
| "quay": "24B", | |
| "ships": 15, | |
| "reward": 0.173333, | |
| "submitted": true, | |
| "feasible": false, | |
| "cost": null, | |
| "optimal_cost": 229, | |
| "naive_cost": 734, | |
| "turns": 6, | |
| "checks": 4, | |
| "seconds": 730.5, | |
| "input_tokens": 37465, | |
| "output_tokens": 102519, | |
| "end_reason": "submitted", | |
| "errors": 0 | |
| }, | |
| { | |
| "model": "hf:Qwen/Qwen3.8-2.4T-A95B:together", | |
| "task_id": "dock-36A-w05x1-busy-0", | |
| "split": "eval", | |
| "difficulty": "busy", | |
| "quay": "36A", | |
| "ships": 27, | |
| "reward": 0.942882, | |
| "submitted": true, | |
| "feasible": true, | |
| "cost": 409, | |
| "optimal_cost": 404, | |
| "naive_cost": 791, | |
| "turns": 7, | |
| "checks": 4, | |
| "seconds": 542.1, | |
| "input_tokens": 78800, | |
| "output_tokens": 64863, | |
| "end_reason": "submitted", | |
| "errors": 0 | |
| }, | |
| { | |
| "model": "hf:Qwen/Qwen3.8-2.4T-A95B:together", | |
| "task_id": "dock-36A-w05x2-busy-0", | |
| "split": "eval", | |
| "difficulty": "busy", | |
| "quay": "36A", | |
| "ships": 51, | |
| "reward": 0.608252, | |
| "submitted": true, | |
| "feasible": true, | |
| "cost": 314, | |
| "optimal_cost": 277, | |
| "naive_cost": 1831, | |
| "turns": 9, | |
| "checks": 6, | |
| "seconds": 695.9, | |
| "input_tokens": 184248, | |
| "output_tokens": 75536, | |
| "end_reason": "submitted", | |
| "errors": 0 | |
| }, | |
| { | |
| "model": "hf:Qwen/Qwen3.8-2.4T-A95B:together", | |
| "task_id": "dock-36A-w05x2-extreme-0", | |
| "split": "eval", | |
| "difficulty": "extreme", | |
| "quay": "36A", | |
| "ships": 58, | |
| "reward": 0.0, | |
| "submitted": false, | |
| "feasible": false, | |
| "cost": null, | |
| "optimal_cost": 1594, | |
| "naive_cost": 4448, | |
| "turns": 2, | |
| "checks": 0, | |
| "seconds": 406.0, | |
| "input_tokens": 17113, | |
| "output_tokens": 64000, | |
| "end_reason": "no_tool_call", | |
| "errors": 0 | |
| }, | |
| { | |
| "model": "hf:Qwen/Qwen3.8-2.4T-A95B:together", | |
| "task_id": "dock-36A-w05x2-storm-0", | |
| "split": "eval", | |
| "difficulty": "storm", | |
| "quay": "36A", | |
| "ships": 59, | |
| "reward": 0.0, | |
| "submitted": false, | |
| "feasible": false, | |
| "cost": null, | |
| "optimal_cost": 290, | |
| "naive_cost": 4214, | |
| "turns": 3, | |
| "checks": 0, | |
| "seconds": 529.9, | |
| "input_tokens": 29744, | |
| "output_tokens": 64052, | |
| "end_reason": "no_tool_call", | |
| "errors": 0 | |
| }, | |
| { | |
| "model": "hf:Qwen/Qwen3.8-2.4T-A95B:together", | |
| "task_id": "dock-36A-w06x1-busy-0", | |
| "split": "eval", | |
| "difficulty": "busy", | |
| "quay": "36A", | |
| "ships": 25, | |
| "reward": 0.743972, | |
| "submitted": true, | |
| "feasible": true, | |
| "cost": 67, | |
| "optimal_cost": 40, | |
| "naive_cost": 824, | |
| "turns": 10, | |
| "checks": 7, | |
| "seconds": 1890.5, | |
| "input_tokens": 163968, | |
| "output_tokens": 156541, | |
| "end_reason": "submitted", | |
| "errors": 0 | |
| }, | |
| { | |
| "model": "hf:Qwen/Qwen3.8-2.4T-A95B:together", | |
| "task_id": "dock-36A-w06x1-standard-0", | |
| "split": "eval", | |
| "difficulty": "standard", | |
| "quay": "36A", | |
| "ships": 24, | |
| "reward": 0.986087, | |
| "submitted": true, | |
| "feasible": true, | |
| "cost": 15, | |
| "optimal_cost": 14, | |
| "naive_cost": 591, | |
| "turns": 4, | |
| "checks": 1, | |
| "seconds": 421.6, | |
| "input_tokens": 28552, | |
| "output_tokens": 84163, | |
| "end_reason": "submitted", | |
| "errors": 0 | |
| }, | |
| { | |
| "model": "hf:Qwen/Qwen3.8-2.4T-A95B:together", | |
| "task_id": "dock-36A-w10x1-standard-0", | |
| "split": "eval", | |
| "difficulty": "standard", | |
| "quay": "36A", | |
| "ships": 32, | |
| "reward": 1.0, | |
| "submitted": true, | |
| "feasible": true, | |
| "cost": 39, | |
| "optimal_cost": 39, | |
| "naive_cost": 606, | |
| "turns": 3, | |
| "checks": 1, | |
| "seconds": 181.0, | |
| "input_tokens": 20377, | |
| "output_tokens": 35420, | |
| "end_reason": "submitted", | |
| "errors": 0 | |
| }, | |
| { | |
| "model": "hf:Qwen/Qwen3.8-2.4T-A95B:together", | |
| "task_id": "dock-36A-w15x1-busy-0", | |
| "split": "eval", | |
| "difficulty": "busy", | |
| "quay": "36A", | |
| "ships": 29, | |
| "reward": 0.0, | |
| "submitted": false, | |
| "feasible": false, | |
| "cost": null, | |
| "optimal_cost": 153, | |
| "naive_cost": 1963, | |
| "turns": 2, | |
| "checks": 0, | |
| "seconds": 483.1, | |
| "input_tokens": 12254, | |
| "output_tokens": 64000, | |
| "end_reason": "no_tool_call", | |
| "errors": 0 | |
| }, | |
| { | |
| "model": "hf:Qwen/Qwen3.8-2.4T-A95B:together", | |
| "task_id": "dock-36A-w15x1-standard-0", | |
| "split": "eval", | |
| "difficulty": "standard", | |
| "quay": "36A", | |
| "ships": 30, | |
| "reward": 0.0, | |
| "submitted": false, | |
| "feasible": false, | |
| "cost": null, | |
| "optimal_cost": 119, | |
| "naive_cost": 2613, | |
| "turns": 3, | |
| "checks": 0, | |
| "seconds": 434.9, | |
| "input_tokens": 19399, | |
| "output_tokens": 64064, | |
| "end_reason": "no_tool_call", | |
| "errors": 0 | |
| }, | |
| { | |
| "model": "hf:Qwen/Qwen3.8-2.4T-A95B:together", | |
| "task_id": "dock-36A-w15x2-busy-0", | |
| "split": "eval", | |
| "difficulty": "busy", | |
| "quay": "36A", | |
| "ships": 55, | |
| "reward": 0.0, | |
| "submitted": false, | |
| "feasible": false, | |
| "cost": null, | |
| "optimal_cost": 368, | |
| "naive_cost": 3372, | |
| "turns": 2, | |
| "checks": 0, | |
| "seconds": 842.9, | |
| "input_tokens": 15782, | |
| "output_tokens": 64000, | |
| "end_reason": "no_tool_call", | |
| "errors": 0 | |
| }, | |
| { | |
| "model": "hf:Qwen/Qwen3.8-2.4T-A95B:together", | |
| "task_id": "dock-36A-w15x2-storm-0", | |
| "split": "eval", | |
| "difficulty": "storm", | |
| "quay": "36A", | |
| "ships": 56, | |
| "reward": 0.0, | |
| "submitted": false, | |
| "feasible": false, | |
| "cost": null, | |
| "optimal_cost": 369, | |
| "naive_cost": 6062, | |
| "turns": 2, | |
| "checks": 0, | |
| "seconds": 464.5, | |
| "input_tokens": 14354, | |
| "output_tokens": 64000, | |
| "end_reason": "no_tool_call", | |
| "errors": 0 | |
| }, | |
| { | |
| "model": "hf:Qwen/Qwen3.8-2.4T-A95B:together", | |
| "task_id": "dock-36A-w16x1-standard-0", | |
| "split": "eval", | |
| "difficulty": "standard", | |
| "quay": "36A", | |
| "ships": 27, | |
| "reward": 0.185185, | |
| "submitted": true, | |
| "feasible": false, | |
| "cost": null, | |
| "optimal_cost": 223, | |
| "naive_cost": 1083, | |
| "turns": 6, | |
| "checks": 4, | |
| "seconds": 1829.4, | |
| "input_tokens": 65446, | |
| "output_tokens": 117268, | |
| "end_reason": "submitted", | |
| "errors": 0 | |
| }, | |
| { | |
| "model": "hf:Qwen/Qwen3.8-2.4T-A95B:together", | |
| "task_id": "dock-36A-w16x2-busy-0", | |
| "split": "eval", | |
| "difficulty": "busy", | |
| "quay": "36A", | |
| "ships": 55, | |
| "reward": 0.200442, | |
| "submitted": true, | |
| "feasible": true, | |
| "cost": 853, | |
| "optimal_cost": 403, | |
| "naive_cost": 2046, | |
| "turns": 10, | |
| "checks": 8, | |
| "seconds": 703.9, | |
| "input_tokens": 245650, | |
| "output_tokens": 124106, | |
| "end_reason": "submitted", | |
| "errors": 0 | |
| }, | |
| { | |
| "model": "hf:Qwen/Qwen3.8-2.4T-A95B:together", | |
| "task_id": "dock-36A-w16x2-storm-0", | |
| "split": "eval", | |
| "difficulty": "storm", | |
| "quay": "36A", | |
| "ships": 57, | |
| "reward": 0.0, | |
| "submitted": false, | |
| "feasible": false, | |
| "cost": null, | |
| "optimal_cost": 108, | |
| "naive_cost": 6938, | |
| "turns": 2, | |
| "checks": 0, | |
| "seconds": 453.6, | |
| "input_tokens": 15312, | |
| "output_tokens": 64000, | |
| "end_reason": "no_tool_call", | |
| "errors": 0 | |
| }, | |
| { | |
| "model": "hf:Qwen/Qwen3.8-2.4T-A95B:together", | |
| "task_id": "dock-36A-w16x2-storm-1", | |
| "split": "eval", | |
| "difficulty": "storm", | |
| "quay": "36A", | |
| "ships": 55, | |
| "reward": 0.0, | |
| "submitted": false, | |
| "feasible": false, | |
| "cost": null, | |
| "optimal_cost": 627, | |
| "naive_cost": 6819, | |
| "turns": 2, | |
| "checks": 0, | |
| "seconds": 382.2, | |
| "input_tokens": 14228, | |
| "output_tokens": 64000, | |
| "end_reason": "no_tool_call", | |
| "errors": 0 | |
| }, | |
| { | |
| "model": "hf:Qwen/Qwen3.8-2.4T-A95B:together", | |
| "task_id": "dock-36A-w17x1-busy-0", | |
| "split": "eval", | |
| "difficulty": "busy", | |
| "quay": "36A", | |
| "ships": 28, | |
| "reward": 0.600896, | |
| "submitted": true, | |
| "feasible": true, | |
| "cost": 190, | |
| "optimal_cost": 152, | |
| "naive_cost": 1521, | |
| "turns": 11, | |
| "checks": 8, | |
| "seconds": 591.5, | |
| "input_tokens": 178562, | |
| "output_tokens": 104335, | |
| "end_reason": "submitted", | |
| "errors": 0 | |
| }, | |
| { | |
| "model": "hf:Qwen/Qwen3.8-2.4T-A95B:together", | |
| "task_id": "dock-36A-w17x1-standard-0", | |
| "split": "eval", | |
| "difficulty": "standard", | |
| "quay": "36A", | |
| "ships": 29, | |
| "reward": 0.823041, | |
| "submitted": true, | |
| "feasible": true, | |
| "cost": 115, | |
| "optimal_cost": 100, | |
| "naive_cost": 1794, | |
| "turns": 8, | |
| "checks": 5, | |
| "seconds": 473.2, | |
| "input_tokens": 104656, | |
| "output_tokens": 69826, | |
| "end_reason": "submitted", | |
| "errors": 0 | |
| }, | |
| { | |
| "model": "hf:Qwen/Qwen3.8-2.4T-A95B:together", | |
| "task_id": "dock-36A-w35x1-standard-0", | |
| "split": "eval", | |
| "difficulty": "standard", | |
| "quay": "36A", | |
| "ships": 19, | |
| "reward": 0.168421, | |
| "submitted": true, | |
| "feasible": false, | |
| "cost": null, | |
| "optimal_cost": 10, | |
| "naive_cost": 274, | |
| "turns": 6, | |
| "checks": 4, | |
| "seconds": 570.5, | |
| "input_tokens": 52619, | |
| "output_tokens": 84900, | |
| "end_reason": "submitted", | |
| "errors": 0 | |
| }, | |
| { | |
| "model": "hf:Qwen/Qwen3.8-2.4T-A95B:together", | |
| "task_id": "dock-36A-w35x2-extreme-0", | |
| "split": "eval", | |
| "difficulty": "extreme", | |
| "quay": "36A", | |
| "ships": 39, | |
| "reward": 0.0, | |
| "submitted": false, | |
| "feasible": false, | |
| "cost": null, | |
| "optimal_cost": 754, | |
| "naive_cost": 3430, | |
| "turns": 2, | |
| "checks": 0, | |
| "seconds": 386.5, | |
| "input_tokens": 14086, | |
| "output_tokens": 64000, | |
| "end_reason": "no_tool_call", | |
| "errors": 0 | |
| }, | |
| { | |
| "model": "hf:Qwen/Qwen3.8-2.4T-A95B:together", | |
| "task_id": "dock-36A-w35x2-extreme-1", | |
| "split": "eval", | |
| "difficulty": "extreme", | |
| "quay": "36A", | |
| "ships": 35, | |
| "reward": 0.0, | |
| "submitted": false, | |
| "feasible": false, | |
| "cost": null, | |
| "optimal_cost": 1315, | |
| "naive_cost": 3041, | |
| "turns": 3, | |
| "checks": 0, | |
| "seconds": 870.3, | |
| "input_tokens": 24619, | |
| "output_tokens": 64174, | |
| "end_reason": "no_tool_call", | |
| "errors": 0 | |
| }, | |
| { | |
| "model": "hf:Qwen/Qwen3.8-2.4T-A95B:together", | |
| "task_id": "dock-36A-w35x2-storm-0", | |
| "split": "eval", | |
| "difficulty": "storm", | |
| "quay": "36A", | |
| "ships": 33, | |
| "reward": 0.255251, | |
| "submitted": true, | |
| "feasible": true, | |
| "cost": 249, | |
| "optimal_cost": 102, | |
| "naive_cost": 670, | |
| "turns": 5, | |
| "checks": 2, | |
| "seconds": 586.6, | |
| "input_tokens": 51170, | |
| "output_tokens": 71362, | |
| "end_reason": "submitted", | |
| "errors": 0 | |
| }, | |
| { | |
| "model": "hf:Qwen/Qwen3.8-2.4T-A95B:together", | |
| "task_id": "dock-36A-w35x3-extreme-0", | |
| "split": "eval", | |
| "difficulty": "extreme", | |
| "quay": "36A", | |
| "ships": 59, | |
| "reward": 0.183051, | |
| "submitted": true, | |
| "feasible": false, | |
| "cost": null, | |
| "optimal_cost": 953, | |
| "naive_cost": 5403, | |
| "turns": 11, | |
| "checks": 8, | |
| "seconds": 1362.6, | |
| "input_tokens": 327043, | |
| "output_tokens": 236251, | |
| "end_reason": "submitted", | |
| "errors": 0 | |
| }, | |
| { | |
| "model": "hf:Qwen/Qwen3.8-2.4T-A95B:together", | |
| "task_id": "dock-36A-w36x2-busy-0", | |
| "split": "eval", | |
| "difficulty": "busy", | |
| "quay": "36A", | |
| "ships": 37, | |
| "reward": 0.0, | |
| "submitted": false, | |
| "feasible": false, | |
| "cost": null, | |
| "optimal_cost": 3143, | |
| "naive_cost": 9374, | |
| "turns": 2, | |
| "checks": 0, | |
| "seconds": 387.3, | |
| "input_tokens": 13840, | |
| "output_tokens": 64000, | |
| "end_reason": "no_tool_call", | |
| "errors": 0 | |
| }, | |
| { | |
| "model": "hf:Qwen/Qwen3.8-2.4T-A95B:together", | |
| "task_id": "dock-36A-w36x2-extreme-0", | |
| "split": "eval", | |
| "difficulty": "extreme", | |
| "quay": "36A", | |
| "ships": 42, | |
| "reward": 0.0, | |
| "submitted": false, | |
| "feasible": false, | |
| "cost": null, | |
| "optimal_cost": 320, | |
| "naive_cost": 4753, | |
| "turns": 2, | |
| "checks": 0, | |
| "seconds": 815.6, | |
| "input_tokens": 15079, | |
| "output_tokens": 64000, | |
| "end_reason": "no_tool_call", | |
| "errors": 0 | |
| }, | |
| { | |
| "model": "hf:Qwen/Qwen3.8-2.4T-A95B:together", | |
| "task_id": "dock-36A-w36x2-storm-0", | |
| "split": "eval", | |
| "difficulty": "storm", | |
| "quay": "36A", | |
| "ships": 37, | |
| "reward": 1.0, | |
| "submitted": true, | |
| "feasible": true, | |
| "cost": 28, | |
| "optimal_cost": 28, | |
| "naive_cost": 864, | |
| "turns": 3, | |
| "checks": 1, | |
| "seconds": 235.3, | |
| "input_tokens": 21715, | |
| "output_tokens": 38649, | |
| "end_reason": "submitted", | |
| "errors": 0 | |
| }, | |
| { | |
| "model": "hf:Qwen/Qwen3.8-2.4T-A95B:together", | |
| "task_id": "dock-36A-w37x1-busy-0", | |
| "split": "eval", | |
| "difficulty": "busy", | |
| "quay": "36A", | |
| "ships": 23, | |
| "reward": 1.0, | |
| "submitted": true, | |
| "feasible": true, | |
| "cost": 36, | |
| "optimal_cost": 36, | |
| "naive_cost": 598, | |
| "turns": 3, | |
| "checks": 1, | |
| "seconds": 154.2, | |
| "input_tokens": 15996, | |
| "output_tokens": 22172, | |
| "end_reason": "submitted", | |
| "errors": 0 | |
| }, | |
| { | |
| "model": "hf:Qwen/Qwen3.8-2.4T-A95B:together", | |
| "task_id": "dock-36A-w37x1-standard-0", | |
| "split": "eval", | |
| "difficulty": "standard", | |
| "quay": "36A", | |
| "ships": 22, | |
| "reward": 0.88409, | |
| "submitted": true, | |
| "feasible": true, | |
| "cost": 24, | |
| "optimal_cost": 15, | |
| "naive_cost": 429, | |
| "turns": 4, | |
| "checks": 2, | |
| "seconds": 349.1, | |
| "input_tokens": 23783, | |
| "output_tokens": 55127, | |
| "end_reason": "submitted", | |
| "errors": 0 | |
| }, | |
| { | |
| "model": "hf:Qwen/Qwen3.8-27B:cerebras|ovhcloud", | |
| "task_id": "dock-24B-w06x1-busy-0", | |
| "split": "eval", | |
| "difficulty": "busy", | |
| "quay": "24B", | |
| "ships": 17, | |
| "reward": 0.630913, | |
| "submitted": true, | |
| "feasible": true, | |
| "cost": 130, | |
| "optimal_cost": 87, | |
| "naive_cost": 658, | |
| "turns": 6, | |
| "checks": 3, | |
| "seconds": 60.4, | |
| "input_tokens": 52975, | |
| "output_tokens": 54239, | |
| "end_reason": "submitted", | |
| "errors": 0 | |
| }, | |
| { | |
| "model": "hf:Qwen/Qwen3.8-27B:cerebras|ovhcloud", | |
| "task_id": "dock-24B-w06x1-standard-0", | |
| "split": "eval", | |
| "difficulty": "standard", | |
| "quay": "24B", | |
| "ships": 16, | |
| "reward": 0.930481, | |
| "submitted": true, | |
| "feasible": true, | |
| "cost": 119, | |
| "optimal_cost": 114, | |
| "naive_cost": 311, | |
| "turns": 6, | |
| "checks": 4, | |
| "seconds": 70.6, | |
| "input_tokens": 37402, | |
| "output_tokens": 74508, | |
| "end_reason": "submitted", | |
| "errors": 0 | |
| }, | |
| { | |
| "model": "hf:Qwen/Qwen3.8-27B:cerebras|ovhcloud", | |
| "task_id": "dock-24B-w06x2-busy-0", | |
| "split": "eval", | |
| "difficulty": "busy", | |
| "quay": "24B", | |
| "ships": 31, | |
| "reward": 0.92946, | |
| "submitted": true, | |
| "feasible": true, | |
| "cost": 622, | |
| "optimal_cost": 616, | |
| "naive_cost": 1181, | |
| "turns": 7, | |
| "checks": 5, | |
| "seconds": 55.6, | |
| "input_tokens": 94554, | |
| "output_tokens": 57039, | |
| "end_reason": "submitted", | |
| "errors": 0 | |
| }, | |
| { | |
| "model": "hf:Qwen/Qwen3.8-27B:cerebras|ovhcloud", | |
| "task_id": "dock-24B-w06x2-extreme-0", | |
| "split": "eval", | |
| "difficulty": "extreme", | |
| "quay": "24B", | |
| "ships": 32, | |
| "reward": 0.0, | |
| "submitted": false, | |
| "feasible": false, | |
| "cost": null, | |
| "optimal_cost": 231, | |
| "naive_cost": 1090, | |
| "turns": 2, | |
| "checks": 0, | |
| "seconds": 56.3, | |
| "input_tokens": 13630, | |
| "output_tokens": 64000, | |
| "end_reason": "no_tool_call", | |
| "errors": 0 | |
| }, | |
| { | |
| "model": "hf:Qwen/Qwen3.8-27B:cerebras|ovhcloud", | |
| "task_id": "dock-24B-w06x2-storm-0", | |
| "split": "eval", | |
| "difficulty": "storm", | |
| "quay": "24B", | |
| "ships": 32, | |
| "reward": 0.452337, | |
| "submitted": true, | |
| "feasible": true, | |
| "cost": 259, | |
| "optimal_cost": 154, | |
| "naive_cost": 1259, | |
| "turns": 8, | |
| "checks": 5, | |
| "seconds": 337.0, | |
| "input_tokens": 107190, | |
| "output_tokens": 152448, | |
| "end_reason": "submitted", | |
| "errors": 0 | |
| }, | |
| { | |
| "model": "hf:Qwen/Qwen3.8-27B:cerebras|ovhcloud", | |
| "task_id": "dock-24B-w07x1-busy-0", | |
| "split": "eval", | |
| "difficulty": "busy", | |
| "quay": "24B", | |
| "ships": 17, | |
| "reward": 0.082353, | |
| "submitted": true, | |
| "feasible": false, | |
| "cost": null, | |
| "optimal_cost": 226, | |
| "naive_cost": 641, | |
| "turns": 5, | |
| "checks": 1, | |
| "seconds": 131.2, | |
| "input_tokens": 45812, | |
| "output_tokens": 110700, | |
| "end_reason": "submitted", | |
| "errors": 0 | |
| }, | |
| { | |
| "model": "hf:Qwen/Qwen3.8-27B:cerebras|ovhcloud", | |
| "task_id": "dock-24B-w15x2-extreme-0", | |
| "split": "eval", | |
| "difficulty": "extreme", | |
| "quay": "24B", | |
| "ships": 24, | |
| "reward": 0.0, | |
| "submitted": false, | |
| "feasible": false, | |
| "cost": null, | |
| "optimal_cost": 1009, | |
| "naive_cost": 3964, | |
| "turns": 2, | |
| "checks": 0, | |
| "seconds": 57.9, | |
| "input_tokens": 12705, | |
| "output_tokens": 64000, | |
| "end_reason": "no_tool_call", | |
| "errors": 0 | |
| }, | |
| { | |
| "model": "hf:Qwen/Qwen3.8-27B:cerebras|ovhcloud", | |
| "task_id": "dock-24B-w16x1-busy-0", | |
| "split": "eval", | |
| "difficulty": "busy", | |
| "quay": "24B", | |
| "ships": 16, | |
| "reward": 0.0, | |
| "submitted": false, | |
| "feasible": false, | |
| "cost": null, | |
| "optimal_cost": 1349, | |
| "naive_cost": 4069, | |
| "turns": 2, | |
| "checks": 0, | |
| "seconds": 59.8, | |
| "input_tokens": 11300, | |
| "output_tokens": 64000, | |
| "end_reason": "no_tool_call", | |
| "errors": 0 | |
| }, | |
| { | |
| "model": "hf:Qwen/Qwen3.8-27B:cerebras|ovhcloud", | |
| "task_id": "dock-24B-w16x2-extreme-0", | |
| "split": "eval", | |
| "difficulty": "extreme", | |
| "quay": "24B", | |
| "ships": 28, | |
| "reward": 0.0, | |
| "submitted": false, | |
| "feasible": false, | |
| "cost": null, | |
| "optimal_cost": 466, | |
| "naive_cost": 1485, | |
| "turns": 7, | |
| "checks": 3, | |
| "seconds": 280.3, | |
| "input_tokens": 106224, | |
| "output_tokens": 176631, | |
| "end_reason": "no_tool_call", | |
| "errors": 0 | |
| }, | |
| { | |
| "model": "hf:Qwen/Qwen3.8-27B:cerebras|ovhcloud", | |
| "task_id": "dock-24B-w16x2-storm-0", | |
| "split": "eval", | |
| "difficulty": "storm", | |
| "quay": "24B", | |
| "ships": 27, | |
| "reward": 0.17037, | |
| "submitted": true, | |
| "feasible": false, | |
| "cost": null, | |
| "optimal_cost": 201, | |
| "naive_cost": 1569, | |
| "turns": 5, | |
| "checks": 2, | |
| "seconds": 82.8, | |
| "input_tokens": 55634, | |
| "output_tokens": 91071, | |
| "end_reason": "submitted", | |
| "errors": 0 | |
| }, | |
| { | |
| "model": "hf:Qwen/Qwen3.8-27B:cerebras|ovhcloud", | |
| "task_id": "dock-24B-w16x2-storm-1", | |
| "split": "eval", | |
| "difficulty": "storm", | |
| "quay": "24B", | |
| "ships": 26, | |
| "reward": 0.0, | |
| "submitted": false, | |
| "feasible": false, | |
| "cost": null, | |
| "optimal_cost": 1187, | |
| "naive_cost": 2403, | |
| "turns": 3, | |
| "checks": 0, | |
| "seconds": 128.6, | |
| "input_tokens": 21674, | |
| "output_tokens": 64030, | |
| "end_reason": "no_tool_call", | |
| "errors": 0 | |
| }, | |
| { | |
| "model": "hf:Qwen/Qwen3.8-27B:cerebras|ovhcloud", | |
| "task_id": "dock-24B-w35x1-busy-0", | |
| "split": "eval", | |
| "difficulty": "busy", | |
| "quay": "24B", | |
| "ships": 14, | |
| "reward": 0.0, | |
| "submitted": false, | |
| "feasible": false, | |
| "cost": null, | |
| "optimal_cost": 1223, | |
| "naive_cost": 2892, | |
| "turns": 2, | |
| "checks": 0, | |
| "seconds": 56.8, | |
| "input_tokens": 11169, | |
| "output_tokens": 64000, | |
| "end_reason": "no_tool_call", | |
| "errors": 0 | |
| }, | |
| { | |
| "model": "hf:Qwen/Qwen3.8-27B:cerebras|ovhcloud", | |
| "task_id": "dock-24B-w35x2-busy-0", | |
| "split": "eval", | |
| "difficulty": "busy", | |
| "quay": "24B", | |
| "ships": 24, | |
| "reward": 0.485738, | |
| "submitted": true, | |
| "feasible": true, | |
| "cost": 284, | |
| "optimal_cost": 162, | |
| "naive_cost": 1443, | |
| "turns": 7, | |
| "checks": 4, | |
| "seconds": 134.4, | |
| "input_tokens": 99350, | |
| "output_tokens": 153915, | |
| "end_reason": "submitted", | |
| "errors": 0 | |
| }, | |
| { | |
| "model": "hf:Qwen/Qwen3.8-27B:cerebras|ovhcloud", | |
| "task_id": "dock-24B-w35x2-extreme-0", | |
| "split": "eval", | |
| "difficulty": "extreme", | |
| "quay": "24B", | |
| "ships": 24, | |
| "reward": 0.0, | |
| "submitted": false, | |
| "feasible": false, | |
| "cost": null, | |
| "optimal_cost": 1792, | |
| "naive_cost": 4682, | |
| "turns": 2, | |
| "checks": 0, | |
| "seconds": 56.8, | |
| "input_tokens": 12363, | |
| "output_tokens": 64000, | |
| "end_reason": "no_tool_call", | |
| "errors": 0 | |
| }, | |
| { | |
| "model": "hf:Qwen/Qwen3.8-27B:cerebras|ovhcloud", | |
| "task_id": "dock-24B-w35x2-extreme-1", | |
| "split": "eval", | |
| "difficulty": "extreme", | |
| "quay": "24B", | |
| "ships": 25, | |
| "reward": 0.0, | |
| "submitted": false, | |
| "feasible": false, | |
| "cost": null, | |
| "optimal_cost": 283, | |
| "naive_cost": 1356, | |
| "turns": 2, | |
| "checks": 0, | |
| "seconds": 119.9, | |
| "input_tokens": 12483, | |
| "output_tokens": 64000, | |
| "end_reason": "no_tool_call", | |
| "errors": 0 | |
| }, | |
| { | |
| "model": "hf:Qwen/Qwen3.8-27B:cerebras|ovhcloud", | |
| "task_id": "dock-24B-w35x2-storm-0", | |
| "split": "eval", | |
| "difficulty": "storm", | |
| "quay": "24B", | |
| "ships": 24, | |
| "reward": 0.0, | |
| "submitted": false, | |
| "feasible": false, | |
| "cost": null, | |
| "optimal_cost": 2591, | |
| "naive_cost": 4835, | |
| "turns": 2, | |
| "checks": 0, | |
| "seconds": 60.7, | |
| "input_tokens": 13053, | |
| "output_tokens": 64000, | |
| "end_reason": "no_tool_call", | |
| "errors": 0 | |
| }, | |
| { | |
| "model": "hf:Qwen/Qwen3.8-27B:cerebras|ovhcloud", | |
| "task_id": "dock-24B-w35x2-storm-1", | |
| "split": "eval", | |
| "difficulty": "storm", | |
| "quay": "24B", | |
| "ships": 24, | |
| "reward": 0.0, | |
| "submitted": false, | |
| "feasible": false, | |
| "cost": null, | |
| "optimal_cost": 312, | |
| "naive_cost": 2701, | |
| "turns": 4, | |
| "checks": 1, | |
| "seconds": 158.0, | |
| "input_tokens": 40445, | |
| "output_tokens": 104164, | |
| "end_reason": "no_tool_call", | |
| "errors": 0 | |
| }, | |
| { | |
| "model": "hf:Qwen/Qwen3.8-27B:cerebras|ovhcloud", | |
| "task_id": "dock-24B-w35x3-extreme-0", | |
| "split": "eval", | |
| "difficulty": "extreme", | |
| "quay": "24B", | |
| "ships": 39, | |
| "reward": 0.0, | |
| "submitted": false, | |
| "feasible": false, | |
| "cost": null, | |
| "optimal_cost": 1106, | |
| "naive_cost": 4303, | |
| "turns": 3, | |
| "checks": 0, | |
| "seconds": 58.7, | |
| "input_tokens": 26810, | |
| "output_tokens": 64044, | |
| "end_reason": "no_tool_call", | |
| "errors": 0 | |
| }, | |
| { | |
| "model": "hf:Qwen/Qwen3.8-27B:cerebras|ovhcloud", | |
| "task_id": "dock-24B-w35x3-extreme-1", | |
| "split": "eval", | |
| "difficulty": "extreme", | |
| "quay": "24B", | |
| "ships": 38, | |
| "reward": 0.0, | |
| "submitted": false, | |
| "feasible": false, | |
| "cost": null, | |
| "optimal_cost": 865, | |
| "naive_cost": 2611, | |
| "turns": 2, | |
| "checks": 0, | |
| "seconds": 116.1, | |
| "input_tokens": 13124, | |
| "output_tokens": 64000, | |
| "end_reason": "no_tool_call", | |
| "errors": 0 | |
| }, | |
| { | |
| "model": "hf:Qwen/Qwen3.8-27B:cerebras|ovhcloud", | |
| "task_id": "dock-24B-w36x2-extreme-0", | |
| "split": "eval", | |
| "difficulty": "extreme", | |
| "quay": "24B", | |
| "ships": 24, | |
| "reward": 0.0, | |
| "submitted": false, | |
| "feasible": false, | |
| "cost": null, | |
| "optimal_cost": 648, | |
| "naive_cost": 1736, | |
| "turns": 2, | |
| "checks": 0, | |
| "seconds": 68.5, | |
| "input_tokens": 13231, | |
| "output_tokens": 64000, | |
| "end_reason": "no_tool_call", | |
| "errors": 0 | |
| }, | |
| { | |
| "model": "hf:Qwen/Qwen3.8-27B:cerebras|ovhcloud", | |
| "task_id": "dock-24B-w36x2-storm-0", | |
| "split": "eval", | |
| "difficulty": "storm", | |
| "quay": "24B", | |
| "ships": 23, | |
| "reward": 0.0, | |
| "submitted": false, | |
| "feasible": false, | |
| "cost": null, | |
| "optimal_cost": 756, | |
| "naive_cost": 1931, | |
| "turns": 3, | |
| "checks": 0, | |
| "seconds": 118.1, | |
| "input_tokens": 20981, | |
| "output_tokens": 64043, | |
| "end_reason": "no_tool_call", | |
| "errors": 0 | |
| }, | |
| { | |
| "model": "hf:Qwen/Qwen3.8-27B:cerebras|ovhcloud", | |
| "task_id": "dock-24B-w36x2-storm-1", | |
| "split": "eval", | |
| "difficulty": "storm", | |
| "quay": "24B", | |
| "ships": 23, | |
| "reward": 1.0, | |
| "submitted": true, | |
| "feasible": true, | |
| "cost": 244, | |
| "optimal_cost": 244, | |
| "naive_cost": 549, | |
| "turns": 3, | |
| "checks": 1, | |
| "seconds": 46.2, | |
| "input_tokens": 17054, | |
| "output_tokens": 57929, | |
| "end_reason": "submitted", | |
| "errors": 0 | |
| }, | |
| { | |
| "model": "hf:Qwen/Qwen3.8-27B:cerebras|ovhcloud", | |
| "task_id": "dock-24B-w37x1-standard-0", | |
| "split": "eval", | |
| "difficulty": "standard", | |
| "quay": "24B", | |
| "ships": 15, | |
| "reward": 0.753933, | |
| "submitted": true, | |
| "feasible": true, | |
| "cost": 263, | |
| "optimal_cost": 229, | |
| "naive_cost": 734, | |
| "turns": 9, | |
| "checks": 6, | |
| "seconds": 385.6, | |
| "input_tokens": 187633, | |
| "output_tokens": 143747, | |
| "end_reason": "submitted", | |
| "errors": 0 | |
| }, | |
| { | |
| "model": "hf:Qwen/Qwen3.8-27B:cerebras|ovhcloud", | |
| "task_id": "dock-36A-w05x1-busy-0", | |
| "split": "eval", | |
| "difficulty": "busy", | |
| "quay": "36A", | |
| "ships": 27, | |
| "reward": 0.976644, | |
| "submitted": true, | |
| "feasible": true, | |
| "cost": 406, | |
| "optimal_cost": 404, | |
| "naive_cost": 791, | |
| "turns": 4, | |
| "checks": 2, | |
| "seconds": 49.1, | |
| "input_tokens": 35394, | |
| "output_tokens": 48666, | |
| "end_reason": "submitted", | |
| "errors": 0 | |
| }, | |
| { | |
| "model": "hf:Qwen/Qwen3.8-27B:cerebras|ovhcloud", | |
| "task_id": "dock-36A-w05x2-busy-0", | |
| "split": "eval", | |
| "difficulty": "busy", | |
| "quay": "36A", | |
| "ships": 51, | |
| "reward": 0.0, | |
| "submitted": false, | |
| "feasible": false, | |
| "cost": null, | |
| "optimal_cost": 277, | |
| "naive_cost": 1831, | |
| "turns": 5, | |
| "checks": 1, | |
| "seconds": 176.0, | |
| "input_tokens": 69707, | |
| "output_tokens": 108209, | |
| "end_reason": "no_tool_call", | |
| "errors": 0 | |
| }, | |
| { | |
| "model": "hf:Qwen/Qwen3.8-27B:cerebras|ovhcloud", | |
| "task_id": "dock-36A-w05x2-extreme-0", | |
| "split": "eval", | |
| "difficulty": "extreme", | |
| "quay": "36A", | |
| "ships": 58, | |
| "reward": 0.034483, | |
| "submitted": true, | |
| "feasible": false, | |
| "cost": null, | |
| "optimal_cost": 1594, | |
| "naive_cost": 4448, | |
| "turns": 4, | |
| "checks": 1, | |
| "seconds": 45.6, | |
| "input_tokens": 56576, | |
| "output_tokens": 45384, | |
| "end_reason": "submitted", | |
| "errors": 0 | |
| }, | |
| { | |
| "model": "hf:Qwen/Qwen3.8-27B:cerebras|ovhcloud", | |
| "task_id": "dock-36A-w05x2-storm-0", | |
| "split": "eval", | |
| "difficulty": "storm", | |
| "quay": "36A", | |
| "ships": 59, | |
| "reward": 0.0, | |
| "submitted": false, | |
| "feasible": false, | |
| "cost": null, | |
| "optimal_cost": 290, | |
| "naive_cost": 4214, | |
| "turns": 2, | |
| "checks": 0, | |
| "seconds": 82.2, | |
| "input_tokens": 16813, | |
| "output_tokens": 64000, | |
| "end_reason": "no_tool_call", | |
| "errors": 0 | |
| }, | |
| { | |
| "model": "hf:Qwen/Qwen3.8-27B:cerebras|ovhcloud", | |
| "task_id": "dock-36A-w06x1-busy-0", | |
| "split": "eval", | |
| "difficulty": "busy", | |
| "quay": "36A", | |
| "ships": 25, | |
| "reward": 0.0, | |
| "submitted": false, | |
| "feasible": false, | |
| "cost": null, | |
| "optimal_cost": 40, | |
| "naive_cost": 824, | |
| "turns": 2, | |
| "checks": 0, | |
| "seconds": 116.7, | |
| "input_tokens": 11206, | |
| "output_tokens": 64000, | |
| "end_reason": "no_tool_call", | |
| "errors": 0 | |
| }, | |
| { | |
| "model": "hf:Qwen/Qwen3.8-27B:cerebras|ovhcloud", | |
| "task_id": "dock-36A-w06x1-standard-0", | |
| "split": "eval", | |
| "difficulty": "standard", | |
| "quay": "36A", | |
| "ships": 24, | |
| "reward": 0.0, | |
| "submitted": false, | |
| "feasible": false, | |
| "cost": null, | |
| "optimal_cost": 14, | |
| "naive_cost": 591, | |
| "turns": 2, | |
| "checks": 0, | |
| "seconds": 59.1, | |
| "input_tokens": 11044, | |
| "output_tokens": 64000, | |
| "end_reason": "no_tool_call", | |
| "errors": 0 | |
| }, | |
| { | |
| "model": "hf:Qwen/Qwen3.8-27B:cerebras|ovhcloud", | |
| "task_id": "dock-36A-w10x1-standard-0", | |
| "split": "eval", | |
| "difficulty": "standard", | |
| "quay": "36A", | |
| "ships": 32, | |
| "reward": 0.0, | |
| "submitted": false, | |
| "feasible": false, | |
| "cost": null, | |
| "optimal_cost": 39, | |
| "naive_cost": 606, | |
| "turns": 3, | |
| "checks": 0, | |
| "seconds": 119.9, | |
| "input_tokens": 20742, | |
| "output_tokens": 64043, | |
| "end_reason": "no_tool_call", | |
| "errors": 0 | |
| }, | |
| { | |
| "model": "hf:Qwen/Qwen3.8-27B:cerebras|ovhcloud", | |
| "task_id": "dock-36A-w15x1-busy-0", | |
| "split": "eval", | |
| "difficulty": "busy", | |
| "quay": "36A", | |
| "ships": 29, | |
| "reward": 0.0, | |
| "submitted": false, | |
| "feasible": false, | |
| "cost": null, | |
| "optimal_cost": 153, | |
| "naive_cost": 1963, | |
| "turns": 2, | |
| "checks": 0, | |
| "seconds": 123.1, | |
| "input_tokens": 12434, | |
| "output_tokens": 64000, | |
| "end_reason": "no_tool_call", | |
| "errors": 0 | |
| }, | |
| { | |
| "model": "hf:Qwen/Qwen3.8-27B:cerebras|ovhcloud", | |
| "task_id": "dock-36A-w15x1-standard-0", | |
| "split": "eval", | |
| "difficulty": "standard", | |
| "quay": "36A", | |
| "ships": 30, | |
| "reward": 0.0, | |
| "submitted": false, | |
| "feasible": false, | |
| "cost": null, | |
| "optimal_cost": 119, | |
| "naive_cost": 2613, | |
| "turns": 2, | |
| "checks": 0, | |
| "seconds": 59.6, | |
| "input_tokens": 11837, | |
| "output_tokens": 64000, | |
| "end_reason": "no_tool_call", | |
| "errors": 0 | |
| }, | |
| { | |
| "model": "hf:Qwen/Qwen3.8-27B:cerebras|ovhcloud", | |
| "task_id": "dock-36A-w15x2-busy-0", | |
| "split": "eval", | |
| "difficulty": "busy", | |
| "quay": "36A", | |
| "ships": 55, | |
| "reward": 0.0, | |
| "submitted": false, | |
| "feasible": false, | |
| "cost": null, | |
| "optimal_cost": 368, | |
| "naive_cost": 3372, | |
| "turns": 2, | |
| "checks": 0, | |
| "seconds": 60.0, | |
| "input_tokens": 16026, | |
| "output_tokens": 64000, | |
| "end_reason": "no_tool_call", | |
| "errors": 0 | |
| }, | |
| { | |
| "model": "hf:Qwen/Qwen3.8-27B:cerebras|ovhcloud", | |
| "task_id": "dock-36A-w15x2-storm-0", | |
| "split": "eval", | |
| "difficulty": "storm", | |
| "quay": "36A", | |
| "ships": 56, | |
| "reward": 0.0, | |
| "submitted": false, | |
| "feasible": false, | |
| "cost": null, | |
| "optimal_cost": 369, | |
| "naive_cost": 6062, | |
| "turns": 5, | |
| "checks": 1, | |
| "seconds": 191.0, | |
| "input_tokens": 83276, | |
| "output_tokens": 114285, | |
| "end_reason": "no_tool_call", | |
| "errors": 0 | |
| }, | |
| { | |
| "model": "hf:Qwen/Qwen3.8-27B:cerebras|ovhcloud", | |
| "task_id": "dock-36A-w16x1-standard-0", | |
| "split": "eval", | |
| "difficulty": "standard", | |
| "quay": "36A", | |
| "ships": 27, | |
| "reward": 0.0, | |
| "submitted": false, | |
| "feasible": false, | |
| "cost": null, | |
| "optimal_cost": 223, | |
| "naive_cost": 1083, | |
| "turns": 2, | |
| "checks": 0, | |
| "seconds": 127.0, | |
| "input_tokens": 11975, | |
| "output_tokens": 64000, | |
| "end_reason": "no_tool_call", | |
| "errors": 0 | |
| }, | |
| { | |
| "model": "hf:Qwen/Qwen3.8-27B:cerebras|ovhcloud", | |
| "task_id": "dock-36A-w16x2-busy-0", | |
| "split": "eval", | |
| "difficulty": "busy", | |
| "quay": "36A", | |
| "ships": 55, | |
| "reward": 0.0, | |
| "submitted": false, | |
| "feasible": false, | |
| "cost": null, | |
| "optimal_cost": 403, | |
| "naive_cost": 2046, | |
| "turns": 2, | |
| "checks": 0, | |
| "seconds": 59.1, | |
| "input_tokens": 15868, | |
| "output_tokens": 64000, | |
| "end_reason": "no_tool_call", | |
| "errors": 0 | |
| }, | |
| { | |
| "model": "hf:Qwen/Qwen3.8-27B:cerebras|ovhcloud", | |
| "task_id": "dock-36A-w16x2-storm-0", | |
| "split": "eval", | |
| "difficulty": "storm", | |
| "quay": "36A", | |
| "ships": 57, | |
| "reward": 0.0, | |
| "submitted": false, | |
| "feasible": false, | |
| "cost": null, | |
| "optimal_cost": 108, | |
| "naive_cost": 6938, | |
| "turns": 2, | |
| "checks": 0, | |
| "seconds": 115.5, | |
| "input_tokens": 15654, | |
| "output_tokens": 64000, | |
| "end_reason": "no_tool_call", | |
| "errors": 0 | |
| }, | |
| { | |
| "model": "hf:Qwen/Qwen3.8-27B:cerebras|ovhcloud", | |
| "task_id": "dock-36A-w16x2-storm-1", | |
| "split": "eval", | |
| "difficulty": "storm", | |
| "quay": "36A", | |
| "ships": 55, | |
| "reward": 0.0, | |
| "submitted": false, | |
| "feasible": false, | |
| "cost": null, | |
| "optimal_cost": 627, | |
| "naive_cost": 6819, | |
| "turns": 4, | |
| "checks": 0, | |
| "seconds": 112.6, | |
| "input_tokens": 49607, | |
| "output_tokens": 96123, | |
| "end_reason": "no_tool_call", | |
| "errors": 0 | |
| }, | |
| { | |
| "model": "hf:Qwen/Qwen3.8-27B:cerebras|ovhcloud", | |
| "task_id": "dock-36A-w17x1-busy-0", | |
| "split": "eval", | |
| "difficulty": "busy", | |
| "quay": "36A", | |
| "ships": 28, | |
| "reward": 0.0, | |
| "submitted": false, | |
| "feasible": false, | |
| "cost": null, | |
| "optimal_cost": 152, | |
| "naive_cost": 1521, | |
| "turns": 2, | |
| "checks": 0, | |
| "seconds": 55.5, | |
| "input_tokens": 12848, | |
| "output_tokens": 64000, | |
| "end_reason": "no_tool_call", | |
| "errors": 0 | |
| }, | |
| { | |
| "model": "hf:Qwen/Qwen3.8-27B:cerebras|ovhcloud", | |
| "task_id": "dock-36A-w17x1-standard-0", | |
| "split": "eval", | |
| "difficulty": "standard", | |
| "quay": "36A", | |
| "ships": 29, | |
| "reward": 0.0, | |
| "submitted": false, | |
| "feasible": false, | |
| "cost": null, | |
| "optimal_cost": 100, | |
| "naive_cost": 1794, | |
| "turns": 2, | |
| "checks": 0, | |
| "seconds": 63.6, | |
| "input_tokens": 12576, | |
| "output_tokens": 64000, | |
| "end_reason": "no_tool_call", | |
| "errors": 0 | |
| }, | |
| { | |
| "model": "hf:Qwen/Qwen3.8-27B:cerebras|ovhcloud", | |
| "task_id": "dock-36A-w35x1-standard-0", | |
| "split": "eval", | |
| "difficulty": "standard", | |
| "quay": "36A", | |
| "ships": 19, | |
| "reward": 1.0, | |
| "submitted": true, | |
| "feasible": true, | |
| "cost": 10, | |
| "optimal_cost": 10, | |
| "naive_cost": 274, | |
| "turns": 4, | |
| "checks": 3, | |
| "seconds": 173.2, | |
| "input_tokens": 25264, | |
| "output_tokens": 40228, | |
| "end_reason": "submitted", | |
| "errors": 0 | |
| }, | |
| { | |
| "model": "hf:Qwen/Qwen3.8-27B:cerebras|ovhcloud", | |
| "task_id": "dock-36A-w35x2-extreme-0", | |
| "split": "eval", | |
| "difficulty": "extreme", | |
| "quay": "36A", | |
| "ships": 39, | |
| "reward": 0.0, | |
| "submitted": false, | |
| "feasible": false, | |
| "cost": null, | |
| "optimal_cost": 754, | |
| "naive_cost": 3430, | |
| "turns": 2, | |
| "checks": 0, | |
| "seconds": 124.2, | |
| "input_tokens": 14517, | |
| "output_tokens": 64000, | |
| "end_reason": "no_tool_call", | |
| "errors": 0 | |
| }, | |
| { | |
| "model": "hf:Qwen/Qwen3.8-27B:cerebras|ovhcloud", | |
| "task_id": "dock-36A-w35x2-extreme-1", | |
| "split": "eval", | |
| "difficulty": "extreme", | |
| "quay": "36A", | |
| "ships": 35, | |
| "reward": 0.0, | |
| "submitted": false, | |
| "feasible": false, | |
| "cost": null, | |
| "optimal_cost": 1315, | |
| "naive_cost": 3041, | |
| "turns": 2, | |
| "checks": 0, | |
| "seconds": 142.6, | |
| "input_tokens": 14664, | |
| "output_tokens": 64000, | |
| "end_reason": "no_tool_call", | |
| "errors": 0 | |
| }, | |
| { | |
| "model": "hf:Qwen/Qwen3.8-27B:cerebras|ovhcloud", | |
| "task_id": "dock-36A-w35x2-storm-0", | |
| "split": "eval", | |
| "difficulty": "storm", | |
| "quay": "36A", | |
| "ships": 33, | |
| "reward": 0.522312, | |
| "submitted": true, | |
| "feasible": true, | |
| "cost": 152, | |
| "optimal_cost": 102, | |
| "naive_cost": 670, | |
| "turns": 7, | |
| "checks": 5, | |
| "seconds": 98.0, | |
| "input_tokens": 95421, | |
| "output_tokens": 102326, | |
| "end_reason": "submitted", | |
| "errors": 0 | |
| }, | |
| { | |
| "model": "hf:Qwen/Qwen3.8-27B:cerebras|ovhcloud", | |
| "task_id": "dock-36A-w35x3-extreme-0", | |
| "split": "eval", | |
| "difficulty": "extreme", | |
| "quay": "36A", | |
| "ships": 59, | |
| "reward": 0.0, | |
| "submitted": false, | |
| "feasible": false, | |
| "cost": null, | |
| "optimal_cost": 953, | |
| "naive_cost": 5403, | |
| "turns": 2, | |
| "checks": 0, | |
| "seconds": 96.3, | |
| "input_tokens": 17168, | |
| "output_tokens": 64000, | |
| "end_reason": "no_tool_call", | |
| "errors": 0 | |
| }, | |
| { | |
| "model": "hf:Qwen/Qwen3.8-27B:cerebras|ovhcloud", | |
| "task_id": "dock-36A-w36x2-busy-0", | |
| "split": "eval", | |
| "difficulty": "busy", | |
| "quay": "36A", | |
| "ships": 37, | |
| "reward": 0.0, | |
| "submitted": false, | |
| "feasible": false, | |
| "cost": null, | |
| "optimal_cost": 3143, | |
| "naive_cost": 9374, | |
| "turns": 2, | |
| "checks": 0, | |
| "seconds": 114.2, | |
| "input_tokens": 14526, | |
| "output_tokens": 64000, | |
| "end_reason": "no_tool_call", | |
| "errors": 0 | |
| }, | |
| { | |
| "model": "hf:Qwen/Qwen3.8-27B:cerebras|ovhcloud", | |
| "task_id": "dock-36A-w36x2-extreme-0", | |
| "split": "eval", | |
| "difficulty": "extreme", | |
| "quay": "36A", | |
| "ships": 42, | |
| "reward": 0.0, | |
| "submitted": false, | |
| "feasible": false, | |
| "cost": null, | |
| "optimal_cost": 320, | |
| "naive_cost": 4753, | |
| "turns": 4, | |
| "checks": 0, | |
| "seconds": 111.4, | |
| "input_tokens": 48052, | |
| "output_tokens": 116906, | |
| "end_reason": "no_tool_call", | |
| "errors": 0 | |
| }, | |
| { | |
| "model": "hf:Qwen/Qwen3.8-27B:cerebras|ovhcloud", | |
| "task_id": "dock-36A-w36x2-storm-0", | |
| "split": "eval", | |
| "difficulty": "storm", | |
| "quay": "36A", | |
| "ships": 37, | |
| "reward": 0.707789, | |
| "submitted": true, | |
| "feasible": true, | |
| "cost": 53, | |
| "optimal_cost": 28, | |
| "naive_cost": 864, | |
| "turns": 4, | |
| "checks": 1, | |
| "seconds": 134.5, | |
| "input_tokens": 41841, | |
| "output_tokens": 82705, | |
| "end_reason": "submitted", | |
| "errors": 0 | |
| }, | |
| { | |
| "model": "hf:Qwen/Qwen3.8-27B:cerebras|ovhcloud", | |
| "task_id": "dock-36A-w37x1-busy-0", | |
| "split": "eval", | |
| "difficulty": "busy", | |
| "quay": "36A", | |
| "ships": 23, | |
| "reward": 1.0, | |
| "submitted": true, | |
| "feasible": true, | |
| "cost": 36, | |
| "optimal_cost": 36, | |
| "naive_cost": 598, | |
| "turns": 4, | |
| "checks": 2, | |
| "seconds": 39.2, | |
| "input_tokens": 24915, | |
| "output_tokens": 25994, | |
| "end_reason": "submitted", | |
| "errors": 0 | |
| }, | |
| { | |
| "model": "hf:Qwen/Qwen3.8-27B:cerebras|ovhcloud", | |
| "task_id": "dock-36A-w37x1-standard-0", | |
| "split": "eval", | |
| "difficulty": "standard", | |
| "quay": "36A", | |
| "ships": 22, | |
| "reward": 0.88409, | |
| "submitted": true, | |
| "feasible": true, | |
| "cost": 24, | |
| "optimal_cost": 15, | |
| "naive_cost": 429, | |
| "turns": 3, | |
| "checks": 1, | |
| "seconds": 41.0, | |
| "input_tokens": 22530, | |
| "output_tokens": 46421, | |
| "end_reason": "submitted", | |
| "errors": 0 | |
| }, | |
| { | |
| "model": "hf:zai-org/GLM-5.3-Flash:baseten", | |
| "task_id": "dock-24B-w06x1-busy-0", | |
| "split": "eval", | |
| "difficulty": "busy", | |
| "quay": "24B", | |
| "ships": 17, | |
| "reward": 0.863522, | |
| "submitted": true, | |
| "feasible": true, | |
| "cost": 100, | |
| "optimal_cost": 87, | |
| "naive_cost": 658, | |
| "turns": 3, | |
| "checks": 0, | |
| "seconds": 163.1, | |
| "input_tokens": 14133, | |
| "output_tokens": 32524, | |
| "end_reason": "submitted", | |
| "errors": 0 | |
| }, | |
| { | |
| "model": "hf:zai-org/GLM-5.3-Flash:baseten", | |
| "task_id": "dock-24B-w06x1-standard-0", | |
| "split": "eval", | |
| "difficulty": "standard", | |
| "quay": "24B", | |
| "ships": 16, | |
| "reward": 1.0, | |
| "submitted": true, | |
| "feasible": true, | |
| "cost": 114, | |
| "optimal_cost": 114, | |
| "naive_cost": 311, | |
| "turns": 4, | |
| "checks": 2, | |
| "seconds": 52.2, | |
| "input_tokens": 16552, | |
| "output_tokens": 12452, | |
| "end_reason": "submitted", | |
| "errors": 0 | |
| }, | |
| { | |
| "model": "hf:zai-org/GLM-5.3-Flash:baseten", | |
| "task_id": "dock-24B-w06x2-busy-0", | |
| "split": "eval", | |
| "difficulty": "busy", | |
| "quay": "24B", | |
| "ships": 31, | |
| "reward": 1.0, | |
| "submitted": true, | |
| "feasible": true, | |
| "cost": 616, | |
| "optimal_cost": 616, | |
| "naive_cost": 1181, | |
| "turns": 8, | |
| "checks": 7, | |
| "seconds": 242.6, | |
| "input_tokens": 83900, | |
| "output_tokens": 51090, | |
| "end_reason": "submitted", | |
| "errors": 0 | |
| }, | |
| { | |
| "model": "hf:zai-org/GLM-5.3-Flash:baseten", | |
| "task_id": "dock-24B-w06x2-extreme-0", | |
| "split": "eval", | |
| "difficulty": "extreme", | |
| "quay": "24B", | |
| "ships": 32, | |
| "reward": 0.19375, | |
| "submitted": true, | |
| "feasible": false, | |
| "cost": null, | |
| "optimal_cost": 231, | |
| "naive_cost": 1090, | |
| "turns": 8, | |
| "checks": 6, | |
| "seconds": 666.1, | |
| "input_tokens": 88951, | |
| "output_tokens": 146517, | |
| "end_reason": "submitted", | |
| "errors": 0 | |
| }, | |
| { | |
| "model": "hf:zai-org/GLM-5.3-Flash:baseten", | |
| "task_id": "dock-24B-w06x2-storm-0", | |
| "split": "eval", | |
| "difficulty": "storm", | |
| "quay": "24B", | |
| "ships": 32, | |
| "reward": 0.732732, | |
| "submitted": true, | |
| "feasible": true, | |
| "cost": 191, | |
| "optimal_cost": 154, | |
| "naive_cost": 1259, | |
| "turns": 3, | |
| "checks": 1, | |
| "seconds": 226.6, | |
| "input_tokens": 19686, | |
| "output_tokens": 48058, | |
| "end_reason": "submitted", | |
| "errors": 0 | |
| }, | |
| { | |
| "model": "hf:zai-org/GLM-5.3-Flash:baseten", | |
| "task_id": "dock-24B-w07x1-busy-0", | |
| "split": "eval", | |
| "difficulty": "busy", | |
| "quay": "24B", | |
| "ships": 17, | |
| "reward": 0.989333, | |
| "submitted": true, | |
| "feasible": true, | |
| "cost": 227, | |
| "optimal_cost": 226, | |
| "naive_cost": 641, | |
| "turns": 7, | |
| "checks": 5, | |
| "seconds": 428.0, | |
| "input_tokens": 45324, | |
| "output_tokens": 94030, | |
| "end_reason": "submitted", | |
| "errors": 0 | |
| }, | |
| { | |
| "model": "hf:zai-org/GLM-5.3-Flash:baseten", | |
| "task_id": "dock-24B-w15x2-extreme-0", | |
| "split": "eval", | |
| "difficulty": "extreme", | |
| "quay": "24B", | |
| "ships": 24, | |
| "reward": 0.0, | |
| "submitted": false, | |
| "feasible": false, | |
| "cost": null, | |
| "optimal_cost": 1009, | |
| "naive_cost": 3964, | |
| "turns": 3, | |
| "checks": 0, | |
| "seconds": 269.1, | |
| "input_tokens": 18966, | |
| "output_tokens": 64017, | |
| "end_reason": "no_tool_call", | |
| "errors": 0 | |
| }, | |
| { | |
| "model": "hf:zai-org/GLM-5.3-Flash:baseten", | |
| "task_id": "dock-24B-w16x1-busy-0", | |
| "split": "eval", | |
| "difficulty": "busy", | |
| "quay": "24B", | |
| "ships": 16, | |
| "reward": 0.864622, | |
| "submitted": true, | |
| "feasible": true, | |
| "cost": 1415, | |
| "optimal_cost": 1349, | |
| "naive_cost": 4069, | |
| "turns": 3, | |
| "checks": 1, | |
| "seconds": 264.3, | |
| "input_tokens": 16438, | |
| "output_tokens": 56124, | |
| "end_reason": "submitted", | |
| "errors": 0 | |
| }, | |
| { | |
| "model": "hf:zai-org/GLM-5.3-Flash:baseten", | |
| "task_id": "dock-24B-w16x2-extreme-0", | |
| "split": "eval", | |
| "difficulty": "extreme", | |
| "quay": "24B", | |
| "ships": 28, | |
| "reward": 0.185714, | |
| "submitted": true, | |
| "feasible": false, | |
| "cost": null, | |
| "optimal_cost": 466, | |
| "naive_cost": 1485, | |
| "turns": 6, | |
| "checks": 3, | |
| "seconds": 367.1, | |
| "input_tokens": 58856, | |
| "output_tokens": 83776, | |
| "end_reason": "submitted", | |
| "errors": 0 | |
| }, | |
| { | |
| "model": "hf:zai-org/GLM-5.3-Flash:baseten", | |
| "task_id": "dock-24B-w16x2-storm-0", | |
| "split": "eval", | |
| "difficulty": "storm", | |
| "quay": "24B", | |
| "ships": 27, | |
| "reward": 0.783994, | |
| "submitted": true, | |
| "feasible": true, | |
| "cost": 232, | |
| "optimal_cost": 201, | |
| "naive_cost": 1569, | |
| "turns": 3, | |
| "checks": 1, | |
| "seconds": 195.5, | |
| "input_tokens": 15905, | |
| "output_tokens": 38645, | |
| "end_reason": "submitted", | |
| "errors": 0 | |
| }, | |
| { | |
| "model": "hf:zai-org/GLM-5.3-Flash:baseten", | |
| "task_id": "dock-24B-w16x2-storm-1", | |
| "split": "eval", | |
| "difficulty": "storm", | |
| "quay": "24B", | |
| "ships": 26, | |
| "reward": 0.752488, | |
| "submitted": true, | |
| "feasible": true, | |
| "cost": 1259, | |
| "optimal_cost": 1187, | |
| "naive_cost": 2403, | |
| "turns": 4, | |
| "checks": 1, | |
| "seconds": 429.2, | |
| "input_tokens": 27871, | |
| "output_tokens": 88094, | |
| "end_reason": "submitted", | |
| "errors": 0 | |
| }, | |
| { | |
| "model": "hf:zai-org/GLM-5.3-Flash:baseten", | |
| "task_id": "dock-24B-w35x1-busy-0", | |
| "split": "eval", | |
| "difficulty": "busy", | |
| "quay": "24B", | |
| "ships": 14, | |
| "reward": 0.0, | |
| "submitted": false, | |
| "feasible": false, | |
| "cost": null, | |
| "optimal_cost": 1223, | |
| "naive_cost": 2892, | |
| "turns": 2, | |
| "checks": 0, | |
| "seconds": 326.0, | |
| "input_tokens": 9142, | |
| "output_tokens": 64000, | |
| "end_reason": "no_tool_call", | |
| "errors": 0 | |
| }, | |
| { | |
| "model": "hf:zai-org/GLM-5.3-Flash:baseten", | |
| "task_id": "dock-24B-w35x2-busy-0", | |
| "split": "eval", | |
| "difficulty": "busy", | |
| "quay": "24B", | |
| "ships": 24, | |
| "reward": 0.166667, | |
| "submitted": true, | |
| "feasible": false, | |
| "cost": null, | |
| "optimal_cost": 162, | |
| "naive_cost": 1443, | |
| "turns": 3, | |
| "checks": 0, | |
| "seconds": 176.3, | |
| "input_tokens": 16598, | |
| "output_tokens": 32701, | |
| "end_reason": "submitted", | |
| "errors": 0 | |
| }, | |
| { | |
| "model": "hf:zai-org/GLM-5.3-Flash:baseten", | |
| "task_id": "dock-24B-w35x2-extreme-0", | |
| "split": "eval", | |
| "difficulty": "extreme", | |
| "quay": "24B", | |
| "ships": 24, | |
| "reward": 0.350663, | |
| "submitted": true, | |
| "feasible": true, | |
| "cost": 1888, | |
| "optimal_cost": 1792, | |
| "naive_cost": 4682, | |
| "turns": 3, | |
| "checks": 2, | |
| "seconds": 186.1, | |
| "input_tokens": 19435, | |
| "output_tokens": 39815, | |
| "end_reason": "submitted", | |
| "errors": 0 | |
| }, | |
| { | |
| "model": "hf:zai-org/GLM-5.3-Flash:baseten", | |
| "task_id": "dock-24B-w35x2-extreme-1", | |
| "split": "eval", | |
| "difficulty": "extreme", | |
| "quay": "24B", | |
| "ships": 25, | |
| "reward": 1.0, | |
| "submitted": true, | |
| "feasible": true, | |
| "cost": 283, | |
| "optimal_cost": 283, | |
| "naive_cost": 1356, | |
| "turns": 4, | |
| "checks": 2, | |
| "seconds": 273.8, | |
| "input_tokens": 25348, | |
| "output_tokens": 60650, | |
| "end_reason": "submitted", | |
| "errors": 0 | |
| }, | |
| { | |
| "model": "hf:zai-org/GLM-5.3-Flash:baseten", | |
| "task_id": "dock-24B-w35x2-storm-0", | |
| "split": "eval", | |
| "difficulty": "storm", | |
| "quay": "24B", | |
| "ships": 24, | |
| "reward": 0.175, | |
| "submitted": true, | |
| "feasible": false, | |
| "cost": null, | |
| "optimal_cost": 2591, | |
| "naive_cost": 4835, | |
| "turns": 5, | |
| "checks": 3, | |
| "seconds": 496.8, | |
| "input_tokens": 40566, | |
| "output_tokens": 97265, | |
| "end_reason": "submitted", | |
| "errors": 0 | |
| }, | |
| { | |
| "model": "hf:zai-org/GLM-5.3-Flash:baseten", | |
| "task_id": "dock-24B-w35x2-storm-1", | |
| "split": "eval", | |
| "difficulty": "storm", | |
| "quay": "24B", | |
| "ships": 24, | |
| "reward": 0.830502, | |
| "submitted": true, | |
| "feasible": true, | |
| "cost": 332, | |
| "optimal_cost": 312, | |
| "naive_cost": 2701, | |
| "turns": 6, | |
| "checks": 4, | |
| "seconds": 243.7, | |
| "input_tokens": 54227, | |
| "output_tokens": 51851, | |
| "end_reason": "submitted", | |
| "errors": 0 | |
| }, | |
| { | |
| "model": "hf:zai-org/GLM-5.3-Flash:baseten", | |
| "task_id": "dock-24B-w35x3-extreme-0", | |
| "split": "eval", | |
| "difficulty": "extreme", | |
| "quay": "24B", | |
| "ships": 39, | |
| "reward": 0.184615, | |
| "submitted": true, | |
| "feasible": false, | |
| "cost": null, | |
| "optimal_cost": 1106, | |
| "naive_cost": 4303, | |
| "turns": 5, | |
| "checks": 2, | |
| "seconds": 447.6, | |
| "input_tokens": 52239, | |
| "output_tokens": 101377, | |
| "end_reason": "submitted", | |
| "errors": 0 | |
| }, | |
| { | |
| "model": "hf:zai-org/GLM-5.3-Flash:baseten", | |
| "task_id": "dock-24B-w35x3-extreme-1", | |
| "split": "eval", | |
| "difficulty": "extreme", | |
| "quay": "24B", | |
| "ships": 38, | |
| "reward": 0.184211, | |
| "submitted": true, | |
| "feasible": false, | |
| "cost": null, | |
| "optimal_cost": 865, | |
| "naive_cost": 2611, | |
| "turns": 4, | |
| "checks": 2, | |
| "seconds": 388.5, | |
| "input_tokens": 35205, | |
| "output_tokens": 82493, | |
| "end_reason": "submitted", | |
| "errors": 0 | |
| }, | |
| { | |
| "model": "hf:zai-org/GLM-5.3-Flash:baseten", | |
| "task_id": "dock-24B-w36x2-extreme-0", | |
| "split": "eval", | |
| "difficulty": "extreme", | |
| "quay": "24B", | |
| "ships": 24, | |
| "reward": 0.834491, | |
| "submitted": true, | |
| "feasible": true, | |
| "cost": 683, | |
| "optimal_cost": 648, | |
| "naive_cost": 1736, | |
| "turns": 7, | |
| "checks": 4, | |
| "seconds": 284.6, | |
| "input_tokens": 73012, | |
| "output_tokens": 65326, | |
| "end_reason": "submitted", | |
| "errors": 0 | |
| }, | |
| { | |
| "model": "hf:zai-org/GLM-5.3-Flash:baseten", | |
| "task_id": "dock-24B-w36x2-storm-0", | |
| "split": "eval", | |
| "difficulty": "storm", | |
| "quay": "24B", | |
| "ships": 23, | |
| "reward": 0.182609, | |
| "submitted": true, | |
| "feasible": false, | |
| "cost": null, | |
| "optimal_cost": 756, | |
| "naive_cost": 1931, | |
| "turns": 4, | |
| "checks": 2, | |
| "seconds": 388.3, | |
| "input_tokens": 27964, | |
| "output_tokens": 73690, | |
| "end_reason": "submitted", | |
| "errors": 0 | |
| }, | |
| { | |
| "model": "hf:zai-org/GLM-5.3-Flash:baseten", | |
| "task_id": "dock-24B-w36x2-storm-1", | |
| "split": "eval", | |
| "difficulty": "storm", | |
| "quay": "24B", | |
| "ships": 23, | |
| "reward": 1.0, | |
| "submitted": true, | |
| "feasible": true, | |
| "cost": 244, | |
| "optimal_cost": 244, | |
| "naive_cost": 549, | |
| "turns": 5, | |
| "checks": 3, | |
| "seconds": 178.7, | |
| "input_tokens": 31471, | |
| "output_tokens": 34491, | |
| "end_reason": "submitted", | |
| "errors": 0 | |
| }, | |
| { | |
| "model": "hf:zai-org/GLM-5.3-Flash:baseten", | |
| "task_id": "dock-24B-w37x1-standard-0", | |
| "split": "eval", | |
| "difficulty": "standard", | |
| "quay": "24B", | |
| "ships": 15, | |
| "reward": 1.0, | |
| "submitted": true, | |
| "feasible": true, | |
| "cost": 229, | |
| "optimal_cost": 229, | |
| "naive_cost": 734, | |
| "turns": 3, | |
| "checks": 1, | |
| "seconds": 174.4, | |
| "input_tokens": 14890, | |
| "output_tokens": 39788, | |
| "end_reason": "submitted", | |
| "errors": 0 | |
| }, | |
| { | |
| "model": "hf:zai-org/GLM-5.3-Flash:baseten", | |
| "task_id": "dock-36A-w05x1-busy-0", | |
| "split": "eval", | |
| "difficulty": "busy", | |
| "quay": "36A", | |
| "ships": 27, | |
| "reward": 1.0, | |
| "submitted": true, | |
| "feasible": true, | |
| "cost": 404, | |
| "optimal_cost": 404, | |
| "naive_cost": 791, | |
| "turns": 4, | |
| "checks": 2, | |
| "seconds": 161.6, | |
| "input_tokens": 22552, | |
| "output_tokens": 33022, | |
| "end_reason": "submitted", | |
| "errors": 0 | |
| }, | |
| { | |
| "model": "hf:zai-org/GLM-5.3-Flash:baseten", | |
| "task_id": "dock-36A-w05x2-busy-0", | |
| "split": "eval", | |
| "difficulty": "busy", | |
| "quay": "36A", | |
| "ships": 51, | |
| "reward": 0.376889, | |
| "submitted": true, | |
| "feasible": true, | |
| "cost": 360, | |
| "optimal_cost": 277, | |
| "naive_cost": 1831, | |
| "turns": 9, | |
| "checks": 7, | |
| "seconds": 521.3, | |
| "input_tokens": 157156, | |
| "output_tokens": 122976, | |
| "end_reason": "submitted", | |
| "errors": 0 | |
| }, | |
| { | |
| "model": "hf:zai-org/GLM-5.3-Flash:baseten", | |
| "task_id": "dock-36A-w05x2-extreme-0", | |
| "split": "eval", | |
| "difficulty": "extreme", | |
| "quay": "36A", | |
| "ships": 58, | |
| "reward": 0.0, | |
| "submitted": false, | |
| "feasible": false, | |
| "cost": null, | |
| "optimal_cost": 1594, | |
| "naive_cost": 4448, | |
| "turns": 2, | |
| "checks": 0, | |
| "seconds": 282.5, | |
| "input_tokens": 14077, | |
| "output_tokens": 64000, | |
| "end_reason": "no_tool_call", | |
| "errors": 0 | |
| }, | |
| { | |
| "model": "hf:zai-org/GLM-5.3-Flash:baseten", | |
| "task_id": "dock-36A-w05x2-storm-0", | |
| "split": "eval", | |
| "difficulty": "storm", | |
| "quay": "36A", | |
| "ships": 59, | |
| "reward": 0.0, | |
| "submitted": false, | |
| "feasible": false, | |
| "cost": null, | |
| "optimal_cost": 290, | |
| "naive_cost": 4214, | |
| "turns": 4, | |
| "checks": 1, | |
| "seconds": 591.9, | |
| "input_tokens": 44048, | |
| "output_tokens": 124142, | |
| "end_reason": "no_tool_call", | |
| "errors": 0 | |
| }, | |
| { | |
| "model": "hf:zai-org/GLM-5.3-Flash:baseten", | |
| "task_id": "dock-36A-w06x1-busy-0", | |
| "split": "eval", | |
| "difficulty": "busy", | |
| "quay": "36A", | |
| "ships": 25, | |
| "reward": 0.184, | |
| "submitted": true, | |
| "feasible": false, | |
| "cost": null, | |
| "optimal_cost": 40, | |
| "naive_cost": 824, | |
| "turns": 4, | |
| "checks": 1, | |
| "seconds": 191.5, | |
| "input_tokens": 27265, | |
| "output_tokens": 46641, | |
| "end_reason": "submitted", | |
| "errors": 0 | |
| }, | |
| { | |
| "model": "hf:zai-org/GLM-5.3-Flash:baseten", | |
| "task_id": "dock-36A-w06x1-standard-0", | |
| "split": "eval", | |
| "difficulty": "standard", | |
| "quay": "36A", | |
| "ships": 24, | |
| "reward": 0.986087, | |
| "submitted": true, | |
| "feasible": true, | |
| "cost": 15, | |
| "optimal_cost": 14, | |
| "naive_cost": 591, | |
| "turns": 3, | |
| "checks": 1, | |
| "seconds": 220.2, | |
| "input_tokens": 12820, | |
| "output_tokens": 45427, | |
| "end_reason": "submitted", | |
| "errors": 0 | |
| }, | |
| { | |
| "model": "hf:zai-org/GLM-5.3-Flash:baseten", | |
| "task_id": "dock-36A-w10x1-standard-0", | |
| "split": "eval", | |
| "difficulty": "standard", | |
| "quay": "36A", | |
| "ships": 32, | |
| "reward": 0.933374, | |
| "submitted": true, | |
| "feasible": true, | |
| "cost": 44, | |
| "optimal_cost": 39, | |
| "naive_cost": 606, | |
| "turns": 6, | |
| "checks": 4, | |
| "seconds": 264.1, | |
| "input_tokens": 48728, | |
| "output_tokens": 51745, | |
| "end_reason": "submitted", | |
| "errors": 0 | |
| }, | |
| { | |
| "model": "hf:zai-org/GLM-5.3-Flash:baseten", | |
| "task_id": "dock-36A-w15x1-busy-0", | |
| "split": "eval", | |
| "difficulty": "busy", | |
| "quay": "36A", | |
| "ships": 29, | |
| "reward": 0.0, | |
| "submitted": false, | |
| "feasible": false, | |
| "cost": null, | |
| "optimal_cost": 153, | |
| "naive_cost": 1963, | |
| "turns": 2, | |
| "checks": 0, | |
| "seconds": 273.7, | |
| "input_tokens": 10335, | |
| "output_tokens": 64000, | |
| "end_reason": "no_tool_call", | |
| "errors": 0 | |
| }, | |
| { | |
| "model": "hf:zai-org/GLM-5.3-Flash:baseten", | |
| "task_id": "dock-36A-w15x1-standard-0", | |
| "split": "eval", | |
| "difficulty": "standard", | |
| "quay": "36A", | |
| "ships": 30, | |
| "reward": 0.0, | |
| "submitted": false, | |
| "feasible": false, | |
| "cost": null, | |
| "optimal_cost": 119, | |
| "naive_cost": 2613, | |
| "turns": 2, | |
| "checks": 0, | |
| "seconds": 325.5, | |
| "input_tokens": 9523, | |
| "output_tokens": 64000, | |
| "end_reason": "no_tool_call", | |
| "errors": 0 | |
| }, | |
| { | |
| "model": "hf:zai-org/GLM-5.3-Flash:baseten", | |
| "task_id": "dock-36A-w15x2-busy-0", | |
| "split": "eval", | |
| "difficulty": "busy", | |
| "quay": "36A", | |
| "ships": 55, | |
| "reward": 0.0, | |
| "submitted": false, | |
| "feasible": false, | |
| "cost": null, | |
| "optimal_cost": 368, | |
| "naive_cost": 3372, | |
| "turns": 2, | |
| "checks": 0, | |
| "seconds": 264.3, | |
| "input_tokens": 12126, | |
| "output_tokens": 64000, | |
| "end_reason": "no_tool_call", | |
| "errors": 0 | |
| }, | |
| { | |
| "model": "hf:zai-org/GLM-5.3-Flash:baseten", | |
| "task_id": "dock-36A-w15x2-storm-0", | |
| "split": "eval", | |
| "difficulty": "storm", | |
| "quay": "36A", | |
| "ships": 56, | |
| "reward": 0.0, | |
| "submitted": false, | |
| "feasible": false, | |
| "cost": null, | |
| "optimal_cost": 369, | |
| "naive_cost": 6062, | |
| "turns": 2, | |
| "checks": 0, | |
| "seconds": 358.4, | |
| "input_tokens": 12299, | |
| "output_tokens": 64000, | |
| "end_reason": "no_tool_call", | |
| "errors": 0 | |
| }, | |
| { | |
| "model": "hf:zai-org/GLM-5.3-Flash:baseten", | |
| "task_id": "dock-36A-w16x1-standard-0", | |
| "split": "eval", | |
| "difficulty": "standard", | |
| "quay": "36A", | |
| "ships": 27, | |
| "reward": 0.162963, | |
| "submitted": true, | |
| "feasible": false, | |
| "cost": null, | |
| "optimal_cost": 223, | |
| "naive_cost": 1083, | |
| "turns": 6, | |
| "checks": 3, | |
| "seconds": 245.4, | |
| "input_tokens": 50700, | |
| "output_tokens": 50909, | |
| "end_reason": "submitted", | |
| "errors": 0 | |
| }, | |
| { | |
| "model": "hf:zai-org/GLM-5.3-Flash:baseten", | |
| "task_id": "dock-36A-w16x2-busy-0", | |
| "split": "eval", | |
| "difficulty": "busy", | |
| "quay": "36A", | |
| "ships": 55, | |
| "reward": 0.0, | |
| "submitted": false, | |
| "feasible": false, | |
| "cost": null, | |
| "optimal_cost": 403, | |
| "naive_cost": 2046, | |
| "turns": 2, | |
| "checks": 0, | |
| "seconds": 296.1, | |
| "input_tokens": 12317, | |
| "output_tokens": 64000, | |
| "end_reason": "no_tool_call", | |
| "errors": 0 | |
| }, | |
| { | |
| "model": "hf:zai-org/GLM-5.3-Flash:baseten", | |
| "task_id": "dock-36A-w16x2-storm-0", | |
| "split": "eval", | |
| "difficulty": "storm", | |
| "quay": "36A", | |
| "ships": 57, | |
| "reward": 0.0, | |
| "submitted": false, | |
| "feasible": false, | |
| "cost": null, | |
| "optimal_cost": 108, | |
| "naive_cost": 6938, | |
| "turns": 2, | |
| "checks": 0, | |
| "seconds": 310.3, | |
| "input_tokens": 13068, | |
| "output_tokens": 64000, | |
| "end_reason": "no_tool_call", | |
| "errors": 0 | |
| }, | |
| { | |
| "model": "hf:zai-org/GLM-5.3-Flash:baseten", | |
| "task_id": "dock-36A-w16x2-storm-1", | |
| "split": "eval", | |
| "difficulty": "storm", | |
| "quay": "36A", | |
| "ships": 55, | |
| "reward": 0.0, | |
| "submitted": false, | |
| "feasible": false, | |
| "cost": null, | |
| "optimal_cost": 627, | |
| "naive_cost": 6819, | |
| "turns": 2, | |
| "checks": 0, | |
| "seconds": 300.1, | |
| "input_tokens": 13018, | |
| "output_tokens": 64000, | |
| "end_reason": "no_tool_call", | |
| "errors": 0 | |
| }, | |
| { | |
| "model": "hf:zai-org/GLM-5.3-Flash:baseten", | |
| "task_id": "dock-36A-w17x1-busy-0", | |
| "split": "eval", | |
| "difficulty": "busy", | |
| "quay": "36A", | |
| "ships": 28, | |
| "reward": 0.867002, | |
| "submitted": true, | |
| "feasible": true, | |
| "cost": 162, | |
| "optimal_cost": 152, | |
| "naive_cost": 1521, | |
| "turns": 5, | |
| "checks": 2, | |
| "seconds": 415.9, | |
| "input_tokens": 38332, | |
| "output_tokens": 91870, | |
| "end_reason": "submitted", | |
| "errors": 0 | |
| }, | |
| { | |
| "model": "hf:zai-org/GLM-5.3-Flash:baseten", | |
| "task_id": "dock-36A-w17x1-standard-0", | |
| "split": "eval", | |
| "difficulty": "standard", | |
| "quay": "36A", | |
| "ships": 29, | |
| "reward": 0.936036, | |
| "submitted": true, | |
| "feasible": true, | |
| "cost": 105, | |
| "optimal_cost": 100, | |
| "naive_cost": 1794, | |
| "turns": 4, | |
| "checks": 2, | |
| "seconds": 157.7, | |
| "input_tokens": 23176, | |
| "output_tokens": 29724, | |
| "end_reason": "submitted", | |
| "errors": 0 | |
| }, | |
| { | |
| "model": "hf:zai-org/GLM-5.3-Flash:baseten", | |
| "task_id": "dock-36A-w35x1-standard-0", | |
| "split": "eval", | |
| "difficulty": "standard", | |
| "quay": "36A", | |
| "ships": 19, | |
| "reward": 1.0, | |
| "submitted": true, | |
| "feasible": true, | |
| "cost": 10, | |
| "optimal_cost": 10, | |
| "naive_cost": 274, | |
| "turns": 3, | |
| "checks": 1, | |
| "seconds": 88.9, | |
| "input_tokens": 11713, | |
| "output_tokens": 18038, | |
| "end_reason": "submitted", | |
| "errors": 0 | |
| }, | |
| { | |
| "model": "hf:zai-org/GLM-5.3-Flash:baseten", | |
| "task_id": "dock-36A-w35x2-extreme-0", | |
| "split": "eval", | |
| "difficulty": "extreme", | |
| "quay": "36A", | |
| "ships": 39, | |
| "reward": 0.0, | |
| "submitted": false, | |
| "feasible": false, | |
| "cost": null, | |
| "optimal_cost": 754, | |
| "naive_cost": 3430, | |
| "turns": 2, | |
| "checks": 0, | |
| "seconds": 248.1, | |
| "input_tokens": 11788, | |
| "output_tokens": 64000, | |
| "end_reason": "no_tool_call", | |
| "errors": 0 | |
| }, | |
| { | |
| "model": "hf:zai-org/GLM-5.3-Flash:baseten", | |
| "task_id": "dock-36A-w35x2-extreme-1", | |
| "split": "eval", | |
| "difficulty": "extreme", | |
| "quay": "36A", | |
| "ships": 35, | |
| "reward": 0.188571, | |
| "submitted": true, | |
| "feasible": false, | |
| "cost": null, | |
| "optimal_cost": 1315, | |
| "naive_cost": 3041, | |
| "turns": 3, | |
| "checks": 1, | |
| "seconds": 270.0, | |
| "input_tokens": 22824, | |
| "output_tokens": 63522, | |
| "end_reason": "submitted", | |
| "errors": 0 | |
| }, | |
| { | |
| "model": "hf:zai-org/GLM-5.3-Flash:baseten", | |
| "task_id": "dock-36A-w35x2-storm-0", | |
| "split": "eval", | |
| "difficulty": "storm", | |
| "quay": "36A", | |
| "ships": 33, | |
| "reward": 0.522312, | |
| "submitted": true, | |
| "feasible": true, | |
| "cost": 152, | |
| "optimal_cost": 102, | |
| "naive_cost": 670, | |
| "turns": 6, | |
| "checks": 3, | |
| "seconds": 343.2, | |
| "input_tokens": 61323, | |
| "output_tokens": 74227, | |
| "end_reason": "submitted", | |
| "errors": 0 | |
| }, | |
| { | |
| "model": "hf:zai-org/GLM-5.3-Flash:baseten", | |
| "task_id": "dock-36A-w35x3-extreme-0", | |
| "split": "eval", | |
| "difficulty": "extreme", | |
| "quay": "36A", | |
| "ships": 59, | |
| "reward": 0.0, | |
| "submitted": false, | |
| "feasible": false, | |
| "cost": null, | |
| "optimal_cost": 953, | |
| "naive_cost": 5403, | |
| "turns": 2, | |
| "checks": 0, | |
| "seconds": 253.8, | |
| "input_tokens": 13920, | |
| "output_tokens": 64000, | |
| "end_reason": "no_tool_call", | |
| "errors": 0 | |
| }, | |
| { | |
| "model": "hf:zai-org/GLM-5.3-Flash:baseten", | |
| "task_id": "dock-36A-w36x2-busy-0", | |
| "split": "eval", | |
| "difficulty": "busy", | |
| "quay": "36A", | |
| "ships": 37, | |
| "reward": 0.194595, | |
| "submitted": true, | |
| "feasible": false, | |
| "cost": null, | |
| "optimal_cost": 3143, | |
| "naive_cost": 9374, | |
| "turns": 4, | |
| "checks": 2, | |
| "seconds": 309.9, | |
| "input_tokens": 35788, | |
| "output_tokens": 67843, | |
| "end_reason": "submitted", | |
| "errors": 0 | |
| }, | |
| { | |
| "model": "hf:zai-org/GLM-5.3-Flash:baseten", | |
| "task_id": "dock-36A-w36x2-extreme-0", | |
| "split": "eval", | |
| "difficulty": "extreme", | |
| "quay": "36A", | |
| "ships": 42, | |
| "reward": 0.0, | |
| "submitted": false, | |
| "feasible": false, | |
| "cost": null, | |
| "optimal_cost": 320, | |
| "naive_cost": 4753, | |
| "turns": 3, | |
| "checks": 0, | |
| "seconds": 279.6, | |
| "input_tokens": 23670, | |
| "output_tokens": 64017, | |
| "end_reason": "no_tool_call", | |
| "errors": 0 | |
| }, | |
| { | |
| "model": "hf:zai-org/GLM-5.3-Flash:baseten", | |
| "task_id": "dock-36A-w36x2-storm-0", | |
| "split": "eval", | |
| "difficulty": "storm", | |
| "quay": "36A", | |
| "ships": 37, | |
| "reward": 1.0, | |
| "submitted": true, | |
| "feasible": true, | |
| "cost": 28, | |
| "optimal_cost": 28, | |
| "naive_cost": 864, | |
| "turns": 4, | |
| "checks": 2, | |
| "seconds": 272.5, | |
| "input_tokens": 29444, | |
| "output_tokens": 66481, | |
| "end_reason": "submitted", | |
| "errors": 0 | |
| }, | |
| { | |
| "model": "hf:zai-org/GLM-5.3-Flash:baseten", | |
| "task_id": "dock-36A-w37x1-busy-0", | |
| "split": "eval", | |
| "difficulty": "busy", | |
| "quay": "36A", | |
| "ships": 23, | |
| "reward": 1.0, | |
| "submitted": true, | |
| "feasible": true, | |
| "cost": 36, | |
| "optimal_cost": 36, | |
| "naive_cost": 598, | |
| "turns": 3, | |
| "checks": 1, | |
| "seconds": 86.1, | |
| "input_tokens": 13381, | |
| "output_tokens": 21213, | |
| "end_reason": "submitted", | |
| "errors": 0 | |
| }, | |
| { | |
| "model": "hf:zai-org/GLM-5.3-Flash:baseten", | |
| "task_id": "dock-36A-w37x1-standard-0", | |
| "split": "eval", | |
| "difficulty": "standard", | |
| "quay": "36A", | |
| "ships": 22, | |
| "reward": 0.88409, | |
| "submitted": true, | |
| "feasible": true, | |
| "cost": 24, | |
| "optimal_cost": 15, | |
| "naive_cost": 429, | |
| "turns": 4, | |
| "checks": 2, | |
| "seconds": 112.0, | |
| "input_tokens": 19586, | |
| "output_tokens": 24380, | |
| "end_reason": "submitted", | |
| "errors": 0 | |
| }, | |
| { | |
| "model": "hf:zai-org/GLM-5.3:together", | |
| "task_id": "dock-24B-w06x1-busy-0", | |
| "split": "eval", | |
| "difficulty": "busy", | |
| "quay": "24B", | |
| "ships": 17, | |
| "reward": 1.0, | |
| "submitted": true, | |
| "feasible": true, | |
| "cost": 87, | |
| "optimal_cost": 87, | |
| "naive_cost": 658, | |
| "turns": 5, | |
| "checks": 3, | |
| "seconds": 279.9, | |
| "input_tokens": 33701, | |
| "output_tokens": 46553, | |
| "end_reason": "submitted", | |
| "errors": 0 | |
| }, | |
| { | |
| "model": "hf:zai-org/GLM-5.3:together", | |
| "task_id": "dock-24B-w06x1-standard-0", | |
| "split": "eval", | |
| "difficulty": "standard", | |
| "quay": "24B", | |
| "ships": 16, | |
| "reward": 1.0, | |
| "submitted": true, | |
| "feasible": true, | |
| "cost": 114, | |
| "optimal_cost": 114, | |
| "naive_cost": 311, | |
| "turns": 3, | |
| "checks": 2, | |
| "seconds": 169.7, | |
| "input_tokens": 12969, | |
| "output_tokens": 32543, | |
| "end_reason": "submitted", | |
| "errors": 0 | |
| }, | |
| { | |
| "model": "hf:zai-org/GLM-5.3:together", | |
| "task_id": "dock-24B-w06x2-busy-0", | |
| "split": "eval", | |
| "difficulty": "busy", | |
| "quay": "24B", | |
| "ships": 31, | |
| "reward": 0.940769, | |
| "submitted": true, | |
| "feasible": true, | |
| "cost": 621, | |
| "optimal_cost": 616, | |
| "naive_cost": 1181, | |
| "turns": 6, | |
| "checks": 4, | |
| "seconds": 585.4, | |
| "input_tokens": 57355, | |
| "output_tokens": 77336, | |
| "end_reason": "submitted", | |
| "errors": 0 | |
| }, | |
| { | |
| "model": "hf:zai-org/GLM-5.3:together", | |
| "task_id": "dock-24B-w06x2-extreme-0", | |
| "split": "eval", | |
| "difficulty": "extreme", | |
| "quay": "24B", | |
| "ships": 32, | |
| "reward": 0.0, | |
| "submitted": false, | |
| "feasible": false, | |
| "cost": null, | |
| "optimal_cost": 231, | |
| "naive_cost": 1090, | |
| "turns": 2, | |
| "checks": 0, | |
| "seconds": 418.6, | |
| "input_tokens": 10631, | |
| "output_tokens": 64000, | |
| "end_reason": "no_tool_call", | |
| "errors": 0 | |
| }, | |
| { | |
| "model": "hf:zai-org/GLM-5.3:together", | |
| "task_id": "dock-24B-w06x2-storm-0", | |
| "split": "eval", | |
| "difficulty": "storm", | |
| "quay": "24B", | |
| "ships": 32, | |
| "reward": 0.908913, | |
| "submitted": true, | |
| "feasible": true, | |
| "cost": 165, | |
| "optimal_cost": 154, | |
| "naive_cost": 1259, | |
| "turns": 12, | |
| "checks": 9, | |
| "seconds": 1412.8, | |
| "input_tokens": 213246, | |
| "output_tokens": 200699, | |
| "end_reason": "submitted", | |
| "errors": 0 | |
| }, | |
| { | |
| "model": "hf:zai-org/GLM-5.3:together", | |
| "task_id": "dock-24B-w07x1-busy-0", | |
| "split": "eval", | |
| "difficulty": "busy", | |
| "quay": "24B", | |
| "ships": 17, | |
| "reward": 1.0, | |
| "submitted": true, | |
| "feasible": true, | |
| "cost": 226, | |
| "optimal_cost": 226, | |
| "naive_cost": 641, | |
| "turns": 3, | |
| "checks": 1, | |
| "seconds": 341.8, | |
| "input_tokens": 16557, | |
| "output_tokens": 53373, | |
| "end_reason": "submitted", | |
| "errors": 0 | |
| }, | |
| { | |
| "model": "hf:zai-org/GLM-5.3:together", | |
| "task_id": "dock-24B-w15x2-extreme-0", | |
| "split": "eval", | |
| "difficulty": "extreme", | |
| "quay": "24B", | |
| "ships": 24, | |
| "reward": 0.0, | |
| "submitted": false, | |
| "feasible": false, | |
| "cost": null, | |
| "optimal_cost": 1009, | |
| "naive_cost": 3964, | |
| "turns": 2, | |
| "checks": 0, | |
| "seconds": 366.5, | |
| "input_tokens": 10761, | |
| "output_tokens": 64000, | |
| "end_reason": "no_tool_call", | |
| "errors": 0 | |
| }, | |
| { | |
| "model": "hf:zai-org/GLM-5.3:together", | |
| "task_id": "dock-24B-w16x1-busy-0", | |
| "split": "eval", | |
| "difficulty": "busy", | |
| "quay": "24B", | |
| "ships": 16, | |
| "reward": 0.0, | |
| "submitted": false, | |
| "feasible": false, | |
| "cost": null, | |
| "optimal_cost": 1349, | |
| "naive_cost": 4069, | |
| "turns": 2, | |
| "checks": 0, | |
| "seconds": 511.9, | |
| "input_tokens": 8811, | |
| "output_tokens": 64000, | |
| "end_reason": "no_tool_call", | |
| "errors": 0 | |
| }, | |
| { | |
| "model": "hf:zai-org/GLM-5.3:together", | |
| "task_id": "dock-24B-w16x2-extreme-0", | |
| "split": "eval", | |
| "difficulty": "extreme", | |
| "quay": "24B", | |
| "ships": 28, | |
| "reward": 1.0, | |
| "submitted": true, | |
| "feasible": true, | |
| "cost": 466, | |
| "optimal_cost": 466, | |
| "naive_cost": 1485, | |
| "turns": 6, | |
| "checks": 4, | |
| "seconds": 715.0, | |
| "input_tokens": 57013, | |
| "output_tokens": 126199, | |
| "end_reason": "submitted", | |
| "errors": 0 | |
| }, | |
| { | |
| "model": "hf:zai-org/GLM-5.3:together", | |
| "task_id": "dock-24B-w16x2-storm-0", | |
| "split": "eval", | |
| "difficulty": "storm", | |
| "quay": "24B", | |
| "ships": 27, | |
| "reward": 0.0, | |
| "submitted": false, | |
| "feasible": false, | |
| "cost": null, | |
| "optimal_cost": 201, | |
| "naive_cost": 1569, | |
| "turns": 2, | |
| "checks": 0, | |
| "seconds": 408.3, | |
| "input_tokens": 9933, | |
| "output_tokens": 64000, | |
| "end_reason": "no_tool_call", | |
| "errors": 0 | |
| }, | |
| { | |
| "model": "hf:zai-org/GLM-5.3:together", | |
| "task_id": "dock-24B-w16x2-storm-1", | |
| "split": "eval", | |
| "difficulty": "storm", | |
| "quay": "24B", | |
| "ships": 26, | |
| "reward": 0.0, | |
| "submitted": false, | |
| "feasible": false, | |
| "cost": null, | |
| "optimal_cost": 1187, | |
| "naive_cost": 2403, | |
| "turns": 2, | |
| "checks": 0, | |
| "seconds": 441.2, | |
| "input_tokens": 9921, | |
| "output_tokens": 64000, | |
| "end_reason": "no_tool_call", | |
| "errors": 0 | |
| }, | |
| { | |
| "model": "hf:zai-org/GLM-5.3:together", | |
| "task_id": "dock-24B-w35x1-busy-0", | |
| "split": "eval", | |
| "difficulty": "busy", | |
| "quay": "24B", | |
| "ships": 14, | |
| "reward": 0.0, | |
| "submitted": false, | |
| "feasible": false, | |
| "cost": null, | |
| "optimal_cost": 1223, | |
| "naive_cost": 2892, | |
| "turns": 2, | |
| "checks": 0, | |
| "seconds": 532.1, | |
| "input_tokens": 8245, | |
| "output_tokens": 64000, | |
| "end_reason": "no_tool_call", | |
| "errors": 0 | |
| }, | |
| { | |
| "model": "hf:zai-org/GLM-5.3:together", | |
| "task_id": "dock-24B-w35x2-busy-0", | |
| "split": "eval", | |
| "difficulty": "busy", | |
| "quay": "24B", | |
| "ships": 24, | |
| "reward": 0.0, | |
| "submitted": false, | |
| "feasible": false, | |
| "cost": null, | |
| "optimal_cost": 162, | |
| "naive_cost": 1443, | |
| "turns": 2, | |
| "checks": 0, | |
| "seconds": 563.4, | |
| "input_tokens": 9657, | |
| "output_tokens": 64000, | |
| "end_reason": "no_tool_call", | |
| "errors": 0 | |
| }, | |
| { | |
| "model": "hf:zai-org/GLM-5.3:together", | |
| "task_id": "dock-24B-w35x2-extreme-0", | |
| "split": "eval", | |
| "difficulty": "extreme", | |
| "quay": "24B", | |
| "ships": 24, | |
| "reward": 0.0, | |
| "submitted": false, | |
| "feasible": false, | |
| "cost": null, | |
| "optimal_cost": 1792, | |
| "naive_cost": 4682, | |
| "turns": 2, | |
| "checks": 0, | |
| "seconds": 362.5, | |
| "input_tokens": 10406, | |
| "output_tokens": 64000, | |
| "end_reason": "no_tool_call", | |
| "errors": 0 | |
| }, | |
| { | |
| "model": "hf:zai-org/GLM-5.3:together", | |
| "task_id": "dock-24B-w35x2-extreme-1", | |
| "split": "eval", | |
| "difficulty": "extreme", | |
| "quay": "24B", | |
| "ships": 25, | |
| "reward": 1.0, | |
| "submitted": true, | |
| "feasible": true, | |
| "cost": 283, | |
| "optimal_cost": 283, | |
| "naive_cost": 1356, | |
| "turns": 5, | |
| "checks": 3, | |
| "seconds": 528.3, | |
| "input_tokens": 39612, | |
| "output_tokens": 91102, | |
| "end_reason": "submitted", | |
| "errors": 0 | |
| }, | |
| { | |
| "model": "hf:zai-org/GLM-5.3:together", | |
| "task_id": "dock-24B-w35x2-storm-0", | |
| "split": "eval", | |
| "difficulty": "storm", | |
| "quay": "24B", | |
| "ships": 24, | |
| "reward": 0.0, | |
| "submitted": false, | |
| "feasible": false, | |
| "cost": null, | |
| "optimal_cost": 2591, | |
| "naive_cost": 4835, | |
| "turns": 2, | |
| "checks": 0, | |
| "seconds": 418.6, | |
| "input_tokens": 9989, | |
| "output_tokens": 64000, | |
| "end_reason": "no_tool_call", | |
| "errors": 0 | |
| }, | |
| { | |
| "model": "hf:zai-org/GLM-5.3:together", | |
| "task_id": "dock-24B-w35x2-storm-1", | |
| "split": "eval", | |
| "difficulty": "storm", | |
| "quay": "24B", | |
| "ships": 24, | |
| "reward": 0.0, | |
| "submitted": false, | |
| "feasible": false, | |
| "cost": null, | |
| "optimal_cost": 312, | |
| "naive_cost": 2701, | |
| "turns": 2, | |
| "checks": 0, | |
| "seconds": 356.0, | |
| "input_tokens": 9996, | |
| "output_tokens": 64000, | |
| "end_reason": "no_tool_call", | |
| "errors": 0 | |
| }, | |
| { | |
| "model": "hf:zai-org/GLM-5.3:together", | |
| "task_id": "dock-24B-w35x3-extreme-0", | |
| "split": "eval", | |
| "difficulty": "extreme", | |
| "quay": "24B", | |
| "ships": 39, | |
| "reward": 0.0, | |
| "submitted": false, | |
| "feasible": false, | |
| "cost": null, | |
| "optimal_cost": 1106, | |
| "naive_cost": 4303, | |
| "turns": 2, | |
| "checks": 0, | |
| "seconds": 359.0, | |
| "input_tokens": 11791, | |
| "output_tokens": 64000, | |
| "end_reason": "no_tool_call", | |
| "errors": 0 | |
| }, | |
| { | |
| "model": "hf:zai-org/GLM-5.3:together", | |
| "task_id": "dock-24B-w35x3-extreme-1", | |
| "split": "eval", | |
| "difficulty": "extreme", | |
| "quay": "24B", | |
| "ships": 38, | |
| "reward": 0.0, | |
| "submitted": false, | |
| "feasible": false, | |
| "cost": null, | |
| "optimal_cost": 865, | |
| "naive_cost": 2611, | |
| "turns": 2, | |
| "checks": 0, | |
| "seconds": 357.8, | |
| "input_tokens": 11515, | |
| "output_tokens": 64000, | |
| "end_reason": "no_tool_call", | |
| "errors": 0 | |
| }, | |
| { | |
| "model": "hf:zai-org/GLM-5.3:together", | |
| "task_id": "dock-24B-w36x2-extreme-0", | |
| "split": "eval", | |
| "difficulty": "extreme", | |
| "quay": "24B", | |
| "ships": 24, | |
| "reward": 0.0, | |
| "submitted": false, | |
| "feasible": false, | |
| "cost": null, | |
| "optimal_cost": 648, | |
| "naive_cost": 1736, | |
| "turns": 2, | |
| "checks": 0, | |
| "seconds": 357.3, | |
| "input_tokens": 10461, | |
| "output_tokens": 64000, | |
| "end_reason": "no_tool_call", | |
| "errors": 0 | |
| }, | |
| { | |
| "model": "hf:zai-org/GLM-5.3:together", | |
| "task_id": "dock-24B-w36x2-storm-0", | |
| "split": "eval", | |
| "difficulty": "storm", | |
| "quay": "24B", | |
| "ships": 23, | |
| "reward": 0.0, | |
| "submitted": false, | |
| "feasible": false, | |
| "cost": null, | |
| "optimal_cost": 756, | |
| "naive_cost": 1931, | |
| "turns": 2, | |
| "checks": 0, | |
| "seconds": 386.4, | |
| "input_tokens": 9547, | |
| "output_tokens": 64000, | |
| "end_reason": "no_tool_call", | |
| "errors": 0 | |
| }, | |
| { | |
| "model": "hf:zai-org/GLM-5.3:together", | |
| "task_id": "dock-24B-w36x2-storm-1", | |
| "split": "eval", | |
| "difficulty": "storm", | |
| "quay": "24B", | |
| "ships": 23, | |
| "reward": 1.0, | |
| "submitted": true, | |
| "feasible": true, | |
| "cost": 244, | |
| "optimal_cost": 244, | |
| "naive_cost": 549, | |
| "turns": 3, | |
| "checks": 1, | |
| "seconds": 270.3, | |
| "input_tokens": 16592, | |
| "output_tokens": 42027, | |
| "end_reason": "submitted", | |
| "errors": 0 | |
| }, | |
| { | |
| "model": "hf:zai-org/GLM-5.3:together", | |
| "task_id": "dock-24B-w37x1-standard-0", | |
| "split": "eval", | |
| "difficulty": "standard", | |
| "quay": "24B", | |
| "ships": 15, | |
| "reward": 0.810539, | |
| "submitted": true, | |
| "feasible": true, | |
| "cost": 254, | |
| "optimal_cost": 229, | |
| "naive_cost": 734, | |
| "turns": 3, | |
| "checks": 1, | |
| "seconds": 288.7, | |
| "input_tokens": 15004, | |
| "output_tokens": 44273, | |
| "end_reason": "submitted", | |
| "errors": 0 | |
| }, | |
| { | |
| "model": "hf:zai-org/GLM-5.3:together", | |
| "task_id": "dock-36A-w05x1-busy-0", | |
| "split": "eval", | |
| "difficulty": "busy", | |
| "quay": "36A", | |
| "ships": 27, | |
| "reward": 1.0, | |
| "submitted": true, | |
| "feasible": true, | |
| "cost": 404, | |
| "optimal_cost": 404, | |
| "naive_cost": 791, | |
| "turns": 6, | |
| "checks": 3, | |
| "seconds": 200.7, | |
| "input_tokens": 49363, | |
| "output_tokens": 41769, | |
| "end_reason": "submitted", | |
| "errors": 0 | |
| }, | |
| { | |
| "model": "hf:zai-org/GLM-5.3:together", | |
| "task_id": "dock-36A-w05x2-busy-0", | |
| "split": "eval", | |
| "difficulty": "busy", | |
| "quay": "36A", | |
| "ships": 51, | |
| "reward": 0.196078, | |
| "submitted": true, | |
| "feasible": false, | |
| "cost": null, | |
| "optimal_cost": 277, | |
| "naive_cost": 1831, | |
| "turns": 6, | |
| "checks": 2, | |
| "seconds": 381.8, | |
| "input_tokens": 77540, | |
| "output_tokens": 88568, | |
| "end_reason": "submitted", | |
| "errors": 0 | |
| }, | |
| { | |
| "model": "hf:zai-org/GLM-5.3:together", | |
| "task_id": "dock-36A-w05x2-extreme-0", | |
| "split": "eval", | |
| "difficulty": "extreme", | |
| "quay": "36A", | |
| "ships": 58, | |
| "reward": 0.0, | |
| "submitted": false, | |
| "feasible": false, | |
| "cost": null, | |
| "optimal_cost": 1594, | |
| "naive_cost": 4448, | |
| "turns": 2, | |
| "checks": 0, | |
| "seconds": 457.5, | |
| "input_tokens": 13829, | |
| "output_tokens": 64000, | |
| "end_reason": "no_tool_call", | |
| "errors": 0 | |
| }, | |
| { | |
| "model": "hf:zai-org/GLM-5.3:together", | |
| "task_id": "dock-36A-w05x2-storm-0", | |
| "split": "eval", | |
| "difficulty": "storm", | |
| "quay": "36A", | |
| "ships": 59, | |
| "reward": 0.0, | |
| "submitted": false, | |
| "feasible": false, | |
| "cost": null, | |
| "optimal_cost": 290, | |
| "naive_cost": 4214, | |
| "turns": 3, | |
| "checks": 0, | |
| "seconds": 381.5, | |
| "input_tokens": 24999, | |
| "output_tokens": 71068, | |
| "end_reason": "no_tool_call", | |
| "errors": 0 | |
| }, | |
| { | |
| "model": "hf:zai-org/GLM-5.3:together", | |
| "task_id": "dock-36A-w06x1-busy-0", | |
| "split": "eval", | |
| "difficulty": "busy", | |
| "quay": "36A", | |
| "ships": 25, | |
| "reward": 0.0, | |
| "submitted": false, | |
| "feasible": false, | |
| "cost": null, | |
| "optimal_cost": 40, | |
| "naive_cost": 824, | |
| "turns": 2, | |
| "checks": 0, | |
| "seconds": 285.8, | |
| "input_tokens": 9394, | |
| "output_tokens": 64000, | |
| "end_reason": "no_tool_call", | |
| "errors": 0 | |
| }, | |
| { | |
| "model": "hf:zai-org/GLM-5.3:together", | |
| "task_id": "dock-36A-w06x1-standard-0", | |
| "split": "eval", | |
| "difficulty": "standard", | |
| "quay": "36A", | |
| "ships": 24, | |
| "reward": 0.986087, | |
| "submitted": true, | |
| "feasible": true, | |
| "cost": 15, | |
| "optimal_cost": 14, | |
| "naive_cost": 591, | |
| "turns": 3, | |
| "checks": 1, | |
| "seconds": 262.7, | |
| "input_tokens": 15662, | |
| "output_tokens": 52453, | |
| "end_reason": "submitted", | |
| "errors": 0 | |
| }, | |
| { | |
| "model": "hf:zai-org/GLM-5.3:together", | |
| "task_id": "dock-36A-w10x1-standard-0", | |
| "split": "eval", | |
| "difficulty": "standard", | |
| "quay": "36A", | |
| "ships": 32, | |
| "reward": 0.933374, | |
| "submitted": true, | |
| "feasible": true, | |
| "cost": 44, | |
| "optimal_cost": 39, | |
| "naive_cost": 606, | |
| "turns": 3, | |
| "checks": 1, | |
| "seconds": 170.8, | |
| "input_tokens": 17150, | |
| "output_tokens": 38255, | |
| "end_reason": "submitted", | |
| "errors": 0 | |
| }, | |
| { | |
| "model": "hf:zai-org/GLM-5.3:together", | |
| "task_id": "dock-36A-w15x1-busy-0", | |
| "split": "eval", | |
| "difficulty": "busy", | |
| "quay": "36A", | |
| "ships": 29, | |
| "reward": 0.0, | |
| "submitted": false, | |
| "feasible": false, | |
| "cost": null, | |
| "optimal_cost": 153, | |
| "naive_cost": 1963, | |
| "turns": 2, | |
| "checks": 0, | |
| "seconds": 283.6, | |
| "input_tokens": 9870, | |
| "output_tokens": 64000, | |
| "end_reason": "no_tool_call", | |
| "errors": 0 | |
| }, | |
| { | |
| "model": "hf:zai-org/GLM-5.3:together", | |
| "task_id": "dock-36A-w15x1-standard-0", | |
| "split": "eval", | |
| "difficulty": "standard", | |
| "quay": "36A", | |
| "ships": 30, | |
| "reward": 0.0, | |
| "submitted": false, | |
| "feasible": false, | |
| "cost": null, | |
| "optimal_cost": 119, | |
| "naive_cost": 2613, | |
| "turns": 2, | |
| "checks": 0, | |
| "seconds": 518.4, | |
| "input_tokens": 9342, | |
| "output_tokens": 64000, | |
| "end_reason": "no_tool_call", | |
| "errors": 0 | |
| }, | |
| { | |
| "model": "hf:zai-org/GLM-5.3:together", | |
| "task_id": "dock-36A-w15x2-busy-0", | |
| "split": "eval", | |
| "difficulty": "busy", | |
| "quay": "36A", | |
| "ships": 55, | |
| "reward": 0.0, | |
| "submitted": false, | |
| "feasible": false, | |
| "cost": null, | |
| "optimal_cost": 368, | |
| "naive_cost": 3372, | |
| "turns": 2, | |
| "checks": 0, | |
| "seconds": 288.1, | |
| "input_tokens": 12449, | |
| "output_tokens": 64000, | |
| "end_reason": "no_tool_call", | |
| "errors": 0 | |
| }, | |
| { | |
| "model": "hf:zai-org/GLM-5.3:together", | |
| "task_id": "dock-36A-w15x2-storm-0", | |
| "split": "eval", | |
| "difficulty": "storm", | |
| "quay": "36A", | |
| "ships": 56, | |
| "reward": 0.0, | |
| "submitted": false, | |
| "feasible": false, | |
| "cost": null, | |
| "optimal_cost": 369, | |
| "naive_cost": 6062, | |
| "turns": 2, | |
| "checks": 0, | |
| "seconds": 350.3, | |
| "input_tokens": 12524, | |
| "output_tokens": 64000, | |
| "end_reason": "no_tool_call", | |
| "errors": 0 | |
| }, | |
| { | |
| "model": "hf:zai-org/GLM-5.3:together", | |
| "task_id": "dock-36A-w16x1-standard-0", | |
| "split": "eval", | |
| "difficulty": "standard", | |
| "quay": "36A", | |
| "ships": 27, | |
| "reward": 0.0, | |
| "submitted": false, | |
| "feasible": false, | |
| "cost": null, | |
| "optimal_cost": 223, | |
| "naive_cost": 1083, | |
| "turns": 2, | |
| "checks": 0, | |
| "seconds": 515.1, | |
| "input_tokens": 9206, | |
| "output_tokens": 64000, | |
| "end_reason": "no_tool_call", | |
| "errors": 0 | |
| }, | |
| { | |
| "model": "hf:zai-org/GLM-5.3:together", | |
| "task_id": "dock-36A-w16x2-busy-0", | |
| "split": "eval", | |
| "difficulty": "busy", | |
| "quay": "36A", | |
| "ships": 55, | |
| "reward": 0.0, | |
| "submitted": false, | |
| "feasible": false, | |
| "cost": null, | |
| "optimal_cost": 403, | |
| "naive_cost": 2046, | |
| "turns": 3, | |
| "checks": 0, | |
| "seconds": 423.8, | |
| "input_tokens": 23053, | |
| "output_tokens": 64019, | |
| "end_reason": "no_tool_call", | |
| "errors": 0 | |
| }, | |
| { | |
| "model": "hf:zai-org/GLM-5.3:together", | |
| "task_id": "dock-36A-w16x2-storm-0", | |
| "split": "eval", | |
| "difficulty": "storm", | |
| "quay": "36A", | |
| "ships": 57, | |
| "reward": 0.0, | |
| "submitted": false, | |
| "feasible": false, | |
| "cost": null, | |
| "optimal_cost": 108, | |
| "naive_cost": 6938, | |
| "turns": 2, | |
| "checks": 0, | |
| "seconds": 644.9, | |
| "input_tokens": 12362, | |
| "output_tokens": 64000, | |
| "end_reason": "no_tool_call", | |
| "errors": 0 | |
| }, | |
| { | |
| "model": "hf:zai-org/GLM-5.3:together", | |
| "task_id": "dock-36A-w16x2-storm-1", | |
| "split": "eval", | |
| "difficulty": "storm", | |
| "quay": "36A", | |
| "ships": 55, | |
| "reward": 0.0, | |
| "submitted": false, | |
| "feasible": false, | |
| "cost": null, | |
| "optimal_cost": 627, | |
| "naive_cost": 6819, | |
| "turns": 2, | |
| "checks": 0, | |
| "seconds": 943.3, | |
| "input_tokens": 12504, | |
| "output_tokens": 64000, | |
| "end_reason": "no_tool_call", | |
| "errors": 0 | |
| }, | |
| { | |
| "model": "hf:zai-org/GLM-5.3:together", | |
| "task_id": "dock-36A-w17x1-busy-0", | |
| "split": "eval", | |
| "difficulty": "busy", | |
| "quay": "36A", | |
| "ships": 28, | |
| "reward": 0.0, | |
| "submitted": false, | |
| "feasible": false, | |
| "cost": null, | |
| "optimal_cost": 152, | |
| "naive_cost": 1521, | |
| "turns": 2, | |
| "checks": 0, | |
| "seconds": 414.2, | |
| "input_tokens": 9465, | |
| "output_tokens": 64000, | |
| "end_reason": "no_tool_call", | |
| "errors": 0 | |
| }, | |
| { | |
| "model": "hf:zai-org/GLM-5.3:together", | |
| "task_id": "dock-36A-w17x1-standard-0", | |
| "split": "eval", | |
| "difficulty": "standard", | |
| "quay": "36A", | |
| "ships": 29, | |
| "reward": 0.0, | |
| "submitted": false, | |
| "feasible": false, | |
| "cost": null, | |
| "optimal_cost": 100, | |
| "naive_cost": 1794, | |
| "turns": 2, | |
| "checks": 0, | |
| "seconds": 488.6, | |
| "input_tokens": 9490, | |
| "output_tokens": 64000, | |
| "end_reason": "no_tool_call", | |
| "errors": 0 | |
| }, | |
| { | |
| "model": "hf:zai-org/GLM-5.3:together", | |
| "task_id": "dock-36A-w35x1-standard-0", | |
| "split": "eval", | |
| "difficulty": "standard", | |
| "quay": "36A", | |
| "ships": 19, | |
| "reward": 1.0, | |
| "submitted": true, | |
| "feasible": true, | |
| "cost": 10, | |
| "optimal_cost": 10, | |
| "naive_cost": 274, | |
| "turns": 3, | |
| "checks": 1, | |
| "seconds": 390.9, | |
| "input_tokens": 16016, | |
| "output_tokens": 39790, | |
| "end_reason": "submitted", | |
| "errors": 0 | |
| }, | |
| { | |
| "model": "hf:zai-org/GLM-5.3:together", | |
| "task_id": "dock-36A-w35x2-extreme-0", | |
| "split": "eval", | |
| "difficulty": "extreme", | |
| "quay": "36A", | |
| "ships": 39, | |
| "reward": 0.0, | |
| "submitted": false, | |
| "feasible": false, | |
| "cost": null, | |
| "optimal_cost": 754, | |
| "naive_cost": 3430, | |
| "turns": 2, | |
| "checks": 0, | |
| "seconds": 385.4, | |
| "input_tokens": 11782, | |
| "output_tokens": 64000, | |
| "end_reason": "no_tool_call", | |
| "errors": 0 | |
| }, | |
| { | |
| "model": "hf:zai-org/GLM-5.3:together", | |
| "task_id": "dock-36A-w35x2-extreme-1", | |
| "split": "eval", | |
| "difficulty": "extreme", | |
| "quay": "36A", | |
| "ships": 35, | |
| "reward": 0.0, | |
| "submitted": false, | |
| "feasible": false, | |
| "cost": null, | |
| "optimal_cost": 1315, | |
| "naive_cost": 3041, | |
| "turns": 2, | |
| "checks": 0, | |
| "seconds": 393.3, | |
| "input_tokens": 11579, | |
| "output_tokens": 64000, | |
| "end_reason": "no_tool_call", | |
| "errors": 0 | |
| }, | |
| { | |
| "model": "hf:zai-org/GLM-5.3:together", | |
| "task_id": "dock-36A-w35x2-storm-0", | |
| "split": "eval", | |
| "difficulty": "storm", | |
| "quay": "36A", | |
| "ships": 33, | |
| "reward": 0.0, | |
| "submitted": false, | |
| "feasible": false, | |
| "cost": null, | |
| "optimal_cost": 102, | |
| "naive_cost": 670, | |
| "turns": 2, | |
| "checks": 0, | |
| "seconds": 939.4, | |
| "input_tokens": 10202, | |
| "output_tokens": 64000, | |
| "end_reason": "no_tool_call", | |
| "errors": 0 | |
| }, | |
| { | |
| "model": "hf:zai-org/GLM-5.3:together", | |
| "task_id": "dock-36A-w35x3-extreme-0", | |
| "split": "eval", | |
| "difficulty": "extreme", | |
| "quay": "36A", | |
| "ships": 59, | |
| "reward": 0.0, | |
| "submitted": false, | |
| "feasible": false, | |
| "cost": null, | |
| "optimal_cost": 953, | |
| "naive_cost": 5403, | |
| "turns": 2, | |
| "checks": 0, | |
| "seconds": 369.5, | |
| "input_tokens": 13724, | |
| "output_tokens": 64000, | |
| "end_reason": "no_tool_call", | |
| "errors": 0 | |
| }, | |
| { | |
| "model": "hf:zai-org/GLM-5.3:together", | |
| "task_id": "dock-36A-w36x2-busy-0", | |
| "split": "eval", | |
| "difficulty": "busy", | |
| "quay": "36A", | |
| "ships": 37, | |
| "reward": 0.0, | |
| "submitted": false, | |
| "feasible": false, | |
| "cost": null, | |
| "optimal_cost": 3143, | |
| "naive_cost": 9374, | |
| "turns": 2, | |
| "checks": 0, | |
| "seconds": 414.6, | |
| "input_tokens": 11330, | |
| "output_tokens": 64000, | |
| "end_reason": "no_tool_call", | |
| "errors": 0 | |
| }, | |
| { | |
| "model": "hf:zai-org/GLM-5.3:together", | |
| "task_id": "dock-36A-w36x2-extreme-0", | |
| "split": "eval", | |
| "difficulty": "extreme", | |
| "quay": "36A", | |
| "ships": 42, | |
| "reward": 0.0, | |
| "submitted": false, | |
| "feasible": false, | |
| "cost": null, | |
| "optimal_cost": 320, | |
| "naive_cost": 4753, | |
| "turns": 2, | |
| "checks": 0, | |
| "seconds": 383.0, | |
| "input_tokens": 12256, | |
| "output_tokens": 64000, | |
| "end_reason": "no_tool_call", | |
| "errors": 0 | |
| }, | |
| { | |
| "model": "hf:zai-org/GLM-5.3:together", | |
| "task_id": "dock-36A-w36x2-storm-0", | |
| "split": "eval", | |
| "difficulty": "storm", | |
| "quay": "36A", | |
| "ships": 37, | |
| "reward": 1.0, | |
| "submitted": true, | |
| "feasible": true, | |
| "cost": 28, | |
| "optimal_cost": 28, | |
| "naive_cost": 864, | |
| "turns": 4, | |
| "checks": 2, | |
| "seconds": 640.8, | |
| "input_tokens": 32861, | |
| "output_tokens": 43773, | |
| "end_reason": "submitted", | |
| "errors": 0 | |
| }, | |
| { | |
| "model": "hf:zai-org/GLM-5.3:together", | |
| "task_id": "dock-36A-w37x1-busy-0", | |
| "split": "eval", | |
| "difficulty": "busy", | |
| "quay": "36A", | |
| "ships": 23, | |
| "reward": 1.0, | |
| "submitted": true, | |
| "feasible": true, | |
| "cost": 36, | |
| "optimal_cost": 36, | |
| "naive_cost": 598, | |
| "turns": 2, | |
| "checks": 1, | |
| "seconds": 230.6, | |
| "input_tokens": 8965, | |
| "output_tokens": 26917, | |
| "end_reason": "submitted", | |
| "errors": 0 | |
| }, | |
| { | |
| "model": "hf:zai-org/GLM-5.3:together", | |
| "task_id": "dock-36A-w37x1-standard-0", | |
| "split": "eval", | |
| "difficulty": "standard", | |
| "quay": "36A", | |
| "ships": 22, | |
| "reward": 0.88409, | |
| "submitted": true, | |
| "feasible": true, | |
| "cost": 24, | |
| "optimal_cost": 15, | |
| "naive_cost": 429, | |
| "turns": 3, | |
| "checks": 1, | |
| "seconds": 366.7, | |
| "input_tokens": 16459, | |
| "output_tokens": 34329, | |
| "end_reason": "submitted", | |
| "errors": 0 | |
| }, | |
| { | |
| "model": "openai:gpt-6.1-sol", | |
| "task_id": "dock-24B-w06x1-busy-0", | |
| "split": "eval", | |
| "difficulty": "busy", | |
| "quay": "24B", | |
| "ships": 17, | |
| "reward": 1.0, | |
| "submitted": true, | |
| "feasible": true, | |
| "cost": 87, | |
| "optimal_cost": 87, | |
| "naive_cost": 658, | |
| "turns": 2, | |
| "checks": 1, | |
| "seconds": 108.5, | |
| "input_tokens": 7303, | |
| "output_tokens": 4154, | |
| "end_reason": "submitted", | |
| "errors": 0 | |
| }, | |
| { | |
| "model": "openai:gpt-6.1-sol", | |
| "task_id": "dock-24B-w06x1-standard-0", | |
| "split": "eval", | |
| "difficulty": "standard", | |
| "quay": "24B", | |
| "ships": 16, | |
| "reward": 1.0, | |
| "submitted": true, | |
| "feasible": true, | |
| "cost": 114, | |
| "optimal_cost": 114, | |
| "naive_cost": 311, | |
| "turns": 2, | |
| "checks": 1, | |
| "seconds": 30.6, | |
| "input_tokens": 5745, | |
| "output_tokens": 1475, | |
| "end_reason": "submitted", | |
| "errors": 0 | |
| }, | |
| { | |
| "model": "openai:gpt-6.1-sol", | |
| "task_id": "dock-24B-w06x2-busy-0", | |
| "split": "eval", | |
| "difficulty": "busy", | |
| "quay": "24B", | |
| "ships": 31, | |
| "reward": 0.940769, | |
| "submitted": true, | |
| "feasible": true, | |
| "cost": 621, | |
| "optimal_cost": 616, | |
| "naive_cost": 1181, | |
| "turns": 3, | |
| "checks": 1, | |
| "seconds": 62.8, | |
| "input_tokens": 15441, | |
| "output_tokens": 2755, | |
| "end_reason": "submitted", | |
| "errors": 0 | |
| }, | |
| { | |
| "model": "openai:gpt-6.1-sol", | |
| "task_id": "dock-24B-w06x2-extreme-0", | |
| "split": "eval", | |
| "difficulty": "extreme", | |
| "quay": "24B", | |
| "ships": 32, | |
| "reward": 1.0, | |
| "submitted": true, | |
| "feasible": true, | |
| "cost": 231, | |
| "optimal_cost": 231, | |
| "naive_cost": 1090, | |
| "turns": 3, | |
| "checks": 2, | |
| "seconds": 115.5, | |
| "input_tokens": 19452, | |
| "output_tokens": 4963, | |
| "end_reason": "submitted", | |
| "errors": 0 | |
| }, | |
| { | |
| "model": "openai:gpt-6.1-sol", | |
| "task_id": "dock-24B-w06x2-storm-0", | |
| "split": "eval", | |
| "difficulty": "storm", | |
| "quay": "24B", | |
| "ships": 32, | |
| "reward": 0.95723, | |
| "submitted": true, | |
| "feasible": true, | |
| "cost": 159, | |
| "optimal_cost": 154, | |
| "naive_cost": 1259, | |
| "turns": 3, | |
| "checks": 2, | |
| "seconds": 111.8, | |
| "input_tokens": 19255, | |
| "output_tokens": 4858, | |
| "end_reason": "submitted", | |
| "errors": 0 | |
| }, | |
| { | |
| "model": "openai:gpt-6.1-sol", | |
| "task_id": "dock-24B-w07x1-busy-0", | |
| "split": "eval", | |
| "difficulty": "busy", | |
| "quay": "24B", | |
| "ships": 17, | |
| "reward": 1.0, | |
| "submitted": true, | |
| "feasible": true, | |
| "cost": 226, | |
| "optimal_cost": 226, | |
| "naive_cost": 641, | |
| "turns": 3, | |
| "checks": 2, | |
| "seconds": 87.7, | |
| "input_tokens": 13578, | |
| "output_tokens": 3607, | |
| "end_reason": "submitted", | |
| "errors": 0 | |
| }, | |
| { | |
| "model": "openai:gpt-6.1-sol", | |
| "task_id": "dock-24B-w15x2-extreme-0", | |
| "split": "eval", | |
| "difficulty": "extreme", | |
| "quay": "24B", | |
| "ships": 24, | |
| "reward": 0.973512, | |
| "submitted": true, | |
| "feasible": true, | |
| "cost": 1014, | |
| "optimal_cost": 1009, | |
| "naive_cost": 3964, | |
| "turns": 3, | |
| "checks": 2, | |
| "seconds": 241.1, | |
| "input_tokens": 24096, | |
| "output_tokens": 9562, | |
| "end_reason": "submitted", | |
| "errors": 0 | |
| }, | |
| { | |
| "model": "openai:gpt-6.1-sol", | |
| "task_id": "dock-24B-w16x1-busy-0", | |
| "split": "eval", | |
| "difficulty": "busy", | |
| "quay": "24B", | |
| "ships": 16, | |
| "reward": 1.0, | |
| "submitted": true, | |
| "feasible": true, | |
| "cost": 1349, | |
| "optimal_cost": 1349, | |
| "naive_cost": 4069, | |
| "turns": 6, | |
| "checks": 4, | |
| "seconds": 260.6, | |
| "input_tokens": 48230, | |
| "output_tokens": 10261, | |
| "end_reason": "submitted", | |
| "errors": 0 | |
| }, | |
| { | |
| "model": "openai:gpt-6.1-sol", | |
| "task_id": "dock-24B-w16x2-extreme-0", | |
| "split": "eval", | |
| "difficulty": "extreme", | |
| "quay": "24B", | |
| "ships": 28, | |
| "reward": 1.0, | |
| "submitted": true, | |
| "feasible": true, | |
| "cost": 466, | |
| "optimal_cost": 466, | |
| "naive_cost": 1485, | |
| "turns": 5, | |
| "checks": 4, | |
| "seconds": 220.7, | |
| "input_tokens": 49792, | |
| "output_tokens": 9095, | |
| "end_reason": "submitted", | |
| "errors": 0 | |
| }, | |
| { | |
| "model": "openai:gpt-6.1-sol", | |
| "task_id": "dock-24B-w16x2-storm-0", | |
| "split": "eval", | |
| "difficulty": "storm", | |
| "quay": "24B", | |
| "ships": 27, | |
| "reward": 1.0, | |
| "submitted": true, | |
| "feasible": true, | |
| "cost": 201, | |
| "optimal_cost": 201, | |
| "naive_cost": 1569, | |
| "turns": 3, | |
| "checks": 2, | |
| "seconds": 127.3, | |
| "input_tokens": 18205, | |
| "output_tokens": 5316, | |
| "end_reason": "submitted", | |
| "errors": 0 | |
| }, | |
| { | |
| "model": "openai:gpt-6.1-sol", | |
| "task_id": "dock-24B-w16x2-storm-1", | |
| "split": "eval", | |
| "difficulty": "storm", | |
| "quay": "24B", | |
| "ships": 26, | |
| "reward": 0.921826, | |
| "submitted": true, | |
| "feasible": true, | |
| "cost": 1207, | |
| "optimal_cost": 1187, | |
| "naive_cost": 2403, | |
| "turns": 3, | |
| "checks": 2, | |
| "seconds": 240.0, | |
| "input_tokens": 20718, | |
| "output_tokens": 9536, | |
| "end_reason": "submitted", | |
| "errors": 0 | |
| }, | |
| { | |
| "model": "openai:gpt-6.1-sol", | |
| "task_id": "dock-24B-w35x1-busy-0", | |
| "split": "eval", | |
| "difficulty": "busy", | |
| "quay": "24B", | |
| "ships": 14, | |
| "reward": 0.941763, | |
| "submitted": true, | |
| "feasible": true, | |
| "cost": 1236, | |
| "optimal_cost": 1223, | |
| "naive_cost": 2892, | |
| "turns": 6, | |
| "checks": 5, | |
| "seconds": 259.5, | |
| "input_tokens": 51475, | |
| "output_tokens": 9979, | |
| "end_reason": "submitted", | |
| "errors": 0 | |
| }, | |
| { | |
| "model": "openai:gpt-6.1-sol", | |
| "task_id": "dock-24B-w35x2-busy-0", | |
| "split": "eval", | |
| "difficulty": "busy", | |
| "quay": "24B", | |
| "ships": 24, | |
| "reward": 0.980001, | |
| "submitted": true, | |
| "feasible": true, | |
| "cost": 165, | |
| "optimal_cost": 162, | |
| "naive_cost": 1443, | |
| "turns": 3, | |
| "checks": 2, | |
| "seconds": 151.8, | |
| "input_tokens": 18813, | |
| "output_tokens": 5804, | |
| "end_reason": "submitted", | |
| "errors": 0 | |
| }, | |
| { | |
| "model": "openai:gpt-6.1-sol", | |
| "task_id": "dock-24B-w35x2-extreme-0", | |
| "split": "eval", | |
| "difficulty": "extreme", | |
| "quay": "24B", | |
| "ships": 24, | |
| "reward": 1.0, | |
| "submitted": true, | |
| "feasible": true, | |
| "cost": 1792, | |
| "optimal_cost": 1792, | |
| "naive_cost": 4682, | |
| "turns": 2, | |
| "checks": 1, | |
| "seconds": 54.2, | |
| "input_tokens": 8627, | |
| "output_tokens": 2395, | |
| "end_reason": "submitted", | |
| "errors": 0 | |
| }, | |
| { | |
| "model": "openai:gpt-6.1-sol", | |
| "task_id": "dock-24B-w35x2-extreme-1", | |
| "split": "eval", | |
| "difficulty": "extreme", | |
| "quay": "24B", | |
| "ships": 25, | |
| "reward": 1.0, | |
| "submitted": true, | |
| "feasible": true, | |
| "cost": 283, | |
| "optimal_cost": 283, | |
| "naive_cost": 1356, | |
| "turns": 3, | |
| "checks": 2, | |
| "seconds": 93.3, | |
| "input_tokens": 16799, | |
| "output_tokens": 3882, | |
| "end_reason": "submitted", | |
| "errors": 0 | |
| }, | |
| { | |
| "model": "openai:gpt-6.1-sol", | |
| "task_id": "dock-24B-w35x2-storm-0", | |
| "split": "eval", | |
| "difficulty": "storm", | |
| "quay": "24B", | |
| "ships": 24, | |
| "reward": 0.981066, | |
| "submitted": true, | |
| "feasible": true, | |
| "cost": 2599, | |
| "optimal_cost": 2591, | |
| "naive_cost": 4835, | |
| "turns": 3, | |
| "checks": 2, | |
| "seconds": 222.4, | |
| "input_tokens": 21154, | |
| "output_tokens": 9111, | |
| "end_reason": "submitted", | |
| "errors": 0 | |
| }, | |
| { | |
| "model": "openai:gpt-6.1-sol", | |
| "task_id": "dock-24B-w35x2-storm-1", | |
| "split": "eval", | |
| "difficulty": "storm", | |
| "quay": "24B", | |
| "ships": 24, | |
| "reward": 1.0, | |
| "submitted": true, | |
| "feasible": true, | |
| "cost": 312, | |
| "optimal_cost": 312, | |
| "naive_cost": 2701, | |
| "turns": 3, | |
| "checks": 2, | |
| "seconds": 142.2, | |
| "input_tokens": 17751, | |
| "output_tokens": 5645, | |
| "end_reason": "submitted", | |
| "errors": 0 | |
| }, | |
| { | |
| "model": "openai:gpt-6.1-sol", | |
| "task_id": "dock-24B-w35x3-extreme-0", | |
| "split": "eval", | |
| "difficulty": "extreme", | |
| "quay": "24B", | |
| "ships": 39, | |
| "reward": 0.986167, | |
| "submitted": true, | |
| "feasible": true, | |
| "cost": 1109, | |
| "optimal_cost": 1106, | |
| "naive_cost": 4303, | |
| "turns": 4, | |
| "checks": 3, | |
| "seconds": 272.0, | |
| "input_tokens": 41018, | |
| "output_tokens": 10496, | |
| "end_reason": "submitted", | |
| "errors": 0 | |
| }, | |
| { | |
| "model": "openai:gpt-6.1-sol", | |
| "task_id": "dock-24B-w35x3-extreme-1", | |
| "split": "eval", | |
| "difficulty": "extreme", | |
| "quay": "24B", | |
| "ships": 38, | |
| "reward": 0.619963, | |
| "submitted": true, | |
| "feasible": true, | |
| "cost": 1010, | |
| "optimal_cost": 865, | |
| "naive_cost": 2611, | |
| "turns": 5, | |
| "checks": 4, | |
| "seconds": 311.2, | |
| "input_tokens": 64563, | |
| "output_tokens": 13117, | |
| "end_reason": "submitted", | |
| "errors": 0 | |
| }, | |
| { | |
| "model": "openai:gpt-6.1-sol", | |
| "task_id": "dock-24B-w36x2-extreme-0", | |
| "split": "eval", | |
| "difficulty": "extreme", | |
| "quay": "24B", | |
| "ships": 24, | |
| "reward": 0.989474, | |
| "submitted": true, | |
| "feasible": true, | |
| "cost": 650, | |
| "optimal_cost": 648, | |
| "naive_cost": 1736, | |
| "turns": 3, | |
| "checks": 2, | |
| "seconds": 243.1, | |
| "input_tokens": 21971, | |
| "output_tokens": 9185, | |
| "end_reason": "submitted", | |
| "errors": 0 | |
| }, | |
| { | |
| "model": "openai:gpt-6.1-sol", | |
| "task_id": "dock-24B-w36x2-storm-0", | |
| "split": "eval", | |
| "difficulty": "storm", | |
| "quay": "24B", | |
| "ships": 23, | |
| "reward": 1.0, | |
| "submitted": true, | |
| "feasible": true, | |
| "cost": 756, | |
| "optimal_cost": 756, | |
| "naive_cost": 1931, | |
| "turns": 3, | |
| "checks": 2, | |
| "seconds": 152.5, | |
| "input_tokens": 16908, | |
| "output_tokens": 6596, | |
| "end_reason": "submitted", | |
| "errors": 0 | |
| }, | |
| { | |
| "model": "openai:gpt-6.1-sol", | |
| "task_id": "dock-24B-w36x2-storm-1", | |
| "split": "eval", | |
| "difficulty": "storm", | |
| "quay": "24B", | |
| "ships": 23, | |
| "reward": 1.0, | |
| "submitted": true, | |
| "feasible": true, | |
| "cost": 244, | |
| "optimal_cost": 244, | |
| "naive_cost": 549, | |
| "turns": 2, | |
| "checks": 1, | |
| "seconds": 50.9, | |
| "input_tokens": 7471, | |
| "output_tokens": 2226, | |
| "end_reason": "submitted", | |
| "errors": 0 | |
| }, | |
| { | |
| "model": "openai:gpt-6.1-sol", | |
| "task_id": "dock-24B-w37x1-standard-0", | |
| "split": "eval", | |
| "difficulty": "standard", | |
| "quay": "24B", | |
| "ships": 15, | |
| "reward": 1.0, | |
| "submitted": true, | |
| "feasible": true, | |
| "cost": 229, | |
| "optimal_cost": 229, | |
| "naive_cost": 734, | |
| "turns": 3, | |
| "checks": 1, | |
| "seconds": 72.8, | |
| "input_tokens": 11266, | |
| "output_tokens": 2845, | |
| "end_reason": "submitted", | |
| "errors": 0 | |
| }, | |
| { | |
| "model": "openai:gpt-6.1-sol", | |
| "task_id": "dock-36A-w05x1-busy-0", | |
| "split": "eval", | |
| "difficulty": "busy", | |
| "quay": "36A", | |
| "ships": 27, | |
| "reward": 1.0, | |
| "submitted": true, | |
| "feasible": true, | |
| "cost": 404, | |
| "optimal_cost": 404, | |
| "naive_cost": 791, | |
| "turns": 3, | |
| "checks": 1, | |
| "seconds": 57.4, | |
| "input_tokens": 13815, | |
| "output_tokens": 2597, | |
| "end_reason": "submitted", | |
| "errors": 0 | |
| }, | |
| { | |
| "model": "openai:gpt-6.1-sol", | |
| "task_id": "dock-36A-w05x2-busy-0", | |
| "split": "eval", | |
| "difficulty": "busy", | |
| "quay": "36A", | |
| "ships": 51, | |
| "reward": 0.489001, | |
| "submitted": true, | |
| "feasible": true, | |
| "cost": 333, | |
| "optimal_cost": 277, | |
| "naive_cost": 1831, | |
| "turns": 3, | |
| "checks": 2, | |
| "seconds": 85.4, | |
| "input_tokens": 22560, | |
| "output_tokens": 4617, | |
| "end_reason": "submitted", | |
| "errors": 0 | |
| }, | |
| { | |
| "model": "openai:gpt-6.1-sol", | |
| "task_id": "dock-36A-w05x2-extreme-0", | |
| "split": "eval", | |
| "difficulty": "extreme", | |
| "quay": "36A", | |
| "ships": 58, | |
| "reward": 0.765127, | |
| "submitted": true, | |
| "feasible": true, | |
| "cost": 1651, | |
| "optimal_cost": 1594, | |
| "naive_cost": 4448, | |
| "turns": 11, | |
| "checks": 9, | |
| "seconds": 746.2, | |
| "input_tokens": 359835, | |
| "output_tokens": 32924, | |
| "end_reason": "submitted", | |
| "errors": 0 | |
| }, | |
| { | |
| "model": "openai:gpt-6.1-sol", | |
| "task_id": "dock-36A-w05x2-storm-0", | |
| "split": "eval", | |
| "difficulty": "storm", | |
| "quay": "36A", | |
| "ships": 59, | |
| "reward": 0.807658, | |
| "submitted": true, | |
| "feasible": true, | |
| "cost": 312, | |
| "optimal_cost": 290, | |
| "naive_cost": 4214, | |
| "turns": 7, | |
| "checks": 5, | |
| "seconds": 679.9, | |
| "input_tokens": 173233, | |
| "output_tokens": 28457, | |
| "end_reason": "submitted", | |
| "errors": 0 | |
| }, | |
| { | |
| "model": "openai:gpt-6.1-sol", | |
| "task_id": "dock-36A-w06x1-busy-0", | |
| "split": "eval", | |
| "difficulty": "busy", | |
| "quay": "36A", | |
| "ships": 25, | |
| "reward": 1.0, | |
| "submitted": true, | |
| "feasible": true, | |
| "cost": 40, | |
| "optimal_cost": 40, | |
| "naive_cost": 824, | |
| "turns": 2, | |
| "checks": 1, | |
| "seconds": 121.5, | |
| "input_tokens": 10157, | |
| "output_tokens": 5044, | |
| "end_reason": "submitted", | |
| "errors": 0 | |
| }, | |
| { | |
| "model": "openai:gpt-6.1-sol", | |
| "task_id": "dock-36A-w06x1-standard-0", | |
| "split": "eval", | |
| "difficulty": "standard", | |
| "quay": "36A", | |
| "ships": 24, | |
| "reward": 0.986087, | |
| "submitted": true, | |
| "feasible": true, | |
| "cost": 15, | |
| "optimal_cost": 14, | |
| "naive_cost": 591, | |
| "turns": 3, | |
| "checks": 1, | |
| "seconds": 55.9, | |
| "input_tokens": 12253, | |
| "output_tokens": 2318, | |
| "end_reason": "submitted", | |
| "errors": 0 | |
| }, | |
| { | |
| "model": "openai:gpt-6.1-sol", | |
| "task_id": "dock-36A-w10x1-standard-0", | |
| "split": "eval", | |
| "difficulty": "standard", | |
| "quay": "36A", | |
| "ships": 32, | |
| "reward": 1.0, | |
| "submitted": true, | |
| "feasible": true, | |
| "cost": 39, | |
| "optimal_cost": 39, | |
| "naive_cost": 606, | |
| "turns": 2, | |
| "checks": 1, | |
| "seconds": 34.7, | |
| "input_tokens": 7528, | |
| "output_tokens": 1846, | |
| "end_reason": "submitted", | |
| "errors": 0 | |
| }, | |
| { | |
| "model": "openai:gpt-6.1-sol", | |
| "task_id": "dock-36A-w15x1-busy-0", | |
| "split": "eval", | |
| "difficulty": "busy", | |
| "quay": "36A", | |
| "ships": 29, | |
| "reward": 0.513285, | |
| "submitted": true, | |
| "feasible": true, | |
| "cost": 228, | |
| "optimal_cost": 153, | |
| "naive_cost": 1963, | |
| "turns": 8, | |
| "checks": 7, | |
| "seconds": 439.3, | |
| "input_tokens": 135957, | |
| "output_tokens": 17945, | |
| "end_reason": "submitted", | |
| "errors": 0 | |
| }, | |
| { | |
| "model": "openai:gpt-6.1-sol", | |
| "task_id": "dock-36A-w15x1-standard-0", | |
| "split": "eval", | |
| "difficulty": "standard", | |
| "quay": "36A", | |
| "ships": 30, | |
| "reward": 1.0, | |
| "submitted": true, | |
| "feasible": true, | |
| "cost": 119, | |
| "optimal_cost": 119, | |
| "naive_cost": 2613, | |
| "turns": 3, | |
| "checks": 2, | |
| "seconds": 96.0, | |
| "input_tokens": 16961, | |
| "output_tokens": 4481, | |
| "end_reason": "submitted", | |
| "errors": 0 | |
| }, | |
| { | |
| "model": "openai:gpt-6.1-sol", | |
| "task_id": "dock-36A-w15x2-busy-0", | |
| "split": "eval", | |
| "difficulty": "busy", | |
| "quay": "36A", | |
| "ships": 55, | |
| "reward": 1.0, | |
| "submitted": true, | |
| "feasible": true, | |
| "cost": 368, | |
| "optimal_cost": 368, | |
| "naive_cost": 3372, | |
| "turns": 4, | |
| "checks": 2, | |
| "seconds": 172.0, | |
| "input_tokens": 40423, | |
| "output_tokens": 7799, | |
| "end_reason": "submitted", | |
| "errors": 0 | |
| }, | |
| { | |
| "model": "openai:gpt-6.1-sol", | |
| "task_id": "dock-36A-w15x2-storm-0", | |
| "split": "eval", | |
| "difficulty": "storm", | |
| "quay": "36A", | |
| "ships": 56, | |
| "reward": 0.590632, | |
| "submitted": true, | |
| "feasible": true, | |
| "cost": 469, | |
| "optimal_cost": 369, | |
| "naive_cost": 6062, | |
| "turns": 6, | |
| "checks": 5, | |
| "seconds": 489.1, | |
| "input_tokens": 119808, | |
| "output_tokens": 20738, | |
| "end_reason": "submitted", | |
| "errors": 0 | |
| }, | |
| { | |
| "model": "openai:gpt-6.1-sol", | |
| "task_id": "dock-36A-w16x1-standard-0", | |
| "split": "eval", | |
| "difficulty": "standard", | |
| "quay": "36A", | |
| "ships": 27, | |
| "reward": 1.0, | |
| "submitted": true, | |
| "feasible": true, | |
| "cost": 223, | |
| "optimal_cost": 223, | |
| "naive_cost": 1083, | |
| "turns": 4, | |
| "checks": 2, | |
| "seconds": 117.9, | |
| "input_tokens": 24726, | |
| "output_tokens": 4942, | |
| "end_reason": "submitted", | |
| "errors": 0 | |
| }, | |
| { | |
| "model": "openai:gpt-6.1-sol", | |
| "task_id": "dock-36A-w16x2-busy-0", | |
| "split": "eval", | |
| "difficulty": "busy", | |
| "quay": "36A", | |
| "ships": 55, | |
| "reward": 0.631792, | |
| "submitted": true, | |
| "feasible": true, | |
| "cost": 440, | |
| "optimal_cost": 403, | |
| "naive_cost": 2046, | |
| "turns": 5, | |
| "checks": 3, | |
| "seconds": 222.6, | |
| "input_tokens": 66227, | |
| "output_tokens": 9751, | |
| "end_reason": "submitted", | |
| "errors": 0 | |
| }, | |
| { | |
| "model": "openai:gpt-6.1-sol", | |
| "task_id": "dock-36A-w16x2-storm-0", | |
| "split": "eval", | |
| "difficulty": "storm", | |
| "quay": "36A", | |
| "ships": 57, | |
| "reward": 0.392055, | |
| "submitted": true, | |
| "feasible": true, | |
| "cost": 225, | |
| "optimal_cost": 108, | |
| "naive_cost": 6938, | |
| "turns": 4, | |
| "checks": 2, | |
| "seconds": 314.4, | |
| "input_tokens": 54613, | |
| "output_tokens": 12285, | |
| "end_reason": "submitted", | |
| "errors": 0 | |
| }, | |
| { | |
| "model": "openai:gpt-6.1-sol", | |
| "task_id": "dock-36A-w16x2-storm-1", | |
| "split": "eval", | |
| "difficulty": "storm", | |
| "quay": "36A", | |
| "ships": 55, | |
| "reward": 0.903738, | |
| "submitted": true, | |
| "feasible": true, | |
| "cost": 642, | |
| "optimal_cost": 627, | |
| "naive_cost": 6819, | |
| "turns": 4, | |
| "checks": 3, | |
| "seconds": 562.7, | |
| "input_tokens": 74500, | |
| "output_tokens": 22946, | |
| "end_reason": "submitted", | |
| "errors": 0 | |
| }, | |
| { | |
| "model": "openai:gpt-6.1-sol", | |
| "task_id": "dock-36A-w17x1-busy-0", | |
| "split": "eval", | |
| "difficulty": "busy", | |
| "quay": "36A", | |
| "ships": 28, | |
| "reward": 1.0, | |
| "submitted": true, | |
| "feasible": true, | |
| "cost": 152, | |
| "optimal_cost": 152, | |
| "naive_cost": 1521, | |
| "turns": 3, | |
| "checks": 2, | |
| "seconds": 107.6, | |
| "input_tokens": 17490, | |
| "output_tokens": 4716, | |
| "end_reason": "submitted", | |
| "errors": 0 | |
| }, | |
| { | |
| "model": "openai:gpt-6.1-sol", | |
| "task_id": "dock-36A-w17x1-standard-0", | |
| "split": "eval", | |
| "difficulty": "standard", | |
| "quay": "36A", | |
| "ships": 29, | |
| "reward": 1.0, | |
| "submitted": true, | |
| "feasible": true, | |
| "cost": 100, | |
| "optimal_cost": 100, | |
| "naive_cost": 1794, | |
| "turns": 3, | |
| "checks": 1, | |
| "seconds": 53.0, | |
| "input_tokens": 14019, | |
| "output_tokens": 2414, | |
| "end_reason": "submitted", | |
| "errors": 0 | |
| }, | |
| { | |
| "model": "openai:gpt-6.1-sol", | |
| "task_id": "dock-36A-w35x1-standard-0", | |
| "split": "eval", | |
| "difficulty": "standard", | |
| "quay": "36A", | |
| "ships": 19, | |
| "reward": 1.0, | |
| "submitted": true, | |
| "feasible": true, | |
| "cost": 10, | |
| "optimal_cost": 10, | |
| "naive_cost": 274, | |
| "turns": 3, | |
| "checks": 1, | |
| "seconds": 34.3, | |
| "input_tokens": 11664, | |
| "output_tokens": 1512, | |
| "end_reason": "submitted", | |
| "errors": 0 | |
| }, | |
| { | |
| "model": "openai:gpt-6.1-sol", | |
| "task_id": "dock-36A-w35x2-extreme-0", | |
| "split": "eval", | |
| "difficulty": "extreme", | |
| "quay": "36A", | |
| "ships": 39, | |
| "reward": 0.700308, | |
| "submitted": true, | |
| "feasible": true, | |
| "cost": 800, | |
| "optimal_cost": 754, | |
| "naive_cost": 3430, | |
| "turns": 4, | |
| "checks": 3, | |
| "seconds": 300.6, | |
| "input_tokens": 48021, | |
| "output_tokens": 12417, | |
| "end_reason": "submitted", | |
| "errors": 0 | |
| }, | |
| { | |
| "model": "openai:gpt-6.1-sol", | |
| "task_id": "dock-36A-w35x2-extreme-1", | |
| "split": "eval", | |
| "difficulty": "extreme", | |
| "quay": "36A", | |
| "ships": 35, | |
| "reward": 0.240178, | |
| "submitted": true, | |
| "feasible": true, | |
| "cost": 1487, | |
| "optimal_cost": 1315, | |
| "naive_cost": 3041, | |
| "turns": 6, | |
| "checks": 5, | |
| "seconds": 325.6, | |
| "input_tokens": 88139, | |
| "output_tokens": 14092, | |
| "end_reason": "submitted", | |
| "errors": 0 | |
| }, | |
| { | |
| "model": "openai:gpt-6.1-sol", | |
| "task_id": "dock-36A-w35x2-storm-0", | |
| "split": "eval", | |
| "difficulty": "storm", | |
| "quay": "36A", | |
| "ships": 33, | |
| "reward": 0.680833, | |
| "submitted": true, | |
| "feasible": true, | |
| "cost": 130, | |
| "optimal_cost": 102, | |
| "naive_cost": 670, | |
| "turns": 3, | |
| "checks": 2, | |
| "seconds": 66.1, | |
| "input_tokens": 17043, | |
| "output_tokens": 3197, | |
| "end_reason": "submitted", | |
| "errors": 0 | |
| }, | |
| { | |
| "model": "openai:gpt-6.1-sol", | |
| "task_id": "dock-36A-w35x3-extreme-0", | |
| "split": "eval", | |
| "difficulty": "extreme", | |
| "quay": "36A", | |
| "ships": 59, | |
| "reward": 0.758625, | |
| "submitted": true, | |
| "feasible": true, | |
| "cost": 1011, | |
| "optimal_cost": 953, | |
| "naive_cost": 5403, | |
| "turns": 5, | |
| "checks": 4, | |
| "seconds": 354.9, | |
| "input_tokens": 81273, | |
| "output_tokens": 15676, | |
| "end_reason": "submitted", | |
| "errors": 0 | |
| }, | |
| { | |
| "model": "openai:gpt-6.1-sol", | |
| "task_id": "dock-36A-w36x2-busy-0", | |
| "split": "eval", | |
| "difficulty": "busy", | |
| "quay": "36A", | |
| "ships": 37, | |
| "reward": 1.0, | |
| "submitted": true, | |
| "feasible": true, | |
| "cost": 3143, | |
| "optimal_cost": 3143, | |
| "naive_cost": 9374, | |
| "turns": 4, | |
| "checks": 3, | |
| "seconds": 230.6, | |
| "input_tokens": 37050, | |
| "output_tokens": 9508, | |
| "end_reason": "submitted", | |
| "errors": 0 | |
| }, | |
| { | |
| "model": "openai:gpt-6.1-sol", | |
| "task_id": "dock-36A-w36x2-extreme-0", | |
| "split": "eval", | |
| "difficulty": "extreme", | |
| "quay": "36A", | |
| "ships": 42, | |
| "reward": 0.64788, | |
| "submitted": true, | |
| "feasible": true, | |
| "cost": 387, | |
| "optimal_cost": 320, | |
| "naive_cost": 4753, | |
| "turns": 4, | |
| "checks": 3, | |
| "seconds": 340.1, | |
| "input_tokens": 51244, | |
| "output_tokens": 14359, | |
| "end_reason": "submitted", | |
| "errors": 0 | |
| }, | |
| { | |
| "model": "openai:gpt-6.1-sol", | |
| "task_id": "dock-36A-w36x2-storm-0", | |
| "split": "eval", | |
| "difficulty": "storm", | |
| "quay": "36A", | |
| "ships": 37, | |
| "reward": 1.0, | |
| "submitted": true, | |
| "feasible": true, | |
| "cost": 28, | |
| "optimal_cost": 28, | |
| "naive_cost": 864, | |
| "turns": 2, | |
| "checks": 1, | |
| "seconds": 59.2, | |
| "input_tokens": 9764, | |
| "output_tokens": 2837, | |
| "end_reason": "submitted", | |
| "errors": 0 | |
| }, | |
| { | |
| "model": "openai:gpt-6.1-sol", | |
| "task_id": "dock-36A-w37x1-busy-0", | |
| "split": "eval", | |
| "difficulty": "busy", | |
| "quay": "36A", | |
| "ships": 23, | |
| "reward": 1.0, | |
| "submitted": true, | |
| "feasible": true, | |
| "cost": 36, | |
| "optimal_cost": 36, | |
| "naive_cost": 598, | |
| "turns": 2, | |
| "checks": 1, | |
| "seconds": 28.9, | |
| "input_tokens": 6594, | |
| "output_tokens": 1409, | |
| "end_reason": "submitted", | |
| "errors": 0 | |
| }, | |
| { | |
| "model": "openai:gpt-6.1-sol", | |
| "task_id": "dock-36A-w37x1-standard-0", | |
| "split": "eval", | |
| "difficulty": "standard", | |
| "quay": "36A", | |
| "ships": 22, | |
| "reward": 1.0, | |
| "submitted": true, | |
| "feasible": true, | |
| "cost": 15, | |
| "optimal_cost": 15, | |
| "naive_cost": 429, | |
| "turns": 2, | |
| "checks": 1, | |
| "seconds": 51.3, | |
| "input_tokens": 7074, | |
| "output_tokens": 2276, | |
| "end_reason": "submitted", | |
| "errors": 0 | |
| } | |
| ] | |
| } |
Xet Storage Details
- Size:
- 136 kB
- Xet hash:
- 71f59f00089577828cb38cbc13ad991d9d7834bee28eb833bd5fd557f8f38775
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.