| <!DOCTYPE html> |
| <html lang="en"> |
| <head> |
| <meta charset="UTF-8"> |
| <meta name="viewport" content="width=device-width,initial-scale=1"> |
| <title>Trajectory Efficiency Stats</title> |
| <style> |
| * { box-sizing: border-box; margin: 0; padding: 0; } |
| body { font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', sans-serif; |
| background: #f8fafc; color: #1e293b; font-size: 13px; } |
| h1 { font-size: 1.2rem; font-weight: 700; } |
| h2 { font-size: 0.95rem; font-weight: 700; margin: 24px 0 6px; color: #1e293b; } |
| .page-header { padding: 14px 28px 10px; border-bottom: 1px solid #e2e8f0; |
| background: white; } |
| .subtitle { font-size: 0.75rem; color: #64748b; margin-top: 4px; } |
| .wrap { padding: 16px 28px 40px; } |
| .note { font-size: 0.72rem; color: #64748b; margin-bottom: 8px; } |
| |
| table.grid { border-collapse: collapse; font-size: 0.75rem; margin-bottom: 4px; } |
| table.grid th, table.grid td { border: 1px solid #e2e8f0; padding: 6px 12px; } |
| table.grid thead th { background: #f1f5f9; font-weight: 700; text-align: center; } |
| table.grid td.row-hdr { font-weight: 700; background: #f8fafc; |
| font-family: monospace; text-align: left; } |
| table.grid td.cell { text-align: center; vertical-align: top; min-width: 120px; } |
| .cell-main { font-size: 0.85rem; font-weight: 700; font-family: monospace; } |
| .cell-med { font-size: 0.70rem; color: #475569; font-family: monospace; } |
| .cell-iqr { font-size: 0.65rem; color: #94a3b8; font-family: monospace; } |
| .cell-n { font-size: 0.64rem; color: #94a3b8; } |
| .na { color: #cbd5e1; font-style: italic; } |
| |
| |
| .sr-hi { background: #dcfce7; } |
| .sr-mid { background: #fef9c3; } |
| .sr-lo { background: #fee2e2; } |
| </style> |
| </head> |
| <body> |
| <div class="page-header"> |
| <h1>Trajectory Efficiency Statistics</h1> |
| <div class="subtitle"> |
| Rows: model · Cols: language · Cells: median / [p25–p75] |
| </div> |
| </div> |
| <div class="wrap"><h2>Solve rate (% trajectories that ever passed all tests)</h2><table class="grid"><thead><tr><th>model</th><th>python</th><th>java</th><th>rust</th><th>ocaml</th></tr></thead><tbody><tr><td class="row-hdr">gemma</td><td class="cell sr-hi"><div class="cell-main">84%</div><div class="cell-n">84/100</div></td><td class="cell sr-hi"><div class="cell-main">82%</div><div class="cell-n">82/100</div></td><td class="cell sr-hi"><div class="cell-main">86%</div><div class="cell-n">86/100</div></td><td class="cell sr-mid"><div class="cell-main">62%</div><div class="cell-n">62/100</div></td></tr><tr><td class="row-hdr">qwen</td><td class="cell sr-hi"><div class="cell-main">82%</div><div class="cell-n">82/100</div></td><td class="cell sr-hi"><div class="cell-main">76%</div><div class="cell-n">76/100</div></td><td class="cell sr-hi"><div class="cell-main">81%</div><div class="cell-n">81/100</div></td><td class="cell sr-hi"><div class="cell-main">70%</div><div class="cell-n">70/100</div></td></tr><tr><td class="row-hdr">gpt55</td><td class="cell sr-hi"><div class="cell-main">93%</div><div class="cell-n">93/100</div></td><td class="cell sr-hi"><div class="cell-main">93%</div><div class="cell-n">93/100</div></td><td class="cell sr-hi"><div class="cell-main">92%</div><div class="cell-n">92/100</div></td><td class="cell sr-hi"><div class="cell-main">89%</div><div class="cell-n">89/100</div></td></tr><tr><td class="row-hdr">sonnet46</td><td class="cell sr-hi"><div class="cell-main">84%</div><div class="cell-n">84/100</div></td><td class="cell sr-hi"><div class="cell-main">86%</div><div class="cell-n">86/100</div></td><td class="cell sr-hi"><div class="cell-main">86%</div><div class="cell-n">86/100</div></td><td class="cell sr-hi"><div class="cell-main">86%</div><div class="cell-n">86/100</div></td></tr><tr><td class="row-hdr">all (merged)</td><td class="cell sr-hi"><div class="cell-main">123%</div><div class="cell-n">491/400</div></td><td class="cell sr-hi"><div class="cell-main">103%</div><div class="cell-n">413/400</div></td><td class="cell sr-hi"><div class="cell-main">105%</div><div class="cell-n">421/400</div></td><td class="cell sr-hi"><div class="cell-main">112%</div><div class="cell-n">447/400</div></td></tr></tbody></table><h2>Assistant turns to first correct solution (solved only)</h2><table class="grid"><thead><tr><th>model</th><th>python</th><th>java</th><th>rust</th><th>ocaml</th></tr></thead><tbody><tr><td class="row-hdr">gemma</td><td class="cell"><div class="cell-main">1</div><div class="cell-iqr">[1–1]</div><div class="cell-n">(84/100 solved)</div></td><td class="cell"><div class="cell-main">1</div><div class="cell-iqr">[1–1]</div><div class="cell-n">(82/100 solved)</div></td><td class="cell"><div class="cell-main">1</div><div class="cell-iqr">[1–1]</div><div class="cell-n">(86/100 solved)</div></td><td class="cell"><div class="cell-main">3</div><div class="cell-iqr">[1–7]</div><div class="cell-n">(62/100 solved)</div></td></tr><tr><td class="row-hdr">qwen</td><td class="cell"><div class="cell-main">1</div><div class="cell-iqr">[1–3]</div><div class="cell-n">(82/100 solved)</div></td><td class="cell"><div class="cell-main">1</div><div class="cell-iqr">[1–4]</div><div class="cell-n">(76/100 solved)</div></td><td class="cell"><div class="cell-main">3</div><div class="cell-iqr">[1–5]</div><div class="cell-n">(81/100 solved)</div></td><td class="cell"><div class="cell-main">5</div><div class="cell-iqr">[1–12]</div><div class="cell-n">(70/100 solved)</div></td></tr><tr><td class="row-hdr">gpt55</td><td class="cell"><div class="cell-main">2</div><div class="cell-iqr">[2–2]</div><div class="cell-n">(93/100 solved)</div></td><td class="cell"><div class="cell-main">2</div><div class="cell-iqr">[2–2]</div><div class="cell-n">(93/100 solved)</div></td><td class="cell"><div class="cell-main">3</div><div class="cell-iqr">[2–3]</div><div class="cell-n">(92/100 solved)</div></td><td class="cell"><div class="cell-main">2</div><div class="cell-iqr">[2–3]</div><div class="cell-n">(89/100 solved)</div></td></tr><tr><td class="row-hdr">sonnet46</td><td class="cell"><div class="cell-main">2</div><div class="cell-iqr">[1–3]</div><div class="cell-n">(84/100 solved)</div></td><td class="cell"><div class="cell-main">1</div><div class="cell-iqr">[1–2]</div><div class="cell-n">(86/100 solved)</div></td><td class="cell"><div class="cell-main">1</div><div class="cell-iqr">[1–3]</div><div class="cell-n">(86/100 solved)</div></td><td class="cell"><div class="cell-main">2</div><div class="cell-iqr">[1–3]</div><div class="cell-n">(86/100 solved)</div></td></tr><tr><td class="row-hdr">all (merged)</td><td class="cell"><div class="cell-main">1</div><div class="cell-iqr">[1–2]</div><div class="cell-n">(491/400 solved)</div></td><td class="cell"><div class="cell-main">1</div><div class="cell-iqr">[1–2]</div><div class="cell-n">(413/400 solved)</div></td><td class="cell"><div class="cell-main">2</div><div class="cell-iqr">[1–3]</div><div class="cell-n">(421/400 solved)</div></td><td class="cell"><div class="cell-main">3</div><div class="cell-iqr">[1–5]</div><div class="cell-n">(447/400 solved)</div></td></tr></tbody></table><h2>Extra turns worked after first correct solution (solved only)</h2><table class="grid"><thead><tr><th>model</th><th>python</th><th>java</th><th>rust</th><th>ocaml</th></tr></thead><tbody><tr><td class="row-hdr">gemma</td><td class="cell"><div class="cell-main">3</div><div class="cell-iqr">[2–9]</div><div class="cell-n">(84/100 solved)</div></td><td class="cell"><div class="cell-main">2</div><div class="cell-iqr">[2–4]</div><div class="cell-n">(82/100 solved)</div></td><td class="cell"><div class="cell-main">4</div><div class="cell-iqr">[2–6]</div><div class="cell-n">(86/100 solved)</div></td><td class="cell"><div class="cell-main">3</div><div class="cell-iqr">[2–11]</div><div class="cell-n">(62/100 solved)</div></td></tr><tr><td class="row-hdr">qwen</td><td class="cell"><div class="cell-main">3</div><div class="cell-iqr">[2–5]</div><div class="cell-n">(82/100 solved)</div></td><td class="cell"><div class="cell-main">3</div><div class="cell-iqr">[2–5]</div><div class="cell-n">(76/100 solved)</div></td><td class="cell"><div class="cell-main">4</div><div class="cell-iqr">[3–5]</div><div class="cell-n">(81/100 solved)</div></td><td class="cell"><div class="cell-main">4</div><div class="cell-iqr">[3–6]</div><div class="cell-n">(70/100 solved)</div></td></tr><tr><td class="row-hdr">gpt55</td><td class="cell"><div class="cell-main">1</div><div class="cell-iqr">[1–1]</div><div class="cell-n">(93/100 solved)</div></td><td class="cell"><div class="cell-main">1</div><div class="cell-iqr">[1–1]</div><div class="cell-n">(93/100 solved)</div></td><td class="cell"><div class="cell-main">1</div><div class="cell-iqr">[1–1]</div><div class="cell-n">(92/100 solved)</div></td><td class="cell"><div class="cell-main">1</div><div class="cell-iqr">[1–1]</div><div class="cell-n">(89/100 solved)</div></td></tr><tr><td class="row-hdr">sonnet46</td><td class="cell"><div class="cell-main">1</div><div class="cell-iqr">[1–2]</div><div class="cell-n">(84/100 solved)</div></td><td class="cell"><div class="cell-main">1</div><div class="cell-iqr">[1–2]</div><div class="cell-n">(86/100 solved)</div></td><td class="cell"><div class="cell-main">2</div><div class="cell-iqr">[1–2]</div><div class="cell-n">(86/100 solved)</div></td><td class="cell"><div class="cell-main">1</div><div class="cell-iqr">[1–2]</div><div class="cell-n">(86/100 solved)</div></td></tr><tr><td class="row-hdr">all (merged)</td><td class="cell"><div class="cell-main">2</div><div class="cell-iqr">[1–3]</div><div class="cell-n">(491/400 solved)</div></td><td class="cell"><div class="cell-main">2</div><div class="cell-iqr">[1–2]</div><div class="cell-n">(413/400 solved)</div></td><td class="cell"><div class="cell-main">2</div><div class="cell-iqr">[1–4]</div><div class="cell-n">(421/400 solved)</div></td><td class="cell"><div class="cell-main">1</div><div class="cell-iqr">[1–3]</div><div class="cell-n">(447/400 solved)</div></td></tr></tbody></table><h2>Total assistant turns (solved only)</h2><table class="grid"><thead><tr><th>model</th><th>python</th><th>java</th><th>rust</th><th>ocaml</th></tr></thead><tbody><tr><td class="row-hdr">gemma</td><td class="cell"><div class="cell-main">4</div><div class="cell-iqr">[3–11]</div><div class="cell-n">(84/100 solved)</div></td><td class="cell"><div class="cell-main">3</div><div class="cell-iqr">[3–5]</div><div class="cell-n">(82/100 solved)</div></td><td class="cell"><div class="cell-main">5</div><div class="cell-iqr">[3–9]</div><div class="cell-n">(86/100 solved)</div></td><td class="cell"><div class="cell-main">11</div><div class="cell-iqr">[4–21]</div><div class="cell-n">(62/100 solved)</div></td></tr><tr><td class="row-hdr">qwen</td><td class="cell"><div class="cell-main">5</div><div class="cell-iqr">[3–8]</div><div class="cell-n">(82/100 solved)</div></td><td class="cell"><div class="cell-main">6</div><div class="cell-iqr">[3–8]</div><div class="cell-n">(76/100 solved)</div></td><td class="cell"><div class="cell-main">7</div><div class="cell-iqr">[6–9]</div><div class="cell-n">(81/100 solved)</div></td><td class="cell"><div class="cell-main">11</div><div class="cell-iqr">[6–21]</div><div class="cell-n">(70/100 solved)</div></td></tr><tr><td class="row-hdr">gpt55</td><td class="cell"><div class="cell-main">3</div><div class="cell-iqr">[3–3]</div><div class="cell-n">(93/100 solved)</div></td><td class="cell"><div class="cell-main">3</div><div class="cell-iqr">[3–3]</div><div class="cell-n">(93/100 solved)</div></td><td class="cell"><div class="cell-main">4</div><div class="cell-iqr">[3–4]</div><div class="cell-n">(92/100 solved)</div></td><td class="cell"><div class="cell-main">3</div><div class="cell-iqr">[3–4]</div><div class="cell-n">(89/100 solved)</div></td></tr><tr><td class="row-hdr">sonnet46</td><td class="cell"><div class="cell-main">3</div><div class="cell-iqr">[2–5]</div><div class="cell-n">(84/100 solved)</div></td><td class="cell"><div class="cell-main">3</div><div class="cell-iqr">[2–5]</div><div class="cell-n">(86/100 solved)</div></td><td class="cell"><div class="cell-main">3</div><div class="cell-iqr">[2–5]</div><div class="cell-n">(86/100 solved)</div></td><td class="cell"><div class="cell-main">3</div><div class="cell-iqr">[2–6]</div><div class="cell-n">(86/100 solved)</div></td></tr><tr><td class="row-hdr">all (merged)</td><td class="cell"><div class="cell-main">3</div><div class="cell-iqr">[3–5]</div><div class="cell-n">(491/400 solved)</div></td><td class="cell"><div class="cell-main">3</div><div class="cell-iqr">[3–5]</div><div class="cell-n">(413/400 solved)</div></td><td class="cell"><div class="cell-main">4</div><div class="cell-iqr">[3–6]</div><div class="cell-n">(421/400 solved)</div></td><td class="cell"><div class="cell-main">4</div><div class="cell-iqr">[3–8]</div><div class="cell-n">(447/400 solved)</div></td></tr></tbody></table><h2>Output tokens to first correct solution (solved only)</h2><table class="grid"><thead><tr><th>model</th><th>python</th><th>java</th><th>rust</th><th>ocaml</th></tr></thead><tbody><tr><td class="row-hdr">gemma</td><td class="cell"><div class="cell-main">1966</div><div class="cell-iqr">[678–4819]</div><div class="cell-n">(84/100 solved)</div></td><td class="cell"><div class="cell-main">2498</div><div class="cell-iqr">[796–4667]</div><div class="cell-n">(82/100 solved)</div></td><td class="cell"><div class="cell-main">2026</div><div class="cell-iqr">[925–5202]</div><div class="cell-n">(86/100 solved)</div></td><td class="cell"><div class="cell-main">2495</div><div class="cell-iqr">[954–8064]</div><div class="cell-n">(62/100 solved)</div></td></tr><tr><td class="row-hdr">qwen</td><td class="cell"><div class="cell-main">1756</div><div class="cell-iqr">[582–4864]</div><div class="cell-n">(82/100 solved)</div></td><td class="cell"><div class="cell-main">2138</div><div class="cell-iqr">[920–5971]</div><div class="cell-n">(76/100 solved)</div></td><td class="cell"><div class="cell-main">2409</div><div class="cell-iqr">[712–7023]</div><div class="cell-n">(81/100 solved)</div></td><td class="cell"><div class="cell-main">3170</div><div class="cell-iqr">[893–10563]</div><div class="cell-n">(70/100 solved)</div></td></tr><tr><td class="row-hdr">gpt55</td><td class="cell"><div class="cell-main">606</div><div class="cell-iqr">[259–1676]</div><div class="cell-n">(93/100 solved)</div></td><td class="cell"><div class="cell-main">841</div><div class="cell-iqr">[426–2327]</div><div class="cell-n">(93/100 solved)</div></td><td class="cell"><div class="cell-main">782</div><div class="cell-iqr">[429–1981]</div><div class="cell-n">(92/100 solved)</div></td><td class="cell"><div class="cell-main">910</div><div class="cell-iqr">[336–2108]</div><div class="cell-n">(89/100 solved)</div></td></tr><tr><td class="row-hdr">sonnet46</td><td class="cell"><div class="cell-main">878</div><div class="cell-iqr">[151–4111]</div><div class="cell-n">(84/100 solved)</div></td><td class="cell"><div class="cell-main">1149</div><div class="cell-iqr">[249–4785]</div><div class="cell-n">(86/100 solved)</div></td><td class="cell"><div class="cell-main">1134</div><div class="cell-iqr">[258–4147]</div><div class="cell-n">(86/100 solved)</div></td><td class="cell"><div class="cell-main">1484</div><div class="cell-iqr">[232–4900]</div><div class="cell-n">(86/100 solved)</div></td></tr><tr><td class="row-hdr">all (merged)</td><td class="cell"><div class="cell-main">1109</div><div class="cell-iqr">[309–3891]</div><div class="cell-n">(491/400 solved)</div></td><td class="cell"><div class="cell-main">1314</div><div class="cell-iqr">[451–4131]</div><div class="cell-n">(413/400 solved)</div></td><td class="cell"><div class="cell-main">1334</div><div class="cell-iqr">[478–4736]</div><div class="cell-n">(421/400 solved)</div></td><td class="cell"><div class="cell-main">1397</div><div class="cell-iqr">[430–4915]</div><div class="cell-n">(447/400 solved)</div></td></tr></tbody></table><h2>Extra output tokens after first correct solution (solved only)</h2><table class="grid"><thead><tr><th>model</th><th>python</th><th>java</th><th>rust</th><th>ocaml</th></tr></thead><tbody><tr><td class="row-hdr">gemma</td><td class="cell"><div class="cell-main">3628</div><div class="cell-iqr">[657–31732]</div><div class="cell-n">(84/100 solved)</div></td><td class="cell"><div class="cell-main">4835</div><div class="cell-iqr">[449–14247]</div><div class="cell-n">(82/100 solved)</div></td><td class="cell"><div class="cell-main">7486</div><div class="cell-iqr">[616–22915]</div><div class="cell-n">(86/100 solved)</div></td><td class="cell"><div class="cell-main">4338</div><div class="cell-iqr">[82–18909]</div><div class="cell-n">(62/100 solved)</div></td></tr><tr><td class="row-hdr">qwen</td><td class="cell"><div class="cell-main">294</div><div class="cell-iqr">[158–632]</div><div class="cell-n">(82/100 solved)</div></td><td class="cell"><div class="cell-main">286</div><div class="cell-iqr">[152–550]</div><div class="cell-n">(76/100 solved)</div></td><td class="cell"><div class="cell-main">376</div><div class="cell-iqr">[248–878]</div><div class="cell-n">(81/100 solved)</div></td><td class="cell"><div class="cell-main">402</div><div class="cell-iqr">[225–833]</div><div class="cell-n">(70/100 solved)</div></td></tr><tr><td class="row-hdr">gpt55</td><td class="cell"><div class="cell-main">42</div><div class="cell-iqr">[37–45]</div><div class="cell-n">(93/100 solved)</div></td><td class="cell"><div class="cell-main">42</div><div class="cell-iqr">[38–46]</div><div class="cell-n">(93/100 solved)</div></td><td class="cell"><div class="cell-main">44</div><div class="cell-iqr">[40–46]</div><div class="cell-n">(92/100 solved)</div></td><td class="cell"><div class="cell-main">44</div><div class="cell-iqr">[40–47]</div><div class="cell-n">(89/100 solved)</div></td></tr><tr><td class="row-hdr">sonnet46</td><td class="cell"><div class="cell-main">114</div><div class="cell-iqr">[83–151]</div><div class="cell-n">(84/100 solved)</div></td><td class="cell"><div class="cell-main">96</div><div class="cell-iqr">[83–137]</div><div class="cell-n">(86/100 solved)</div></td><td class="cell"><div class="cell-main">136</div><div class="cell-iqr">[83–137]</div><div class="cell-n">(86/100 solved)</div></td><td class="cell"><div class="cell-main">136</div><div class="cell-iqr">[83–187]</div><div class="cell-n">(86/100 solved)</div></td></tr><tr><td class="row-hdr">all (merged)</td><td class="cell"><div class="cell-main">107</div><div class="cell-iqr">[43–348]</div><div class="cell-n">(491/400 solved)</div></td><td class="cell"><div class="cell-main">114</div><div class="cell-iqr">[59–363]</div><div class="cell-n">(413/400 solved)</div></td><td class="cell"><div class="cell-main">137</div><div class="cell-iqr">[81–468]</div><div class="cell-n">(421/400 solved)</div></td><td class="cell"><div class="cell-main">83</div><div class="cell-iqr">[42–275]</div><div class="cell-n">(447/400 solved)</div></td></tr></tbody></table><h2>Total output tokens (solved only)</h2><table class="grid"><thead><tr><th>model</th><th>python</th><th>java</th><th>rust</th><th>ocaml</th></tr></thead><tbody><tr><td class="row-hdr">gemma</td><td class="cell"><div class="cell-main">7187</div><div class="cell-iqr">[1601–40723]</div><div class="cell-n">(84/100 solved)</div></td><td class="cell"><div class="cell-main">7935</div><div class="cell-iqr">[1288–22195]</div><div class="cell-n">(82/100 solved)</div></td><td class="cell"><div class="cell-main">11540</div><div class="cell-iqr">[1847–34840]</div><div class="cell-n">(86/100 solved)</div></td><td class="cell"><div class="cell-main">12709</div><div class="cell-iqr">[1754–29416]</div><div class="cell-n">(62/100 solved)</div></td></tr><tr><td class="row-hdr">qwen</td><td class="cell"><div class="cell-main">2172</div><div class="cell-iqr">[924–7393]</div><div class="cell-n">(82/100 solved)</div></td><td class="cell"><div class="cell-main">2514</div><div class="cell-iqr">[1176–7029]</div><div class="cell-n">(76/100 solved)</div></td><td class="cell"><div class="cell-main">3083</div><div class="cell-iqr">[1301–7676]</div><div class="cell-n">(81/100 solved)</div></td><td class="cell"><div class="cell-main">3664</div><div class="cell-iqr">[1247–12449]</div><div class="cell-n">(70/100 solved)</div></td></tr><tr><td class="row-hdr">gpt55</td><td class="cell"><div class="cell-main">681</div><div class="cell-iqr">[300–1729]</div><div class="cell-n">(93/100 solved)</div></td><td class="cell"><div class="cell-main">912</div><div class="cell-iqr">[467–2366]</div><div class="cell-n">(93/100 solved)</div></td><td class="cell"><div class="cell-main">858</div><div class="cell-iqr">[474–2026]</div><div class="cell-n">(92/100 solved)</div></td><td class="cell"><div class="cell-main">978</div><div class="cell-iqr">[379–2163]</div><div class="cell-n">(89/100 solved)</div></td></tr><tr><td class="row-hdr">sonnet46</td><td class="cell"><div class="cell-main">1042</div><div class="cell-iqr">[261–4238]</div><div class="cell-n">(84/100 solved)</div></td><td class="cell"><div class="cell-main">1407</div><div class="cell-iqr">[339–5081]</div><div class="cell-n">(86/100 solved)</div></td><td class="cell"><div class="cell-main">1324</div><div class="cell-iqr">[359–4286]</div><div class="cell-n">(86/100 solved)</div></td><td class="cell"><div class="cell-main">1863</div><div class="cell-iqr">[335–5341]</div><div class="cell-n">(86/100 solved)</div></td></tr><tr><td class="row-hdr">all (merged)</td><td class="cell"><div class="cell-main">1399</div><div class="cell-iqr">[399–5204]</div><div class="cell-n">(491/400 solved)</div></td><td class="cell"><div class="cell-main">1638</div><div class="cell-iqr">[594–6724]</div><div class="cell-n">(413/400 solved)</div></td><td class="cell"><div class="cell-main">1909</div><div class="cell-iqr">[652–7472]</div><div class="cell-n">(421/400 solved)</div></td><td class="cell"><div class="cell-main">1736</div><div class="cell-iqr">[530–6404]</div><div class="cell-n">(447/400 solved)</div></td></tr></tbody></table></div> |
| </body> |
| </html> |
