| { |
| "replay_floor_s_per_turn": 0.05, |
| "latency": { |
| "tsmc": [ |
| { |
| "arm": "our 4B stack, vLLM on an H200", |
| "run": "spd4-4b-h200--tsmc_core_agentic", |
| "protocol": "single", |
| "n": 15, |
| "worked": 1.0, |
| "p50_worked_s": null, |
| "p50_common_s": 3.2, |
| "turns_common": 7.0, |
| "p25_s": 3.6, |
| "p50_s": 5.6, |
| "p75_s": 8.0, |
| "turns_per_episode": 12.8, |
| "s_per_turn": 0.44, |
| "s_per_question_amortised": null, |
| "protocol_source": "summary.json" |
| }, |
| { |
| "arm": "our 4B stack, vLLM on an L40S", |
| "run": "spd4-4b-l40s--tsmc_core_agentic", |
| "protocol": "single", |
| "n": 15, |
| "worked": 1.0, |
| "p50_worked_s": null, |
| "p50_common_s": 10.2, |
| "turns_common": 7.0, |
| "p25_s": 11.3, |
| "p50_s": 18.2, |
| "p75_s": 22.2, |
| "turns_per_episode": 13.7, |
| "s_per_turn": 1.38, |
| "s_per_question_amortised": null, |
| "protocol_source": "summary.json" |
| }, |
| { |
| "arm": "our 4B stack, MLX on a MacBook M4 Pro", |
| "run": "spd4-4b-mac-mlx--tsmc_core_agentic", |
| "protocol": "single", |
| "n": 15, |
| "worked": 1.0, |
| "p50_worked_s": null, |
| "p50_common_s": 30.0, |
| "turns_common": 7.0, |
| "p25_s": 30.4, |
| "p50_s": 50.4, |
| "p75_s": 79.8, |
| "turns_per_episode": 13.6, |
| "s_per_turn": 4.22, |
| "s_per_question_amortised": null, |
| "protocol_source": "summary.json" |
| }, |
| { |
| "arm": "gpt-5-mini + our tools", |
| "run": "spd5-gpt5mini-native--tsmc_core_agentic", |
| "protocol": "single", |
| "n": 15, |
| "worked": 1.0, |
| "p50_worked_s": null, |
| "p50_common_s": 36.3, |
| "turns_common": 16.5, |
| "p25_s": 31.3, |
| "p50_s": 39.5, |
| "p75_s": 53.4, |
| "turns_per_episode": 17.3, |
| "s_per_turn": 2.29, |
| "s_per_question_amortised": null, |
| "protocol_source": "summary.json" |
| }, |
| { |
| "arm": "gpt-5.6-sol + our tools", |
| "run": "spd5-gpt56sol-native--tsmc_core_agentic", |
| "protocol": "single", |
| "n": 15, |
| "worked": 1.0, |
| "p50_worked_s": null, |
| "p50_common_s": 29.0, |
| "turns_common": 12.5, |
| "p25_s": 24.4, |
| "p50_s": 31.3, |
| "p75_s": 35.9, |
| "turns_per_episode": 14.1, |
| "s_per_turn": 2.28, |
| "s_per_question_amortised": null, |
| "protocol_source": "summary.json" |
| }, |
| { |
| "arm": "gpt-6-astra + our tools", |
| "run": "spd5-gpt6astra-label--tsmc_core_agentic", |
| "protocol": "single", |
| "n": 15, |
| "worked": 1.0, |
| "p50_worked_s": null, |
| "p50_common_s": 14.1, |
| "turns_common": 6.0, |
| "p25_s": 10.1, |
| "p50_s": 14.7, |
| "p75_s": 21.4, |
| "turns_per_episode": 7.7, |
| "s_per_turn": 2.32, |
| "s_per_question_amortised": null, |
| "protocol_source": "summary.json" |
| }, |
| { |
| "arm": "gemini-3.7-flash + our tools", |
| "run": "spd5-gemini37flash-label--tsmc_core_agentic", |
| "protocol": "single", |
| "n": 15, |
| "worked": 1.0, |
| "p50_worked_s": null, |
| "p50_common_s": 23.7, |
| "turns_common": 9.5, |
| "p25_s": 25.6, |
| "p50_s": 29.8, |
| "p75_s": 37.9, |
| "turns_per_episode": 14.7, |
| "s_per_turn": 2.18, |
| "s_per_question_amortised": null, |
| "protocol_source": "summary.json" |
| }, |
| { |
| "arm": "gemini-3.8-flash + our tools", |
| "run": "gemini38flash-agent-t20-label--tsmc_core_agentic", |
| "protocol": "single", |
| "n": 15, |
| "worked": 1.0, |
| "p50_worked_s": null, |
| "p50_common_s": 21.2, |
| "turns_common": 9.0, |
| "p25_s": 34.1, |
| "p50_s": 36.9, |
| "p75_s": 38.4, |
| "turns_per_episode": 16.6, |
| "s_per_turn": 1.92, |
| "s_per_question_amortised": null, |
| "protocol_source": "summary.json" |
| }, |
| { |
| "arm": "claude-opus-5.5 + our tools", |
| "run": "opus55-agent-t20-3tool--tsmc_core_agentic", |
| "protocol": "single", |
| "n": 15, |
| "worked": 1.0, |
| "p50_worked_s": null, |
| "p50_common_s": 26.2, |
| "turns_common": 6.0, |
| "p25_s": 21.8, |
| "p50_s": 38.3, |
| "p75_s": 56.0, |
| "turns_per_episode": 7.7, |
| "s_per_turn": 4.78, |
| "s_per_question_amortised": null, |
| "protocol_source": "summary.json" |
| }, |
| { |
| "arm": "deepseek-v4-pro + our tools", |
| "run": "dsv4pro-agent-t20-3tool--tsmc_core_agentic", |
| "protocol": "single", |
| "n": 15, |
| "worked": 1.0, |
| "p50_worked_s": null, |
| "p50_common_s": 49.2, |
| "turns_common": 12.5, |
| "p25_s": 66.1, |
| "p50_s": 78.0, |
| "p75_s": 87.8, |
| "turns_per_episode": 18.2, |
| "s_per_turn": 4.06, |
| "s_per_question_amortised": null, |
| "protocol_source": "summary.json" |
| }, |
| { |
| "arm": "qwen3.5-397b-a17b + our tools", |
| "run": "qwen397-agent-t20-3tool--tsmc_core_agentic", |
| "protocol": "single", |
| "n": 15, |
| "worked": 1.0, |
| "p50_worked_s": null, |
| "p50_common_s": 45.8, |
| "turns_common": 12.5, |
| "p25_s": 47.7, |
| "p50_s": 57.1, |
| "p75_s": 66.3, |
| "turns_per_episode": 18.1, |
| "s_per_turn": 2.92, |
| "s_per_question_amortised": null, |
| "protocol_source": "summary.json" |
| }, |
| { |
| "arm": "gemini-3.7-flash, closed book", |
| "run": "spd2-gemini-cb--tsmc_core__closed_book", |
| "protocol": "single", |
| "n": 15, |
| "worked": 1.0, |
| "p50_worked_s": null, |
| "p50_common_s": null, |
| "turns_common": null, |
| "p25_s": 5.9, |
| "p50_s": 7.5, |
| "p75_s": 10.3, |
| "turns_per_episode": 1.0, |
| "s_per_turn": 7.54, |
| "s_per_question_amortised": null, |
| "protocol_source": "summary.json" |
| }, |
| { |
| "arm": "gpt-5.6-sol, closed book", |
| "run": "spd2-gpt56sol-cb--tsmc_core__closed_book", |
| "protocol": "single", |
| "n": 15, |
| "worked": 1.0, |
| "p50_worked_s": null, |
| "p50_common_s": null, |
| "turns_common": null, |
| "p25_s": 5.9, |
| "p50_s": 8.6, |
| "p75_s": 21.3, |
| "turns_per_episode": 1.0, |
| "s_per_turn": 8.58, |
| "s_per_question_amortised": null, |
| "protocol_source": "summary.json" |
| }, |
| { |
| "arm": "gpt-6-astra, closed book", |
| "run": "spd2-gpt6astra-cb--tsmc_core__closed_book", |
| "protocol": "single", |
| "n": 15, |
| "worked": 1.0, |
| "p50_worked_s": null, |
| "p50_common_s": null, |
| "turns_common": null, |
| "p25_s": 5.8, |
| "p50_s": 7.8, |
| "p75_s": 11.4, |
| "turns_per_episode": 1.0, |
| "s_per_turn": 7.83, |
| "s_per_question_amortised": null, |
| "protocol_source": "summary.json" |
| }, |
| { |
| "arm": "gemini-3.8-flash, closed book", |
| "run": "spd2-gemini38-cb--tsmc_core__closed_book", |
| "protocol": "single", |
| "n": 15, |
| "worked": 1.0, |
| "p50_worked_s": null, |
| "p50_common_s": null, |
| "turns_common": null, |
| "p25_s": 5.7, |
| "p50_s": 7.9, |
| "p75_s": 14.5, |
| "turns_per_episode": 1.0, |
| "s_per_turn": 7.92, |
| "s_per_question_amortised": null, |
| "protocol_source": "summary.json" |
| }, |
| { |
| "arm": "gpt-5-mini, closed book", |
| "run": "spd5-gpt5mini-cb--tsmc_core__closed_book", |
| "protocol": "single", |
| "n": 15, |
| "worked": 1.0, |
| "p50_worked_s": null, |
| "p50_common_s": null, |
| "turns_common": null, |
| "p25_s": 5.5, |
| "p50_s": 9.0, |
| "p75_s": 12.0, |
| "turns_per_episode": 1.0, |
| "s_per_turn": 8.97, |
| "s_per_question_amortised": null, |
| "protocol_source": "summary.json" |
| }, |
| { |
| "arm": "claude-opus-5.5, closed book", |
| "run": "spd5-opus55-cb--tsmc_core__closed_book", |
| "protocol": "single", |
| "n": 15, |
| "worked": 1.0, |
| "p50_worked_s": null, |
| "p50_common_s": null, |
| "turns_common": null, |
| "p25_s": 8.5, |
| "p50_s": 9.4, |
| "p75_s": 11.9, |
| "turns_per_episode": 1.0, |
| "s_per_turn": 9.42, |
| "s_per_question_amortised": null, |
| "protocol_source": "summary.json" |
| }, |
| { |
| "arm": "qwen3.5-397b-a17b, closed book", |
| "run": "spd5-qwen397-cb--tsmc_core__closed_book", |
| "protocol": "single", |
| "n": 15, |
| "worked": 1.0, |
| "p50_worked_s": null, |
| "p50_common_s": null, |
| "turns_common": null, |
| "p25_s": 51.3, |
| "p50_s": 123.2, |
| "p75_s": 168.0, |
| "turns_per_episode": 1.0, |
| "s_per_turn": 123.19, |
| "s_per_question_amortised": null, |
| "protocol_source": "summary.json" |
| }, |
| { |
| "arm": "deepseek-v4-pro, closed book", |
| "run": "spd5-dsv4pro-cb--tsmc_core__closed_book", |
| "protocol": "single", |
| "n": 15, |
| "worked": 1.0, |
| "p50_worked_s": null, |
| "p50_common_s": null, |
| "turns_common": null, |
| "p25_s": 10.5, |
| "p50_s": 24.8, |
| "p75_s": 146.0, |
| "turns_per_episode": 1.0, |
| "s_per_turn": 24.78, |
| "s_per_question_amortised": null, |
| "protocol_source": "summary.json" |
| } |
| ] |
| }, |
| "common_correct": { |
| "tsmc": [ |
| "hard-0001", |
| "medium-0004" |
| ] |
| }, |
| "throughput": [ |
| { |
| "arm": "our policy, 4B stack, local vLLM on a GH200", |
| "run": "flash20cap1024-s0--tsmc_core_agentic", |
| "protocol": "batched", |
| "n": 195, |
| "worked": 1.0, |
| "p50_worked_s": null, |
| "p50_common_s": null, |
| "turns_common": null, |
| "p25_s": 65.9, |
| "p50_s": 119.9, |
| "p75_s": 147.3, |
| "turns_per_episode": 13.3, |
| "s_per_turn": 8.23, |
| "s_per_question_amortised": 0.89, |
| "protocol_source": "summary.json" |
| } |
| ], |
| "pool": { |
| "l40s": [ |
| { |
| "arm": "width 1 (control)", |
| "run": "poollat-l40s-w1--tsmc_core_agentic", |
| "width": 1, |
| "n_questions": 15, |
| "n_episodes": 15, |
| "p25_s": 11.7, |
| "p50_s": 18.1, |
| "p75_s": 20.5, |
| "questions_per_min": 3.32, |
| "turns_slowest_draw": 13.6, |
| "s_per_turn": 1.45 |
| }, |
| { |
| "arm": "width 8", |
| "run": "poollat-l40s-w8--tsmc_core_agentic", |
| "width": 8, |
| "n_questions": 15, |
| "n_episodes": 120, |
| "p25_s": 45.7, |
| "p50_s": 59.8, |
| "p75_s": 71.6, |
| "questions_per_min": 1.0, |
| "turns_slowest_draw": 18.4, |
| "s_per_turn": 3.51 |
| }, |
| { |
| "arm": "width 17", |
| "run": "poollat-l40s-w17--tsmc_core_agentic", |
| "width": 17, |
| "n_questions": 15, |
| "n_episodes": 255, |
| "p25_s": 66.6, |
| "p50_s": 88.2, |
| "p75_s": 97.3, |
| "questions_per_min": 0.68, |
| "turns_slowest_draw": 19.9, |
| "s_per_turn": 3.92 |
| }, |
| { |
| "arm": "width 34 (the shipped pool)", |
| "run": "poollat-l40s-w34--tsmc_core_agentic", |
| "width": 34, |
| "n_questions": 15, |
| "n_episodes": 510, |
| "p25_s": 85.7, |
| "p50_s": 109.5, |
| "p75_s": 133.6, |
| "questions_per_min": 0.55, |
| "turns_slowest_draw": 20.0, |
| "s_per_turn": 5.35 |
| } |
| ], |
| "gh200": [ |
| { |
| "arm": "greedy, shipped policy (Figure 7 protocol)", |
| "run": "flash20cap1024-s0--tsmc_core_agentic/latency", |
| "width": 1, |
| "n_questions": 15, |
| "n_episodes": 15, |
| "p25_s": 3.3, |
| "p50_s": 5.5, |
| "p75_s": 7.0, |
| "questions_per_min": 10.93, |
| "turns_slowest_draw": 11.8, |
| "s_per_turn": 0.5 |
| }, |
| { |
| "arm": "greedy (Figure 7 protocol)", |
| "run": "poollat-gh200-greedy--tsmc_core_agentic", |
| "width": 1, |
| "n_questions": 15, |
| "n_episodes": 15, |
| "p25_s": 4.7, |
| "p50_s": 5.9, |
| "p75_s": 7.0, |
| "questions_per_min": 10.17, |
| "turns_slowest_draw": 13.5, |
| "s_per_turn": 0.5 |
| }, |
| { |
| "arm": "width 1, sampled", |
| "run": "poollat-gh200-w1--tsmc_core_agentic", |
| "width": 1, |
| "n_questions": 15, |
| "n_episodes": 15, |
| "p25_s": 4.4, |
| "p50_s": 6.0, |
| "p75_s": 7.3, |
| "questions_per_min": 9.99, |
| "turns_slowest_draw": 13.5, |
| "s_per_turn": 0.56 |
| }, |
| { |
| "arm": "width 8", |
| "run": "poollat-gh200-w8--tsmc_core_agentic", |
| "width": 8, |
| "n_questions": 15, |
| "n_episodes": 120, |
| "p25_s": 13.3, |
| "p50_s": 19.8, |
| "p75_s": 23.9, |
| "questions_per_min": 3.02, |
| "turns_slowest_draw": 17.7, |
| "s_per_turn": 1.35 |
| }, |
| { |
| "arm": "width 17", |
| "run": "poollat-gh200-w17--tsmc_core_agentic", |
| "width": 17, |
| "n_questions": 15, |
| "n_episodes": 255, |
| "p25_s": 23.4, |
| "p50_s": 28.7, |
| "p75_s": 35.5, |
| "questions_per_min": 2.09, |
| "turns_slowest_draw": 18.8, |
| "s_per_turn": 1.72 |
| }, |
| { |
| "arm": "width 34 (the shipped pool)", |
| "run": "poollat-gh200-w34--tsmc_core_agentic", |
| "width": 34, |
| "n_questions": 15, |
| "n_episodes": 510, |
| "p25_s": 28.1, |
| "p50_s": 41.4, |
| "p75_s": 44.3, |
| "questions_per_min": 1.45, |
| "turns_slowest_draw": 19.8, |
| "s_per_turn": 1.85 |
| } |
| ], |
| "mac (4-bit)": [ |
| { |
| "arm": "width 1", |
| "run": "poollat-q4-w1--tsmc_core_agentic", |
| "width": 1, |
| "n_questions": 15, |
| "n_episodes": 15, |
| "p25_s": 15.1, |
| "p50_s": 25.7, |
| "p75_s": 35.0, |
| "questions_per_min": 2.34, |
| "turns_slowest_draw": 15.1, |
| "s_per_turn": 1.77 |
| }, |
| { |
| "arm": "width 4", |
| "run": "poollat-q4-w4--tsmc_core_agentic", |
| "width": 4, |
| "n_questions": 15, |
| "n_episodes": 60, |
| "p25_s": 71.1, |
| "p50_s": 80.9, |
| "p75_s": 95.2, |
| "questions_per_min": 0.74, |
| "turns_slowest_draw": 18.0, |
| "s_per_turn": 4.08 |
| }, |
| { |
| "arm": "width 8, no prefix reuse", |
| "run": "poollat-q4nc-w8--tsmc_core_agentic", |
| "width": 8, |
| "n_questions": 15, |
| "n_episodes": 120, |
| "p25_s": 245.0, |
| "p50_s": 539.3, |
| "p75_s": 784.9, |
| "questions_per_min": 0.11, |
| "turns_slowest_draw": 19.1, |
| "s_per_turn": 27.01 |
| } |
| ] |
| }, |
| "cost": { |
| "gpu_usd_per_hour": { |
| "gh200": 2.29 |
| }, |
| "api_usd_per_mtok": { |
| "gemini-3.7-flash": [ |
| 0.75, |
| 3.75 |
| ], |
| "gpt-5.6-sol": [ |
| 4.0, |
| 20.0 |
| ], |
| "gpt-6-astra": [ |
| 10.0, |
| 50.0 |
| ], |
| "gemini-3.8-flash": [ |
| 0.75, |
| 3.75 |
| ], |
| "gpt-5-mini": [ |
| 0.25, |
| 2.0 |
| ], |
| "claude-opus-5.5": [ |
| 4.0, |
| 20.0 |
| ], |
| "qwen3.5-397b-a17b": [ |
| 0.55, |
| 3.5 |
| ], |
| "deepseek-v4-pro": [ |
| 0.21, |
| 0.42 |
| ] |
| }, |
| "quoted": { |
| "gemini-3.7-flash": "2026-09-16", |
| "gpt-5.6-sol": "2026-09-16", |
| "gpt-6-astra": "2026-09-16", |
| "gemini-3.8-flash": "2026-09-16", |
| "gpt-5-mini": "2026-10-03", |
| "claude-opus-5.5": "2026-10-02", |
| "qwen3.5-397b-a17b": "2026-10-02", |
| "deepseek-v4-pro": "2026-10-02" |
| }, |
| "ours": [ |
| { |
| "arm": "greedy, shipped policy (Figure 7 protocol)", |
| "card": "gh200", |
| "protocol": "single", |
| "s_per_question": 5.5, |
| "usd_per_1k": 3.5, |
| "questions_per_usd": 285.8, |
| "gated_acc": 0.839 |
| }, |
| { |
| "arm": "greedy (Figure 7 protocol)", |
| "card": "gh200", |
| "protocol": "single", |
| "s_per_question": 5.9, |
| "usd_per_1k": 3.75, |
| "questions_per_usd": 266.4, |
| "gated_acc": 0.733 |
| }, |
| { |
| "arm": "width 8", |
| "card": "gh200", |
| "protocol": "single", |
| "s_per_question": 19.8, |
| "usd_per_1k": 12.6, |
| "questions_per_usd": 79.4, |
| "gated_acc": 0.78 |
| }, |
| { |
| "arm": "width 17", |
| "card": "gh200", |
| "protocol": "single", |
| "s_per_question": 28.7, |
| "usd_per_1k": 18.26, |
| "questions_per_usd": 54.8, |
| "gated_acc": 0.79 |
| }, |
| { |
| "arm": "width 34 (the shipped pool)", |
| "card": "gh200", |
| "protocol": "single", |
| "s_per_question": 41.4, |
| "usd_per_1k": 26.34, |
| "questions_per_usd": 38.0, |
| "gated_acc": 0.79 |
| }, |
| { |
| "arm": "our policy, 4B stack, local vLLM on a GH200 (bulk)", |
| "card": "gh200", |
| "protocol": "batched", |
| "s_per_question": 0.89, |
| "usd_per_1k": 0.57, |
| "questions_per_usd": 1766.4, |
| "gated_acc": 0.839 |
| } |
| ], |
| "api": [ |
| { |
| "arm": "gpt-5-mini, closed book", |
| "model": "gpt-5-mini", |
| "prefill_tok": 68, |
| "generated_tok_floor": 99, |
| "usd_per_1k": 0.21, |
| "usd_per_1k_at_observed_output": 2.66, |
| "questions_per_usd": 4651.2, |
| "gated_acc": 0.205 |
| }, |
| { |
| "arm": "claude-opus-5.5, closed book", |
| "model": "claude-opus-5.5", |
| "prefill_tok": 68, |
| "generated_tok_floor": 266, |
| "usd_per_1k": 5.59, |
| "usd_per_1k_at_observed_output": 26.69, |
| "questions_per_usd": 178.8, |
| "gated_acc": 0.226 |
| }, |
| { |
| "arm": "qwen3.5-397b-a17b, closed book", |
| "model": "qwen3.5-397b-a17b", |
| "prefill_tok": 68, |
| "generated_tok_floor": 207, |
| "usd_per_1k": 0.76, |
| "usd_per_1k_at_observed_output": 4.66, |
| "questions_per_usd": 1312.5, |
| "gated_acc": 0.154 |
| }, |
| { |
| "arm": "deepseek-v4-pro, closed book", |
| "model": "deepseek-v4-pro", |
| "prefill_tok": 68, |
| "generated_tok_floor": 101, |
| "usd_per_1k": 0.06, |
| "usd_per_1k_at_observed_output": 0.57, |
| "questions_per_usd": 17636.7, |
| "gated_acc": 0.2 |
| }, |
| { |
| "arm": "gemini-3.7-flash, closed book", |
| "model": "gemini-3.7-flash", |
| "prefill_tok": 66, |
| "generated_tok_floor": 93, |
| "usd_per_1k": 0.4, |
| "usd_per_1k_at_observed_output": 5.0, |
| "questions_per_usd": 2511.0, |
| "gated_acc": 0.2 |
| }, |
| { |
| "arm": "gpt-5.6-sol, closed book", |
| "model": "gpt-5.6-sol", |
| "prefill_tok": 66, |
| "generated_tok_floor": 53, |
| "usd_per_1k": 1.32, |
| "usd_per_1k_at_observed_output": 26.68, |
| "questions_per_usd": 755.3, |
| "gated_acc": 0.226 |
| }, |
| { |
| "arm": "gpt-6-astra, closed book", |
| "model": "gpt-6-astra", |
| "prefill_tok": 66, |
| "generated_tok_floor": 65, |
| "usd_per_1k": 3.91, |
| "usd_per_1k_at_observed_output": 66.71, |
| "questions_per_usd": 255.8, |
| "gated_acc": 0.21 |
| }, |
| { |
| "arm": "gemini-3.8-flash, closed book", |
| "model": "gemini-3.8-flash", |
| "prefill_tok": 66, |
| "generated_tok_floor": 89, |
| "usd_per_1k": 0.38, |
| "usd_per_1k_at_observed_output": 5.0, |
| "questions_per_usd": 2609.3, |
| "gated_acc": 0.19 |
| }, |
| { |
| "arm": "gpt-5-mini + our tools", |
| "model": "gpt-5-mini", |
| "run": "gpt5mini-agent-t20-3tool-native-f16k--tsmc_core_agentic", |
| "source": "provider usage", |
| "prefill_tok": 33779, |
| "generated_tok": 2470, |
| "reasoning_tok": 1802, |
| "cached_prefill_share": 0.35, |
| "usd_per_1k": 13.39, |
| "questions_per_usd": 74.7, |
| "gated_acc": 0.549 |
| }, |
| { |
| "arm": "gpt-5.6-sol + our tools", |
| "model": "gpt-5.6-sol", |
| "run": "gpt56sol-agent-t20-3tool-native--tsmc_core_agentic", |
| "source": "provider usage", |
| "prefill_tok": 31663, |
| "generated_tok": 1045, |
| "reasoning_tok": 673, |
| "cached_prefill_share": 0.79, |
| "usd_per_1k": 147.56, |
| "questions_per_usd": 6.8, |
| "gated_acc": 0.785 |
| }, |
| { |
| "arm": "gpt-6-astra + our tools", |
| "model": "gpt-6-astra", |
| "run": "gpt6astra-agent-t20-label--tsmc_core_agentic", |
| "source": "provider usage", |
| "prefill_tok": 18750, |
| "generated_tok": 445, |
| "reasoning_tok": 86, |
| "cached_prefill_share": 0.72, |
| "usd_per_1k": 209.76, |
| "questions_per_usd": 4.8, |
| "gated_acc": 0.877 |
| }, |
| { |
| "arm": "gemini-3.7-flash + our tools", |
| "model": "gemini-3.7-flash", |
| "run": "gemini37flash-agent-t20-3tool-label--tsmc_core_agentic", |
| "source": "provider usage", |
| "prefill_tok": 28278, |
| "generated_tok": 1532, |
| "reasoning_tok": 693, |
| "cached_prefill_share": 0.01, |
| "usd_per_1k": 26.95, |
| "questions_per_usd": 37.1, |
| "gated_acc": 0.687 |
| }, |
| { |
| "arm": "gemini-3.8-flash + our tools", |
| "model": "gemini-3.8-flash", |
| "run": "gemini38flash-agent-t20-label--tsmc_core_agentic", |
| "source": "provider usage", |
| "prefill_tok": 36570, |
| "generated_tok": 2431, |
| "reasoning_tok": 1224, |
| "cached_prefill_share": 0.02, |
| "usd_per_1k": 36.54, |
| "questions_per_usd": 27.4, |
| "gated_acc": 0.641 |
| }, |
| { |
| "arm": "claude-opus-5.5 + our tools", |
| "model": "claude-opus-5.5", |
| "run": "opus55-agent-t20-3tool--tsmc_core_agentic", |
| "source": "provider usage", |
| "prefill_tok": 18413, |
| "generated_tok": 1154, |
| "reasoning_tok": 427, |
| "cached_prefill_share": 0.0, |
| "usd_per_1k": 96.73, |
| "questions_per_usd": 10.3, |
| "gated_acc": 0.846 |
| }, |
| { |
| "arm": "deepseek-v4-pro + our tools", |
| "model": "deepseek-v4-pro", |
| "run": "dsv4pro-agent-t20-3tool--tsmc_core_agentic", |
| "source": "provider usage", |
| "prefill_tok": 41651, |
| "generated_tok": 4702, |
| "reasoning_tok": 4112, |
| "cached_prefill_share": 0.61, |
| "usd_per_1k": 10.72, |
| "questions_per_usd": 93.3, |
| "gated_acc": 0.59 |
| }, |
| { |
| "arm": "qwen3.5-397b-a17b + our tools", |
| "model": "qwen3.5-397b-a17b", |
| "run": "qwen397-agent-t20-3tool--tsmc_core_agentic", |
| "source": "provider usage", |
| "prefill_tok": 44483, |
| "generated_tok": 2883, |
| "reasoning_tok": 2400, |
| "cached_prefill_share": 0.89, |
| "usd_per_1k": 34.56, |
| "questions_per_usd": 28.9, |
| "gated_acc": 0.605 |
| } |
| ] |
| }, |
| "pool_not_measured": [ |
| "poollat-h200-w1--tsmc_core_agentic", |
| "poollat-h200-w17--tsmc_core_agentic", |
| "poollat-h200-w34--tsmc_core_agentic", |
| "poollat-h200-w8--tsmc_core_agentic" |
| ] |
| } |
