{
 "replay_floor_s_per_turn": 0.05,
 "latency": {
  "tsmc": [
   {
    "arm": "our 4B stack, vLLM on an H200",
    "run": "spd4-4b-h200--tsmc_core_agentic",
    "protocol": "single",
    "n": 15,
    "worked": 1.0,
    "p50_worked_s": null,
    "p50_common_s": 3.2,
    "turns_common": 7.0,
    "p25_s": 3.6,
    "p50_s": 5.6,
    "p75_s": 8.0,
    "turns_per_episode": 12.8,
    "s_per_turn": 0.44,
    "s_per_question_amortised": null,
    "protocol_source": "summary.json"
   },
   {
    "arm": "our 4B stack, vLLM on an L40S",
    "run": "spd4-4b-l40s--tsmc_core_agentic",
    "protocol": "single",
    "n": 15,
    "worked": 1.0,
    "p50_worked_s": null,
    "p50_common_s": 10.2,
    "turns_common": 7.0,
    "p25_s": 11.3,
    "p50_s": 18.2,
    "p75_s": 22.2,
    "turns_per_episode": 13.7,
    "s_per_turn": 1.38,
    "s_per_question_amortised": null,
    "protocol_source": "summary.json"
   },
   {
    "arm": "our 4B stack, MLX on a MacBook M4 Pro",
    "run": "spd4-4b-mac-mlx--tsmc_core_agentic",
    "protocol": "single",
    "n": 15,
    "worked": 1.0,
    "p50_worked_s": null,
    "p50_common_s": 30.0,
    "turns_common": 7.0,
    "p25_s": 30.4,
    "p50_s": 50.4,
    "p75_s": 79.8,
    "turns_per_episode": 13.6,
    "s_per_turn": 4.22,
    "s_per_question_amortised": null,
    "protocol_source": "summary.json"
   },
   {
    "arm": "gpt-5-mini + our tools",
    "run": "spd5-gpt5mini-native--tsmc_core_agentic",
    "protocol": "single",
    "n": 15,
    "worked": 1.0,
    "p50_worked_s": null,
    "p50_common_s": 36.3,
    "turns_common": 16.5,
    "p25_s": 31.3,
    "p50_s": 39.5,
    "p75_s": 53.4,
    "turns_per_episode": 17.3,
    "s_per_turn": 2.29,
    "s_per_question_amortised": null,
    "protocol_source": "summary.json"
   },
   {
    "arm": "gpt-5.6-sol + our tools",
    "run": "spd5-gpt56sol-native--tsmc_core_agentic",
    "protocol": "single",
    "n": 15,
    "worked": 1.0,
    "p50_worked_s": null,
    "p50_common_s": 29.0,
    "turns_common": 12.5,
    "p25_s": 24.4,
    "p50_s": 31.3,
    "p75_s": 35.9,
    "turns_per_episode": 14.1,
    "s_per_turn": 2.28,
    "s_per_question_amortised": null,
    "protocol_source": "summary.json"
   },
   {
    "arm": "gpt-6-astra + our tools",
    "run": "spd5-gpt6astra-label--tsmc_core_agentic",
    "protocol": "single",
    "n": 15,
    "worked": 1.0,
    "p50_worked_s": null,
    "p50_common_s": 14.1,
    "turns_common": 6.0,
    "p25_s": 10.1,
    "p50_s": 14.7,
    "p75_s": 21.4,
    "turns_per_episode": 7.7,
    "s_per_turn": 2.32,
    "s_per_question_amortised": null,
    "protocol_source": "summary.json"
   },
   {
    "arm": "gemini-3.7-flash + our tools",
    "run": "spd5-gemini37flash-label--tsmc_core_agentic",
    "protocol": "single",
    "n": 15,
    "worked": 1.0,
    "p50_worked_s": null,
    "p50_common_s": 23.7,
    "turns_common": 9.5,
    "p25_s": 25.6,
    "p50_s": 29.8,
    "p75_s": 37.9,
    "turns_per_episode": 14.7,
    "s_per_turn": 2.18,
    "s_per_question_amortised": null,
    "protocol_source": "summary.json"
   },
   {
    "arm": "gemini-3.8-flash + our tools",
    "run": "gemini38flash-agent-t20-label--tsmc_core_agentic",
    "protocol": "single",
    "n": 15,
    "worked": 1.0,
    "p50_worked_s": null,
    "p50_common_s": 21.2,
    "turns_common": 9.0,
    "p25_s": 34.1,
    "p50_s": 36.9,
    "p75_s": 38.4,
    "turns_per_episode": 16.6,
    "s_per_turn": 1.92,
    "s_per_question_amortised": null,
    "protocol_source": "summary.json"
   },
   {
    "arm": "claude-opus-5.5 + our tools",
    "run": "opus55-agent-t20-3tool--tsmc_core_agentic",
    "protocol": "single",
    "n": 15,
    "worked": 1.0,
    "p50_worked_s": null,
    "p50_common_s": 26.2,
    "turns_common": 6.0,
    "p25_s": 21.8,
    "p50_s": 38.3,
    "p75_s": 56.0,
    "turns_per_episode": 7.7,
    "s_per_turn": 4.78,
    "s_per_question_amortised": null,
    "protocol_source": "summary.json"
   },
   {
    "arm": "deepseek-v4-pro + our tools",
    "run": "dsv4pro-agent-t20-3tool--tsmc_core_agentic",
    "protocol": "single",
    "n": 15,
    "worked": 1.0,
    "p50_worked_s": null,
    "p50_common_s": 49.2,
    "turns_common": 12.5,
    "p25_s": 66.1,
    "p50_s": 78.0,
    "p75_s": 87.8,
    "turns_per_episode": 18.2,
    "s_per_turn": 4.06,
    "s_per_question_amortised": null,
    "protocol_source": "summary.json"
   },
   {
    "arm": "qwen3.5-397b-a17b + our tools",
    "run": "qwen397-agent-t20-3tool--tsmc_core_agentic",
    "protocol": "single",
    "n": 15,
    "worked": 1.0,
    "p50_worked_s": null,
    "p50_common_s": 45.8,
    "turns_common": 12.5,
    "p25_s": 47.7,
    "p50_s": 57.1,
    "p75_s": 66.3,
    "turns_per_episode": 18.1,
    "s_per_turn": 2.92,
    "s_per_question_amortised": null,
    "protocol_source": "summary.json"
   },
   {
    "arm": "gemini-3.7-flash, closed book",
    "run": "spd2-gemini-cb--tsmc_core__closed_book",
    "protocol": "single",
    "n": 15,
    "worked": 1.0,
    "p50_worked_s": null,
    "p50_common_s": null,
    "turns_common": null,
    "p25_s": 5.9,
    "p50_s": 7.5,
    "p75_s": 10.3,
    "turns_per_episode": 1.0,
    "s_per_turn": 7.54,
    "s_per_question_amortised": null,
    "protocol_source": "summary.json"
   },
   {
    "arm": "gpt-5.6-sol, closed book",
    "run": "spd2-gpt56sol-cb--tsmc_core__closed_book",
    "protocol": "single",
    "n": 15,
    "worked": 1.0,
    "p50_worked_s": null,
    "p50_common_s": null,
    "turns_common": null,
    "p25_s": 5.9,
    "p50_s": 8.6,
    "p75_s": 21.3,
    "turns_per_episode": 1.0,
    "s_per_turn": 8.58,
    "s_per_question_amortised": null,
    "protocol_source": "summary.json"
   },
   {
    "arm": "gpt-6-astra, closed book",
    "run": "spd2-gpt6astra-cb--tsmc_core__closed_book",
    "protocol": "single",
    "n": 15,
    "worked": 1.0,
    "p50_worked_s": null,
    "p50_common_s": null,
    "turns_common": null,
    "p25_s": 5.8,
    "p50_s": 7.8,
    "p75_s": 11.4,
    "turns_per_episode": 1.0,
    "s_per_turn": 7.83,
    "s_per_question_amortised": null,
    "protocol_source": "summary.json"
   },
   {
    "arm": "gemini-3.8-flash, closed book",
    "run": "spd2-gemini38-cb--tsmc_core__closed_book",
    "protocol": "single",
    "n": 15,
    "worked": 1.0,
    "p50_worked_s": null,
    "p50_common_s": null,
    "turns_common": null,
    "p25_s": 5.7,
    "p50_s": 7.9,
    "p75_s": 14.5,
    "turns_per_episode": 1.0,
    "s_per_turn": 7.92,
    "s_per_question_amortised": null,
    "protocol_source": "summary.json"
   },
   {
    "arm": "gpt-5-mini, closed book",
    "run": "spd5-gpt5mini-cb--tsmc_core__closed_book",
    "protocol": "single",
    "n": 15,
    "worked": 1.0,
    "p50_worked_s": null,
    "p50_common_s": null,
    "turns_common": null,
    "p25_s": 5.5,
    "p50_s": 9.0,
    "p75_s": 12.0,
    "turns_per_episode": 1.0,
    "s_per_turn": 8.97,
    "s_per_question_amortised": null,
    "protocol_source": "summary.json"
   },
   {
    "arm": "claude-opus-5.5, closed book",
    "run": "spd5-opus55-cb--tsmc_core__closed_book",
    "protocol": "single",
    "n": 15,
    "worked": 1.0,
    "p50_worked_s": null,
    "p50_common_s": null,
    "turns_common": null,
    "p25_s": 8.5,
    "p50_s": 9.4,
    "p75_s": 11.9,
    "turns_per_episode": 1.0,
    "s_per_turn": 9.42,
    "s_per_question_amortised": null,
    "protocol_source": "summary.json"
   },
   {
    "arm": "qwen3.5-397b-a17b, closed book",
    "run": "spd5-qwen397-cb--tsmc_core__closed_book",
    "protocol": "single",
    "n": 15,
    "worked": 1.0,
    "p50_worked_s": null,
    "p50_common_s": null,
    "turns_common": null,
    "p25_s": 51.3,
    "p50_s": 123.2,
    "p75_s": 168.0,
    "turns_per_episode": 1.0,
    "s_per_turn": 123.19,
    "s_per_question_amortised": null,
    "protocol_source": "summary.json"
   },
   {
    "arm": "deepseek-v4-pro, closed book",
    "run": "spd5-dsv4pro-cb--tsmc_core__closed_book",
    "protocol": "single",
    "n": 15,
    "worked": 1.0,
    "p50_worked_s": null,
    "p50_common_s": null,
    "turns_common": null,
    "p25_s": 10.5,
    "p50_s": 24.8,
    "p75_s": 146.0,
    "turns_per_episode": 1.0,
    "s_per_turn": 24.78,
    "s_per_question_amortised": null,
    "protocol_source": "summary.json"
   }
  ]
 },
 "common_correct": {
  "tsmc": [
   "hard-0001",
   "medium-0004"
  ]
 },
 "throughput": [
  {
   "arm": "our policy, 4B stack, local vLLM on a GH200",
   "run": "flash20cap1024-s0--tsmc_core_agentic",
   "protocol": "batched",
   "n": 195,
   "worked": 1.0,
   "p50_worked_s": null,
   "p50_common_s": null,
   "turns_common": null,
   "p25_s": 65.9,
   "p50_s": 119.9,
   "p75_s": 147.3,
   "turns_per_episode": 13.3,
   "s_per_turn": 8.23,
   "s_per_question_amortised": 0.89,
   "protocol_source": "summary.json"
  }
 ],
 "pool": {
  "l40s": [
   {
    "arm": "width 1 (control)",
    "run": "poollat-l40s-w1--tsmc_core_agentic",
    "width": 1,
    "n_questions": 15,
    "n_episodes": 15,
    "p25_s": 11.7,
    "p50_s": 18.1,
    "p75_s": 20.5,
    "questions_per_min": 3.32,
    "turns_slowest_draw": 13.6,
    "s_per_turn": 1.45
   },
   {
    "arm": "width 8",
    "run": "poollat-l40s-w8--tsmc_core_agentic",
    "width": 8,
    "n_questions": 15,
    "n_episodes": 120,
    "p25_s": 45.7,
    "p50_s": 59.8,
    "p75_s": 71.6,
    "questions_per_min": 1.0,
    "turns_slowest_draw": 18.4,
    "s_per_turn": 3.51
   },
   {
    "arm": "width 17",
    "run": "poollat-l40s-w17--tsmc_core_agentic",
    "width": 17,
    "n_questions": 15,
    "n_episodes": 255,
    "p25_s": 66.6,
    "p50_s": 88.2,
    "p75_s": 97.3,
    "questions_per_min": 0.68,
    "turns_slowest_draw": 19.9,
    "s_per_turn": 3.92
   },
   {
    "arm": "width 34 (the shipped pool)",
    "run": "poollat-l40s-w34--tsmc_core_agentic",
    "width": 34,
    "n_questions": 15,
    "n_episodes": 510,
    "p25_s": 85.7,
    "p50_s": 109.5,
    "p75_s": 133.6,
    "questions_per_min": 0.55,
    "turns_slowest_draw": 20.0,
    "s_per_turn": 5.35
   }
  ],
  "gh200": [
   {
    "arm": "greedy, shipped policy (Figure 7 protocol)",
    "run": "flash20cap1024-s0--tsmc_core_agentic/latency",
    "width": 1,
    "n_questions": 15,
    "n_episodes": 15,
    "p25_s": 3.3,
    "p50_s": 5.5,
    "p75_s": 7.0,
    "questions_per_min": 10.93,
    "turns_slowest_draw": 11.8,
    "s_per_turn": 0.5
   },
   {
    "arm": "greedy (Figure 7 protocol)",
    "run": "poollat-gh200-greedy--tsmc_core_agentic",
    "width": 1,
    "n_questions": 15,
    "n_episodes": 15,
    "p25_s": 4.7,
    "p50_s": 5.9,
    "p75_s": 7.0,
    "questions_per_min": 10.17,
    "turns_slowest_draw": 13.5,
    "s_per_turn": 0.5
   },
   {
    "arm": "width 1, sampled",
    "run": "poollat-gh200-w1--tsmc_core_agentic",
    "width": 1,
    "n_questions": 15,
    "n_episodes": 15,
    "p25_s": 4.4,
    "p50_s": 6.0,
    "p75_s": 7.3,
    "questions_per_min": 9.99,
    "turns_slowest_draw": 13.5,
    "s_per_turn": 0.56
   },
   {
    "arm": "width 8",
    "run": "poollat-gh200-w8--tsmc_core_agentic",
    "width": 8,
    "n_questions": 15,
    "n_episodes": 120,
    "p25_s": 13.3,
    "p50_s": 19.8,
    "p75_s": 23.9,
    "questions_per_min": 3.02,
    "turns_slowest_draw": 17.7,
    "s_per_turn": 1.35
   },
   {
    "arm": "width 17",
    "run": "poollat-gh200-w17--tsmc_core_agentic",
    "width": 17,
    "n_questions": 15,
    "n_episodes": 255,
    "p25_s": 23.4,
    "p50_s": 28.7,
    "p75_s": 35.5,
    "questions_per_min": 2.09,
    "turns_slowest_draw": 18.8,
    "s_per_turn": 1.72
   },
   {
    "arm": "width 34 (the shipped pool)",
    "run": "poollat-gh200-w34--tsmc_core_agentic",
    "width": 34,
    "n_questions": 15,
    "n_episodes": 510,
    "p25_s": 28.1,
    "p50_s": 41.4,
    "p75_s": 44.3,
    "questions_per_min": 1.45,
    "turns_slowest_draw": 19.8,
    "s_per_turn": 1.85
   }
  ],
  "mac (4-bit)": [
   {
    "arm": "width 1",
    "run": "poollat-q4-w1--tsmc_core_agentic",
    "width": 1,
    "n_questions": 15,
    "n_episodes": 15,
    "p25_s": 15.1,
    "p50_s": 25.7,
    "p75_s": 35.0,
    "questions_per_min": 2.34,
    "turns_slowest_draw": 15.1,
    "s_per_turn": 1.77
   },
   {
    "arm": "width 4",
    "run": "poollat-q4-w4--tsmc_core_agentic",
    "width": 4,
    "n_questions": 15,
    "n_episodes": 60,
    "p25_s": 71.1,
    "p50_s": 80.9,
    "p75_s": 95.2,
    "questions_per_min": 0.74,
    "turns_slowest_draw": 18.0,
    "s_per_turn": 4.08
   },
   {
    "arm": "width 8, no prefix reuse",
    "run": "poollat-q4nc-w8--tsmc_core_agentic",
    "width": 8,
    "n_questions": 15,
    "n_episodes": 120,
    "p25_s": 245.0,
    "p50_s": 539.3,
    "p75_s": 784.9,
    "questions_per_min": 0.11,
    "turns_slowest_draw": 19.1,
    "s_per_turn": 27.01
   }
  ]
 },
 "cost": {
  "gpu_usd_per_hour": {
   "gh200": 2.29
  },
  "api_usd_per_mtok": {
   "gemini-3.7-flash": [
    0.75,
    3.75
   ],
   "gpt-5.6-sol": [
    4.0,
    20.0
   ],
   "gpt-6-astra": [
    10.0,
    50.0
   ],
   "gemini-3.8-flash": [
    0.75,
    3.75
   ],
   "gpt-5-mini": [
    0.25,
    2.0
   ],
   "claude-opus-5.5": [
    4.0,
    20.0
   ],
   "qwen3.5-397b-a17b": [
    0.55,
    3.5
   ],
   "deepseek-v4-pro": [
    0.21,
    0.42
   ]
  },
  "quoted": {
   "gemini-3.7-flash": "2026-09-16",
   "gpt-5.6-sol": "2026-09-16",
   "gpt-6-astra": "2026-09-16",
   "gemini-3.8-flash": "2026-09-16",
   "gpt-5-mini": "2026-10-03",
   "claude-opus-5.5": "2026-10-02",
   "qwen3.5-397b-a17b": "2026-10-02",
   "deepseek-v4-pro": "2026-10-02"
  },
  "ours": [
   {
    "arm": "greedy, shipped policy (Figure 7 protocol)",
    "card": "gh200",
    "protocol": "single",
    "s_per_question": 5.5,
    "usd_per_1k": 3.5,
    "questions_per_usd": 285.8,
    "gated_acc": 0.839
   },
   {
    "arm": "greedy (Figure 7 protocol)",
    "card": "gh200",
    "protocol": "single",
    "s_per_question": 5.9,
    "usd_per_1k": 3.75,
    "questions_per_usd": 266.4,
    "gated_acc": 0.733
   },
   {
    "arm": "width 8",
    "card": "gh200",
    "protocol": "single",
    "s_per_question": 19.8,
    "usd_per_1k": 12.6,
    "questions_per_usd": 79.4,
    "gated_acc": 0.78
   },
   {
    "arm": "width 17",
    "card": "gh200",
    "protocol": "single",
    "s_per_question": 28.7,
    "usd_per_1k": 18.26,
    "questions_per_usd": 54.8,
    "gated_acc": 0.79
   },
   {
    "arm": "width 34 (the shipped pool)",
    "card": "gh200",
    "protocol": "single",
    "s_per_question": 41.4,
    "usd_per_1k": 26.34,
    "questions_per_usd": 38.0,
    "gated_acc": 0.79
   },
   {
    "arm": "our policy, 4B stack, local vLLM on a GH200 (bulk)",
    "card": "gh200",
    "protocol": "batched",
    "s_per_question": 0.89,
    "usd_per_1k": 0.57,
    "questions_per_usd": 1766.4,
    "gated_acc": 0.839
   }
  ],
  "api": [
   {
    "arm": "gpt-5-mini, closed book",
    "model": "gpt-5-mini",
    "prefill_tok": 68,
    "generated_tok_floor": 99,
    "usd_per_1k": 0.21,
    "usd_per_1k_at_observed_output": 2.66,
    "questions_per_usd": 4651.2,
    "gated_acc": 0.205
   },
   {
    "arm": "claude-opus-5.5, closed book",
    "model": "claude-opus-5.5",
    "prefill_tok": 68,
    "generated_tok_floor": 266,
    "usd_per_1k": 5.59,
    "usd_per_1k_at_observed_output": 26.69,
    "questions_per_usd": 178.8,
    "gated_acc": 0.226
   },
   {
    "arm": "qwen3.5-397b-a17b, closed book",
    "model": "qwen3.5-397b-a17b",
    "prefill_tok": 68,
    "generated_tok_floor": 207,
    "usd_per_1k": 0.76,
    "usd_per_1k_at_observed_output": 4.66,
    "questions_per_usd": 1312.5,
    "gated_acc": 0.154
   },
   {
    "arm": "deepseek-v4-pro, closed book",
    "model": "deepseek-v4-pro",
    "prefill_tok": 68,
    "generated_tok_floor": 101,
    "usd_per_1k": 0.06,
    "usd_per_1k_at_observed_output": 0.57,
    "questions_per_usd": 17636.7,
    "gated_acc": 0.2
   },
   {
    "arm": "gemini-3.7-flash, closed book",
    "model": "gemini-3.7-flash",
    "prefill_tok": 66,
    "generated_tok_floor": 93,
    "usd_per_1k": 0.4,
    "usd_per_1k_at_observed_output": 5.0,
    "questions_per_usd": 2511.0,
    "gated_acc": 0.2
   },
   {
    "arm": "gpt-5.6-sol, closed book",
    "model": "gpt-5.6-sol",
    "prefill_tok": 66,
    "generated_tok_floor": 53,
    "usd_per_1k": 1.32,
    "usd_per_1k_at_observed_output": 26.68,
    "questions_per_usd": 755.3,
    "gated_acc": 0.226
   },
   {
    "arm": "gpt-6-astra, closed book",
    "model": "gpt-6-astra",
    "prefill_tok": 66,
    "generated_tok_floor": 65,
    "usd_per_1k": 3.91,
    "usd_per_1k_at_observed_output": 66.71,
    "questions_per_usd": 255.8,
    "gated_acc": 0.21
   },
   {
    "arm": "gemini-3.8-flash, closed book",
    "model": "gemini-3.8-flash",
    "prefill_tok": 66,
    "generated_tok_floor": 89,
    "usd_per_1k": 0.38,
    "usd_per_1k_at_observed_output": 5.0,
    "questions_per_usd": 2609.3,
    "gated_acc": 0.19
   },
   {
    "arm": "gpt-5-mini + our tools",
    "model": "gpt-5-mini",
    "run": "gpt5mini-agent-t20-3tool-native-f16k--tsmc_core_agentic",
    "source": "provider usage",
    "prefill_tok": 33779,
    "generated_tok": 2470,
    "reasoning_tok": 1802,
    "cached_prefill_share": 0.35,
    "usd_per_1k": 13.39,
    "questions_per_usd": 74.7,
    "gated_acc": 0.549
   },
   {
    "arm": "gpt-5.6-sol + our tools",
    "model": "gpt-5.6-sol",
    "run": "gpt56sol-agent-t20-3tool-native--tsmc_core_agentic",
    "source": "provider usage",
    "prefill_tok": 31663,
    "generated_tok": 1045,
    "reasoning_tok": 673,
    "cached_prefill_share": 0.79,
    "usd_per_1k": 147.56,
    "questions_per_usd": 6.8,
    "gated_acc": 0.785
   },
   {
    "arm": "gpt-6-astra + our tools",
    "model": "gpt-6-astra",
    "run": "gpt6astra-agent-t20-label--tsmc_core_agentic",
    "source": "provider usage",
    "prefill_tok": 18750,
    "generated_tok": 445,
    "reasoning_tok": 86,
    "cached_prefill_share": 0.72,
    "usd_per_1k": 209.76,
    "questions_per_usd": 4.8,
    "gated_acc": 0.877
   },
   {
    "arm": "gemini-3.7-flash + our tools",
    "model": "gemini-3.7-flash",
    "run": "gemini37flash-agent-t20-3tool-label--tsmc_core_agentic",
    "source": "provider usage",
    "prefill_tok": 28278,
    "generated_tok": 1532,
    "reasoning_tok": 693,
    "cached_prefill_share": 0.01,
    "usd_per_1k": 26.95,
    "questions_per_usd": 37.1,
    "gated_acc": 0.687
   },
   {
    "arm": "gemini-3.8-flash + our tools",
    "model": "gemini-3.8-flash",
    "run": "gemini38flash-agent-t20-label--tsmc_core_agentic",
    "source": "provider usage",
    "prefill_tok": 36570,
    "generated_tok": 2431,
    "reasoning_tok": 1224,
    "cached_prefill_share": 0.02,
    "usd_per_1k": 36.54,
    "questions_per_usd": 27.4,
    "gated_acc": 0.641
   },
   {
    "arm": "claude-opus-5.5 + our tools",
    "model": "claude-opus-5.5",
    "run": "opus55-agent-t20-3tool--tsmc_core_agentic",
    "source": "provider usage",
    "prefill_tok": 18413,
    "generated_tok": 1154,
    "reasoning_tok": 427,
    "cached_prefill_share": 0.0,
    "usd_per_1k": 96.73,
    "questions_per_usd": 10.3,
    "gated_acc": 0.846
   },
   {
    "arm": "deepseek-v4-pro + our tools",
    "model": "deepseek-v4-pro",
    "run": "dsv4pro-agent-t20-3tool--tsmc_core_agentic",
    "source": "provider usage",
    "prefill_tok": 41651,
    "generated_tok": 4702,
    "reasoning_tok": 4112,
    "cached_prefill_share": 0.61,
    "usd_per_1k": 10.72,
    "questions_per_usd": 93.3,
    "gated_acc": 0.59
   },
   {
    "arm": "qwen3.5-397b-a17b + our tools",
    "model": "qwen3.5-397b-a17b",
    "run": "qwen397-agent-t20-3tool--tsmc_core_agentic",
    "source": "provider usage",
    "prefill_tok": 44483,
    "generated_tok": 2883,
    "reasoning_tok": 2400,
    "cached_prefill_share": 0.89,
    "usd_per_1k": 34.56,
    "questions_per_usd": 28.9,
    "gated_acc": 0.605
   }
  ]
 },
 "pool_not_measured": [
  "poollat-h200-w1--tsmc_core_agentic",
  "poollat-h200-w17--tsmc_core_agentic",
  "poollat-h200-w34--tsmc_core_agentic",
  "poollat-h200-w8--tsmc_core_agentic"
 ]
}