scripts/tokens_reported.py

#!/usr/bin/env python
"""Claim 4 - Token cost ~3x (Table 3), REPORTED-NOT-RUN.

Token totals are a byproduct of the L1+Tool vs L1-CoT LLM rollouts (the runner
logs usage). Those rollouts were NOT run here (vLLM/driver ABI mismatch). This
script transcribes the paper's reported Table 3 token/performance numbers and
reproduces the *arithmetic* of the ~3x token-cost claim, clearly labelled
REPORTED-NOT-RUN.

Paper: DynaSchedBench (arXiv 2605.27566), Table 3.
"""
from __future__ import annotations

import json
from pathlib import Path

OUT = Path("/home/ubuntu/samuel/dynasched-repro/outputs/tokens_reported.json")

# Table 3 (paper): tokens in millions (1e6) and mean Cmax gap (%).
TABLE3 = {
    "L1 Direct": {"tokens_M": 3.752, "mean_gap_pct": 2.0},
    "L1 CoT": {"tokens_M": 4.016, "mean_gap_pct": 1.9},
    "L1+Tool": {"tokens_M": 12.614, "mean_gap_pct": 2.0},
    "L2 CoT": {"tokens_M": 6.883, "mean_gap_pct": 0.7},
    "L2 Reflection": {"tokens_M": 14.470, "mean_gap_pct": 1.7},
    "L2 BestOfN": {"tokens_M": 7.770, "mean_gap_pct": 2.7},
    "L3 CoT": {"tokens_M": 8.620, "mean_gap_pct": 1.7},
}


def main():
    tool = TABLE3["L1+Tool"]
    cot = TABLE3["L1 CoT"]
    ratio = tool["tokens_M"] / cot["tokens_M"]

    print("=" * 62)
    print("CLAIM 4 - TOKEN COST ~3x (Table 3)   [REPORTED-NOT-RUN]")
    print("=" * 62)
    print("LLM slice not run (vLLM/driver ABI mismatch); paper values below.\n")
    print(f"{'Config':<16}{'Tokens(M)':>11}{'Mean gap %':>12}")
    for k, v in TABLE3.items():
        print(f"{k:<16}{v['tokens_M']:>11.3f}{v['mean_gap_pct']:>12.1f}")
    print()
    print(
        f"Token-cost ratio  L1+Tool / L1 CoT = "
        f"{tool['tokens_M']:.3f} / {cot['tokens_M']:.3f} = {ratio:.2f}x"
    )
    print(
        f"  L1+Tool spends {ratio:.2f}x the tokens of L1 CoT "
        f"({tool['mean_gap_pct']:.1f}% gap vs {cot['mean_gap_pct']:.1f}% gap)"
    )
    print(
        "  => ~3x token cost for equal-or-worse performance (paper: 12.614M vs 4.016M)."
    )

    OUT.write_text(
        json.dumps(
            {
                "status": "REPORTED-NOT-RUN",
                "reason": "Token usage requires L1+Tool / L1 CoT rollouts (vLLM/driver ABI mismatch).",
                "table3_reported": TABLE3,
                "token_ratio_tool_over_cot": ratio,
            },
            indent=2,
        )
    )
    print(f"\nWrote {OUT}")


if __name__ == "__main__":
    main()