#!/usr/bin/env python3 """Exact CPU audit of ablation trends under alternative aggregations. This is deliberately separate from reproduce.py's printed-average check. It parses the pinned source tables, keeps the two-decimal values as exact hundredths, and tests the endpoint trend after omitting each benchmark in turn, as well as benchmark-wise monotonicity across the three ablation levels. It does not claim to recreate the unavailable 32B training runs. """ from __future__ import annotations import json import sys from decimal import Decimal from pathlib import Path ROOT = Path(__file__).resolve().parents[1] sys.path.insert(0, str(ROOT)) from reproduce import row_values, source_files, table_block # noqa: E402 def cents(values: list[float]) -> list[int]: """Convert source's two-decimal scores to exact integer hundredths.""" out = [int((Decimal(str(value)) * 100).to_integral_exact()) for value in values] if any(value < 0 for value in out): raise AssertionError(out) return out def summarize(levels: list[int], rows: list[list[int]]) -> dict: if len(levels) != 3 or any(len(row) != 8 for row in rows): raise AssertionError((levels, rows)) benchmark_rows = [row[:7] for row in rows] averages = [sum(row) / 7 / 100 for row in benchmark_rows] leave_one_out = [] for level, row in zip(levels, benchmark_rows): leave_one_out.append({ "level": level, "means_omit_benchmark": [ (sum(row) - row[index]) / 6 / 100 for index in range(7) ], }) endpoint_deltas = [ (benchmark_rows[-1][index] - benchmark_rows[0][index]) / 100 for index in range(7) ] loo_endpoint_deltas = [ (sum(benchmark_rows[-1]) - benchmark_rows[-1][index] - sum(benchmark_rows[0]) + benchmark_rows[0][index]) / 6 / 100 for index in range(7) ] monotone_benchmarks = sum( benchmark_rows[0][index] <= benchmark_rows[1][index] <= benchmark_rows[2][index] for index in range(7) ) adjacent_positive = [ sum(benchmark_rows[level + 1][index] > benchmark_rows[level][index] for index in range(7)) for level in range(2) ] if not all(delta > 0 for delta in loo_endpoint_deltas): raise AssertionError({"levels": levels, "loo_endpoint_deltas": loo_endpoint_deltas}) return { "levels": levels, "benchmark_hundredths": benchmark_rows, "arithmetic_means_percent": averages, "leave_one_out_means_percent": leave_one_out, "endpoint_deltas_percent": endpoint_deltas, "leave_one_out_endpoint_deltas_percent": loo_endpoint_deltas, "positive_leave_one_out_endpoints": len([d for d in loo_endpoint_deltas if d > 0]), "monotone_benchmarks": monotone_benchmarks, "adjacent_positive_benchmark_counts": adjacent_positive, } def main() -> None: files = source_files() cell_block = table_block(files["src/results.tex"], "tab:cell_count_results") table_block_text = table_block(files["src/results.tex"], "tab:table_count_results") cell_rows = [ cents(row_values(cell_block, r"0$\sim$30 &", 8)), cents(row_values(cell_block, r"0$\sim$100 &", 8)), cents(row_values(cell_block, r"0$\sim$300+ &", 8)), ] table_rows = [ cents(row_values(table_block_text, r"$|-$ 1 &", 8)), cents(row_values(table_block_text, r"1$\sim$5 &", 8)), cents(row_values(table_block_text, r"1$\sim$30 &", 8)), ] result = { "claim": 6, "source_tables": ["tab:cell_count_results", "tab:table_count_results"], "cell_count": summarize([30, 100, 300], cell_rows), "table_count": summarize([1, 5, 30], table_rows), "scope_boundary": "released table rows only; no TableLong checkpoint or trained-condition rerun", } if result["cell_count"]["positive_leave_one_out_endpoints"] != 7: raise AssertionError(result) if result["table_count"]["positive_leave_one_out_endpoints"] != 7: raise AssertionError(result) print(json.dumps(result, indent=2, sort_keys=True)) if __name__ == "__main__": main()