| |
| """Exact CPU audit of ablation trends under alternative aggregations. |
| |
| This is deliberately separate from reproduce.py's printed-average check. It |
| parses the pinned source tables, keeps the two-decimal values as exact |
| hundredths, and tests the endpoint trend after omitting each benchmark in |
| turn, as well as benchmark-wise monotonicity across the three ablation levels. |
| It does not claim to recreate the unavailable 32B training runs. |
| """ |
|
|
| from __future__ import annotations |
|
|
| import json |
| import sys |
| from decimal import Decimal |
| from pathlib import Path |
|
|
| ROOT = Path(__file__).resolve().parents[1] |
| sys.path.insert(0, str(ROOT)) |
|
|
| from reproduce import row_values, source_files, table_block |
|
|
|
|
| def cents(values: list[float]) -> list[int]: |
| """Convert source's two-decimal scores to exact integer hundredths.""" |
| out = [int((Decimal(str(value)) * 100).to_integral_exact()) for value in values] |
| if any(value < 0 for value in out): |
| raise AssertionError(out) |
| return out |
|
|
|
|
| def summarize(levels: list[int], rows: list[list[int]]) -> dict: |
| if len(levels) != 3 or any(len(row) != 8 for row in rows): |
| raise AssertionError((levels, rows)) |
| benchmark_rows = [row[:7] for row in rows] |
| averages = [sum(row) / 7 / 100 for row in benchmark_rows] |
| leave_one_out = [] |
| for level, row in zip(levels, benchmark_rows): |
| leave_one_out.append({ |
| "level": level, |
| "means_omit_benchmark": [ |
| (sum(row) - row[index]) / 6 / 100 for index in range(7) |
| ], |
| }) |
| endpoint_deltas = [ |
| (benchmark_rows[-1][index] - benchmark_rows[0][index]) / 100 |
| for index in range(7) |
| ] |
| loo_endpoint_deltas = [ |
| (sum(benchmark_rows[-1]) - benchmark_rows[-1][index] |
| - sum(benchmark_rows[0]) + benchmark_rows[0][index]) / 6 / 100 |
| for index in range(7) |
| ] |
| monotone_benchmarks = sum( |
| benchmark_rows[0][index] <= benchmark_rows[1][index] <= benchmark_rows[2][index] |
| for index in range(7) |
| ) |
| adjacent_positive = [ |
| sum(benchmark_rows[level + 1][index] > benchmark_rows[level][index] for index in range(7)) |
| for level in range(2) |
| ] |
| if not all(delta > 0 for delta in loo_endpoint_deltas): |
| raise AssertionError({"levels": levels, "loo_endpoint_deltas": loo_endpoint_deltas}) |
| return { |
| "levels": levels, |
| "benchmark_hundredths": benchmark_rows, |
| "arithmetic_means_percent": averages, |
| "leave_one_out_means_percent": leave_one_out, |
| "endpoint_deltas_percent": endpoint_deltas, |
| "leave_one_out_endpoint_deltas_percent": loo_endpoint_deltas, |
| "positive_leave_one_out_endpoints": len([d for d in loo_endpoint_deltas if d > 0]), |
| "monotone_benchmarks": monotone_benchmarks, |
| "adjacent_positive_benchmark_counts": adjacent_positive, |
| } |
|
|
|
|
| def main() -> None: |
| files = source_files() |
| cell_block = table_block(files["src/results.tex"], "tab:cell_count_results") |
| table_block_text = table_block(files["src/results.tex"], "tab:table_count_results") |
| cell_rows = [ |
| cents(row_values(cell_block, r"0$\sim$30 &", 8)), |
| cents(row_values(cell_block, r"0$\sim$100 &", 8)), |
| cents(row_values(cell_block, r"0$\sim$300+ &", 8)), |
| ] |
| table_rows = [ |
| cents(row_values(table_block_text, r"$|-$ 1 &", 8)), |
| cents(row_values(table_block_text, r"1$\sim$5 &", 8)), |
| cents(row_values(table_block_text, r"1$\sim$30 &", 8)), |
| ] |
| result = { |
| "claim": 6, |
| "source_tables": ["tab:cell_count_results", "tab:table_count_results"], |
| "cell_count": summarize([30, 100, 300], cell_rows), |
| "table_count": summarize([1, 5, 30], table_rows), |
| "scope_boundary": "released table rows only; no TableLong checkpoint or trained-condition rerun", |
| } |
| if result["cell_count"]["positive_leave_one_out_endpoints"] != 7: |
| raise AssertionError(result) |
| if result["table_count"]["positive_leave_one_out_endpoints"] != 7: |
| raise AssertionError(result) |
| print(json.dumps(result, indent=2, sort_keys=True)) |
|
|
|
|
| if __name__ == "__main__": |
| main() |
|
|
