File size: 4,183 Bytes
f22db1c | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 | #!/usr/bin/env python3
"""Exact CPU audit of ablation trends under alternative aggregations.
This is deliberately separate from reproduce.py's printed-average check. It
parses the pinned source tables, keeps the two-decimal values as exact
hundredths, and tests the endpoint trend after omitting each benchmark in
turn, as well as benchmark-wise monotonicity across the three ablation levels.
It does not claim to recreate the unavailable 32B training runs.
"""
from __future__ import annotations
import json
import sys
from decimal import Decimal
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT))
from reproduce import row_values, source_files, table_block # noqa: E402
def cents(values: list[float]) -> list[int]:
"""Convert source's two-decimal scores to exact integer hundredths."""
out = [int((Decimal(str(value)) * 100).to_integral_exact()) for value in values]
if any(value < 0 for value in out):
raise AssertionError(out)
return out
def summarize(levels: list[int], rows: list[list[int]]) -> dict:
if len(levels) != 3 or any(len(row) != 8 for row in rows):
raise AssertionError((levels, rows))
benchmark_rows = [row[:7] for row in rows]
averages = [sum(row) / 7 / 100 for row in benchmark_rows]
leave_one_out = []
for level, row in zip(levels, benchmark_rows):
leave_one_out.append({
"level": level,
"means_omit_benchmark": [
(sum(row) - row[index]) / 6 / 100 for index in range(7)
],
})
endpoint_deltas = [
(benchmark_rows[-1][index] - benchmark_rows[0][index]) / 100
for index in range(7)
]
loo_endpoint_deltas = [
(sum(benchmark_rows[-1]) - benchmark_rows[-1][index]
- sum(benchmark_rows[0]) + benchmark_rows[0][index]) / 6 / 100
for index in range(7)
]
monotone_benchmarks = sum(
benchmark_rows[0][index] <= benchmark_rows[1][index] <= benchmark_rows[2][index]
for index in range(7)
)
adjacent_positive = [
sum(benchmark_rows[level + 1][index] > benchmark_rows[level][index] for index in range(7))
for level in range(2)
]
if not all(delta > 0 for delta in loo_endpoint_deltas):
raise AssertionError({"levels": levels, "loo_endpoint_deltas": loo_endpoint_deltas})
return {
"levels": levels,
"benchmark_hundredths": benchmark_rows,
"arithmetic_means_percent": averages,
"leave_one_out_means_percent": leave_one_out,
"endpoint_deltas_percent": endpoint_deltas,
"leave_one_out_endpoint_deltas_percent": loo_endpoint_deltas,
"positive_leave_one_out_endpoints": len([d for d in loo_endpoint_deltas if d > 0]),
"monotone_benchmarks": monotone_benchmarks,
"adjacent_positive_benchmark_counts": adjacent_positive,
}
def main() -> None:
files = source_files()
cell_block = table_block(files["src/results.tex"], "tab:cell_count_results")
table_block_text = table_block(files["src/results.tex"], "tab:table_count_results")
cell_rows = [
cents(row_values(cell_block, r"0$\sim$30 &", 8)),
cents(row_values(cell_block, r"0$\sim$100 &", 8)),
cents(row_values(cell_block, r"0$\sim$300+ &", 8)),
]
table_rows = [
cents(row_values(table_block_text, r"$|-$ 1 &", 8)),
cents(row_values(table_block_text, r"1$\sim$5 &", 8)),
cents(row_values(table_block_text, r"1$\sim$30 &", 8)),
]
result = {
"claim": 6,
"source_tables": ["tab:cell_count_results", "tab:table_count_results"],
"cell_count": summarize([30, 100, 300], cell_rows),
"table_count": summarize([1, 5, 30], table_rows),
"scope_boundary": "released table rows only; no TableLong checkpoint or trained-condition rerun",
}
if result["cell_count"]["positive_leave_one_out_endpoints"] != 7:
raise AssertionError(result)
if result["table_count"]["positive_leave_one_out_endpoints"] != 7:
raise AssertionError(result)
print(json.dumps(result, indent=2, sort_keys=True))
if __name__ == "__main__":
main()
|