"""Build evaluator-facing figures and check statistical rerun reproducibility.""" import hashlib import html import json import math from pathlib import Path RAW_PATH = Path( ".openresearch/artifacts/cumulative/raw/hf_run_2e812c37.json" ) FIGURE_PREFIX = "reports/reproduction/images" def _svg(title: str, subtitle: str, body: str, height: int = 470) -> str: return f''' {html.escape(title)} {html.escape(subtitle)} {body} ''' def _bar_svg(rows: list[dict]) -> str: values = [] labels = [] colors = [] for row in rows: label = f'{row["activation"]} n={row["width"]}' values.extend( [ 100 * row["comparison"]["diagonal_relative_shift"], 100 * row["comparison"]["offdiagonal_relative_shift"], ] ) labels.extend([f"{label} diag", f"{label} offdiag"]) colors.extend(["#167d70", "#d95d39"]) maximum = max(values) * 1.12 body = '' for tick in range(0, 10, 2): x = 245 + 600 * tick / maximum body += f'' body += f'{tick}%' for index, (label, value, color) in enumerate(zip(labels, values, colors)): y = 114 + index * 34 width = 600 * value / maximum body += f'{html.escape(label)}' body += f'' body += f'{value:.3f}%' one_percent = 245 + 600 / maximum body += f'' body += f'1% precommitted diagonal equivalence margin' body += 'diagonal cancellation target' body += 'off-diagonal negative control' return _svg( "Scale invariance cancels only the diagonal correction", "Five million networks per activation and width; absolute relative shifts", body, ) def _claim4_svg(rows: list[dict]) -> str: body = '' body += '' widths = [row["width"] for row in rows] x_min, x_max = min(widths), max(widths) fractions = [] for row in rows: measured = row["mean"][0] - row["source_infinite_width_prediction"][0] predicted = ( row["source_first_order_prediction"][0] - row["source_infinite_width_prediction"][0] ) fractions.append(measured / predicted) for tick in [0, 0.5, 1.0, 1.5]: y = 390 - tick * 180 body += f'' body += f'{tick:.1f}' body += '' body += 'paper first-order correction = 1' for width, fraction in zip(widths, fractions): x = 110 + 720 * (width - x_min) / (x_max - x_min) y = 390 - fraction * 180 body += f'' body += f'{width}' body += f'{fraction:.2f}' body += 'hidden width n' body += 'measured / predicted correction' return _svg( "Finite-width GeLU means follow the 1/n recursion correction", "Four-layer source architecture; 100,000 initializations at each width", body, ) def _claim5_svg(verifier: dict) -> str: colors = {"low": "#355c9a", "critical": "#167d70", "high": "#d95d39"} body = '' body += '' for level in range(-1, 5): y = 390 - (level + 1) * 48 body += f'' body += f'10^{level}' for name in ["low", "critical", "high"]: means = verifier["summaries"][name]["mean"] points = [] for depth_index, row in enumerate(means, start=1): value = max(row[0] / depth_index, 1e-2) x = 95 + 755 * (depth_index - 1) / 29 y = 390 - (math.log10(value) + 1) * 48 points.append(f"{x:.1f},{y:.1f}") point_text = " ".join(points) body += f'' body += 'depth' body += 'mean diagonal NTK / depth (log scale)' for index, name in enumerate(["low", "critical", "high"]): x = 315 + index * 125 body += f'' body += f'{name}' return _svg( "Only critical initialization stays linear through depth 30", "Width 200, 1,000 networks per regime, source gradient-stability observable", body, height=480, ) def _claim2_svg(verifier: dict) -> str: diagrams = verifier["diagrams"] body = '' for index, diagram in enumerate(diagrams): x = 95 + index * 160 quadratic = diagram["correction_vertex"]["name"] in {"K1", "Theta1"} order = 2 if quadratic else 4 color = "#355c9a" if quadratic else "#d95d39" body += f'' body += f'{order}' body += f'D{index + 1}' body += f'{html.escape(diagram["id"])}' body += '' body += 'independently summed recursion coefficient matches the closed form' body += 'blue: quadratic vertex · red: quartic vertex · injected sign error exits nonzero' return _svg( "The first-order mean recursion has exactly five diagrams", "Machine-enumerated quadratic and quartic contributions, checked independently", body, ) def _paired_z(current_rows: list, previous_rows: list) -> dict: z_values = [] for current, previous in zip(current_rows, previous_rows): for mean, old_mean, se, old_se in zip( current["mean"], previous["mean"], current["standard_error"], previous["standard_error"], ): denominator = math.sqrt(se * se + old_se * old_se) z_values.append(abs(mean - old_mean) / denominator) maximum = max(z_values) return { "comparison": "independent reruns agree within five combined standard errors", "maximum_combined_standard_error_z": maximum, "threshold": 5.0, "passed": maximum <= 5.0, } def _claim5_rows(verifier: dict) -> list[dict]: rows = [] for name in ["low", "critical", "high"]: summary = verifier["summaries"][name] for mean, standard_error in zip(summary["mean"], summary["standard_error"]): rows.append({"mean": mean, "standard_error": standard_error}) return rows def build_release_artifacts(current: dict) -> dict: snapshot = json.loads(RAW_PATH.read_text()) reproducibility = { "claim3": _paired_z( current["claim3_empirical_verifier"]["rows"], snapshot["claim3"]["empirical"]["rows"], ), "claim4": _paired_z( current["claim4_verifier"]["rows"], snapshot["claim4"]["verifier"]["rows"], ), "claim5": _paired_z( _claim5_rows(current["claim5_verifier"]), _claim5_rows(snapshot["claim5"]["verifier"]), ), } figures = { f"{FIGURE_PREFIX}/claim3_exact_scale.svg": _bar_svg( current["claim3_empirical_verifier"]["rows"] ), f"{FIGURE_PREFIX}/claim4_gelu_correction.svg": _claim4_svg( current["claim4_verifier"]["rows"] ), f"{FIGURE_PREFIX}/claim5_depth_stability.svg": _claim5_svg( current["claim5_verifier"] ), f"{FIGURE_PREFIX}/claim2_five_diagrams.svg": _claim2_svg( current["claim2_verifier"] ), } payloads = [] for path, svg in figures.items(): payloads.append( { "path": path, "sha256": hashlib.sha256(svg.encode()).hexdigest(), "text": svg, } ) fixed_command_matches = current["fixed_command"] == snapshot["fixed_command"] passed = ( snapshot["passed"] and fixed_command_matches and all(item["passed"] for item in reproducibility.values()) and len(payloads) == 4 and all("