"""Static HTML and README rendering for PostTrainBench reproduction. All rendered values trace to a canonical JSON pointer. No manually copied result value appears in the output. Templates are plain Python string formatting to avoid runtime template engine variation. """ from __future__ import annotations from typing import Any from posttrainbench_repro.constants import MODEL_ORDER # --------------------------------------------------------------------------- # Shared CSS # --------------------------------------------------------------------------- _CSS = """\ :root { --bg: #0f172a; --surface: #1e293b; --border: #334155; --text: #e2e8f0; --text-muted: #94a3b8; --accent: #38bdf8; --green: #4ade80; --yellow: #facc15; --red: #f87171; --orange: #fb923c; } * { margin: 0; padding: 0; box-sizing: border-box; } body { font-family: 'Inter', system-ui, -apple-system, sans-serif; background: var(--bg); color: var(--text); line-height: 1.6; padding: 2rem; max-width: 1200px; margin: 0 auto; } h1 { font-size: 1.75rem; margin-bottom: 0.5rem; color: var(--accent); } h2 { font-size: 1.25rem; margin: 1.5rem 0 0.75rem; color: var(--accent); border-bottom: 1px solid var(--border); padding-bottom: 0.25rem; } h3 { font-size: 1rem; margin: 1rem 0 0.5rem; color: var(--text); } table { border-collapse: collapse; width: 100%; margin: 0.5rem 0; } th, td { padding: 0.4rem 0.75rem; text-align: left; border: 1px solid var(--border); font-size: 0.85rem; } th { background: var(--surface); color: var(--accent); font-weight: 600; } td { background: var(--bg); } .card { background: var(--surface); border: 1px solid var(--border); border-radius: 8px; padding: 1rem; margin: 0.75rem 0; } .status { display: inline-block; padding: 0.15rem 0.5rem; border-radius: 4px; font-size: 0.8rem; font-weight: 600; } .status-partial { background: rgba(250, 204, 21, 0.15); color: var(--yellow); border: 1px solid var(--yellow); } .status-unavailable { background: rgba(248, 113, 113, 0.15); color: var(--red); border: 1px solid var(--red); } .status-verified { background: rgba(74, 222, 128, 0.15); color: var(--green); border: 1px solid var(--green); } .ptr { color: var(--text-muted); font-family: monospace; font-size: 0.75rem; } a { color: var(--accent); text-decoration: none; } a:hover { text-decoration: underline; } ul, ol { margin: 0.5rem 0 0.5rem 1.5rem; } li { margin: 0.25rem 0; font-size: 0.9rem; } .limitation { background: rgba(251, 146, 60, 0.1); border-left: 3px solid var(--orange); padding: 0.5rem 0.75rem; margin: 0.25rem 0; font-size: 0.85rem; } .excerpt { background: var(--bg); border: 1px solid var(--border); padding: 0.5rem; margin: 0.25rem 0; font-family: monospace; font-size: 0.8rem; white-space: pre-wrap; } nav { margin-bottom: 1.5rem; } nav a { margin-right: 1rem; } footer { margin-top: 2rem; padding-top: 1rem; border-top: 1px solid var(--border); font-size: 0.8rem; color: var(--text-muted); } """ def _status_span(status: str) -> str: """Render a status badge.""" cls = { "partial-support": "status-partial", "unavailable": "status-unavailable", "verified": "status-verified", }.get(status, "status-partial") return f'{status}' def _ptr(pointer: str) -> str: """Render a JSON pointer reference.""" return f'{pointer}' def _evidence_value(value_html: str, pointer: str) -> str: """Bind one visible value to the exact canonical JSON value it renders.""" return ( '' f"{value_html} {_ptr(pointer)}" ) def _status_value(status: str, pointer: str) -> str: return _evidence_value(_status_span(status), pointer) def _escape(text: str) -> str: """HTML-escape text.""" return ( text .replace("&", "&") .replace("<", "<") .replace(">", ">") .replace('"', """) ) # --------------------------------------------------------------------------- # Coverage matrix HTML # --------------------------------------------------------------------------- def _render_matrix(coverage: dict[str, Any]) -> str: """Render the 4×7 coverage matrix table.""" cell_counts = coverage.get("cell_counts", {}) benchmarks = sorted(cell_counts.keys()) rows = [] rows.append("") rows.append("") for model in MODEL_ORDER: rows.append(f"") rows.append("") for bench in benchmarks: counts = cell_counts[bench] rows.append(f"") for model_index, count in enumerate(counts): pointer = ( f"evidence/coverage.json#/cell_counts/{bench}/{model_index}" ) rows.append( f"" ) rows.append("") rows.append("
Benchmark{_escape(model)}
{bench}{_evidence_value(str(count), pointer)}
") return "\n".join(rows) # --------------------------------------------------------------------------- # Reward-hacking modes HTML # --------------------------------------------------------------------------- def _render_reward_hacking(rh: dict[str, Any]) -> str: """Render the three reward-hacking submodes.""" parts: list[str] = [] # Mode 1: Contamination contam = rh["training_on_test_sets"] contam_base = "evidence/reward_hacking.json#/training_on_test_sets" parts.append('
') parts.append( "

Training on Test Sets " f'{_status_value(contam["status"], contam_base + "/status")}

' ) parts.append( "

Witness: " + _evidence_value( f'{_escape(contam["witness_path"])}', contam_base + "/witness_path", ) + "

" ) parts.append( "

SHA-256: " + _evidence_value( f'{contam["witness_sha256"]}', contam_base + "/witness_sha256", ) + "

" ) parts.append( "

Content: " + _evidence_value( f'{_escape(contam["witness_bytes"])}', contam_base + "/witness_bytes", ) + "

" ) parts.append( "

Type: " + _evidence_value( _escape(contam["observation_type"]), contam_base + "/observation_type", ) + "

" ) parts.append("
") # Mode 2: Instruction-model instr = rh["downloading_instruction_tuned_checkpoint"] instr_base = ( "evidence/reward_hacking.json#/" "downloading_instruction_tuned_checkpoint" ) parts.append('
') parts.append( "

Instruction-Tuned Checkpoint " f'{_status_value(instr["status"], instr_base + "/status")}

' ) parts.append( "

Judgment: " + _evidence_value( f'{_escape(instr["judgment_path"])}', instr_base + "/judgment_path", ) + "

" ) parts.append( "

Judgment SHA-256: " + _evidence_value( f'{instr["judgment_sha256"]}', instr_base + "/judgment_sha256", ) + "

" ) parts.append( "

Content: " + _evidence_value( f'{_escape(instr["judgment_bytes"])}', instr_base + "/judgment_bytes", ) + " (no trailing newline)

" ) parts.append("

Safe Trace Excerpts

") for index, excerpt in enumerate(instr.get("safe_excerpts", [])): excerpt_base = f"{instr_base}/safe_excerpts/{index}" parts.append( '
Record ' + _evidence_value( str(excerpt["record"]), excerpt_base + "/record", ) + " " + _evidence_value( _escape(excerpt["json_pointer"]), excerpt_base + "/json_pointer", ) + ": " + _evidence_value( _escape(str(excerpt["text"])), excerpt_base + "/text", ) + "\nSHA-256: " + _evidence_value( excerpt["sha256"], excerpt_base + "/sha256", ) + "
" ) parts.append( "

Complete trace (" + _evidence_value( f'{instr["trace"]["size"]:,}', instr_base + "/trace/size", ) + " bytes) is " + _evidence_value( "NOT redistributed", instr_base + "/trace/redistributed", ) + ".

" ) parts.append("
") # Mode 3: API misuse api = rh["using_discovered_api_key"] api_base = "evidence/reward_hacking.json#/using_discovered_api_key" parts.append('
') parts.append( "

API Key Discovery " f'{_status_value(api["status"], api_base + "/status")}

' ) parts.append( "

" + _evidence_value( _escape(api["unavailability_reason"]), api_base + "/unavailability_reason", ) + "

" ) parts.append("
") return "\n".join(parts) # --------------------------------------------------------------------------- # Limitations HTML # --------------------------------------------------------------------------- def _render_limitations(claims: dict[str, Any]) -> str: """Render all limitations from both claims.""" parts: list[str] = [] seen: set[str] = set() for claim_key in ["claim_1", "claim_2"]: claim = claims[claim_key] for index, lim in enumerate(claim.get("limitations", [])): if lim not in seen: seen.add(lim) pointer = ( f"evidence/claims.json#/{claim_key}/limitations/{index}" ) parts.append( '
' f"{_evidence_value(_escape(lim), pointer)}
" ) return "\n".join(parts) def _render_claim(claim: dict[str, Any], claim_key: str, title: str) -> str: base = f"evidence/claims.json#/{claim_key}" return f"""\

{title} {_status_value(claim["status"], base + "/status")}

{_evidence_value(_escape(claim["text"]), base + "/text")}

SHA-256: {_evidence_value(f'{claim["sha256"]}', base + "/sha256")}

{_evidence_value(_escape(claim["summary"]), base + "/summary")}

""" def _render_coverage_summary(coverage: dict[str, Any]) -> str: auxiliary = coverage["excluded_auxiliary_data"] values = [ ( "Accepted benchmarks", coverage["accepted_benchmark_count"], "accepted_benchmark_count", ), ( "Accepted models", coverage["accepted_model_count"], "accepted_model_count", ), ("Recognized tasks", coverage["recognized_task_count"], "recognized_task_count"), ("Recognized roots", coverage["recognized_root_count"], "recognized_root_count"), ( "Root/cell pairs", coverage["recognized_root_cell_pairs"], "recognized_root_cell_pairs", ), ("Duplicate pairs", coverage["duplicate_job_pairs"], "duplicate_job_pairs"), ("Missing pairs", coverage["missing_root_cell_pairs"], "missing_root_cell_pairs"), ("Excluded nested task dirs", coverage["excluded_dirs_count"], "excluded_dirs_count"), ] parts = ['
") return "\n".join(parts) def _source_reference_text(reference: dict[str, Any]) -> str: if "path" in reference: line_text = ",".join(str(line) for line in reference["lines"]) return ( f'{reference["commit"]}/{reference["path"]}:line {line_text} ' f'(blob {reference["git_object_sha1"]}, ' f'raw SHA-256 {reference["raw_sha256"]})' ) return ( f'{reference["kind"]}: ' + ", ".join(reference["paths"]) ) def _render_protocol(coverage: dict[str, Any]) -> str: protocol = coverage["protocol"] analysis = protocol["commit_sh_analysis"] references = protocol["source_references"] def source(reference_key: str) -> str: reference = references[reference_key] pointer = ( "evidence/coverage.json#/protocol/source_references/" + reference_key ) return _evidence_value( _escape(_source_reference_text(reference)), pointer, ) def item( label: str, value: Any, value_pointer: str, reference_key: str, ) -> str: return ( f"
  • {label}: " f"{_evidence_value(_escape(str(value)), value_pointer)}" f"
    Source: {source(reference_key)}
  • " ) base = "evidence/coverage.json#/protocol" lines = ['
    ") return "\n".join(lines) def _render_reward_status_summary(reward_hacking: dict[str, Any]) -> str: values = [ ( "Contamination", "training_on_test_sets", ), ( "Instruction model", "downloading_instruction_tuned_checkpoint", ), ( "API misuse", "using_discovered_api_key", ), ] return " · ".join( label + ": " + _status_value( reward_hacking[key]["status"], f"evidence/reward_hacking.json#/{key}/status", ) for label, key in values ) # --------------------------------------------------------------------------- # Index page # --------------------------------------------------------------------------- def render_index_html( provenance: dict[str, Any], coverage: dict[str, Any], reward_hacking: dict[str, Any], claims: dict[str, Any], ) -> str: """Render the landing page (index.html).""" claim1 = claims["claim_1"] claim2 = claims["claim_2"] return f"""\ PostTrainBench Reproduction

    PostTrainBench Reproduction

    Paper: PostTrainBench: Can LLM Agents Automate LLM Post-Training?

    Paper ID: {_evidence_value( f'{_escape(provenance["paper_id"])}', "evidence/provenance.json#/paper_id", )} · Attempt: {_evidence_value( f'{_escape(provenance["attempt_id"])}', "evidence/provenance.json#/attempt_id", )}

    Selected Claims

    {_render_claim(claim1, "claim_1", "Claim 1")} {_render_claim(claim2, "claim_2", "Claim 2")}

    Coverage Matrix (4 Models × 7 Benchmarks)

    {_render_coverage_summary(coverage)} {_render_matrix(coverage)}

    Protocol Audit

    {_render_protocol(coverage)}

    Reward-Hacking Submodes

    {_render_reward_hacking(reward_hacking)}

    Limitations

    {_render_limitations(claims)} """ # --------------------------------------------------------------------------- # Report page # --------------------------------------------------------------------------- def render_report_html( provenance: dict[str, Any], coverage: dict[str, Any], reward_hacking: dict[str, Any], claims: dict[str, Any], ) -> str: """Render the detailed report page.""" claim1 = claims["claim_1"] claim2 = claims["claim_2"] return f"""\ PostTrainBench Reproduction Report

    Reproduction Report: PostTrainBench

    ← Back to summary

    Provenance

    FieldValuePointer
    Paper ID{provenance["paper_id"]}{_ptr("evidence/provenance.json#/paper_id")}
    Attempt ID{provenance["attempt_id"]}{_ptr("evidence/provenance.json#/attempt_id")}
    Snapshot{provenance["assessed_snapshot"][:16]}…{_ptr("evidence/provenance.json#/assessed_snapshot")}
    Challenge Rev{provenance["challenge_revision"][:16]}…{_ptr("evidence/provenance.json#/challenge_revision")}
    Upstream Token{_escape(provenance["upstream_token"][:40])}…{_ptr("evidence/provenance.json#/upstream_token")}
    Source Commit{provenance["source"]["pinned_commit"][:16]}…{_ptr("evidence/provenance.json#/source/pinned_commit")}
    Dataset Rev{provenance["dataset"]["pinned_revision"][:16]}…{_ptr("evidence/provenance.json#/dataset/pinned_revision")}
    Paid API CostUSD {provenance["paid_api_cost_usd"]}{_ptr("evidence/provenance.json#/paid_api_cost_usd")}

    Claim 1: Coverage

    {_render_claim(claim1, "claim_1", "Coverage claim")}

    Coverage Matrix

    {_render_coverage_summary(coverage)} {_render_matrix(coverage)}

    Protocol Controls

    {_render_protocol(coverage)}

    Claim 2: Reward Hacking

    {_render_claim(claim2, "claim_2", "Reward-hacking claim")} {_render_reward_hacking(reward_hacking)}

    Limitations

    {_render_limitations(claims)} """ # --------------------------------------------------------------------------- # Poster page # --------------------------------------------------------------------------- def render_poster_html( provenance: dict[str, Any], coverage: dict[str, Any], reward_hacking: dict[str, Any], claims: dict[str, Any], ) -> str: """Render the poster page.""" claim1 = claims["claim_1"] claim2 = claims["claim_2"] return f"""\ PostTrainBench Reproduction Poster

    PostTrainBench: Released-Artifact Audit

    Paper ID: {_evidence_value( f'{_escape(provenance["paper_id"])}', "evidence/provenance.json#/paper_id", )} · arXiv: {_evidence_value( f'{_escape(provenance["arxiv_id"])}', "evidence/provenance.json#/arxiv_id", )}

    {_render_claim(claim1, "claim_1", "Claim 1")} {_render_claim(claim2, "claim_2", "Claim 2")}

    Coverage Counts

    {_render_coverage_summary(coverage)}

    Coverage Matrix

    {_render_matrix(coverage)}

    Reward-Hacking Statuses

    {_render_reward_status_summary(reward_hacking)}

    Limitations

    {_render_limitations(claims)} """ # --------------------------------------------------------------------------- # README with Space frontmatter # --------------------------------------------------------------------------- def render_readme( provenance: dict[str, Any], claims: dict[str, Any], ) -> str: """Render README.md with Hugging Face Space frontmatter.""" claim1 = claims["claim_1"] claim2 = claims["claim_2"] return f"""\ --- title: PostTrainBench Reproduction emoji: "\U0001f50d" colorFrom: blue colorTo: indigo sdk: static app_file: index.html pinned: false license: mit tags: - icml2026-repro - paper-UnjxMTe57e --- # PostTrainBench Reproduction Deterministic CPU-only released-artifact audit for PostTrainBench (OpenReview: `{provenance["paper_id"]}`, arXiv: `{provenance["arxiv_id"]}`). ## Selected Claims ### Claim 1: {claim1["status"]} {claim1["text"]} {claim1["summary"]} ### Claim 2: {claim2["status"]} {claim2["text"]} {claim2["summary"]} ## Evidence - [Evidence summary](index.html) - [Detailed report](report.html) - [Poster](poster.html) - [Provenance](evidence/provenance.json) - [Coverage](evidence/coverage.json) - [Reward hacking](evidence/reward_hacking.json) - [Claims](evidence/claims.json) - [Manifest](evidence/manifest.json) ## Limitations This is not an official challenge verdict. See [the report](report.html) for the full limitation list. No H100 run is reproduced. A released judge label is not independently established behavioral truth. The API-key submode remains unavailable. ## Licenses - Source repository: {provenance["source"]["license"]} - Dataset: {provenance["dataset"]["license"]} - Paper: {provenance["paper_license"]} - This reproduction: MIT ## Cost Paid API cost: USD {provenance["paid_api_cost_usd"]} """