poster/fill_poster.py

#!/usr/bin/env python3
"""Fill the landscape_4col_neutral posterly template with our reproduction content."""
from pathlib import Path

P = Path(__file__).resolve().parent / "poster.html"
text = P.read_text()


def replace(old, new, count=1):
    global text
    n = text.count(old)
    if n == 0:
        raise SystemExit(f"NOT FOUND (0 occurrences): {old[:80]!r}")
    text = text.replace(old, new, count)


# ---- <title> ----
replace(
    "<title>POSTER TITLE — Venue Year</title>",
    "<title>Reproducing Squirrel Benchmark — ICML 2026 Agent Reproduction</title>",
)

# ---- Header: venue badge ----
replace(
    """    <div class="venue-badge">
      <div class="vb-venue">VENUE</div>
      <div class="vb-year">YEAR</div>
      <div class="vb-tag">POSTER</div>
    </div>""",
    """    <div class="venue-badge">
      <div class="vb-venue">ICML</div>
      <div class="vb-year">2026</div>
      <div class="vb-tag">REPRO</div>
    </div>""",
)

# ---- Header: title block ----
replace(
    """    <div class="title-block">
      <h1 class="title">POSTER TITLE: <span class="accent">Subtitle Keyword</span></h1>
      <div class="subtitle">One-sentence tagline that explains the work in plain English.</div>
      <div class="authors-line">
        <span class="author">Author One</span> &middot;
        <span class="author">Author Two</span> &middot;
        <span class="author">Author Three<sup>&#9993;</sup></span>
        <span class="aff">Lab / Department &middot; Institution &middot; City, Country</span>
      </div>
    </div>""",
    """    <div class="title-block">
      <h1 class="title">Beyond Text-to-SQL: <span class="accent">Reproducing Squirrel Benchmark</span></h1>
      <div class="subtitle">Independently verifying whether LLMs can really debug enterprise ETL SQL (arXiv:2601.18119)</div>
      <div class="authors-line">
        <span class="author">Agent Reproduction<sup>&#9993;</sup></span>
        <span class="aff">ICML-2026-agent-repro Challenge &middot; Hugging Face &times; AlphaXiv Open Reproductions</span>
      </div>
    </div>""",
)

# ---- Header: right block (logo slot removed, QR repointed) ----
replace(
    """    <div class="right-block">
      <div class="logo-slot">
        <!-- Affiliation / lab logo goes here:
             <img src="images/lab-logo.png" alt="Lab">
             For an inline-SVG logo with brand colors, mark it exempt so
             style_check skips its fills:
             <svg data-color-exempt="logo" ...> ... </svg>
             No logo? Delete this entire <div class="logo-slot"> instead of
             leaving it empty (an empty slot just adds a stray gap). The .aff
             text line still carries attribution. -->
      </div>
      <div class="qr-block">
        <!-- TODO: replace with your real QR code. The default below is an inline
             SVG placeholder so the template renders offline. Generate a real one with
             one of:
               $ qrencode -o qr.png -s 12 -m 0 "https://your-url"     # apt install qrencode
               $ python -c "import qrcode; qrcode.make('https://your-url').save('qr.png')"
             then change src to "qr.png" (export at ≥ 2× rendered size for crisp print).
             The placeholder data-URI carries data-color-exempt="logo" so its
             literal fills are exempt from the color gate. -->
        <img data-color-exempt="logo" src="data:image/svg+xml;utf8,<svg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 100 100'><rect width='100' height='100' fill='%23fff'/><g fill='%23222'><rect x='4' y='4' width='26' height='26'/><rect x='70' y='4' width='26' height='26'/><rect x='4' y='70' width='26' height='26'/></g><g fill='%23fff'><rect x='10' y='10' width='14' height='14'/><rect x='76' y='10' width='14' height='14'/><rect x='10' y='76' width='14' height='14'/></g><g fill='%23222'><rect x='14' y='14' width='6' height='6'/><rect x='80' y='14' width='6' height='6'/><rect x='14' y='80' width='6' height='6'/><rect x='38' y='10' width='4' height='4'/><rect x='48' y='10' width='4' height='4'/><rect x='58' y='10' width='4' height='4'/><rect x='34' y='34' width='4' height='4'/><rect x='44' y='38' width='4' height='4'/><rect x='54' y='34' width='4' height='4'/><rect x='62' y='40' width='4' height='4'/><rect x='38' y='46' width='4' height='4'/><rect x='50' y='52' width='4' height='4'/><rect x='62' y='56' width='4' height='4'/><rect x='34' y='60' width='4' height='4'/><rect x='44' y='66' width='4' height='4'/><rect x='56' y='62' width='4' height='4'/><rect x='80' y='80' width='4' height='4'/><rect x='88' y='86' width='4' height='4'/></g><text x='50' y='54' text-anchor='middle' font-family='sans-serif' font-size='6' font-weight='bold' fill='%23888'>QR</text></svg>" alt="QR placeholder — replace with your real QR before printing">
        <div class="qr-label">TODO: QR label</div>
      </div>
    </div>""",
    """    <div class="right-block">
      <div class="qr-block">
        <img data-color-exempt="logo" src="images/qr.png" alt="QR code linking to the published reproduction logbook">
        <div class="qr-label">Published logbook</div>
      </div>
    </div>""",
)

# ---- Framework banner ----
replace(
    """    <div class="fb-text">
      <!-- Eyebrow word is microcopy, not canon: "Framework" was the old fixed
           default -- reword to the poster's voice (SKILL.md Step 3). -->
      <span class="fb-label">TODO: eyebrow</span>
      &nbsp;<strong>Method name</strong> in one sentence — what it does, what it improves, what's new.
    </div>
    <div class="banner-stats">
      <div class="bs-item"><div class="bs-num">N&times;</div><div class="bs-label">headline<br>improvement</div></div>
      <div class="bs-item"><div class="bs-num">SOTA</div><div class="bs-label">benchmark<br>positioning</div></div>
      <div class="bs-item"><div class="bs-num">key</div><div class="bs-label">property<br>(e.g. proxy-free)</div></div>
      <div class="bs-item"><div class="bs-num">tag</div><div class="bs-label">domain<br>(e.g. offline RL)</div></div>
    </div>""",
    """    <div class="fb-text">
      <span class="fb-label">Reproduction verdict</span>
      &nbsp;All <strong>4 headline claims</strong> of the paper check out verbatim against its own tables and abstract; the real 985-task benchmark and Claude-4-Sonnet are unavailable, so we additionally ran a small <strong>toy</strong> substitute experiment on open models.
    </div>
    <div class="banner-stats">
      <div class="bs-item"><div class="bs-num">4/4</div><div class="bs-label">claims confirmed<br>vs. primary source</div></div>
      <div class="bs-item"><div class="bs-num">20</div><div class="bs-label">toy tasks &times; 4 models<br>HF Inference Providers</div></div>
      <div class="bs-item"><div class="bs-num">&lt;$1</div><div class="bs-label">compute cost<br>(API calls only)</div></div>
      <div class="bs-item"><div class="bs-num">toy</div><div class="bs-label">scale vs. real<br>985-task benchmark</div></div>
    </div>""",
)

# ---- Column 1, card 1: Motivation ----
replace(
    """      <div class="card highlight" data-measure-role="card">
        <div class="section-title"><span class="num">1</span><span class="st-text">Motivation</span></div>
        <p class="body-text">
          TODO: 2-3 sentences on the problem. Use <span class="keyword">accent color</span> for terms you want the eye to land on.
        </p>
        <ul class="mt-3 fs-4">
          <li>TODO: pain point 1.</li>
          <li>TODO: pain point 2.</li>
        </ul>
        <div class="callout mt-4">
          <strong>Q:</strong> TODO: the question your paper answers.
        </div>
      </div>""",
    """      <div class="card highlight" data-measure-role="card" data-logbook-target="executive-summary">
        <div class="section-title"><span class="num">1</span><span class="st-text">Why this reproduction</span></div>
        <p class="body-text">
          Squirrel Benchmark claims enterprise SQL <span class="keyword">debugging</span> (not generation) is where LLMs fail hardest: real ETL scripts run 140+ lines with deep nested joins, yet even the best model tested solves only about a third of tasks.
        </p>
        <ul class="mt-3 fs-4">
          <li>The real 985-task benchmark is <strong>not yet public</strong> (paper: "scheduled for release upon acceptance").</li>
          <li>Claude-4-Sonnet, the best model in Table 2, is <strong>unavailable</strong> in this HF-based challenge.</li>
        </ul>
        <div class="callout mt-4">
          <strong>Q:</strong> can we verify the paper's numbers from its own primary source, and independently corroborate its qualitative findings with a toy substitute experiment?
        </div>
      </div>""",
)

# ---- Column 1, card 2: Claim 1 ----
replace(
    """      <div class="card" data-measure-role="card">
        <div class="section-title"><span class="num">2</span><span class="st-text">Key Insight</span></div>
        <p class="body-text">
          TODO: 2-3 sentences on the central insight.
        </p>
        <div class="figure mt-3">
          <!-- Key-insight figure from the paper:
               <img src="assets/paper_figures/key-figure.png" data-source="paper"
                    data-asset-id="key-figure" class="w-95"> -->
          <div class="caption">
            TODO: caption explaining (a) / (b) / (c) panels in one line each.
          </div>
        </div>
      </div>""",
    """      <div class="card" data-measure-role="card" data-logbook-target="claim-1-sqlbench-scale-469-syntax-516-semantic-debugging-tasks-from-1-000-seed-scripts-across-26-business-scenarios">
        <div class="section-title"><span class="num">2</span><span class="st-text">Claim 1 &middot; Benchmark scale</span></div>
        <p class="body-text">
          Built from <span class="keyword">1,000+ seed SQL scripts</span> across 26 business scenarios via an automated reverse-engineering pipeline &mdash; verbatim-confirmed against Section 3.1 / abstract, no discrepancy.
        </p>
        <div class="keybox mt-3">
          <div class="kb-item"><div class="kb-num">469</div><div class="kb-label">Squirrel-Syntax<br>tasks</div></div>
          <div class="kb-item"><div class="kb-num">516</div><div class="kb-label">Squirrel-Semantic<br>tasks</div></div>
          <div class="kb-item"><div class="kb-num">26</div><div class="kb-label">business<br>scenarios</div></div>
        </div>
      </div>""",
)

# ---- Column 2, card 3: Claim 4 (using the real complexity-threshold equation, Eq. 2) ----
replace(
    """      <div class="card" data-measure-role="card">
        <div class="section-title"><span class="num">3</span><span class="st-text">Method Step 1</span></div>
        <p class="body-text">TODO: how step 1 works.</p>
        <div class="eqn">
          <span class="label">Core equation</span>
          $$f(x) \\;=\\; \\text{TODO}$$
        </div>
        <p class="body-text">TODO: interpretation.</p>
      </div>""",
    """      <div class="card" data-measure-role="card" data-logbook-target="claim-4-sqlbench-script-complexity-420-tokens-and-17-34-21-62-functions-per-script-on-average">
        <div class="section-title"><span class="num">3</span><span class="st-text">Claim 4 &middot; Script complexity</span></div>
        <p class="body-text">Seed scripts are filtered by a composite complexity score before being kept (paper Eq. 2):</p>
        <div class="eqn">
          <span class="label">Seed complexity filter</span>
          $$\\mathcal{C}(q) = \\alpha\\big(D_{\\text{AST}}(q) + W_{\\text{AST}}(q)\\big) + \\beta L(q)$$
        </div>
        <p class="body-text">Resulting scripts average <strong>420-497 tokens</strong> and <strong>17.34-21.62 functions</strong> each &mdash; verbatim-confirmed against Table 1, an order of magnitude above Spider 1.0's 18.5 tokens/query.</p>
      </div>""",
)

# ---- Column 2, card 4: Claim 2 (KEY, using the GM equation, Eq. 6) ----
replace(
    """      <div class="card highlight" data-measure-role="card">
        <div class="section-title"><span class="num">4</span><span class="st-text">Method Step 2&nbsp;<span class="key-mark">&#9733; KEY</span></span></div>
        <p class="body-text">TODO: how step 2 works.</p>
        <div class="eqn eqn--large">
          <span class="label">Step 2 equation</span>
          $$g(x) \\;=\\; \\text{TODO}$$
        </div>
        <div class="callout emph">
          <strong>Theorem.</strong> TODO: one-sentence statement, with $\\varepsilon$ bounds if applicable.
        </div>
      </div>""",
    """      <div class="card highlight" data-measure-role="card" data-logbook-target="claim-2-claude-4-sonnet-best-model-36-46-graph-match-on-squirrel-syntax-32-17-on-squirrel-semantic">
        <div class="section-title"><span class="num">4</span><span class="st-text">Claim 2 &middot; Best model&nbsp;<span class="key-mark">&#9733; KEY</span></span></div>
        <p class="body-text">Graph Match credits structurally-equivalent SQL even when strings differ (Eq. 6) &mdash; it is why EM alone undercounts correct fixes:</p>
        <div class="eqn eqn--large">
          <span class="label">Graph Match score</span>
          $$\\text{GM} = \\tfrac{1}{N}\\textstyle\\sum_i \\mathbf{1}\\big[\\text{Graph}(\\hat q_i) \\cong \\text{Graph}(q_i)\\big]$$
        </div>
        <div class="callout emph">
          <strong>Verbatim-confirmed:</strong> Claude-4-Sonnet is the best of ~30 evaluated models, yet clears only <strong>36.46%</strong> (Syntax) / <strong>32.17%</strong> (Semantic) GM &mdash; it built the benchmark and still can't reliably debug it forward.
        </div>
      </div>""",
)

# ---- Column 3, card 5: Claim 3 (main results table, real Table 2 rows) ----
replace(
    """      <div class="card highlight" data-measure-role="card">
        <div class="section-title"><span class="num">5</span><span class="st-text">Main Result&nbsp;<span class="key-mark">&#9733; Headline</span></span></div>
        <p class="body-text fs-3 text-secondary mb-1">
          TODO: setup line — dataset, # samples, baselines.
        </p>
        <div class="figure">
          <!-- Main-result figure from the paper:
               <img src="assets/paper_figures/main-result.png" data-source="paper"
                    data-asset-id="main-result" class="w-100"> -->
          <div class="caption fs-2">
            <strong>Left:</strong> TODO. <strong>Right:</strong> TODO.
          </div>
        </div>
        <div class="keybox">
          <div class="kb-item"><div class="kb-num">A</div><div class="kb-label">stat 1<br>caption</div></div>
          <div class="kb-item"><div class="kb-num">B</div><div class="kb-label">stat 2<br>caption</div></div>
          <div class="kb-item"><div class="kb-num">C</div><div class="kb-label">stat 3<br>caption</div></div>
        </div>
      </div>""",
    """      <div class="card highlight" data-measure-role="card" data-logbook-target="claim-3-deepseek-v3-and-qwen-2-5-coder-32b-scores-most-llms-below-20-success-rate">
        <div class="section-title"><span class="num">5</span><span class="st-text">Claim 3 &middot; Model rankings&nbsp;<span class="key-mark">&#9733; Headline</span></span></div>
        <p class="body-text fs-3 text-secondary mb-1">
          Table 2, Graph Match (GM) &middot; ~30 LLMs evaluated on Squirrel-Syntax / Squirrel-Semantic
        </p>
        <table class="result-table">
          <thead>
            <tr>
              <th class="method">Model</th>
              <th>Syntax GM &#8593;</th>
              <th>Semantic GM &#8593;</th>
            </tr>
          </thead>
          <tbody>
            <tr class="ours"><td class="method">Claude-4-Sonnet</td><td class="best">36.46</td><td class="best">32.17</td></tr>
            <tr><td class="method">DeepSeek-V3</td><td>30.28</td><td>21.32</td></tr>
            <tr><td class="method">Qwen-2.5-Coder-32B</td><td>20.26</td><td>23.45</td></tr>
            <tr><td class="method">GPT-4o</td><td>4.69</td><td>4.84</td></tr>
          </tbody>
        </table>
        <div class="keybox">
          <div class="kb-item"><div class="kb-num">36.46%</div><div class="kb-label">best Syntax<br>(Claude-4-Sonnet)</div></div>
          <div class="kb-item"><div class="kb-num">32.17%</div><div class="kb-label">best Semantic<br>(Claude-4-Sonnet)</div></div>
          <div class="kb-item"><div class="kb-num">&lt;20%</div><div class="kb-label">most models'<br>GM score</div></div>
        </div>
      </div>""",
)

# ---- Column 3, card 6: Toy substitute experiment ----
replace(
    """      <div class="card" data-measure-role="card">
        <div class="section-title"><span class="num">6</span><span class="st-text">Robustness / Secondary</span></div>
        <p class="body-text">TODO: 1-2 sentences on robustness, ablation, or secondary axis.</p>
        <ul class="mt-3">
          <li><strong>Setting A</strong> — TODO.</li>
          <li><strong>Setting B</strong> — TODO.</li>
        </ul>
        <div class="callout mt-3">
          TODO: bottom-line takeaway from the robustness study.
        </div>
      </div>""",
    """      <div class="card" data-measure-role="card">
        <div class="section-title"><span class="num">6</span><span class="st-text">Our toy substitute experiment</span></div>
        <p class="body-text">985 real tasks and Claude-4-Sonnet unavailable, so we built a 20-task synthetic benchmark (10 domains) and ran 4 open models via HF Inference Providers.</p>
        <ul class="mt-3">
          <li><strong>Syntax (1 bug pattern)</strong> &mdash; 100% GM-proxy for 3/4 models: too easy/homogeneous vs. the real 469-task, taxonomy-diverse split.</li>
          <li><strong>Semantic (3 bug patterns)</strong> &mdash; drops to 50-60% GM-proxy for every model, same ranking as Table 2 (32B &ge; 7B).</li>
        </ul>
        <div class="callout mt-3">
          Directionally reproduces "Semantic &gt; Syntax difficulty" and the model-scale effect &mdash; absolute scores aren't comparable at 20 vs. 985 tasks.
        </div>
      </div>""",
)

# ---- Column 4, card 7: Claim -> Verdict table ----
replace(
    """      <div class="card highlight" data-measure-role="card">
        <div class="section-title"><span class="num">7</span><span class="st-text">Benchmark Table</span></div>
        <p class="body-text fs-2 text-secondary mb-1">
          TODO: setup line — samples, seeds, metric description.
        </p>
        <table class="result-table">
          <thead>
            <tr>
              <th class="method">Method</th>
              <th>Metric A &#8593;</th>
              <th>Metric B &#8593;</th>
              <th>Cost &#8595;</th>
            </tr>
          </thead>
          <tbody>
            <tr class="reference"><td class="method"><em>Reference</em></td><td>x.xx</td><td>x.xx</td><td>x.xx</td></tr>
            <tr><td class="method">Baseline 1</td><td>x.xx</td><td>x.xx</td><td>x.xx</td></tr>
            <tr><td class="method">Baseline 2</td><td>x.xx</td><td>x.xx</td><td>x.xx</td></tr>
            <tr class="ours"><td class="method">Ours</td><td class="best">x.xx</td><td class="best">x.xx</td><td class="best">x.xx</td></tr>
          </tbody>
        </table>
        <p class="body-text mt-2 fs-3">
          TODO: one-sentence interpretation.
        </p>
      </div>""",
    """      <div class="card highlight" data-measure-role="card">
        <div class="section-title"><span class="num">7</span><span class="st-text">Claim-by-claim verdict</span></div>
        <p class="body-text fs-2 text-secondary mb-1">
          Every figure checked against Tables 1-2 / abstract / Section 3-5 of arXiv:2601.18119
        </p>
        <table class="result-table">
          <thead>
            <tr>
              <th class="method">Claim</th>
              <th>Primary source</th>
              <th>Toy experiment</th>
            </tr>
          </thead>
          <tbody>
            <tr><td class="method">1 &middot; Scale</td><td class="best">verbatim</td><td>order-of-magnitude</td></tr>
            <tr><td class="method">2 &middot; Best model</td><td class="best">verbatim</td><td>EM-vs-GM gap replayed</td></tr>
            <tr><td class="method">3 &middot; Rankings</td><td class="best">verbatim</td><td>ranking direction holds</td></tr>
            <tr class="ours"><td class="method">4 &middot; Complexity</td><td class="best">verbatim</td><td class="best">order-of-magnitude</td></tr>
          </tbody>
        </table>
        <p class="body-text mt-2 fs-3">
          Real benchmark/model unavailable &rArr; exact Table 1/2 percentages not independently re-derived, by design.
        </p>
      </div>""",
)

# ---- Column 4, card 8: Where to look next / bundle ----
replace(
    """      <div class="card" data-measure-role="card">
        <div class="section-title"><span class="num">8</span><span class="st-text">Diversity / Extra View</span></div>
        <p class="body-text">TODO: 1-sentence framing.</p>
        <div class="figure">
          <!-- Secondary figure from the paper:
               <img src="assets/paper_figures/secondary-figure.png" data-source="paper"
                    data-asset-id="secondary-figure" class="w-100"> -->
          <div class="caption fs-2">TODO: caption.</div>
        </div>
      </div>""",
    """      <div class="card" data-measure-role="card" data-logbook-target="conclusion">
        <div class="section-title"><span class="num">8</span><span class="st-text">Reproduction bundle</span></div>
        <p class="body-text">Every script, generated task, model response, and eval log is published as a Hugging Face Bucket artifact, linked from the logbook's Conclusion page for anyone to re-run or extend.</p>
        <div class="callout mt-3">
          Scripts: <code>generate_benchmark.py</code> &middot; <code>eval_models.py</code> &middot; <code>metrics.py</code> &mdash; all runnable with just an HF token.
        </div>
      </div>""",
)

# ---- Takeaways strip ----
replace(
    """  <section class="takeaways-strip" data-measure-role="footer-strip">
    <div class="ts-title"><span class="num">9</span> TODO: strip title</div>
    <div class="ts-item"><span class="ts-key">TODO.</span><span class="ts-text">TODO: 1-line.</span></div>
    <div class="ts-item"><span class="ts-key">TODO.</span><span class="ts-text">TODO: 1-line.</span></div>
    <div class="ts-item"><span class="ts-key">TODO.</span><span class="ts-text">TODO: 1-line.</span></div>
    <div class="ts-item"><span class="ts-key">TODO.</span><span class="ts-text">TODO: 1-line.</span></div>
  </section>""",
    """  <section class="takeaways-strip" data-measure-role="footer-strip">
    <div class="ts-title"><span class="num">9</span> Bottom line</div>
    <div class="ts-item"><span class="ts-key">Primary source.</span><span class="ts-text">All 4 claims match Tables 1-2 verbatim, no discrepancy.</span></div>
    <div class="ts-item"><span class="ts-key">Real data unavailable.</span><span class="ts-text">985-task benchmark + Claude-4-Sonnet not accessible here.</span></div>
    <div class="ts-item"><span class="ts-key">Toy substitute.</span><span class="ts-text">20 tasks &times; 4 open models via HF Inference Providers.</span></div>
    <div class="ts-item"><span class="ts-key">Qualitative match.</span><span class="ts-text">Semantic&gt;Syntax difficulty + model-scale effect both hold.</span></div>
  </section>""",
)

# ---- Footer ----
replace(
    """  <div class="footer" data-measure-role="footer">
    <div>
      <strong class="method-name">METHOD NAME</strong> &middot; Venue Year &middot;
      Acknowledgements: TODO.
    </div>
    <div>
      Code: <span class="repo">github.com/&lt;org&gt;/&lt;repo&gt;</span> &nbsp;&middot;&nbsp;
      Contact: <span class="repo">presenter@example.com</span>
    </div>
  </div>""",
    """  <div class="footer" data-measure-role="footer">
    <div>
      <strong class="method-name">SQUIRREL BENCHMARK REPRODUCTION</strong> &middot; ICML 2026 Agent Reproduction Challenge &middot;
      Built with Hugging Face Trackio + posterly.
    </div>
    <div>
      Bundle: <span class="repo">huggingface.co/buckets/Firemedic15/squirrel-sqlbench-repro</span> &nbsp;&middot;&nbsp;
      Paper: <span class="repo">arxiv.org/abs/2601.18119</span>
    </div>
  </div>""",
)

P.write_text(text)
print("poster.html filled successfully")