| """Generate static index.html for the leaderboard from results.csv. |
| |
| Static HF Spaces are free; this replaces the Gradio app (kept as app.py for a |
| future PRO upgrade). Run: python3 build_static.py |
| """ |
| import csv |
| import html |
| from pathlib import Path |
|
|
| HERE = Path(__file__).parent |
| DATASET = "https://huggingface.co/datasets/ModelBrew/sequential-forgetting-benchmark" |
|
|
| SUITES = { |
| "5-domain-realworld": "Suite A — 5 real-world domains · Mistral-7B · 3 seeds", |
| "4-domain-MLCF": "Suite B — Medical→Legal→Code→Finance", |
| "4-domain-MLCF-history": "Suite B history — CL-technique stacks (within-version comparisons only)", |
| "mquake-5skill-vault": "Suite C — MQuAKE 5-skill retention · Qwen3-4B", |
| } |
|
|
| rows = list(csv.DictReader(open(HERE / "results.csv"))) |
|
|
|
|
| def table(subset): |
| if not subset: |
| return "" |
| subset = sorted(subset, key=lambda r: abs(float(r["retention_value_pct"]))) |
| out = ["<div class='tablewrap'><table><thead><tr>"] |
| heads = ["method", "base model", "domains", "seeds", "metric", "value %", "status", "source", "notes"] |
| out += [f"<th>{h}</th>" for h in heads] + ["</tr></thead><tbody>"] |
| for r in subset: |
| src = r["source_file"] |
| link = f"<a href='{DATASET}/blob/main/results/{html.escape(src)}'>{html.escape(src.split('/')[-1])}</a>" |
| status_cls = "ok" if r["status"].startswith("valid") else "bad" |
| cells = [ |
| f"<td><code>{html.escape(r['method'])}</code></td>", |
| f"<td>{html.escape(r['base_model'])}</td>", |
| f"<td>{r['n_domains']}</td>", |
| f"<td>{r['n_seeds']}</td>", |
| f"<td>{html.escape(r['retention_metric'])}</td>", |
| f"<td class='num'>{r['retention_value_pct']}</td>", |
| f"<td><span class='pill {status_cls}'>{html.escape(r['status'])}</span></td>", |
| f"<td>{link}</td>", |
| f"<td class='notes'>{html.escape(r['notes'])}</td>", |
| ] |
| out.append("<tr>" + "".join(cells) + "</tr>") |
| out.append("</tbody></table></div>") |
| return "".join(out) |
|
|
|
|
| sections = [] |
| for suite, label in SUITES.items(): |
| valid = [r for r in rows if r["suite"] == suite and r["status"].startswith("valid")] |
| if valid: |
| sections.append(f"<h2>{html.escape(label)}</h2>{table(valid)}") |
|
|
| disclosed = [r for r in rows if not r["status"].startswith("valid")] |
| sections.append( |
| "<h2>Invalid & incomplete runs (disclosed)</h2>" |
| "<p>Buggy or unfinished runs are <strong>relabeled, not deleted</strong>. Highlight: our early " |
| "O-LoRA arm appeared to win (−2.0% forgetting) until we found a gradient-clipping bug that had " |
| "frozen the model — so O-LoRA is listed as <em>invalid, never validly measured here</em>, not as " |
| "<em>beaten</em>.</p>" + table(disclosed) |
| ) |
|
|
| sections.append( |
| "<h2>Submit your method</h2><ol>" |
| f"<li>Run your method on a suite (<a href='{DATASET}/blob/main/protocol/PROTOCOL.md'>protocol</a>).</li>" |
| f"<li>Score it with <a href='{DATASET}/blob/main/scoring/score.py'><code>scoring/score.py</code></a> " |
| "(<code>--nll</code> or <code>--matrix</code>).</li>" |
| "<li>Open a PR on the dataset repo adding your raw log, a <code>results.csv</code> row, and a provenance line.</li>" |
| "</ol><p>Single-seed submissions are accepted and labeled <code>valid_single_run</code>. If your run " |
| "later turns out buggy, it moves to the disclosed section — that's the deal for everyone, including us.</p>" |
| ) |
|
|
| page = f"""<!doctype html> |
| <html lang="en"><head><meta charset="utf-8"> |
| <meta name="viewport" content="width=device-width, initial-scale=1"> |
| <title>Sequential Forgetting Leaderboard</title> |
| <style> |
| :root {{ --bg:#ffffff; --fg:#1a1a1a; --muted:#6a6a6a; --line:#e3e3e3; --accent:#e17100; |
| --ok-bg:#e8f5e9; --ok-fg:#1b5e20; --bad-bg:#fdecea; --bad-fg:#8c1d18; --code:#f4f4f4; }} |
| @media (prefers-color-scheme: dark) {{ |
| :root {{ --bg:#101010; --fg:#eaeaea; --muted:#9a9a9a; --line:#2b2b2b; --accent:#ff9e2c; |
| --ok-bg:#12331a; --ok-fg:#8fdf9f; --bad-bg:#3a1512; --bad-fg:#ff9d94; --code:#1d1d1d; }} }} |
| body {{ margin:0; background:var(--bg); color:var(--fg); |
| font:16px/1.55 system-ui,-apple-system,Segoe UI,Roboto,sans-serif; }} |
| main {{ max-width:1100px; margin:0 auto; padding:2.5rem 1.25rem 4rem; }} |
| h1 {{ font-size:1.7rem; margin:0 0 .3rem; }} h2 {{ margin:2.2rem 0 .6rem; font-size:1.2rem; }} |
| .sub {{ color:var(--muted); margin:0 0 1.4rem; }} |
| a {{ color:var(--accent); text-decoration:none; }} a:hover {{ text-decoration:underline; }} |
| .tablewrap {{ overflow-x:auto; border:1px solid var(--line); border-radius:8px; }} |
| table {{ border-collapse:collapse; width:100%; font-size:.85rem; }} |
| th,td {{ text-align:left; padding:.5rem .65rem; border-bottom:1px solid var(--line); vertical-align:top; }} |
| th {{ color:var(--muted); font-weight:600; white-space:nowrap; }} |
| tr:last-child td {{ border-bottom:none; }} |
| td.num {{ font-variant-numeric:tabular-nums; font-weight:600; white-space:nowrap; }} |
| td.notes {{ color:var(--muted); min-width:220px; }} |
| code {{ background:var(--code); padding:.1rem .35rem; border-radius:4px; font-size:.82em; }} |
| .pill {{ padding:.12rem .5rem; border-radius:99px; font-size:.75rem; white-space:nowrap; }} |
| .pill.ok {{ background:var(--ok-bg); color:var(--ok-fg); }} |
| .pill.bad {{ background:var(--bad-bg); color:var(--bad-fg); }} |
| footer {{ margin-top:3rem; color:var(--muted); font-size:.85rem; border-top:1px solid var(--line); padding-top:1rem; }} |
| </style></head><body><main> |
| <h1>📉 Sequential Forgetting Leaderboard</h1> |
| <p class="sub">How much does sequential fine-tuning destroy what the model already learned? |
| Lower magnitude = better retention. <strong>Every number links to the raw run file</strong> in the |
| <a href="{DATASET}">benchmark dataset</a>; transcriptions are hand-checked |
| (<a href="{DATASET}/blob/main/results/PROVENANCE.md">provenance</a>). |
| Suites are <strong>not</strong> cross-comparable; each table ranks within its own protocol.</p> |
| {"".join(sections)} |
| <footer>Maintained by <a href="https://modelbrew.ai">ModelBrew</a> — fine-tuning without catastrophic |
| forgetting (patent-pending CRMA adapters). The <code>modular_crma</code> rows are our method; |
| independent replications welcome.</footer> |
| </main></body></html> |
| """ |
|
|
| (HERE / "index.html").write_text(page) |
| print(f"wrote index.html ({len(page)} bytes, {len(rows)} rows)") |
|
|
