Spaces:

ModelBrew
/

forgetting-leaderboard

Running

File size: 6,348 Bytes

0d70486

"""Generate static index.html for the leaderboard from results.csv.

Static HF Spaces are free; this replaces the Gradio app (kept as app.py for a
future PRO upgrade). Run: python3 build_static.py
"""
import csv
import html
from pathlib import Path

HERE = Path(__file__).parent
DATASET = "https://huggingface.co/datasets/ModelBrew/sequential-forgetting-benchmark"

SUITES = {
    "5-domain-realworld": "Suite A — 5 real-world domains · Mistral-7B · 3 seeds",
    "4-domain-MLCF": "Suite B — Medical→Legal→Code→Finance",
    "4-domain-MLCF-history": "Suite B history — CL-technique stacks (within-version comparisons only)",
    "mquake-5skill-vault": "Suite C — MQuAKE 5-skill retention · Qwen3-4B",
}

rows = list(csv.DictReader(open(HERE / "results.csv")))


def table(subset):
    if not subset:
        return ""
    subset = sorted(subset, key=lambda r: abs(float(r["retention_value_pct"])))
    out = ["<div class='tablewrap'><table><thead><tr>"]
    heads = ["method", "base model", "domains", "seeds", "metric", "value %", "status", "source", "notes"]
    out += [f"<th>{h}</th>" for h in heads] + ["</tr></thead><tbody>"]
    for r in subset:
        src = r["source_file"]
        link = f"<a href='{DATASET}/blob/main/results/{html.escape(src)}'>{html.escape(src.split('/')[-1])}</a>"
        status_cls = "ok" if r["status"].startswith("valid") else "bad"
        cells = [
            f"<td><code>{html.escape(r['method'])}</code></td>",
            f"<td>{html.escape(r['base_model'])}</td>",
            f"<td>{r['n_domains']}</td>",
            f"<td>{r['n_seeds']}</td>",
            f"<td>{html.escape(r['retention_metric'])}</td>",
            f"<td class='num'>{r['retention_value_pct']}</td>",
            f"<td><span class='pill {status_cls}'>{html.escape(r['status'])}</span></td>",
            f"<td>{link}</td>",
            f"<td class='notes'>{html.escape(r['notes'])}</td>",
        ]
        out.append("<tr>" + "".join(cells) + "</tr>")
    out.append("</tbody></table></div>")
    return "".join(out)


sections = []
for suite, label in SUITES.items():
    valid = [r for r in rows if r["suite"] == suite and r["status"].startswith("valid")]
    if valid:
        sections.append(f"<h2>{html.escape(label)}</h2>{table(valid)}")

disclosed = [r for r in rows if not r["status"].startswith("valid")]
sections.append(
    "<h2>Invalid &amp; incomplete runs (disclosed)</h2>"
    "<p>Buggy or unfinished runs are <strong>relabeled, not deleted</strong>. Highlight: our early "
    "O-LoRA arm appeared to win (−2.0% forgetting) until we found a gradient-clipping bug that had "
    "frozen the model — so O-LoRA is listed as <em>invalid, never validly measured here</em>, not as "
    "<em>beaten</em>.</p>" + table(disclosed)
)

sections.append(
    "<h2>Submit your method</h2><ol>"
    f"<li>Run your method on a suite (<a href='{DATASET}/blob/main/protocol/PROTOCOL.md'>protocol</a>).</li>"
    f"<li>Score it with <a href='{DATASET}/blob/main/scoring/score.py'><code>scoring/score.py</code></a> "
    "(<code>--nll</code> or <code>--matrix</code>).</li>"
    "<li>Open a PR on the dataset repo adding your raw log, a <code>results.csv</code> row, and a provenance line.</li>"
    "</ol><p>Single-seed submissions are accepted and labeled <code>valid_single_run</code>. If your run "
    "later turns out buggy, it moves to the disclosed section — that's the deal for everyone, including us.</p>"
)

page = f"""<!doctype html>
<html lang="en"><head><meta charset="utf-8">
<meta name="viewport" content="width=device-width, initial-scale=1">
<title>Sequential Forgetting Leaderboard</title>
<style>
:root {{ --bg:#ffffff; --fg:#1a1a1a; --muted:#6a6a6a; --line:#e3e3e3; --accent:#e17100;
         --ok-bg:#e8f5e9; --ok-fg:#1b5e20; --bad-bg:#fdecea; --bad-fg:#8c1d18; --code:#f4f4f4; }}
@media (prefers-color-scheme: dark) {{
  :root {{ --bg:#101010; --fg:#eaeaea; --muted:#9a9a9a; --line:#2b2b2b; --accent:#ff9e2c;
           --ok-bg:#12331a; --ok-fg:#8fdf9f; --bad-bg:#3a1512; --bad-fg:#ff9d94; --code:#1d1d1d; }} }}
body {{ margin:0; background:var(--bg); color:var(--fg);
       font:16px/1.55 system-ui,-apple-system,Segoe UI,Roboto,sans-serif; }}
main {{ max-width:1100px; margin:0 auto; padding:2.5rem 1.25rem 4rem; }}
h1 {{ font-size:1.7rem; margin:0 0 .3rem; }} h2 {{ margin:2.2rem 0 .6rem; font-size:1.2rem; }}
.sub {{ color:var(--muted); margin:0 0 1.4rem; }}
a {{ color:var(--accent); text-decoration:none; }} a:hover {{ text-decoration:underline; }}
.tablewrap {{ overflow-x:auto; border:1px solid var(--line); border-radius:8px; }}
table {{ border-collapse:collapse; width:100%; font-size:.85rem; }}
th,td {{ text-align:left; padding:.5rem .65rem; border-bottom:1px solid var(--line); vertical-align:top; }}
th {{ color:var(--muted); font-weight:600; white-space:nowrap; }}
tr:last-child td {{ border-bottom:none; }}
td.num {{ font-variant-numeric:tabular-nums; font-weight:600; white-space:nowrap; }}
td.notes {{ color:var(--muted); min-width:220px; }}
code {{ background:var(--code); padding:.1rem .35rem; border-radius:4px; font-size:.82em; }}
.pill {{ padding:.12rem .5rem; border-radius:99px; font-size:.75rem; white-space:nowrap; }}
.pill.ok {{ background:var(--ok-bg); color:var(--ok-fg); }}
.pill.bad {{ background:var(--bad-bg); color:var(--bad-fg); }}
footer {{ margin-top:3rem; color:var(--muted); font-size:.85rem; border-top:1px solid var(--line); padding-top:1rem; }}
</style></head><body><main>
<h1>📉 Sequential Forgetting Leaderboard</h1>
<p class="sub">How much does sequential fine-tuning destroy what the model already learned?
Lower magnitude = better retention. <strong>Every number links to the raw run file</strong> in the
<a href="{DATASET}">benchmark dataset</a>; transcriptions are hand-checked
(<a href="{DATASET}/blob/main/results/PROVENANCE.md">provenance</a>).
Suites are <strong>not</strong> cross-comparable; each table ranks within its own protocol.</p>
{"".join(sections)}
<footer>Maintained by <a href="https://modelbrew.ai">ModelBrew</a> — fine-tuning without catastrophic
forgetting (patent-pending CRMA adapters). The <code>modular_crma</code> rows are our method;
independent replications welcome.</footer>
</main></body></html>
"""

(HERE / "index.html").write_text(page)
print(f"wrote index.html ({len(page)} bytes, {len(rows)} rows)")