"""Generate static index.html for the leaderboard from results.csv.
Static HF Spaces are free; this replaces the Gradio app (kept as app.py for a
future PRO upgrade). Run: python3 build_static.py
"""
import csv
import html
from pathlib import Path
HERE = Path(__file__).parent
DATASET = "https://huggingface.co/datasets/ModelBrew/sequential-forgetting-benchmark"
SUITES = {
"5-domain-realworld": "Suite A — 5 real-world domains · Mistral-7B · 3 seeds",
"4-domain-MLCF": "Suite B — Medical→Legal→Code→Finance",
"4-domain-MLCF-history": "Suite B history — CL-technique stacks (within-version comparisons only)",
"mquake-5skill-vault": "Suite C — MQuAKE 5-skill retention · Qwen3-4B",
}
rows = list(csv.DictReader(open(HERE / "results.csv")))
def table(subset):
if not subset:
return ""
subset = sorted(subset, key=lambda r: abs(float(r["retention_value_pct"])))
out = ["
"]
for r in subset:
src = r["source_file"]
link = f"{html.escape(src.split('/')[-1])}"
status_cls = "ok" if r["status"].startswith("valid") else "bad"
cells = [
f"
{html.escape(r['method'])}
",
f"
{html.escape(r['base_model'])}
",
f"
{r['n_domains']}
",
f"
{r['n_seeds']}
",
f"
{html.escape(r['retention_metric'])}
",
f"
{r['retention_value_pct']}
",
f"
{html.escape(r['status'])}
",
f"
{link}
",
f"
{html.escape(r['notes'])}
",
]
out.append("
" + "".join(cells) + "
")
out.append("
")
return "".join(out)
sections = []
for suite, label in SUITES.items():
valid = [r for r in rows if r["suite"] == suite and r["status"].startswith("valid")]
if valid:
sections.append(f"
{html.escape(label)}
{table(valid)}")
disclosed = [r for r in rows if not r["status"].startswith("valid")]
sections.append(
"
Invalid & incomplete runs (disclosed)
"
"
Buggy or unfinished runs are relabeled, not deleted. Highlight: our early "
"O-LoRA arm appeared to win (−2.0% forgetting) until we found a gradient-clipping bug that had "
"frozen the model — so O-LoRA is listed as invalid, never validly measured here, not as "
"beaten.
Open a PR on the dataset repo adding your raw log, a results.csv row, and a provenance line.
"
"
Single-seed submissions are accepted and labeled valid_single_run. If your run "
"later turns out buggy, it moves to the disclosed section — that's the deal for everyone, including us.
"
)
page = f"""
Sequential Forgetting Leaderboard
📉 Sequential Forgetting Leaderboard
How much does sequential fine-tuning destroy what the model already learned?
Lower magnitude = better retention. Every number links to the raw run file in the
benchmark dataset; transcriptions are hand-checked
(provenance).
Suites are not cross-comparable; each table ranks within its own protocol.