File size: 4,003 Bytes
0d70486 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 | """Sequential Forgetting Leaderboard — every number links to a raw run file."""
from pathlib import Path
import gradio as gr
import pandas as pd
HERE = Path(__file__).parent
DATASET_URL = "https://huggingface.co/datasets/ModelBrew/sequential-forgetting-benchmark"
df = pd.read_csv(HERE / "results.csv").fillna("")
SUITE_LABELS = {
"5-domain-realworld": "Suite A — 5 real-world domains · Mistral-7B · 3 seeds",
"4-domain-MLCF": "Suite B — Medical→Legal→Code→Finance",
"4-domain-MLCF-history": "Suite B history — CL-technique stacks (within-version comparisons only)",
"mquake-5skill-vault": "Suite C — MQuAKE 5-skill retention · Qwen3-4B",
}
def linkify(source: str) -> str:
return f"[{source}]({DATASET_URL}/blob/main/results/{source})"
def suite_table(suite: str, valid_only: bool) -> pd.DataFrame:
sub = df[df.suite == suite].copy()
if valid_only:
sub = sub[sub.status.str.startswith("valid")]
else:
sub = sub[~sub.status.str.startswith("valid")]
sub = sub.sort_values("retention_value_pct", key=lambda s: s.abs())
sub["source"] = sub["source_file"].map(linkify)
cols = ["method", "base_model", "n_domains", "n_seeds",
"retention_metric", "retention_value_pct", "status", "source", "notes"]
return sub[cols].reset_index(drop=True)
with gr.Blocks(title="Sequential Forgetting Leaderboard") as demo:
gr.Markdown(
"# 📉 Sequential Forgetting Leaderboard\n"
"How much does sequential fine-tuning destroy what the model already learned? "
"Lower magnitude = better retention. **Every number links to the raw run file** "
f"in the [benchmark dataset]({DATASET_URL}); transcriptions are hand-checked "
f"([provenance]({DATASET_URL}/blob/main/results/PROVENANCE.md)).\n\n"
"Suites are **not** cross-comparable; each table ranks within its own protocol."
)
with gr.Tab("Leaderboard"):
for suite, label in SUITE_LABELS.items():
table = suite_table(suite, valid_only=True)
if len(table):
gr.Markdown(f"### {label}")
gr.Dataframe(table, interactive=False, wrap=True,
datatype=["str"] * len(table.columns))
with gr.Tab("Invalid & incomplete runs (disclosed)"):
gr.Markdown(
"Buggy or unfinished runs are **relabeled, not deleted**. Highlight: our "
"early O-LoRA arm appeared to win (−2.0% forgetting) until we found a "
"gradient-clipping bug that had frozen the model — so O-LoRA is listed as "
"*invalid, never validly measured here*, not as *beaten*."
)
for suite in SUITE_LABELS:
table = suite_table(suite, valid_only=False)
if len(table):
gr.Markdown(f"### {SUITE_LABELS[suite]}")
gr.Dataframe(table, interactive=False, wrap=True,
datatype=["str"] * len(table.columns))
with gr.Tab("Submit your method"):
gr.Markdown(
"1. Run your method on a suite "
f"([protocol]({DATASET_URL}/blob/main/protocol/PROTOCOL.md)).\n"
"2. Score it with "
f"[`scoring/score.py`]({DATASET_URL}/blob/main/scoring/score.py) "
"(`--nll` or `--matrix`).\n"
"3. Open a PR on the dataset repo adding your raw log, a `results.csv` row, "
"and a provenance line.\n\n"
"Single-seed submissions are accepted and labeled `valid_single_run`. "
"If your run later turns out buggy, it moves to the disclosed tab — "
"that's the deal for everyone, including us."
)
gr.Markdown(
"---\nMaintained by [ModelBrew](https://modelbrew.ai) — fine-tuning without "
"catastrophic forgetting (patent-pending CRMA adapters). The `modular_crma` "
"rows are our method; independent replications welcome."
)
if __name__ == "__main__":
demo.launch()
|