Add kind/mode dropdowns to data and model catalog tabs

app.py

@@ -2,26 +2,21 @@
 from __future__ import annotations
 
 import json
+import sys
+from pathlib import Path
 
 import gradio as gr
 import pandas as pd
 from huggingface_hub import hf_hub_download
 
+# Make catalog importable when the dashboard is launched from any directory.
+sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
 
-DATASET_REPO = "stride-influence/stride-applications-data"
-MODEL_REPO = "stride-influence/stride-applications-models"
+from applications.infra.catalog import DataCatalog, ModelCatalog
 
 
-def _parse_contamination_rate(path: str) -> str | None:
-    """Extract contamination rate from a catalog path, e.g. '1pct' → '1%', '0pt5pct' → '0.5%'."""
-    import re
-    m = re.search(r'(\d+)pt(\d+)pct', path)
-    if m:
-        return f"{m.group(1)}.{m.group(2)}%"
-    m = re.search(r'(\d+)pct', path)
-    if m:
-        return f"{m.group(1)}%"
-    return None
+DATASET_REPO = "stride-influence/stride-applications-data"
+MODEL_REPO = "stride-influence/stride-applications-models"
 
 
 def _try_load(repo_id: str, filename: str, repo_type: str):
@@ -36,23 +31,65 @@ def _try_load(repo_id: str, filename: str, repo_type: str):
         return None
 
 
-def load_data_catalog() -> pd.DataFrame:
-    entries = _try_load(DATASET_REPO, "data_catalog.json", "dataset") or []
+DATA_KIND_OPTIONS = ["All", "benchmark_split", "contamination_manifest", "eval_config",
+                     "eval_results", "proxy_corpus", "training_pool"]
+MODEL_MODE_OPTIONS = ["All", "contaminated", "clean"]
+
+
+def load_data_catalog(kind_filter: str = "All") -> pd.DataFrame:
+    try:
+        cat = DataCatalog(repo_id=DATASET_REPO).fetch(verbose=False)
+        entries = [
+            {
+                "path": e.path,
+                "kind": e.kind,
+                "version": e.version,
+                "n_examples": e.n_examples,
+                "n_tokens": e.n_tokens,
+                "seed": e.seed,
+                "status": e.status,
+                "description": e.description,
+            }
+            for e in cat.entries
+        ]
+    except Exception:
+        entries = []
+
     if not entries:
         return pd.DataFrame(
             columns=["path", "kind", "version", "n_examples", "n_tokens", "seed", "status", "description"]
         )
     df = pd.DataFrame(entries)
-    df["contamination_rate"] = df["path"].apply(_parse_contamination_rate)
-    df["path"] = df["path"].apply(
-        lambda p: f'<a href="https://huggingface.co/datasets/{DATASET_REPO}/blob/main/{p}" target="_blank">{p}</a>'
-    )
-    cols = ["path", "kind", "contamination_rate", "version", "n_examples", "seed", "status", "description"]
+    if kind_filter != "All":
+        df = df[df["kind"] == kind_filter]
+    cols = ["path", "kind", "version", "n_examples", "n_tokens", "seed", "status", "description"]
     return df[[c for c in cols if c in df.columns]]
 
 
-def load_model_catalog(show_deleted: bool = False, show_smoke: bool = False) -> pd.DataFrame:
-    entries = _try_load(MODEL_REPO, "model_catalog.json", "model") or []
+def load_model_catalog(show_deleted: bool = False, show_smoke: bool = False,
+                       mode_filter: str = "All") -> pd.DataFrame:
+    try:
+        cat = ModelCatalog(repo_id=MODEL_REPO).fetch(verbose=False)
+        entries = [
+            {
+                "name": e.name,
+                "status": e.status,
+                "mode": e.mode,
+                "benchmark": e.benchmark,
+                "contamination_rate": e.contamination_rate,
+                "contamination_seed": e.contamination_seed,
+                "accuracy_overall": e.accuracy_overall,
+                "accuracy_leaked": e.accuracy_leaked,
+                "accuracy_nonleaked": e.accuracy_nonleaked,
+                "proxy_dataset": e.proxy_dataset,
+                "base_model": e.base_model,
+                "epochs": e._cfg("epochs"),
+            }
+            for e in cat.entries
+        ]
+    except Exception:
+        entries = []
+
     if not entries:
         return pd.DataFrame(
             columns=["name", "status", "mode", "benchmark", "contamination_rate",
@@ -60,29 +97,17 @@ def load_model_catalog(show_deleted: bool = False, show_smoke: bool = False) ->
                      "proxy_dataset", "base_model"]
         )
     df = pd.DataFrame(entries)
-    # Hoist nested config/metrics fields to top-level columns
-    for nested_col, fields in [
-        ("config", ["contamination_rate", "contamination_seed", "lr", "epochs", "base_model", "proxy_dataset"]),
-        ("metrics", ["accuracy_overall", "accuracy_leaked", "accuracy_nonleaked"]),
-    ]:
-        if nested_col in df.columns:
-            nested = df[nested_col].apply(lambda x: x if isinstance(x, dict) else {})
-            for field in fields:
-                if field not in df.columns:
-                    df[field] = nested.apply(lambda x: x.get(field))
     if not show_deleted:
-        # Hide both status=DELETED and physically archived models (deleted/ prefix)
         is_deleted = (df.get("status", pd.Series(["VALID"] * len(df))) == "DELETED") | \
                      df["name"].str.startswith("deleted/")
         df = df[~is_deleted]
     if not show_smoke:
         df = df[~df["name"].str.startswith("smoke/")]
-    df["name"] = df["name"].apply(
-        lambda n: f'<a href="https://huggingface.co/{MODEL_REPO}/tree/main/{n}" target="_blank">{n.split("/")[-1]}</a>'
-    )
-    cols = ["name", "status", "contamination_rate", "contamination_seed",
+    if mode_filter != "All":
+        df = df[df["mode"] == mode_filter]
+    cols = ["name", "status", "mode", "benchmark", "contamination_rate", "contamination_seed",
             "accuracy_overall", "accuracy_leaked", "accuracy_nonleaked",
-            "lr", "epochs", "base_model", "proxy_dataset"]
+            "proxy_dataset", "base_model", "epochs"]
     return df[[c for c in cols if c in df.columns]]
 
 
@@ -104,9 +129,14 @@ def load_queue_status():
     return summary, df
 
 
-def refresh_all(show_deleted: bool, show_smoke: bool):
+def refresh_all(show_deleted: bool, show_smoke: bool, kind_filter: str, mode_filter: str):
     summary, queue_df = load_queue_status()
-    return load_data_catalog(), load_model_catalog(show_deleted, show_smoke), summary, queue_df
+    return (
+        load_data_catalog(kind_filter),
+        load_model_catalog(show_deleted, show_smoke, mode_filter),
+        summary,
+        queue_df,
+    )
 
 
 with gr.Blocks(title="STRIDE Applications") as demo:
@@ -124,20 +154,25 @@ with gr.Blocks(title="STRIDE Applications") as demo:
         show_smoke = gr.Checkbox(label="Show smoke-test models", value=False)
 
     with gr.Tab("Data catalog"):
-        data_tbl = gr.DataFrame(interactive=False, wrap=True, datatype="html")
+        kind_filter = gr.Dropdown(choices=DATA_KIND_OPTIONS, value="All", label="Kind")
+        data_tbl = gr.DataFrame(interactive=False, wrap=True)
 
     with gr.Tab("Model catalog"):
-        model_tbl = gr.DataFrame(interactive=False, wrap=True, datatype="html")
+        mode_filter = gr.Dropdown(choices=MODEL_MODE_OPTIONS, value="All", label="Mode")
+        model_tbl = gr.DataFrame(interactive=False, wrap=True)
 
     with gr.Tab("GPU queue"):
         queue_md = gr.Markdown()
         queue_tbl = gr.DataFrame(interactive=False, wrap=True)
 
+    inputs = [show_deleted, show_smoke, kind_filter, mode_filter]
     outputs = [data_tbl, model_tbl, queue_md, queue_tbl]
-    demo.load(fn=refresh_all, inputs=[show_deleted, show_smoke], outputs=outputs)
-    refresh_btn.click(fn=refresh_all, inputs=[show_deleted, show_smoke], outputs=outputs)
-    show_deleted.change(fn=refresh_all, inputs=[show_deleted, show_smoke], outputs=outputs)
-    show_smoke.change(fn=refresh_all, inputs=[show_deleted, show_smoke], outputs=outputs)
+    demo.load(fn=refresh_all, inputs=inputs, outputs=outputs)
+    refresh_btn.click(fn=refresh_all, inputs=inputs, outputs=outputs)
+    show_deleted.change(fn=refresh_all, inputs=inputs, outputs=outputs)
+    show_smoke.change(fn=refresh_all, inputs=inputs, outputs=outputs)
+    kind_filter.change(fn=refresh_all, inputs=inputs, outputs=outputs)
+    mode_filter.change(fn=refresh_all, inputs=inputs, outputs=outputs)
 
 
 if __name__ == "__main__":