Upload folder using huggingface_hub

app.py

@@ -288,34 +288,20 @@ activation-level probing or SAEs needed. This is what we test on
         detail_btn = gr.Button("Show splits")
         detail_table = gr.Dataframe(headers=["Split", "Accuracy", "N"], interactive=False)
 
-    with gr.Accordion("Eval Sets Info", open=False):
-        gr.Markdown("""
-**Fixed eval sets** (seed=42, cached, deterministic — all models evaluated on identical examples):
-
-| Split Type | Splits | Examples | Description |
-|-----------|--------|----------|-------------|
-| Quirke cascades (add) | S0–S6 | 250 each | Carry cascade depth 0–6 |
-| Quirke cascades (sub) | M0–M5 | 250 each | Borrow cascade depth 0–5 (M6 impossible for 6-digit) |
-| Hot carry chains | C3–C6 | 250 each | Varied answer digits (not just 0s) |
-| Hot borrow chains | B3–B5 | 250 each | Varied answer digits (not just 9s) |
-| Random | add_random, sub_random | 1000 each | Uniform random |
-
-**Total**: 1400 examples (add_sub), 750 examples (add-only)
-""")
+    with gr.Accordion("About This Study", open=False):
+        eval_info_md = gr.Markdown("")
 
     def get_queue_status_text(n_models):
         """Show live queue status from HF-uploaded queue_status.json."""
-        EXPECTED = 90
-
-        # Try to read live queue status
+        # Try to read live queue status (pushed by upload_status daemon)
         try:
             path = hf_hub_download(MODEL_REPO, "queue_status.json",
                                    local_dir="/tmp/hf_dash_cache")
             with open(path) as f:
                 qs = json.load(f)
 
-            total = qs.get("total", EXPECTED)
-            done = qs.get("done", 0)
+            total = qs.get("total", n_models)  # fall back to HF model count
+            done = qs.get("done", n_models)
             failed = qs.get("failed", 0)
             running = qs.get("running", 0)
             pending = qs.get("pending", 0)
@@ -365,6 +351,55 @@ activation-level probing or SAEs needed. This is what we test on
                 f"`{bar}`"
             )
 
+    def build_eval_info(models):
+        """Build eval set description from actual model metadata."""
+        # Try to get eval config from first available model
+        n_per_split = "?"
+        n_digits = 6
+        splits = []
+        total = "?"
+        for m in models:
+            metrics = m.get("metrics", {})
+            for key in ("sft_eval", "sorl_eval"):
+                cfg = metrics.get(key, {}).get("config", {})
+                if cfg.get("n_per_split"):
+                    n_per_split = cfg["n_per_split"]
+                    n_digits = cfg.get("n_digits", 6)
+                    total = metrics[key].get("summary", {}).get("total_examples", "?")
+                    splits = list(metrics[key].get("splits", {}).keys())
+                    break
+            if splits:
+                break
+
+        n_add_cascade = len([s for s in splits if s.startswith("add_S")])
+        n_sub_cascade = len([s for s in splits if s.startswith("sub_M")])
+        n_hot_carry = len([s for s in splits if s.startswith("add_C")])
+        n_hot_borrow = len([s for s in splits if s.startswith("sub_B")])
+
+        return f"""**Replication of [Quirke et al. 2024](https://arxiv.org/abs/2402.02619)** — \
+understanding addition and subtraction in transformers.
+
+We train tiny Qwen3 models (2L/3H/510d, ~8M transformer params) from scratch on \
+{n_digits}-digit arithmetic. SoRL v1 (info-gain loss) adds learnable "abstraction tokens" \
+every K positions. The hypothesis: SoRL externalizes carry/borrow circuits that Quirke \
+found via activation-level analysis as explicit, interpretable tokens.
+
+**Eval**: autoregressive (errors propagate, no teacher forcing). Fixed eval sets (seed=42, cached).
+
+| Split Type | Splits | Examples | Description |
+|-----------|--------|----------|-------------|
+| Carry cascades | S0–S{n_add_cascade - 1} | {n_per_split} each | Carry cascade depth (Quirke §3.2) |
+| Borrow cascades | M0–M{n_sub_cascade - 1} | {n_per_split} each | Borrow cascade depth (Quirke §3.3) |
+| Hot carry chains | C3–C{2 + n_hot_carry} | {n_per_split} each | Cascades with varied answer digits |
+| Hot borrow chains | B3–B{2 + n_hot_borrow} | {n_per_split} each | Borrow cascades with varied digits |
+| Random | add\\_random, sub\\_random | {n_per_split * 4} each | Uniform random |
+
+**Total**: {total} examples per eval. \
+[Paper](https://arxiv.org/abs/2402.02619) · \
+[Models](https://huggingface.co/thoughtworks/arithmetic-sorl) · \
+[Data](https://huggingface.co/datasets/thoughtworks/arithmetic-sorl-data) · \
+[Code](https://github.com/thoughtworks/mod_gpt)"""
+
     def on_refresh(arch):
         models = fetch_all_models()
         df = build_comparison_table(models, arch_filter=arch, enriched_only=False)
@@ -372,6 +407,7 @@ activation-level probing or SAEs needed. This is what we test on
         n_models = len(models)
         summary = f"**{n_models}** models | Arch filter: {arch}"
         q_status = get_queue_status_text(n_models)
+        eval_info = build_eval_info(models)
 
         main_cols = ["Ops", "Data", "Arch", "Baseline", "SoRL", "Config", "B_wandb", "S_wandb"]
 
@@ -380,7 +416,7 @@ activation-level probing or SAEs needed. This is what we test on
                      if (c.startswith("B_") or c.startswith("S_")) and "wandb" not in c]
         hard_df = df[["Ops", "Data", "Config"] + hard_cols] if len(df) > 0 else pd.DataFrame()
 
-        return models, summary, q_status, main_df, hard_df
+        return models, summary, q_status, main_df, hard_df, eval_info
 
     def on_detail(models, name):
         return build_detailed_splits(models, name.strip())
@@ -388,13 +424,13 @@ activation-level probing or SAEs needed. This is what we test on
     refresh_btn.click(
         on_refresh,
         inputs=[arch_filter],
-        outputs=[models_state, summary_text, queue_status, main_table, hard_table],
+        outputs=[models_state, summary_text, queue_status, main_table, hard_table, eval_info_md],
     )
 
     arch_filter.change(
         on_refresh,
         inputs=[arch_filter],
-        outputs=[models_state, summary_text, queue_status, main_table, hard_table],
+        outputs=[models_state, summary_text, queue_status, main_table, hard_table, eval_info_md],
     )
 
     detail_btn.click(on_detail, inputs=[models_state, model_selector], outputs=[detail_table])