Publish the arXiv-linked v1.3 leaderboard

Embeds the bound stability analysis, exposes the 342 optimizer-facing reward maps while keeping the 84-task transfer split out of the reward endpoint, and links the canonical arXiv record.

README.md

@@ -83,15 +83,18 @@ The Space exposes four named endpoints:
 |---|---|
 | `/score_completion` | Score one completion on one official task |
 | `/score_submission` | Score a complete JSON or JSONL artifact supplied as text |
-| `/training_reward` | Query one of 426 anchor-disjoint development reward maps |
+| `/training_reward` | Query one of 342 anchor-disjoint train/validation reward maps |
 | `/score_uploaded_submission` | Score an uploaded artifact and return a report |
 
 Use **Use via API** in the running Space for generated Python, JavaScript, and curl clients. For
 high-throughput RL, use the local deterministic reward function. The linked dataset includes
 ready-to-load SFT, DPO, and GRPO views plus runnable LoRA recipes for Hugging Face Jobs.
 
+The [prospective reward-transfer protocol](https://github.com/josefchen/flavourbench/blob/main/docs/reward-transfer-study.md)
+defines the 84-task transfer split, seeds, six confirmatory contrasts, and multiplicity control.
+
 [Dataset and lab kit](https://huggingface.co/datasets/josefchen/flavourbench)   
-[Paper](https://github.com/josefchen/flavourbench/blob/main/paper/build/flavourbench.pdf)   
+[Paper](https://arxiv.org/abs/2608.20574)   
 [Source](https://github.com/josefchen/flavourbench)
 
 Josef Chen, Independent Researcher<br>
@@ -101,6 +104,11 @@ Erim Hayretci, Imperial College London
 @article{chen2026flavourbench,
   title  = {FlavourBench: Ranking Frontier Language Models with Executable Culinary Ground Truth},
   author = {Chen, Josef and Hayretci, Erim},
-  year   = {2026}
+  journal = {arXiv preprint arXiv:2608.20574},
+  year   = {2026},
+  eprint = {2608.20574},
+  archivePrefix = {arXiv},
+  primaryClass = {cs.CL},
+  url = {https://arxiv.org/abs/2608.20574}
 }
 ```

app.py

@@ -30,7 +30,7 @@ BUNDLE_PATH = Path(
 
 RUST = "#A83D34"
 INK = "#161817"
-PAPER_URL = "https://github.com/josefchen/flavourbench/blob/main/paper/build/flavourbench.pdf"
+PAPER_URL = "https://arxiv.org/abs/2608.20574"
 DATASET_URL = "https://huggingface.co/datasets/josefchen/flavourbench"
 SOURCE_URL = "https://github.com/josefchen/flavourbench"
 DATASET_RESULTS_URL = (
@@ -1140,6 +1140,7 @@ MODELS = BUNDLE["models"]
 TASKS = BUNDLE["tasks"]
 LAB_TASKS = BUNDLE.get("lab_tasks", [])
 PAIRWISE = BUNDLE["pairwise_comparisons"]
+STABILITY = BUNDLE["stability_analysis"]
 MODEL_COUNT = len(MODELS)
 TASK_COUNT = len(TASKS)
 PAIR_COUNT = len(PAIRWISE)
@@ -1658,6 +1659,23 @@ def _insights_html() -> str:
             "mean_difference", float(DISPLAY_MODELS[0]["flavourbench_score"]) - chance
         )
     )
+    stability_rows = []
+    for row in STABILITY["task_count_stability"]:
+        rank = row["metrics"]["rank_spearman"]
+        top_five = row["metrics"]["top_5_overlap"]
+        leader = row["metrics"]["top_1_preserved"]
+        stability_rows.append(
+            "<tr>"
+            f"<td>{int(row['tasks'])}</td>"
+            f"<td>{float(rank['median']):.3f}</td>"
+            f"<td>{float(rank['p2_5']):.3f}–{float(rank['p97_5']):.3f}</td>"
+            f"<td>{float(top_five['median']) * 100:.0f}%</td>"
+            f"<td>{float(leader['mean']) * 100:.1f}%</td>"
+            "</tr>"
+        )
+    variance = STABILITY["variance_partition"]
+    generalizability = float(variance["relative_decision_generalizability_at_534_tasks"])
+    tasks_for_g_90 = int(variance["estimated_balanced_tasks_for_relative_g_0_90"])
     return f"""
     <div class="fb-insight-layout">
       <section>
@@ -1675,6 +1693,29 @@ def _insights_html() -> str:
         </div>
       </aside>
     </div>
+    <section class="fb-family-insight">
+      <h3>Why 534 tasks?</h3>
+      <p>The crossed design's descriptive relative-decision generalizability is
+      <strong>{generalizability:.3f}</strong>; the same variance model estimates
+      <strong>{tasks_for_g_90}</strong> balanced tasks for 0.90. The table below repeatedly takes
+      score-blind, balanced subsets and compares them with the complete point order.</p>
+      <div class="fb-table-wrap">
+        <table class="fb-table">
+          <caption>5,000 family-by-panel stratified subsets at each non-complete task count</caption>
+          <thead><tr>
+            <th scope="col">Tasks</th>
+            <th scope="col">Median rank ρ</th>
+            <th scope="col">Empirical 95%</th>
+            <th scope="col">Top-five overlap</th>
+            <th scope="col">Point leader kept</th>
+          </tr></thead>
+          <tbody>{"".join(stability_rows)}</tbody>
+        </table>
+      </div>
+      <p class="fb-resolution-note">This is a precision diagnostic relative to the complete
+      release, not a post-hoc power claim. The point leader remains unstable in smaller subsets;
+      the simultaneous score bands and rank intervals remain the inferential result.</p>
+    </section>
     <section class="fb-family-insight">
       <h3>Where the leading labs differ</h3>
       <p>Scores are out of 100. Red marks each column leader.</p>
@@ -2417,8 +2458,9 @@ and GRPO recipes in the source repository. The full publication contract is in t
                     sign flips, Holm correction, exact tests against a random legal choice, bootstrap rank intervals,
                     and an independently compiled second panel.</p>
                     <h3>Training boundary</h3>
-                    <p>The 426 SFT, DPO, and GRPO development maps use anchors that do not occur in
-                    the {TASK_COUNT}-task leaderboard. Training cannot query the official test reward maps by accident.</p>
+                    <p>The 342 optimizer-facing SFT, DPO, and GRPO maps use anchors that do not
+                    occur in the 84-task transfer split or the {TASK_COUNT}-task leaderboard.
+                    Training cannot query either evaluation map through the reward endpoint.</p>
                     <aside class="fb-evidence">
                       <strong>Content-addressed release</strong><br>
                       <span class="fb-hash">{BUNDLE["release_artifact_sha256"]}</span><br><br>
@@ -2467,7 +2509,7 @@ pytest -q tests/lab_cli_test.py tests/hf_lab_space_api_test.py
             <span>Erim Hayretci, Imperial College London</span>
           </div>
           <nav class="fb-footer-links" aria-label="Project resources">
-            <a href="https://github.com/josefchen/flavourbench/blob/main/paper/build/flavourbench.pdf">Paper</a>
+            <a href="https://arxiv.org/abs/2608.20574">Paper</a>
             <a href="https://huggingface.co/datasets/josefchen/flavourbench">Dataset</a>
             <a href="https://github.com/josefchen/flavourbench">Source</a>
           </nav>

build_complete_core_space_bundle.py

@@ -11,6 +11,9 @@ from typing import Any
 HERE = Path(__file__).resolve().parent
 DEFAULT_DATASET = HERE.parent / "dataset" / "data-complete-core"
 DEFAULT_LAB_DATASET = HERE.parent / "dataset" / "data-lab"
+DEFAULT_STABILITY = (
+    HERE.parents[1] / "paper/generated/complete-core/complete-core-stability-analysis.json"
+)
 DEFAULT_OUTPUT = HERE / "data-complete-core" / "flavourbench-complete-core-space.json"
 
 
@@ -53,7 +56,12 @@ def _jsonl(path: Path, *, expected_sha256: str, expected_rows: int) -> list[dict
     return rows
 
 
-def build_bundle(*, dataset_directory: Path, lab_dataset_directory: Path) -> dict[str, Any]:
+def build_bundle(
+    *,
+    dataset_directory: Path,
+    lab_dataset_directory: Path,
+    stability_path: Path = DEFAULT_STABILITY,
+) -> dict[str, Any]:
     manifest = _load(dataset_directory / "DATA_MANIFEST.json")
     if (
         not _semantic_valid(manifest)
@@ -92,8 +100,8 @@ def build_bundle(*, dataset_directory: Path, lab_dataset_directory: Path) -> dic
     lab_manifest = _load(lab_dataset_directory / "DATA_MANIFEST.json")
     if (
         not _semantic_valid(lab_manifest)
-        or lab_manifest.get("schema_version") != "flavourbench-lab-dataset-v1"
-        or lab_manifest.get("status") != "development_reward_maps_not_official_test"
+        or lab_manifest.get("schema_version") != "flavourbench-lab-dataset-v2"
+        or lab_manifest.get("status") != "preregistered_transfer_maps_not_official_leaderboard_test"
     ):
         raise CompleteCoreSpaceBuildError("lab dataset manifest failed verification")
     lab_records = {str(row["name"]): row for row in lab_manifest["files"]}
@@ -107,14 +115,21 @@ def build_bundle(*, dataset_directory: Path, lab_dataset_directory: Path) -> dic
         )
 
     lab_tasks = [*lab_rows("train_tasks.jsonl"), *lab_rows("validation_tasks.jsonl")]
+    stability = _load(stability_path)
+    if (
+        not _semantic_valid(stability)
+        or stability.get("schema_version") != "flavourbench-task-count-stability-v1"
+        or stability.get("status") != "retrospective_precision_and_stability_analysis"
+    ):
+        raise CompleteCoreSpaceBuildError("task-count stability analysis failed verification")
     official_ids = {str(row["task_id"]) for row in tasks}
     official_anchors = {str(row["anchor_ingredient"]) for row in tasks}
     lab_ids = {str(row["task_id"]) for row in lab_tasks}
     lab_anchors = {str(row["anchor_ingredient"]) for row in lab_tasks}
     if (
-        len(lab_tasks) != 426
-        or len(lab_ids) != 426
-        or len(lab_anchors) != 426
+        len(lab_tasks) != 342
+        or len(lab_ids) != 342
+        or len(lab_anchors) != 342
         or lab_ids & official_ids
         or lab_anchors & official_anchors
     ):
@@ -160,9 +175,11 @@ def build_bundle(*, dataset_directory: Path, lab_dataset_directory: Path) -> dic
             "task_count": len(lab_tasks),
             "train_tasks": int(lab_manifest["counts"]["train_tasks"]),
             "validation_tasks": int(lab_manifest["counts"]["validation_tasks"]),
+            "evaluation_tasks": int(lab_manifest["counts"]["evaluation_tasks"]),
             "official_anchor_overlap": 0,
         },
         "lab_tasks": lab_tasks,
+        "stability_analysis": stability,
         "primary_observations": compact_observations,
         "pairwise_comparisons": pairwise,
     }
@@ -199,12 +216,14 @@ def main() -> None:
     parser = argparse.ArgumentParser(description="Build the final FlavourBench Space bundle")
     parser.add_argument("--dataset-directory", type=Path, default=DEFAULT_DATASET)
     parser.add_argument("--lab-dataset-directory", type=Path, default=DEFAULT_LAB_DATASET)
+    parser.add_argument("--stability", type=Path, default=DEFAULT_STABILITY)
     parser.add_argument("--output", type=Path, default=DEFAULT_OUTPUT)
     parser.add_argument("--check", action="store_true")
     args = parser.parse_args()
     bundle = build_bundle(
         dataset_directory=args.dataset_directory,
         lab_dataset_directory=args.lab_dataset_directory,
+        stability_path=args.stability,
     )
     payload = _bytes(bundle)
     if args.check:

data-complete-core/flavourbench-complete-core-space.json

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:026299fa7f98e2410033533ccc3e25ccd4b9df3281bb8fa34d2fcbf0678f8290
-size 14662246
+oid sha256:7bb9959b2a73d1c5ef2cafe064805ec874b5f4fa91c05b1d005e4b6e22e4d1ff
+size 14353273