External polish: Success Rate label; App Intents section in task creation; Siri-verb timeline; trajectory/outcome eval naming; drop AI-judge table, family cards, blocked families; trim repetition

app.py

@@ -1174,7 +1174,7 @@ HERO = """
     </div>
     <div class="stat-card">
       <div class="stat-num text-accent">71.4%</div>
-      <div class="stat-label">Documented Run</div>
+      <div class="stat-label">Success Rate</div>
     </div>
   </div>
 
@@ -1311,7 +1311,7 @@ def render_timeline(task_id):
             rows.append(_tl_row("ico-user", "👤", "The user asks",
                 f'<div class="tl-card user">“{html.escape(e.get("command", ""))}”</div>'))
         elif ev == "function_call":
-            rows.append(_tl_row("ico-tool", "🛠️", "Model calls a tool",
+            rows.append(_tl_row("ico-tool", "🛠️", "Siri calls a tool",
                 f'<div class="tl-card"><span class="tool-name-chip">{html.escape(e.get("name", "?"))}</span>'
                 f'{_arg_chips(e.get("arguments"))}</div>'))
         elif ev == "tool_result":
@@ -1320,7 +1320,7 @@ def render_timeline(task_id):
                 "Tool failed" if err else "Tool returns",
                 f'<div class="tl-card{" error" if err else ""}">{html.escape(_clip(e.get("content", "")))}</div>'))
         elif ev == "agent_done":
-            rows.append(_tl_row("ico-done", "🤖", "Assistant's final answer",
+            rows.append(_tl_row("ico-done", "🤖", "Siri's final answer",
                 f'<div class="tl-card done">{html.escape(_clip(e.get("final", ""), 600))}</div>'))
         elif ev == "agent_error":
             rows.append(_tl_row("ico-error", "⛔", "Run error",
@@ -1531,6 +1531,41 @@ TOOLS_HTML = """
 </table>
 """
 
+APP_INTENTS_HTML = """
+<div class="section-h">🧩 These tools are App Intents — the way Siri really acts</div>
+<p class="lead">On a real iPhone, Siri does not tap the screen to get things done. It uses <b>App
+Intents</b> — declared, typed actions an app exposes to the system (create a reminder, add a calendar
+event, send a message). Siri selects the right named action and fills its typed parameters. That is
+exactly how SiriBench works: each of the 11 tools is an App-Intents-style action backed by the real
+system frameworks, so a task is authored by choosing <b>which real actions the model may take</b> — never
+by scripting a user interface.</p>
+<div class="ingredients">
+  <div class="ing">
+    <div class="ing-badge">DECLARE</div>
+    <div class="ing-header"><h4>A typed action</h4></div>
+    <p>Each action has a name and typed parameters — e.g. create a reminder with a <code>title</code>.
+    The model is shown the action's shape and must fill it correctly.</p>
+    <div class="ex"><b>e.g.</b> <code>create_reminder(title:)</code></div>
+  </div>
+  <div class="ing">
+    <div class="ing-badge">CALL</div>
+    <div class="ing-header"><h4>Siri selects &amp; invokes it</h4></div>
+    <p>From the user's request, the on-device model picks the right action and supplies its parameters —
+    the same “choose a tool, fill its arguments” pattern Siri uses in production.</p>
+    <div class="ex"><b>e.g.</b> "remind me to call the dentist" → the reminder action</div>
+  </div>
+  <div class="ing">
+    <div class="ing-badge">EXECUTE</div>
+    <div class="ing-header"><h4>The real app runs it</h4></div>
+    <p>The action runs against the real device store via EventKit / Contacts / Messages, so the effect is
+    genuine and can be re-read afterward to score the outcome.</p>
+    <div class="ex"><b>e.g.</b> a real row appears in the Reminders store</div>
+  </div>
+</div>
+<p class="sub-lead">Because it is App-Intents tool-calling — not screen automation — SiriBench measures the
+same action mechanism Siri uses on device, with only the voice front-end removed.</p>
+"""
+
 CAPTURE_SNIPPET = '''// Every step is one ordered JSON line; the model's transcript is logged too.
 Trajectory.record("function_call", ["name": "create_reminder", "arguments": "{\\"title\\":\\"…\\"}"])
 Trajectory.record("tool_result",   ["tool": "create_reminder", "content": "Created reminder '…'."])
@@ -1555,34 +1590,6 @@ FAMILY_DEFS = {
     "Recipient-conditioned drafting": ("📨", "do", "Draft complete, appropriately-toned text for a specific recipient, keeping every required detail."),
 }
 
-BLOCKED_FAMILIES = [
-    ("🖼️ Visual QA",              "Image input to the model",        "Image input is an iOS 27 / macOS 27 model capability — unblocks once the host Mac is on macOS 27."),
-    ("🧾 Receipt parsing",         "Image input to the model",        "Same iOS 27 / macOS 27 image-input requirement."),
-    ("🎨 Image editing",           "Image input and image output",    "Same iOS 27 / macOS 27 requirement, plus image generation."),
-    ("🥽 Spatial",                 "visionOS (Apple Vision Pro)",     "Requires the headset platform — not available on this setup."),
-    ("🗣️ Speech (TTS / ASR)",     "The speech stack",                "Siri's voice assets are never served inside the Simulator, so speech in/out can't run there."),
-    ("📱 On-screen accessibility QA", "Private accessibility APIs",  "Reading live on-screen elements needs private system interfaces."),
-]
-
-def _family_cards():
-    order, by_cat = [], {}
-    for t in TASKS:
-        if t["cat"] not in by_cat:
-            order.append(t["cat"]); by_cat[t["cat"]] = []
-        by_cat[t["cat"]].append(t)
-    cards = []
-    for cat in order:
-        emoji, kind, definition = FAMILY_DEFS[cat]
-        chip = ('<span class="kind-chip kind-dont">pass = holds back</span>' if kind == "dont"
-                else '<span class="kind-chip kind-do">pass = does it right</span>')
-        chips = "".join(
-            f'<span class="task-chip"><span class="dot dot-{t["result"].lower()}"></span>{t["num"]} · {t["id"]}</span>'
-            for t in by_cat[cat])
-        cards.append(
-            f'<div class="fam-card"><div class="fam-name">{emoji} {cat}{chip}</div>'
-            f'<div class="fam-def">{definition}</div>{chips}</div>')
-    return f'<div class="fam-grid">{"".join(cards)}</div>'
-
 def _catalog_table():
     rows = []
     for t in TASKS:
@@ -1598,94 +1605,49 @@ def _catalog_table():
             '<th>The user asks</th><th>What it tests</th><th>Passes when</th><th>Result</th></tr></thead>'
             f'<tbody>{"".join(rows)}</tbody></table>')
 
-def _blocked_table():
-    rows = "".join(f"<tr><td><b>{f}</b></td><td>{needs}</td><td>{why}</td></tr>"
-                   for f, needs, why in BLOCKED_FAMILIES)
-    return ('<table><thead><tr><th>Blocked family</th><th>Input it needs</th><th>Why it is blocked</th></tr></thead>'
-            f'<tbody>{rows}</tbody></table>')
-
-_n_fams = len(FAMILY_DEFS)
-_n_dont = sum(1 for _, k, _ in FAMILY_DEFS.values() if k == "dont")
 CLASSIFY_HTML = f"""
 <div class="section-h">🗂️ How the {len(TASKS)} tasks are classified</div>
-<p class="lead">Every task belongs to exactly one <b>family</b> — the capability it isolates. The same neutral
-agent (same model, same 11 tools, same instructions) runs all of them, so a family's pass/fail says something
-about the <b>model</b>, not about task-specific prompting. {_n_fams} families cover the {len(TASKS)} tasks:
-capability families where the model must <b>do the right thing</b>, and judgment families where passing means
-it <b>holds back</b> (asks, confirms, resists, or keeps it local).</p>
-{_family_cards()}
-
-<div class="section-h">⚖️ Two ways a task is judged</div>
-<div class="judge-cards">
-  <div class="judge-card"><h4>✅ “Should-do” families</h4>
-    <p>The model must take the right actions <i>and</i> leave the right real end-state. Both are checked:
-    the recorded trajectory must show the required tool calls, and the real Reminders/Calendar/Contacts
-    store (or the model's own final answer) must contain the required result.</p></div>
-  <div class="judge-card"><h4>🚫 “Should-not” families</h4>
-    <p>The rubric is <b>inverted</b>: passing means the model <i>refrained</i>. It must NOT send the ambiguous
-    message, NOT wipe the reminders, NOT obey the injected instruction, NOT escalate trivial math to the web.
-    {_n_dont} of the {_n_fams} families work this way — they measure judgment, not capability.</p></div>
-</div>
+<p class="lead">Every task belongs to exactly one <b>family</b> — the capability it isolates. The same
+neutral agent (same model, same 11 App-Intent tools, same instructions) runs all of them, so a family's
+pass or fail reflects the <b>model</b>, not task-specific prompting. Some families test whether Siri
+<b>does the right thing</b>; others (marked “refrain”) test whether it <b>holds back</b> — asks instead
+of guessing, confirms before deleting, resists an injected instruction, or answers locally instead of
+over-escalating.</p>
 
 <div class="section-h">📋 Full catalog — every task, classified</div>
 {_catalog_table()}
-
-<div class="section-h">🚧 Blocked families (defined, not yet scored)</div>
-<p class="lead">Six further families are designed but can't run on this setup — they need image or speech
-input, a newer OS, or private system interfaces. They unblock as the environment catches up.</p>
-{_blocked_table()}
 """
 
 SCORING_HTML = """
-<div class="section-h">⚖️ How a run is scored — and why there is no AI judge</div>
-<p class="lead">Every verdict is a set of fixed, mechanical checks — never another model's opinion. A task
-passes only when <b>both</b> layers below hold, so the score reflects what the model actually <i>did</i>
-and what actually <i>happened on the device</i>, not how convincing its wording sounds.</p>
-
-<div class="judge-cards">
-  <div class="judge-card"><h4>1️⃣ Process — which tools were called</h4>
-    <p>We read the recorded run for evidence that the right tools fired — and that forbidden ones did
-    <i>not</i>. Example: the dentist lookup <b>must</b> read the calendar and <b>must not</b> create a
-    reminder or an event along the way.</p></div>
-  <div class="judge-card"><h4>2️⃣ Outcome — the real end-state</h4>
-    <p>After the run we re-read the actual Reminders / Calendar / Contacts store on the device, and/or
-    require a specific fact in the model's own final answer ("Friday", "Canberra", the booking code
-    "58291", "36"). The model cannot fake this — it is read from the OS itself.</p></div>
-</div>
+<div class="section-h">⚖️ How a run is scored</div>
+<p class="lead">Every verdict comes from fixed, programmatic checks — never another model's opinion.
+A task passes only when <b>both</b> of the layers below hold, so the score reflects what Siri actually
+<i>did</i> and the <i>real end-state on the device</i>, not how convincing its wording sounds.</p>
 
-<div class="section-h">✅ "Should-do" vs 🚫 "should-not" tasks</div>
 <div class="judge-cards">
-  <div class="judge-card"><h4>✅ Should-do: pass = does it right</h4>
-    <p>Most tasks require the right actions <i>and</i> the right end-state: chain the calendar into a
-    reminder, find the booking code, draft the email with the new deadline. Doing nothing fails.</p></div>
-  <div class="judge-card"><h4>🚫 Should-not: pass = holds back</h4>
-    <p>Four families invert the rubric: passing means the model <i>refrained</i> — asked which Alex
-    instead of texting one, refused to wipe every reminder without confirmation, ignored an instruction
-    hidden in a web page, and answered 15% of 240 locally instead of searching the web. This measures
-    judgment, not capability.</p></div>
+  <div class="judge-card"><h4>1️⃣ Trajectory evaluation — which App Intents were called</h4>
+    <p>The recorded trajectory must show the <b>required</b> tool calls and <b>none</b> of the forbidden
+    ones. Example: the dentist lookup must read the calendar, and must not create a reminder or an event
+    along the way.</p></div>
+  <div class="judge-card"><h4>2️⃣ Task-outcome evaluation — the real end-state</h4>
+    <p>After the run, the real Reminders / Calendar / Contacts store is re-read on the device, and/or a
+    specific fact is required in Siri's final answer (“Friday”, “Canberra”, the booking code “58291”,
+    “36”). This is read from the OS itself, so it cannot be faked.</p></div>
 </div>
 
-<div class="section-h">🤖 Why not just use an AI judge?</div>
-<table>
-  <thead><tr><th>Question</th><th>AI-judge benchmark</th><th>SiriBench</th></tr></thead>
-  <tbody>
-    <tr><td>Where does the verdict come from?</td><td>Another model's opinion of the transcript</td><td class="cap-yes">Mechanical checks + a re-read of the real device</td></tr>
-    <tr><td>Same input, same score?</td><td>Varies with the judge, its prompt, its settings</td><td class="cap-yes">Perfectly repeatable given a transcript</td></tr>
-    <tr><td>Can slick wording fake success?</td><td>Sometimes — persuasion can win</td><td class="cap-yes">No — the right action must actually happen</td></tr>
-    <tr><td>What kind of reason do you get?</td><td>Diffuse ("this seems wrong")</td><td class="cap-yes">Exact — which tool, which fact, which count</td></tr>
-  </tbody>
-</table>
-<p class="sub-lead">This is the same real-state-verification philosophy as AndroidWorld and WebArena:
-success is defined by the observable end-state of the real environment, not by a grader's impression.
-One more gate: if a run hits any model or system error, the task fails outright — an error is never
-mistaken for the model "wisely abstaining."</p>
+<div class="section-h">✅ “Should-do” vs 🚫 “should-not” tasks</div>
+<p class="lead">Most tasks pass by <b>doing the right thing</b> — the required tool calls plus the correct
+end-state. Four families <b>invert</b> the rubric, where passing means Siri <i>held back</i>: it asked
+which Alex instead of texting one, refused to wipe every reminder without confirmation, ignored an
+instruction hidden in a web page, and answered “15% of 240” locally instead of searching the web. A final
+gate fails any run that hit a model or system error, so a crash is never mistaken for wise abstention.</p>
 """
 
 FOOTER_HTML = """
 <div class="foot">
   <b>SiriBench</b> — benchmarking Apple's on-device Foundation Model as a tool-calling iOS agent over the
   real system apps.<br/>This Space replays recorded runs; the model itself runs only on a Mac + iOS
-  Simulator. Documented run: 10 / 14 passed (~3B on-device model, iOS 26.4).
+  Simulator. Success rate: 10 / 14 tasks passed (~3B on-device model, iOS 26.4).
 </div>
 """
 
@@ -1760,9 +1722,9 @@ with gr.Blocks(title="SiriBench — iOS agent tasks & trajectories") as demo:
                         swift = gr.Code(label=None, language=None, lines=16, elem_classes=["swift-code"])
 
                 # The friendly step-by-step timeline parsed from trajectory.jsonl
-                gr.HTML('<div class="control-header" style="margin-top:16px;">🎞️ What the model did — step by step</div>'
-                        '<p class="sub-lead">Parsed from the recorded trajectory: the user\'s request, every tool '
-                        'the model called (with its arguments), what each tool returned, the final answer, and the '
+                gr.HTML('<div class="control-header" style="margin-top:16px;">🎞️ What Siri did — step by step</div>'
+                        '<p class="sub-lead">Parsed from the recorded trajectory: the user\'s request, every App '
+                        'Intent Siri called (with its arguments), what each returned, the final answer, and the '
                         'independent re-read of the real device data.</p>')
                 timeline = gr.HTML()
 
@@ -1808,6 +1770,7 @@ with gr.Blocks(title="SiriBench — iOS agent tasks & trajectories") as demo:
         gr.HTML(WORKED)
         gr.HTML(NEUTRAL)
         gr.HTML(TOOLS_HTML)
+        gr.HTML(APP_INTENTS_HTML)
         gr.HTML('<div class="section-h" style="margin-top:14px">In code — authoring a task (Tasks.swift)</div>'
                 '<p class="lead">Concretely, a task is a few lines of Swift: <code>seed</code> sets up the '
                 'iOS world, <code>prompts</code> is what the user says. This is the exact definition behind '
@@ -1819,7 +1782,7 @@ with gr.Blocks(title="SiriBench — iOS agent tasks & trajectories") as demo:
             "### Scope — why some tasks aren't here yet\n"
             "These run on the **iOS 26.4** text-only on-device model. **Visual tasks** (image QA, "
             "receipt parsing, image editing) need **image input to the model — an iOS 27 / macOS 27 "
-            "capability**. The full list of blocked families is in the **Task classification** tab."
+            "capability**, and unblock once the host Mac is on macOS 27."
         )
 
     with gr.Tab("⚖️ How runs are scored"):