Publish paired SIMPLE instruction revisions with preserved native evidence

app.js

@@ -27,6 +27,14 @@ function instructionPanel(t,e){
 }
 
 
+function visualReviewPanel(e){
+ const r=e.visual_review;if(!r)return '';
+ const labels={consistent_pass:'画面与原生通过一致',confirmed_false_positive:'误通过：画面未完成，但原生 verifier 判定通过',consistent_failure:'画面与原生失败一致'};
+ const label=labels[r.status]||'原生条件与画面复核';
+ const text=e.analysis?.visible_completion||r.review||'';
+ return panel('独立视频复核',`<p><strong>${esc(label)}</strong></p><p>${esc(text)}</p><p class="source-note">原生分数保持不变。${external(e.links.independent_review||e.links.analysis,'完整复核记录')}</p>`,'wide-panel');
+}
+
 function failureBadge(r){return `<span class="failure-badge ${esc(r.category)}">${r.status==='provisional'?'可能 · ':''}${esc(r.label)}</span>`;}
 function failureTable(episodes,id){
  return `<details id="${id}" class="failure-list"><summary>查看全部 ${episodes.length} 项失败说明</summary><div class="table-wrap"><table><thead><tr><th>Task</th><th>分类</th><th>失败原因</th></tr></thead><tbody>${episodes.map(e=>`<tr><td><a href="#task/${esc(e.task_key)}">${esc(data.tasks.find(t=>t.key===e.task_key)?.title || e.task_key)}</a><small>${esc(data.tasks.find(t=>t.key===e.task_key)?.display_key || e.task_key)}</small></td><td>${failureBadge(e.failure_review)}</td><td>${esc(e.failure_review.reason)}</td></tr>`).join('')}</tbody></table></div></details>`;
@@ -70,7 +78,7 @@ function simpleAttemptHistory(t,e){
  if(!t.simple_rerun)return '';
  const history=t.attempt_history||[];
  const current=e.provenance.max_steps||t.planned_protocol.max_control_steps;
- return panel('Attempt history',`<div class="simple-attempt-history"><p>Current result: attempt ${e.provenance.attempt||1} · ${num(current)} action-step limit.</p><p>${esc(t.simple_rerun.note)}</p>${history.map(h=>`<div class="previous-attempt"><p>Previous attempt · ${num(h.max_steps)} action-step limit · native ${h.success?'success':'failure'} · ${num(h.steps)} steps used. Retained outside current-selection totals.</p><div class="downloads">${['video','native_session','verdict','protocol','usage','provenance','episode_record'].filter(k=>h.links[k]).map(k=>external(h.links[k],k.replaceAll('_',' '))).join('')}</div></div>`).join('')}</div>`);
+ return panel('Attempt history',`<div class="simple-attempt-history"><p>Current result: attempt ${e.provenance.attempt||1} · ${num(current)} action-step limit.</p><p>${esc(t.simple_rerun.note)}</p>${history.map(h=>`<div class="previous-attempt"><p>Previous attempt · ${num(h.max_steps)} action-step limit · native ${h.success?'success':'failure'} · ${num(h.steps)} steps used. Retained outside current-selection totals.</p>${h.visual_review?`<p>独立复核：${esc(h.visual_review.review)}</p>`:''}<div class="downloads">${['video','native_session','verdict','protocol','usage','provenance','episode_record','independent_review'].filter(k=>h.links[k]).map(k=>external(h.links[k],k.replaceAll('_',' '))).join('')}</div></div>`).join('')}</div>`);
 }
 function taskHeader(t,e){return `<a class="back" href="#${t.family}">← Back to ${esc(familyName(t.family))}</a><div class="detail-header"><div><div class="slot">${t.family} / ${t.display_slot || t.slot} · ${esc(familyName(t.family))}</div><h2>${esc(t.title)}</h2><div class="detail-meta">Codex ${esc(e?'v'+e.codex_version:'benchmark')} · GPT-6 Astra · high · seed 0 · ${e?'1 measured episode':'1 planned episode'}</div><div class="badge-row">${e?`<span class="badge ${e.success?'':'fail'}">Native: ${outcome(e).toLowerCase()}</span><span class="badge">Execution: ${esc(e.execution.status)}</span><span class="badge">Evidence: ${e.verdict.evidence_valid?'valid':'invalid'}</span><span class="badge">Usage: ${e.usage.audit_complete?'complete':'incomplete'}</span>`:'<span class="badge">Pending · no published result</span>'}</div></div>${e?`<span class="chip">${e.phase==='preflight'?'Preflight record':'Formal evaluation'}</span>`:''}</div>`;}
 function renderPending(t){
@@ -81,7 +89,7 @@ async function renderResult(t,e,epoch){
  const u=e.usage,m=e.media,views=m.view_names||m.recording.views.map(v=>v.name);
  const metrics=[[`${num(e.steps)} / ${num(t.planned_protocol.max_control_steps)}`,'Native control steps',e.simulation_time_s==null?`${m.recording.end_time_s.toFixed(2)} recorded simulation seconds`:`${e.simulation_time_s.toFixed(2)} simulation seconds`],[elapsed(e.wall_time_s),'Agent wall time','8-hour safety limit'],[num(u.accounting==='reported-responses'?u.response_count:(u.request_attempts??u.response_count)),u.accounting==='reported-responses'?'Model responses':'Model requests',u.accounting==='reported-responses'?'Provider-reported responses':`${num(u.reported_responses?.input_tokens)} / ${num(u.request_attempts??u.response_count)} terminal usage reported`],[num(e.call_activity.model_tool_calls),'Model tool calls','Nested RPC count unavailable'],[num(u.input_tokens),'Input tokens',num(u.cached_input_tokens)+' cached'],[num(u.output_tokens),'Output tokens',num(u.reasoning_output_tokens)+' reasoning']];
  const r=e.failure_review;const reviewEvidence=r?`<details><summary>失败判定依据 · ${esc(r.status_label)}</summary><div class="evidence-note"><p>${esc(r.basis)}</p><p>${external(e.links.review_frame,'查看录像末帧')} · ${external(e.links.failure_review,'完整判定记录')}</p>${r.source_links.map(s=>`<p>${external(s.url,s.path)}</p>`).join('')}</div></details>`:'';
- $('#detail').innerHTML=taskHeader(t,e)+simpleAttemptHistory(t,e)+(e.usage_note?`<div class="usage-note"><strong>Usage incomplete</strong><p>${esc(e.usage_note)}</p></div>`:'')+`<div class="video-panel"><div class="view-labels">${views.map(v=>`<span>${esc(v.replaceAll('_',' ').toUpperCase())}</span>`).join('')}</div><video id="episode-video" style="aspect-ratio:${m.width}/${m.height}" controls playsinline preload="metadata" poster="${esc(e.links.poster)}" src="${esc(e.links.video)}"></video><div class="video-toolbar"><span>Native spectator recording · ${m.width} × ${m.height} · ${m.duration_s.toFixed(2)} s</span><label>Playback <select id="playback-speed" aria-label="Video playback speed"><option value="0.25">1× simulation / 0.25× video</option><option value="0.5">2× simulation / 0.5× video</option><option value="1" selected>4× simulation / 1× video</option><option value="2">8× simulation / 2× video</option></select></label></div></div><p class="video-note">${views.length} native ${m.recording.views[0].height}p views captured at ${m.source_fps} FPS; ${m.recording.dropped_samples?num(m.recording.dropped_samples)+' capture samples dropped; ':''}edited to 4× simulation speed and ${m.output_fps} FPS. ${m.terminal_hold_s?`All ${num(m.native_frames)} recorded source frames retained; the true final frame is held for ${m.terminal_hold_s} seconds without advancing simulation. ${e.links.terminal_frame?external(e.links.terminal_frame,"Final recorded frame"):""}`:""} Agent events below use wall time and are not frame-synchronized.</p><div class="detail-grid"><div>${instructionPanel(t,e)}${data.instruction_correction?.task_key===t.key?panel("屋顶方向说明的纠正",`<div class="instruction-correction"><p>${esc(data.instruction_correction.summary)}</p><p>${external(data.instruction_correction.evidence,"网格与原生条件核对")} · ${external(data.instruction_correction.video,"错误措辞回合的视频")} · ${external(data.instruction_correction.session,"完整 session")} · ${external(data.instruction_correction.native_verdict,"原生判定")}</p></div>`):""}${data.supplemental_verifications?.[t.key]?panel('Verifier 与环境不对齐',`<div class="button-verification"><p>${esc(data.supplemental_verifications[t.key].summary)}</p><p>独立物理验证 · 0 次模型调用 · 不计入正式成功率</p><a href="${esc(data.supplemental_verifications[t.key].report)}">查看按下、撤手与回弹记录 →</a></div>`):''}${panel('运行分析',`${r?`<div class="failure-verdict">${failureBadge(r)}<p>${esc(r.reason)}</p></div>`:''}<div class="analysis">${Object.entries(e.analysis).filter(([k])=>k!=='preflight').map(([,v])=>`<p>${esc(v)}</p>`).join('')}</div>${r?.trace_event_ids?`<div class="review-events"><p>查看引用的 session 事件：</p>${r.trace_event_ids.map(id=>`<button class="review-event" data-event-id="${esc(id)}">${esc(id)}</button>`).join('')}</div>`:''}`,'wide-panel')}</div><div>${panel('Episode metrics',`<div class="metrics">${metrics.map(([v,l,s])=>`<div class="metric"><b>${v}</b><span>${l}</span><small>${s}</small></div>`).join('')}</div><p class="metric-note">Native reward: ${num(e.native_reward??(e.success?1:0))} · termination: ${esc(e.verdict.termination)}<br>Uncached input: ${num(u.uncached_input_tokens)} · cost: ${u.cost_usd==null?'unavailable':u.cost_usd}<br>Execution: ${esc(e.execution.status)}${e.execution.reason?' · '+esc(e.execution.reason):''}.</p>`)}${e.phase==='preflight'?`<div class="preflight-note"><strong>Preserved preflight result</strong><p>${esc(e.analysis.preflight)}</p>${external(e.links.provenance,'Build provenance')}</div>`:''}</div></div>`+
+ $('#detail').innerHTML=taskHeader(t,e)+visualReviewPanel(e)+simpleAttemptHistory(t,e)+(e.usage_note?`<div class="usage-note"><strong>Usage incomplete</strong><p>${esc(e.usage_note)}</p></div>`:'')+`<div class="video-panel"><div class="view-labels">${views.map(v=>`<span>${esc(v.replaceAll('_',' ').toUpperCase())}</span>`).join('')}</div><video id="episode-video" style="aspect-ratio:${m.width}/${m.height}" controls playsinline preload="metadata" poster="${esc(e.links.poster)}" src="${esc(e.links.video)}"></video><div class="video-toolbar"><span>Native spectator recording · ${m.width} × ${m.height} · ${m.duration_s.toFixed(2)} s</span><label>Playback <select id="playback-speed" aria-label="Video playback speed"><option value="0.25">1× simulation / 0.25× video</option><option value="0.5">2× simulation / 0.5× video</option><option value="1" selected>4× simulation / 1× video</option><option value="2">8× simulation / 2× video</option></select></label></div></div><p class="video-note">${views.length} native ${m.recording.views[0].height}p views captured at ${m.source_fps} FPS; ${m.recording.dropped_samples?num(m.recording.dropped_samples)+' capture samples dropped; ':''}edited to 4× simulation speed and ${m.output_fps} FPS. ${m.terminal_hold_s?`All ${num(m.native_frames)} recorded source frames retained; the true final frame is held for ${m.terminal_hold_s} seconds without advancing simulation. ${e.links.terminal_frame?external(e.links.terminal_frame,"Final recorded frame"):""}`:""} Agent events below use wall time and are not frame-synchronized.</p><div class="detail-grid"><div>${instructionPanel(t,e)}${data.instruction_correction?.task_key===t.key?panel("屋顶方向说明的纠正",`<div class="instruction-correction"><p>${esc(data.instruction_correction.summary)}</p><p>${external(data.instruction_correction.evidence,"网格与原生条件核对")} · ${external(data.instruction_correction.video,"错误措辞回合的视频")} · ${external(data.instruction_correction.session,"完整 session")} · ${external(data.instruction_correction.native_verdict,"原生判定")}</p></div>`):""}${data.supplemental_verifications?.[t.key]?panel('Verifier 与环境不对齐',`<div class="button-verification"><p>${esc(data.supplemental_verifications[t.key].summary)}</p><p>独立物理验证 · 0 次模型调用 · 不计入正式成功率</p><a href="${esc(data.supplemental_verifications[t.key].report)}">查看按下、撤手与回弹记录 →</a></div>`):''}${panel('运行分析',`${r?`<div class="failure-verdict">${failureBadge(r)}<p>${esc(r.reason)}</p></div>`:''}<div class="analysis">${Object.entries(e.analysis).filter(([k])=>k!=='preflight').map(([,v])=>`<p>${esc(v)}</p>`).join('')}</div>${r?.trace_event_ids?`<div class="review-events"><p>查看引用的 session 事件：</p>${r.trace_event_ids.map(id=>`<button class="review-event" data-event-id="${esc(id)}">${esc(id)}</button>`).join('')}</div>`:''}`,'wide-panel')}</div><div>${panel('Episode metrics',`<div class="metrics">${metrics.map(([v,l,s])=>`<div class="metric"><b>${v}</b><span>${l}</span><small>${s}</small></div>`).join('')}</div><p class="metric-note">Native reward: ${num(e.native_reward??(e.success?1:0))} · termination: ${esc(e.verdict.termination)}<br>Uncached input: ${num(u.uncached_input_tokens)} · cost: ${u.cost_usd==null?'unavailable':u.cost_usd}<br>Execution: ${esc(e.execution.status)}${e.execution.reason?' · '+esc(e.execution.reason):''}.</p>`)}${e.phase==='preflight'?`<div class="preflight-note"><strong>Preserved preflight result</strong><p>${esc(e.analysis.preflight)}</p>${external(e.links.provenance,'Build provenance')}</div>`:''}</div></div>`+
  panel('Session & observations',`<p class="section-note" id="session-counts">Loading session counts… Tool inputs and outputs are expandable; the native session download retains the full event format. Event timestamps are in UTC.</p><div class="timeline-controls"><label><span class="sr-only">Filter session events</span><select id="event-filter"><option value="all">All events</option><option value="tools">Tool results</option><option value="errors">Tool errors</option><option value="assistant">Assistant</option><option value="images">With images</option></select></label><input id="event-search" type="search" aria-label="Search session" placeholder="Search text, tool or event ID…"><span id="event-count" aria-live="polite">Loading…</span></div><div id="timeline"></div><button class="plain-button" id="more-events">Load more events</button><details id="gallery"><summary>Browse agent observation images</summary><div class="image-grid" id="observation-grid"></div></details>`,'wide-panel')+
  panel('Agent resources',e.resources.length?`<p class="section-note">Final tool and memo files written during this episode. Native session edits preserve their development history.</p><div class="resource-toolbar"><label class="sr-only" for="resource-select">Resource file</label><select id="resource-select">${e.resources.map(r=>`<option value="${esc(r.file)}">${esc(r.name)}</option>`).join('')}</select><a id="resource-download" target="_blank" rel="noopener">Open file ↗</a></div><pre id="resource-preview">Loading…</pre>`:'<p>No tool or memo files were saved in this episode. The session and workspace evidence remain available below.','wide-panel')+
  panel('Evidence & provenance',`<p class="section-note">Lightweight public evidence is stored in the Dataset. Host paths are redacted in copies; original event IDs, native verdicts and usage are preserved.</p>${reviewEvidence}<div class="downloads">${Object.entries(e.links).filter(([k])=>!['poster','transcript'].includes(k)).map(([k,v])=>external(v,k.replaceAll('_',' '))).join('')}</div><details><summary>Source revisions and measured build identities</summary><pre>${esc(JSON.stringify(e.provenance,null,2))}</pre></details>`,'wide-panel')+recordingRepairHistory(t,e);

manifest.json

@@ -4,7 +4,7 @@
   "README.md": "c2072205759773d80bbc173f03e69da48abec58ae20a218f87dda4c5d8777bc7",
   "REPORT.md": "7717a341da6488a163b6982f14f8248186a78a62caf657b08cb010330d64d769",
   "THIRD_PARTY_NOTICES.md": "8aa8e3db100d7a570931c5d87f44f78705b42cf17e8147016b35e9aeddb06bba",
-  "app.js": "686a2ef374566407865f85e3cbb0e62e9b8f669be2858cb2a7a2143f0bf4c4de",
+  "app.js": "872d57ccb8fab7c996443d78e5fcb757831bb8100ffd803109f55d4af4854177",
   "attempt-history.json": "928e53e66edf104fa6d27ad7630aae102a25f33b5fe2841c70f1867f29e70bb8",
   "catalog/task06/01-xmove-pick/task.yaml": "8a75a1ad3200a278b9159788433c0398ca16c4b01959086a651af325c2dc4c0c",
   "catalog/task06/02-transfer-between-tables/task.yaml": "4d582260a03b2d6fc3d4516c488b580b8ff926f307968ca9d5a6b961c7a1729e",
@@ -19,9 +19,9 @@
   "catalog/task06/11-push-office-chair/task.yaml": "0e2de1c50619b0c58fd939f58362dfc0630f34d4e19587fe4927a2c4e5c1d785",
   "catalog/task06/12-open-trash-can/task.yaml": "e02833927892caa9c40ed193348aab1858e6ae6208c4c04892dec8eae4ad45f0",
   "comparison.json": "41ac8e73537d3866ad37188ed82e53dae6c51bb1d87414ee6406ee7479b2d777",
-  "data.json": "53dad574486978c895a130fce20c8f134363314cf644596411aca35a22216475",
+  "data.json": "2283435d7f15c50d747eb5e231ede975757bfaa6cba15c6495aa7c1667cec9a6",
   "episodes.csv": "f8262dfb6fc34aa549282d2e4983ffd3c97e3002a9259bff3d562010a2137a5f",
-  "episodes.json": "d34f4753eae5d65cf5421e7379cc67c5b081a7ddea88b3b85708806e4b279584",
+  "episodes.json": "007cf098118cc2c2a111e0480db329d96fd91b23aed0099aae2737d7ba814e6e",
   "failure-reviews.json": "c66a997c9ecd6f7f43f130a8a33cd64c7a5b2049033e9ccde653ee30e4796f23",
   "favicon.svg": "46063791218655f70c828ef71a1232a34fd38181da938338badec929ad226819",
   "index.html": "d29e3b0133b9af6378b8b3ab7a6bc04f8f985eff46a409078c4f8e6a521edfc2",
@@ -32,6 +32,6 @@
   "licenses/robosuite-LICENSE.txt": "177978cbece0a4c454c2aaec5b3f145b39270814874c43109da9e829c39d9cba",
   "protocol.json": "8945b054460c822ed8e443487ec7f8d3ba65bf9c37242372f963d5f5865c6cea",
   "style.css": "dd31b30951c220ebb82bc96224b5e640913945679e42b2e0a42e2c373a329e6c",
-  "summary.json": "e8c846725387c23e26bf7c410589079dce965a96937072346eb8454dfd823adc",
-  "task-index.json": "9df9a672d9b43d9710a46c9e34479bce0e30ea93856c912b755ed670ac118805"
+  "summary.json": "3ffb3acaba725be116e5bf35c3240e100e8827029116d8c0ff73b9718f5500aa",
+  "task-index.json": "54f6e5004261b999d2823a2ac74e71ab913cb9766a52fbdc1703e0ecb6f2360d"
 }

publication-manifest.json

@@ -22,8 +22,8 @@
       "bytes": 3015
     },
     "app.js": {
-      "sha256": "686a2ef374566407865f85e3cbb0e62e9b8f669be2858cb2a7a2143f0bf4c4de",
-      "bytes": 26720
+      "sha256": "872d57ccb8fab7c996443d78e5fcb757831bb8100ffd803109f55d4af4854177",
+      "bytes": 27476
     },
     "attempt-history.json": {
       "sha256": "928e53e66edf104fa6d27ad7630aae102a25f33b5fe2841c70f1867f29e70bb8",
@@ -82,16 +82,16 @@
       "bytes": 157967
     },
     "data.json": {
-      "sha256": "53dad574486978c895a130fce20c8f134363314cf644596411aca35a22216475",
-      "bytes": 1509720
+      "sha256": "2283435d7f15c50d747eb5e231ede975757bfaa6cba15c6495aa7c1667cec9a6",
+      "bytes": 1615510
     },
     "episodes.csv": {
       "sha256": "f8262dfb6fc34aa549282d2e4983ffd3c97e3002a9259bff3d562010a2137a5f",
       "bytes": 8275
     },
     "episodes.json": {
-      "sha256": "d34f4753eae5d65cf5421e7379cc67c5b081a7ddea88b3b85708806e4b279584",
-      "bytes": 1015618
+      "sha256": "007cf098118cc2c2a111e0480db329d96fd91b23aed0099aae2737d7ba814e6e",
+      "bytes": 1044695
     },
     "failure-reviews.json": {
       "sha256": "c66a997c9ecd6f7f43f130a8a33cd64c7a5b2049033e9ccde653ee30e4796f23",
@@ -134,15 +134,15 @@
       "bytes": 16560
     },
     "summary.json": {
-      "sha256": "e8c846725387c23e26bf7c410589079dce965a96937072346eb8454dfd823adc",
-      "bytes": 5827
+      "sha256": "3ffb3acaba725be116e5bf35c3240e100e8827029116d8c0ff73b9718f5500aa",
+      "bytes": 6602
     },
     "task-index.json": {
-      "sha256": "9df9a672d9b43d9710a46c9e34479bce0e30ea93856c912b755ed670ac118805",
-      "bytes": 416923
+      "sha256": "54f6e5004261b999d2823a2ac74e71ab913cb9766a52fbdc1703e0ecb6f2360d",
+      "bytes": 487396
     },
     "manifest.json": {
-      "sha256": "c793ea7c9a9a0cbdf522a8d610ed7ebec6e120b7adb965cd57d33df8e90487a7",
+      "sha256": "1df2a450e95eeb95998bb2b98ec02af22fd5043125426d78b62eebd88cd241ec",
       "bytes": 3481
     }
   }

summary.json

@@ -90,9 +90,9 @@
       "total": 12,
       "completed": 12,
       "pending": 0,
-      "successes": 7,
+      "successes": 8,
       "valid_results": 12,
-      "success_rate": 0.5833333333333334,
+      "success_rate": 0.6666666666666666,
       "usage_complete": 12,
       "control_frequency_hz": 50,
       "max_control_steps": null,
@@ -112,18 +112,18 @@
       },
       "preflight_results": 0,
       "formal_results": 12,
-      "modified_results": 0,
-      "original_results": 12,
-      "input_tokens": 106763761,
-      "cached_input_tokens": 105331968,
-      "output_tokens": 303566
+      "modified_results": 4,
+      "original_results": 8,
+      "input_tokens": 89900808,
+      "cached_input_tokens": 88661760,
+      "output_tokens": 244198
     }
   ],
   "interrupted_attempts": 4,
   "usage_incomplete_tasks": [],
   "execution_incomplete_tasks": [],
   "deferred_tasks": [],
-  "new_evaluations": 12,
+  "new_evaluations": 4,
   "progress": {
     "finished": 84,
     "running": 0,
@@ -198,5 +198,26 @@
         "success": false
       }
     ]
+  },
+  "current_modified_results": 60,
+  "new_modified_results": 4,
+  "simple_instruction_reruns": {
+    "condition": "modified-instructions",
+    "tasks": [
+      "task06/02",
+      "task06/09",
+      "task06/04",
+      "task06/05"
+    ],
+    "new_evaluations": 4,
+    "seed": 0,
+    "max_steps": 15000,
+    "native_verifier_changed": false,
+    "native_successes": 3,
+    "visible_goal_completed": 2,
+    "confirmed_false_positives": 1,
+    "confirmed_false_negatives": 0,
+    "scope": "Single-seed instruction revision, with earlier runs preserved. Native success and independent visible completion are reported separately.",
+    "comparison_limit": "Earlier selected runs have mixed budgets and use an earlier account; this is not a multi-seed causal success-rate estimate."
   }
 }