File size: 8,272 Bytes
dd9cc19 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 | 'use strict';
async function loadModelResults(legacy) {
const response = await fetch('model-source.json');
if (!response.ok) throw Error('Model result version is unavailable.');
const pin = await response.json();
if (!/^[a-f0-9]{40}$/.test(pin.revision) || !/^[a-f0-9]{64}$/.test(pin.sha256)) throw Error('Invalid model result version.');
const root = `https://huggingface.co/datasets/${pin.repo}/resolve/${pin.revision}/`;
const res = await fetch(root + pin.path); if (!res.ok) throw Error('Model results are unavailable.');
const bytes = await res.arrayBuffer();
const digest = Array.from(new Uint8Array(await crypto.subtle.digest('SHA-256', bytes)), x => x.toString(16).padStart(2, '0')).join('');
if (digest !== pin.sha256) throw Error('Model result checksum mismatch.');
const data = JSON.parse(new TextDecoder().decode(bytes));
if (data.schema !== 'chemlab/model-evaluation/1' || data.tasks.length !== 4 || !data.complete) throw Error('Model evaluation is not complete.');
const oldTasks = [...document.querySelector('#tasks').children];
const oldMetrics = [...document.querySelector('.overview-metrics').children];
const mode = el('div', undefined, 'result-modes'); mode.setAttribute('role','group'); mode.setAttribute('aria-label','Result set');
const description = el('p', undefined, 'result-description');
document.querySelector('#overview').before(mode, description);
const buttons = {};
for (const [key,label] of [['model','GPT-6 Astra'],['scripted','Scripted acceptance']]) {
const b = el('button',label); b.type='button'; b.dataset.mode=key; mode.append(b); buttons[key]=b;
b.addEventListener('click',()=>select(key));
}
const modelTasks = data.tasks.map(task => {
const card = nativeCard({native_success:task.native_success},root,task,{video:root+task.media.video,poster:root+task.media.poster});
const jump=el('button','First command','jump-command');jump.type='button';jump.addEventListener('click',()=>{const v=card.querySelector('video');v.currentTime=Math.max(0,task.first_command_time_s-0.5);});card.querySelector('.player-controls').prepend(jump);
if(task.slug.startsWith('02-'))card.querySelector('.split-labels span:last-child').textContent='Native screen + runtime readout';
card.querySelector('.task-metrics').replaceChildren(metric('Model responses',task.model_responses),metric('Instrument commands',task.commands.length),metric('Simulation time',`${task.simulation_seconds.toFixed(1)} s`));
const detail=el('button','Task & model trace'); detail.type='button'; detail.addEventListener('click',()=>showModelTask(task,root));
card.querySelector('.links').replaceChildren(detail); return card;
});
const modelMetrics = [metric('Tasks passed',`${data.tasks.filter(t=>t.native_success).length} / 4`),metric('Evidence valid',`${data.tasks.filter(t=>t.evidence_valid).length} / 4`),metric('Usage complete',`${data.tasks.filter(t=>t.usage.audit_complete).length} / 4`),metric('Attempts per task','1')];
function select(key) {
for(const video of document.querySelectorAll('video')) video.pause();
for(const [k,b]of Object.entries(buttons)) b.setAttribute('aria-pressed',String(k===key));
const model=key==='model';
document.querySelector('#tasks').replaceChildren(...(model?modelTasks:oldTasks));
document.querySelector('.overview-metrics').replaceChildren(...(model?modelMetrics:oldMetrics));
document.querySelector('.hero .context').textContent=model?'Kinex + GPT-6 Astra · high · realtime · seed 0':'Scripted acceptance · realtime · no model calls';
document.querySelector('#task-heading').textContent=model?'Model runs':'Scripted acceptance runs';
description.textContent=model?'Four independent instrument tasks · no robot · one attempt each':'Earlier scripted checks of the instrument pipeline and asset runtime.';
document.documentElement.dataset.resultMode=key;
}
const provenance=document.querySelector('#provenance-content');
const block=el('div');block.append(el('h3','GPT-6 Astra evaluation'));
const limits=el('ul');for(const text of data.limits) limits.append(el('li',text));block.append(limits);
block.append(el('p','Native success, evidence validity and usage completeness are checked separately. Videos contain the original run; command replays validate terminal state and scene bindings, not per-frame trajectory restoration.'));
const links=el('div',undefined,'links');links.append(link('RLE implementation PR ↗','https://github.com/stacx-lab/RLE-Bench-inhouse/pull/6'),link('Model result index ↗',root+pin.path),link('Model file manifest ↗',root+pin.path.replace('index.json','manifest.json')));block.append(links);
provenance.prepend(block);select('model');
}
function showModelTask(task,root){
document.querySelector('#detail-title').textContent=task.title;
details.replaceChildren(el('p',task.instruction,'instruction'));
details.append(el('p',`Original task instruction · seed 0 · attempt 1 · ${task.frames} native frames · ${task.agent_wall_seconds.toFixed(1)} s agent runtime`,'muted'));
const table=el('table');const tr=el('tr');for(const label of ['Condition','Initial','Final','Target'])tr.append(el('th',label));table.append(tr);
for(const g of task.goals){const row=el('tr');for(const value of [labels[g.state]??g.state,measurement(g.initial,g.unit,g.state),measurement(g.final,g.unit,g.state),target(g)])row.append(el('td',value));table.append(row);}
const wrap=el('div',undefined,'table-wrap');wrap.append(table);details.append(wrap);
details.append(el('p',`Native: ${task.native_success?'success':'failure'} · Evidence: ${task.evidence_valid?'valid':'invalid'} · Execution: ${task.execution.status} · Usage: ${task.usage.audit_complete?'complete':'incomplete'}`));
details.append(el('p',`${task.model_responses} model responses · ${task.model_tool_calls} model tool calls · ${task.commands.length} instrument commands · ${task.owner_rpc_attempts} owner RPC attempts`));
details.append(el('p',`Tokens: ${task.usage.fresh_input_tokens.toLocaleString()} fresh input + ${task.usage.cached_input_tokens.toLocaleString()} cached input + ${task.usage.output_tokens.toLocaleString()} output. Provider cost unavailable.`));
details.append(el('h3','Instrument commands'));
for(const c of task.commands){const row=el('div',undefined,'command');row.append(el('strong',`${c.capability} · frame ${c.frame}`),el('pre',JSON.stringify(c.arguments,null,2)),el('span',c.status,'command-status'));details.append(row);}
details.append(el('h3','Assessment'));
details.append(el('p',task.analysis.summary_zh??task.analysis.difficulty,'model-analysis'));
details.append(el('p',`Selection: ${task.analysis.recommendation}. ${task.analysis.rsi_hypothesis}`));
details.append(el('p',task.replay?`Command replay: ${task.replay.passed?'passed':'failed'} · rendered scene: ${task.replay.scene_matched?'matched':'not verified'}`:'Command replay: not run.'));
const links=el('div',undefined,'links');
for(const [k,label]of [['record','Owner state'],['terminal','Native verdict'],['session','Full session'],['usage','Usage audit'],['journal','Owner journal'],['definition','Task YAML'],['media','Video checks']])links.append(link(label+' ↗',root+task.evidence[k]));
if(task.replay)links.append(link('Replay receipt ↗',root+task.evidence.replay));
for(const path of task.media.extra_views)links.append(link('Additional readout video ↗',root+path));details.append(links);
const trace=el('details',undefined,'session-details');trace.append(el('summary','Model conversation & tool I/O'));const body=el('div',undefined,'model-trace');trace.append(body);details.append(trace);
trace.addEventListener('toggle',async()=>{
if(!trace.open||trace.dataset.loaded)return;trace.dataset.loaded='true';body.textContent='Loading conversation…';
try{const r=await fetch(root+task.evidence.trace);if(!r.ok)throw Error('Trace unavailable');const d=await r.json();body.replaceChildren();
for(const m of d.messages){const entry=el('details',undefined,'message');entry.append(el('summary',`${m.role} · ${m.timestamp??''}`),el('pre',JSON.stringify(m.parts,null,2)));body.append(entry);}
}catch(e){body.textContent=e.message;}
});dialog.showModal();
}
|