| 'use strict'; |
| const el=(tag,text,cls)=>{const n=document.createElement(tag);if(text!==undefined)n.textContent=text;if(cls)n.className=cls;return n;}; |
| const dialog=document.querySelector('#detail-dialog'),detail=document.querySelector('#detail-content'); |
| let detailRequest=0; |
| document.querySelector('#close-dialog').addEventListener('click',()=>dialog.close()); |
| dialog.addEventListener('click',event=>{if(event.target===dialog){const r=dialog.getBoundingClientRect();if(event.clientX<r.left||event.clientX>r.right||event.clientY<r.top||event.clientY>r.bottom)dialog.close();}}); |
| dialog.addEventListener('close',()=>{detailRequest++;detail.replaceChildren();}); |
|
|
| function player(episode,root) { |
| const box=el('div',undefined,'player'),video=el('video'); |
| const labels=el('div',undefined,'split-labels'); |
| labels.append(el('span','Overview'),el('span',episode.screen_label)); |
| video.controls=true;video.preload='metadata';video.playsInline=true; |
| video.src=root+episode.video;video.poster=root+episode.poster; |
| video.setAttribute('aria-label',`${episode.title}: overview and instrument screen`); |
| video.style.aspectRatio='32 / 9';video.defaultPlaybackRate=2;video.playbackRate=2; |
| const controls=el('div',undefined,'player-controls'),speeds=el('div',undefined,'playback-speeds'); |
| const firstAction=el('button','First action','first-action');firstAction.type='button'; |
| firstAction.setAttribute('aria-label',`Jump to first instrument action: ${episode.title}`); |
| firstAction.addEventListener('click',()=>{video.currentTime=Math.max(0,episode.first_command_time_s-0.5);}); |
| speeds.setAttribute('role','group');speeds.setAttribute('aria-label',`Playback speed: ${episode.title}`); |
| for(const rate of [1,2,4,8]) { |
| const button=el('button',`${rate}×`);button.type='button';button.dataset.rate=String(rate);button.setAttribute('aria-pressed',String(rate===2)); |
| button.addEventListener('click',()=>{video.defaultPlaybackRate=rate;video.playbackRate=rate;});speeds.append(button); |
| } |
| video.addEventListener('ratechange',()=>{for(const button of speeds.children)button.setAttribute('aria-pressed',String(Number(button.dataset.rate)===video.playbackRate));}); |
| video.addEventListener('play',()=>{for(const other of document.querySelectorAll('video'))if(other!==video)other.pause();}); |
| controls.append(firstAction,speeds);box.append(labels,video,controls);return box; |
| } |
|
|
| function transcriptMessages(trace) { |
| return trace.messages.map(message=>({ |
| ...message, |
| content:message.parts.flatMap(part=>part.type==='tool' |
| ?[{type:'toolCall',name:part.name,arguments:part.input}] |
| :part.type==='text' && part.text?[part]:[]), |
| })).filter(message=>message.content.length); |
| } |
|
|
| function contentBlocks(parent,blocks,base) { |
| for(const block of blocks??[]) { |
| if(block.type==='text')parent.append(el('pre',block.text,'message-text')); |
| if(block.type==='image' && block.path) { |
| const img=el('img');img.src=new URL(block.path,base).href;img.alt='Camera image from the agent session';img.loading='lazy';parent.append(img); |
| } |
| if(block.type==='toolCall') { |
| const call=el('div',undefined,'tool-call');call.append(el('strong',block.name),el('pre',JSON.stringify(block.arguments,null,2)));parent.append(call); |
| } |
| } |
| } |
|
|
| async function showDetails(kind,episode,root,copy) { |
| const request=++detailRequest;document.querySelector('#detail-title').textContent=`${kind} · ${episode.title}`; |
| detail.replaceChildren(el('p','Loading…','muted'));dialog.showModal(); |
| if(kind==='Instruction'){detail.replaceChildren(el('p',copy.instruction,'instruction'));return;} |
| try { |
| const url=new URL(root+episode.links.Transcript,location.href).href,response=await fetch(url);if(!response.ok)throw Error('Session is unavailable.'); |
| const transcript=await response.json();if(request!==detailRequest)return; |
| detail.replaceChildren();const download=el('a','Download session JSON ↗','download');download.href=url;download.target='_blank';download.rel='noopener';detail.append(download); |
| let count=0; |
| for(const message of transcriptMessages(transcript)) { |
| const calls=(message.content??[]).filter(x=>x.type==='toolCall'); |
| if(kind==='Tool calls' && message.role!=='toolResult' && !calls.length)continue; |
| const row=el(message.role==='toolResult'?'details':'section',undefined,'message'); |
| const label=message.role==='toolResult'?`${message.toolName??'Tool'} result${message.isError?' · error':''}`:message.role==='assistant'?'Agent':message.role==='user'?'Task':'Context'; |
| row.append(el(message.role==='toolResult'?'summary':'h3',label));contentBlocks(row,kind==='Tool calls' && message.role==='assistant'?calls:message.content,url); |
| detail.append(row);count+=calls.length; |
| } |
| if(kind==='Tool calls')detail.prepend(el('p',`${count} tool calls`,'muted')); |
| }catch(error){if(request===detailRequest)detail.replaceChildren(el('p',error.message));} |
| } |
|
|
| function renderOverview(episodes) { |
| const section=document.querySelector('#overview'),metrics=section.querySelector('dl'); |
| const successes=episodes.filter(x=>x.task_success===true).length; |
| const costs=episodes.map(x=>x.api_cost),knownCosts=costs.filter(x=>Number.isFinite(x?.amount_usd)); |
| const incomplete=knownCosts.length!==episodes.length || knownCosts.some(x=>x.lower_bound || !x.usage_complete); |
| const total=knownCosts.reduce((sum,x)=>sum+x.amount_usd,0); |
| const money=value=>knownCosts.length?`$${(incomplete?Math.floor(value*100)/100:value).toFixed(2)}`:'Unavailable'; |
| const values=[['Tasks',String(episodes.length)],['Success rate',episodes.length?`${(100*successes/episodes.length).toFixed(1)}%`:'Unavailable'],['Average API cost (est.)',episodes.length?money(total/episodes.length):'Unavailable'],['Total API cost (est.)',money(total)]]; |
| for(const [label,value] of values) { |
| const metric=el('div');metric.append(el('dt',label),el('dd',value,value==='Unavailable'?'unavailable':undefined)); |
| if(label==='Success rate')metric.append(el('span',`${successes} / ${episodes.length} tasks successful`,'overview-detail')); |
| metrics.append(metric); |
| } |
| section.hidden=false; |
| } |
|
|
| function episodeCard(episode,root,copy) { |
| const card=el('article',undefined,'episode');card.dataset.task=episode.task; |
| const heading=el('div',undefined,'card-heading');heading.append(el('h3',episode.title)); |
| heading.append(el('span',episode.task_success===true?'Success':episode.task_success===false?'Fail':'Unavailable',`outcome ${episode.task_success===true?'success':'miss'}`));card.append(heading,player(episode,root)); |
| const body=el('div',undefined,'episode-body');body.append(el('p',copy.instruction,'task-description')); |
| const usage=episode.usage??{},metrics=el('dl',undefined,'usage-metrics'); |
| const count=value=>value==null?'Unavailable':value.toLocaleString(); |
| const ratio=usage.cache_hit_rate; |
| for(const [label,value] of [['Input tokens',count(usage.input_tokens)],['Cached tokens',count(usage.cached_input_tokens)],['Cache hit ratio',ratio==null?'Unavailable':`${(ratio*100).toFixed(2)}%`],['Output tokens',count(usage.output_tokens)]]) { |
| const metric=el('div');metric.append(el('dt',label),el('dd',value,value==='Unavailable'?'unavailable':undefined));metrics.append(metric); |
| } |
| metrics.children[2].title='Cached input tokens ÷ input tokens with reported cache usage'; |
| body.append(metrics); |
| const cost=episode.api_cost; |
| const costRow=el('div',undefined,'api-cost');costRow.append(el('span','API cost (est.)'),el('strong',Number.isFinite(cost?.amount_usd)?`$${cost.amount_usd.toFixed(2)}`:'Unavailable'));costRow.title='Estimated at OpenAI Standard API rates: $10 / $1 / $50 per million uncached input / cached input / output tokens. Actual subscription charge is unknown.';body.append(costRow); |
| const api=episode.api; |
| if(api){const row=el('div',undefined,'api-info');row.append(el('span','API'),el('span',`${api.provider} / ${api.model}`));row.title=`${api.protocol} · ${api.transport.toUpperCase()} · ${api.requests} requests · ${api.responses} responses`;body.append(row);} |
| const links=el('div',undefined,'links'); |
| for(const kind of ['Instruction','Session','Tool calls']) {const button=el('button',kind);button.type='button';button.addEventListener('click',()=>showDetails(kind,episode,root,copy));links.append(button);} |
| if(episode.links['Reusable resources']){const a=el('a','Resources ↓');a.href=root+episode.links['Reusable resources'];a.target='_blank';a.rel='noopener';links.append(a);} |
| body.append(links);card.append(body);return card; |
| } |
|
|
| (async()=>{ |
| const pinResponse=await fetch('model-source.json',{cache:'no-cache'}); |
| if(!pinResponse.ok)throw Error('Demos are unavailable.'); |
| const pin=await pinResponse.json(); |
| const root=`https://huggingface.co/datasets/${pin.repo}/resolve/${pin.revision}/`; |
| const response=await fetch(root+pin.path);if(!response.ok)throw Error('Demos are unavailable.'); |
| const raw=await response.arrayBuffer(),hash=[...new Uint8Array(await crypto.subtle.digest('SHA-256',raw))].map(x=>x.toString(16).padStart(2,'0')).join(''); |
| if(hash!==pin.sha256)throw Error('Could not verify demo data.'); |
| const data=JSON.parse(new TextDecoder().decode(raw)); |
| if(data.schema!=='chemlab/model-evaluation/1' || !data.complete || data.tasks.length!==data.selected_tasks)throw Error('Task results are incomplete.'); |
| const costResponse=await fetch('api-cost.json',{cache:'no-cache'}); |
| if(!costResponse.ok)throw Error('API cost estimates are unavailable.'); |
| const costs=await costResponse.json(); |
| if(costs.model!==data.model || ['repo','revision','path','sha256'].some(key=>costs.source[key]!==pin[key]))throw Error('API cost estimates do not match these results.'); |
| for(const task of data.tasks)if(!Number.isFinite(costs.tasks[task.task]?.amount_usd))throw Error('A task cost estimate is unavailable.'); |
| const [provider,model]=data.model.split('/'); |
| const episodes=data.tasks.map(task=>({ |
| ...task,task_success:task.native_success,video:task.media.video,poster:task.media.poster, |
| screen_label:task.media.extra_views.length?'Native screen + temperature readout':'Native screen', |
| links:{Transcript:task.evidence.trace},api_cost:costs.tasks[task.task], |
| api:{provider,model,protocol:'Responses',transport:data.transport,requests:task.usage.request_attempts,responses:task.usage.response_count}, |
| })); |
| document.querySelector('#summary').textContent='GPT-6 Astra · High reasoning · Real-time instrument control'; |
| renderOverview(episodes); |
| for(const episode of episodes)document.querySelector('#episodes').append(episodeCard(episode,root,episode)); |
| document.querySelector('#progress').textContent=`${episodes.length} / ${data.selected_tasks} tasks have recorded results.`; |
| document.documentElement.dataset.ready='true'; |
| })().catch(error=>{document.querySelector('#summary').textContent=error.message;}); |
|
|
