const $ = (selector) => document.querySelector(selector); const $$ = (selector) => [...document.querySelectorAll(selector)]; const escape = (text) => String(text).replace(/[&<>"']/g, c => ({'&':'&','<':'<','>':'>','"':'"',"'":'''}[c])); const visible = (text) => String(text).replace(/ /g, '·').replace(/\n/g, '↵').replace(/\t/g, '⇥').replace(/\r/g, '␍') || '∅'; const percent = (p) => `${(p * 100).toFixed(p >= .1 ? 1 : p >= .001 ? 2 : 3)}%`; const colorRGB = (value) => { const p = Math.max(0, Math.min(1, value)); return [Math.round(245-238*p),Math.round(251-160*p),Math.round(247-195*p)]; }; const color = value => `rgb(${colorRGB(value).join(',')})`; const ink = value => { const linear=colorRGB(value).map(v=>{const s=v/255;return s<=.04045?s/12.92:((s+.055)/1.055)**2.4;}); return .2126*linear[0]+.7152*linear[1]+.0722*linear[2]>.179?'#000000':'#ffffff'; }; const QWEN_ID = 'Qwen/Qwen3-0.6B'; const state = { model:QWEN_ID, mode:'chat', messages:[], thinking:false, raw:'', revision:0, prepared:null, run:null, runRevision:null, layer:27, head:0, query:0, key:0, lens:'token', busy:false, appendPreview:null, appendVersion:0, status:null, selection:null, matrix:null, matrixCache:new Map(), selectionVersion:0, lensColor:'probability', lensHead:'mean', layerAttention:null, layerAttentionCache:new Map(), layerAttentionVersion:0, }; let prepareTimer, prepareVersion = 0, lastEditor = null, autoStarted = false, busyTimer, statusTimer, appendTimer; const descriptions = { '<|im_start|>':'Starts a message, followed by the role (system, user, or assistant) and a newline.', '<|im_end|>':'Ends a message; also marks the end of an assistant response.', '<|endoftext|>':'Text boundary. Chat messages normally use im_end.', '':'Starts a reasoning block. Added token ID 151667, special=false.', '':'Ends a reasoning block. Added token ID 151668, special=false.', '':'Starts a tool call in the Qwen format.', '':'Ends a tool call.', '':'Starts a tool response.', '':'Ends a tool response.', '':'Beginning of the Gemma input sequence.', '':'End-of-sequence marker.', '':'Starts a Gemma message, followed by user or model and a newline.', '':'Ends a Gemma turn; also stops this checkpoint’s response.', '':'Padding token. Normally excluded from the visible conversation.', '':'Unknown-token marker.', }; const primaryTokens = ['<|im_start|>','<|im_end|>','<|endoftext|>','','','','','','','','','','']; const supportsThinking = () => state.model === QWEN_ID; const kvHead = head => Math.floor(head / (state.run.heads / state.run.kv_heads)); function maskReason(query,key,layer=state.layer) { if(key>query)return 'Hidden by the causal mask'; if(state.run?.layer_types?.[layer]==='sliding_attention' && query-key>=state.run.sliding_window)return 'Outside the sliding attention window'; return ''; } function renderModel(meta) { $('#model-pill').textContent=meta.model===QWEN_ID?'Qwen3-0.6B':'Gemma3-270M'; $('#model-name').textContent=meta.name; $('#footer-model').textContent=`Model laboratory · ${meta.name}`; $('#checkpoint-link').href=`https://huggingface.co/${meta.model}`; $('#empty-model-note').textContent=`Computed from the actual ${meta.name} weights`; if(!meta.layers)return; const group=meta.heads/meta.kv_heads; $('#intro-description').innerHTML=`One context. ${meta.layers} layers. ${meta.layers*meta.heads} attention heads.
Trace how the prediction at each token changes.`; $('#model-shape').textContent=`${meta.layers} LAYERS / ${meta.heads} Q HEADS / ${meta.kv_heads} KV HEADS`; $('#model-description').textContent=`${meta.layers} layers · ${meta.heads} heads · ${meta.native_thinking?'native reasoning switch':'learned Reasoning / Solution format'}`; $('#diagram-layers').textContent=`${meta.layers} layers`; $('#metric-next-label').textContent=`Next token · layer ${meta.layers}`; $('#lens-footnote').textContent=`Input → L01 → L${String(meta.layers).padStart(2,'0')} ↑ · positions start at 0 · predictions refer to the next token`; $('#heads-heading').textContent=`All ${meta.heads} heads in this layer`; $('#attention-description').textContent=`Color shows how much attention the selected position assigns to each token: softmax(QKᵀ / √${meta.attention_scale} + attention mask). Choose one head or the arithmetic mean of all ${meta.heads} query heads after softmax.`; const windowNote=meta.sliding_window?` Sliding layers use a ${meta.sliding_window}-token window; layers ${meta.layer_types.map((type,i)=>type==='full_attention'?i+1:null).filter(Boolean).join(', ')} use full causal attention.`:''; $('#method-model').textContent=`${meta.name} has ${meta.layers} decoder layers, ${meta.heads} query heads, and ${meta.kv_heads} KV heads. With GQA, query head h uses KV head ⌊h / ${group}⌋.${windowNote}`; $('#method-lens').textContent=`After each layer, apply the final RMSNorm and shared language model head, then softmax over all ${meta.vocab_size.toLocaleString('en-US')} tokens. This is a logit lens: early layers were not trained to predict text independently. The final layer gives the standard model prediction.`; $('#method-reasoning').textContent=meta.native_thinking?'Chat uses the checkpoint’s apply_chat_template with enable_thinking. Disabling reasoning adds an empty … block. In Raw mode, you supply the control tokens and prefix. is an added token with special=false.':'Chat uses the checkpoint’s standard Gemma template with user and model. This TinyGSM fine-tune generates ordinary Reasoning / Solution text. It has no special token or native reasoning on/off switch. A system instruction is folded into the first user turn.'; $('#attention-formula').textContent=`A⁽ˡ,ʰ⁾ = softmax(QKᵀ / √${meta.attention_scale} + attention mask)`; if($('#layer-select').options.length!==meta.layers)$('#layer-select').innerHTML=Array.from({length:meta.layers},(_,i)=>``).join(''); if($('#head-select').options.length!==meta.heads+1){ $('#head-select').innerHTML=``+Array.from({length:meta.heads},(_,i)=>``).join(''); $('#head-select').value=state.head; } if($('#lens-head').options.length!==meta.heads+1){ $('#lens-head').innerHTML=``+Array.from({length:meta.heads},(_,i)=>``).join(''); $('#lens-head').value=state.lensHead; } } async function switchModel(model) { if(state.busy||model===state.model)return; clearTimeout(statusTimer);clearTimeout(prepareTimer); state.model=model;state.status=null;state.run=null;state.prepared=null; closeAppendForm();state.runRevision=null; state.selection=null;state.matrix=null;state.matrixCache.clear();state.selectionVersion++;prepareVersion++; state.layerAttention=null;state.layerAttentionCache.clear();state.layerAttentionVersion++;state.lensHead='mean'; state.layer=0;state.head=0;state.query=0;state.key=0;autoStarted=false; if(!supportsThinking())state.thinking=false; $('#run-content').classList.add('hidden');$('#empty-state').classList.remove('hidden'); $('#stale-notice').classList.add('hidden');$('#export').disabled=true; for(const id of ['metric-tokens','metric-next','metric-time'])$(`#${id}`).textContent='—'; $('#metric-input').textContent='Exact model tokenization';$('#metric-prob').textContent='Probability over the full vocabulary'; $('#metric-runtime').textContent='Loading model…';$('#model-description').textContent='Loading model configuration…'; $('#prompt-preview').textContent='Loading tokenizer…';$('#token-count').textContent='— tokens'; $('#special-tokens').replaceChildren();$('#extra-tokens').replaceChildren(); $('#layer-select').replaceChildren();$('#head-select').replaceChildren(); error();renderEditor();updateRunButton();await pollStatus(); } async function api(path, data) { const response = await fetch(path, data === undefined ? {} : {method:'POST', headers:{'Content-Type':'application/json'}, body:JSON.stringify(data)}); if (!response.ok) { let detail; try { detail = (await response.json()).detail; } catch { detail = response.statusText; } throw new Error(typeof detail === 'string' ? detail : 'Check the context format and length.'); } return response.json(); } function payload() { return {model:state.model, mode:state.mode, messages:state.messages, text:state.raw, thinking:supportsThinking()&&state.thinking, max_new_tokens:0}; } function error(message='') { $('#input-error').textContent=message; $('#input-error').classList.toggle('hidden',!message); } function changed() { state.revision++; if (state.run) $('#stale-notice').classList.remove('hidden'); error(); clearTimeout(prepareTimer); // Invalidate an earlier response immediately, before the debounce starts. prepareVersion++; state.prepared=null; $('#token-count').textContent='Counting tokens…'; updateRunButton(); prepareTimer=setTimeout(prepare,300); } function updateRunButton() { $('#model-select').disabled=state.busy; $('#analyze').disabled=state.busy || state.status?.phase!=='ready' || !state.prepared?.fits; $('#analyze span:first-child').textContent=state.busy?'Computing…':'Inspect context'; updateAppendControls(); } function maybeAutoStart() { if(autoStarted||state.status?.phase!=='ready'||!state.prepared?.fits||state.busy||state.run)return; autoStarted=true; analyze(); } async function prepare() { if (!state.status?.tokenizer_ready) return; const version=++prepareVersion; try { const result=await api('/api/prepare',payload()); if(version!==prepareVersion)return; state.prepared=result; $('#prompt-preview').textContent=result.text; $('#token-count').textContent=`${result.count} / ${result.max_tokens} tokens`; if(!result.fits) error(`Context exceeds the ${result.max_tokens}-token limit. Input has not been truncated.`); } catch(exc) { if(version!==prepareVersion)return; state.prepared=null; $('#token-count').textContent='Could not prepare input'; $('#prompt-preview').textContent=exc.message; error(exc.message); } updateRunButton(); maybeAutoStart(); } function renderEditor() { $$('#messages textarea').forEach((el,i)=>{if(state.messages[i])state.messages[i].content=el.value;}); $('#chat-editor').classList.toggle('hidden',state.mode!=='chat'); $('#raw-editor').classList.toggle('hidden',state.mode!=='raw'); $('#raw-text').value=state.raw; $$('[data-mode]').forEach(button=>{const active=button.dataset.mode===state.mode;button.classList.toggle('active',active);button.setAttribute('aria-pressed',active);}); $('#format-label').textContent=state.mode==='chat'?'Official chat template':'Exact text, no wrapper'; $('#thinking').checked=state.thinking; $('#thinking').disabled=state.mode==='raw'||!supportsThinking(); $('#reasoning-switch').classList.toggle('hidden',!supportsThinking()); $('#thinking-note').textContent=!supportsThinking()?'Learned Reasoning / Solution text. No native on/off switch.':state.mode==='raw'?'Controlled by the Raw prefix':state.thinking?'The model can start a block':'An empty block precedes the answer'; $('#messages').innerHTML=state.messages.map((message,i)=>`
`).join(''); lastEditor=null; } function loadTemplate(name) { // Remove old editors before changing message state. $('#messages').innerHTML=''; state.mode='chat';state.thinking=false;state.raw=''; const templates={ question:[{role:'user',content:'What is the capital of France? Answer with one word.'}], math:[{role:'user',content:'Anna has 3 apples. She buys twice as many apples as she originally had. How many apples does Anna have now?'}], system:[{role:'system',content:'Answer in English using one word.'},{role:'user',content:'Which city is the capital of France?'}], dialogue:[{role:'user',content:'My name is Anna. I have a cat named Mars.'},{role:'assistant',content:'Nice to meet you, Anna!'},{role:'user',content:'What is the name of my cat?'}], completion:[],empty:[{role:'user',content:''}], }; state.messages=structuredClone(templates[name]??templates.question); if(name==='math')state.thinking=supportsThinking(); if(name==='completion'){state.mode='raw';state.raw='The capital of France is';} renderEditor();changed(); } async function switchMode(mode) { if(mode===state.mode)return; if(mode==='raw'){ try {const prepared=await api('/api/prepare',payload());state.raw=prepared.text;}catch {state.raw=state.messages.map(m=>m.content).join('\n');} } state.mode=mode;renderEditor();changed(); } function insertToken(token) { const editor=lastEditor?.isConnected?lastEditor:state.mode==='raw'?$('#raw-text'):$('#messages .message:last-child textarea'); if(!editor||editor.disabled)return; editor.focus(); editor.setRangeText(token,editor.selectionStart,editor.selectionEnd,'end'); editor.dispatchEvent(new Event('input',{bubbles:true})); } function renderSpecialTokens(tokens) { const button=t=>``; $('#special-tokens').innerHTML=primaryTokens.map(text=>tokens.find(t=>t.text===text)).filter(Boolean).map(button).join(''); $('#extra-tokens').innerHTML=tokens.filter(t=>!primaryTokens.includes(t.text)).map(button).join(''); } function appendError(message='') { $('#append-error').textContent=message; $('#append-error').classList.toggle('hidden',!message); } function updateAppendControls() { const blocked=state.busy||!state.run||state.run.tokens.length>=(state.run.max_tokens??512); for(const id of ['append-next','write-token']){const button=$(`#${id}`);if(button)button.disabled=blocked;} const preview=state.appendPreview; $('#append-text').disabled=state.busy; $('#append-cancel').disabled=state.busy; $('#append-custom').disabled=blocked||!preview?.fits||preview.runId!==state.run?.id||preview.text!==$('#append-text').value; $('#append-custom').textContent='Append first token →'; } function closeAppendForm() { clearTimeout(appendTimer);state.appendVersion++;state.appendPreview=null; $('#append-form').classList.add('hidden');$('#append-text').value=''; $('#append-preview').replaceChildren();appendError(); if(lastEditor?.id==='append-text')lastEditor=null; $('#write-token')?.setAttribute('aria-expanded','false'); updateAppendControls(); } function openAppendForm() { if(!state.run||state.busy)return; $('#append-form').classList.remove('hidden'); $('#write-token')?.setAttribute('aria-expanded','true'); queueAppendPreview(); $('#append-text').focus(); $('#append-form').scrollIntoView({block:'nearest'}); } function queueAppendPreview() { clearTimeout(appendTimer); const version=++state.appendVersion,run=state.run,text=$('#append-text').value; state.appendPreview=null;appendError();$('#append-preview').replaceChildren(); $('#append-count').textContent=text?'Finding the first token…':'Only the first tokenizer token will be appended. Spaces are preserved.'; updateAppendControls(); if(!run||!text)return; appendTimer=setTimeout(async()=>{ try { const result=await api(`/api/runs/${run.id}/append-preview`,{text}); if(version!==state.appendVersion||state.run!==run)return; state.appendPreview={...result,text,runId:run.id}; $('#append-count').textContent=`First token only · ${result.total} / ${result.max_tokens} after append`; $('#append-preview').innerHTML=result.tokens.map(token=>`${escape(visible(token.text))}`).join(''); if(!result.fits)appendError(`The context has reached the ${result.max_tokens}-token limit.`); } catch(exc){ if(version!==state.appendVersion||state.run!==run)return; $('#append-count').textContent='Could not tokenize this text';appendError(exc.message); } updateAppendControls(); },200); } async function analyze(endpoint='/api/analyze', customPayload=null) { if(state.busy)return; error();state.busy=true;updateRunButton(); const revision=endpoint==='/api/analyze'?state.revision:state.runRevision; const started=performance.now(); $('#busy-notice').classList.remove('hidden'); $('#busy-text').textContent='Running the forward pass…'; $('#busy-time').textContent='0 s'; $('#results').setAttribute('aria-busy','true'); busyTimer=setInterval(()=>{$('#busy-time').textContent=`${((performance.now()-started)/1000).toFixed(0)} s`;},1000); try { const result=await api(endpoint,customPayload??payload()); state.run=result;state.runRevision=revision;state.query=result.tokens.length-1;state.layer=result.layers-1;state.key=state.query;state.head=state.head==='mean'?'mean':Math.min(state.head,result.heads-1); state.selectionVersion++;state.selection=null;state.matrix=null;state.matrixCache.clear(); state.layerAttention=null;state.layerAttentionCache.clear();state.layerAttentionVersion++; closeAppendForm(); renderRun(); $('#stale-notice').classList.toggle('hidden',state.revision===revision); selectPosition(state.query,state.layer); } catch(exc) {error(exc.message);if(!$('#append-form').classList.contains('hidden'))appendError(exc.message);} finally { state.busy=false;clearInterval(busyTimer);$('#busy-notice').classList.add('hidden'); $('#results').setAttribute('aria-busy','false');updateRunButton(); } } function tokenTitle(token,index) { const source=state.run?.token_sources?.[index]; return `Position ${index} · ID ${token.id}\nText: ${token.text}\nBPE: ${token.piece}${token.special?'\nSpecial token':''}${source==='manual'?'\nManually appended':source==='predicted'?'\nAppended from the model prediction':''}`; } function renderRun() { const run=state.run; $('#empty-state').classList.add('hidden');$('#run-content').classList.remove('hidden'); $('#export').disabled=false; $('#run-label').textContent=`ACTUAL FORWARD PASS / ${run.device.toUpperCase()} / FLOAT32`; $('#metric-tokens').textContent=run.tokens.length; $('#metric-input').textContent=`${run.input_count} input${run.added_count?` + ${run.added_count} added`:''}`; const next=run.predictions.at(-1).ids.at(-1), probability=run.predictions.at(-1).probabilities.at(-1); $('#metric-next').textContent=visible(run.vocabulary[next].text); $('#metric-next').title=`ID ${next} · ${run.vocabulary[next].piece}`; $('#metric-prob').textContent=`${percent(probability)} · no sampling temperature`; $('#metric-time').textContent=`${run.elapsed.toFixed(1)} s`; $('#metric-runtime').textContent=`${run.device.toUpperCase()} · eager attention`; $('#context-tokens').innerHTML=run.tokens.map((token,index)=>``).join(''); $('#generated-legend').classList.toggle('hidden',!run.added_count); $('#generation-details').classList.toggle('hidden',!run.added_count); $('#generated-text').textContent=run.continuation_text; $('#generation-note').textContent=`${run.generated_count} predicted · ${run.manual_count} manually added${run.stopped_on_eos?` · ends with ${run.stop_token}`:''}`; renderLens(); if(state.lensColor==='attention')fetchLayerAttention(); } function renderLens() { const run=state.run; if(!run)return; const attention=state.lensColor==='attention',ready=!!state.layerAttention; const amplified=attention&&$('#lens-relative').checked; const focused=document.activeElement; const focusSelector=focused?.dataset.cell?`[data-cell="${focused.dataset.cell}"]`:focused?.dataset.column!==undefined?`[data-column="${focused.dataset.column}"]`:['append-next','write-token'].includes(focused?.id)?`#${focused.id}`:null; updateLensControls(); let html=''; for(let layer=run.layers-1;layer>=0;layer--){ const row=attention?state.layerAttention?.weights[layer]:null; const scale=amplified&&row?Math.max(...row,1e-9):1; html+=`L${String(layer+1).padStart(2,'0')}${layer===run.layers-1?'FINAL':''}`; for(let key=0;key`; } html+=''; } const next=run.predictions.at(-1).ids.at(-1),nextToken=run.vocabulary[next]; const nextTitle=`Append ${visible(nextToken.text)} · ID ${next} · ${percent(run.predictions.at(-1).probabilities.at(-1))}. Recompute all layers.`; html+=`INPUT ↑${run.tokens.map((token,i)=>``).join('')}`; $('#lens-table').innerHTML=html; updateAppendControls(); $('.lens-scroll').dataset.colorMode=state.lensColor; $('.lens-scroll').dataset.labelMode=state.lens; $('.lens-scroll').setAttribute('aria-busy',attention&&!ready); markSelection(); if(focusSelector)$(focusSelector)?.focus({preventScroll:true}); } function updateLensControls() { const attention=state.lensColor==='attention',run=state.run; $$('[data-lens-color]').forEach(button=>{const active=button.dataset.lensColor===state.lensColor;button.classList.toggle('active',active);button.setAttribute('aria-pressed',active);}); $$('.lens-head-control, .lens-attention-setting').forEach(el=>el.classList.toggle('hidden',!attention)); $('#lens-head').value=state.lensHead; $('#lens-legend-label').textContent=attention?'Attention weight':'Prediction probability'; $('#lens-scale-min').textContent=attention&&$('#lens-relative').checked?'Low':'0%'; $('#lens-scale-max').textContent=attention&&$('#lens-relative').checked?'Layer maximum':'100%'; const peak=attention?layerAttentionPeak():null; $('#lens-peak').classList.toggle('hidden',!peak); if(peak){ $('#lens-peak').textContent=`Show peak: #${peak.key} · ${percent(peak.weight)} ↗`; $('#lens-peak').title=`Scroll to layer ${peak.layer+1}, key #${peak.key} ${visible(run.tokens[peak.key].text)}. The query stays at #${state.query}.`; } const attentionLabels=state.lens==='token-only'?'Cells show predicted tokens; hover for attention weights.':state.lens==='probability'?'Labels show attention weights.':'Cells keep their predicted token; percentages show attention weights.'; $('#lens-description').textContent=attention?`Each row shows where the selected query token attends in that layer. ${attentionLabels} Future positions are masked.`:'Input tokens stay at the bottom. Read layers upward: layer 1 → final layer. Click a column to select a token. Use Attention to view its weights across layers.'; if(!attention)$('#lens-status').textContent=state.lens==='token-only'?'Cell color shows the next-token probability. Hover for the exact percentage.':'Cell color and percentage show the next-token probability.'; else if(run){ const heads=state.lensHead==='mean'?`mean of all ${run.heads} heads`:`head ${state.lensHead}`; $('#lens-status').textContent=`${state.layerAttention?'Attention':'Loading attention'} from #${state.query} ${visible(run.tokens[state.query].text)} · ${heads}${$('#lens-relative').checked?' · log colors per layer; percentages stay absolute':''}`; } } function layerAttentionPeak() { let peak=null; state.layerAttention?.weights.forEach((row,layer)=>row.forEach((weight,key)=>{ if(!maskReason(state.query,key,layer)&&(!peak||weight>peak.weight))peak={layer,key,weight}; })); return peak; } async function fetchLayerAttention() { const run=state.run,query=state.query,head=state.lensHead; if(!run||state.lensColor!=='attention')return; const version=++state.layerAttentionVersion,cacheKey=`${run.id}:${query}:${head}`; state.layerAttention=state.layerAttentionCache.get(cacheKey)??null; renderLens(); if(state.layerAttention)return; try { const result=await api(`/api/runs/${run.id}/attention-layers?query=${query}&head=${head}`); if(version!==state.layerAttentionVersion||state.run!==run||state.lensColor!=='attention')return; state.layerAttention=result;state.layerAttentionCache.set(cacheKey,result); if(state.layerAttentionCache.size>24)state.layerAttentionCache.delete(state.layerAttentionCache.keys().next().value); renderLens(); } catch(exc){if(version===state.layerAttentionVersion){error(exc.message);$('#lens-status').textContent=`Attention could not be loaded: ${exc.message}`;$('.lens-scroll').setAttribute('aria-busy','false');}} } function setLensColor(mode) { state.lensColor=mode; if(mode==='attention')fetchLayerAttention(); else{state.layerAttentionVersion++;renderLens();} } function markSelection(reveal=false) { $$('#context-tokens [data-query]').forEach(el=>{const active=Number(el.dataset.query)===state.query;el.classList.toggle('active',active);el.setAttribute('aria-pressed',active);}); $$('#lens-table .active, #lens-table .column-selected, #lens-table .active-header').forEach(el=>el.classList.remove('active','column-selected','active-header')); $(`[data-token-header="${state.query}"]`)?.classList.add('active-header'); $$('#lens-table [data-column]').forEach(el=>el.setAttribute('aria-pressed',Number(el.dataset.column)===state.query)); for(let layer=0;layerview.right-reserve-5||box.leftview.bottom-footer-6)scroller.scrollTop+=box.bottom-view.bottom+footer+10; else if(box.top`
${escape(visible(token.text))}${percent(token.probability)}
`).join(''); renderAttention(); }catch(exc){if(version===state.selectionVersion){error(exc.message);$('#top-predictions').textContent='No data';}} } function setHead(head) { state.head=head==='mean'?'mean':Number(head);$('#head-select').value=state.head; if(state.selection)renderAttention(); fetchMatrix(); if(state.lensColor==='attention'){state.lensHead=String(head);fetchLayerAttention();} } function attentionScale(row) {return $('#relative-scale').checked?Math.max(...row,1e-9):1;} function selectedAttentionRow() { const heads=state.selection?.attention; if(!heads)return null; return state.head==='mean'?heads[0].map((_,key)=>heads.reduce((sum,row)=>sum+row[key],0)/heads.length):heads[state.head]; } function renderAttention() { if(!state.selection)return; const run=state.run,row=selectedAttentionRow(),scale=attentionScale(row); const peak=Math.max(...row),peakIndex=row.indexOf(peak); const entropy=-row.reduce((sum,p)=>sum+(p>0?p*Math.log2(p):0),0); const headLabel=state.head==='mean'?`Mean of all ${run.heads} query heads`:`Q ${String(state.head).padStart(2,'0')} → KV ${String(kvHead(state.head)).padStart(2,'0')}`; $('#head-mapping').textContent=`${headLabel} · GQA ${run.heads}:${run.kv_heads} · ${run.layer_types?.[state.layer]==='sliding_attention'?`Sliding window ${run.sliding_window}`:'Full causal'}`; $('#attention-matrix-heading').textContent=state.head==='mean'?'Attention matrix · mean of all heads':`Attention matrix · head ${state.head}`; $('#attention-query').textContent=`QUERY #${state.query} ${visible(run.tokens[state.query].text)}`; $('#attention-peak').textContent=`Peak: #${peakIndex} · ${percent(peak)} · H ${entropy.toFixed(2)} bit`; $('#attention-tokens').innerHTML=run.tokens.map((token,index)=>{ const normalized=row[index]/scale,masked=maskReason(state.query,index); return ``; }).join(''); $('#attention-scale-label').textContent=$('#relative-scale').checked?'Color relative to row maximum':'Absolute attention weight'; $('#attention-scale-max').textContent=percent(scale); $('#heads-grid').innerHTML=state.selection.attention.map((headRow,h)=>{ const max=Math.max(...headRow),position=headRow.indexOf(max); return ``; }).join(''); $$('.head-strip').forEach(canvas=>{ const h=Number(canvas.dataset.headStrip),headRow=state.selection.attention[h],max=attentionScale(headRow); canvas.width=Math.max(128,run.tokens.length);canvas.height=11; const ctx=canvas.getContext('2d'),width=canvas.width/run.tokens.length; headRow.forEach((p,i)=>{ctx.fillStyle=maskReason(state.query,i)?'#efefeb':color(p/max);ctx.fillRect(i*width,0,Math.ceil(width),11);}); }); drawMatrix();renderEdge(state.query,state.key); } async function fetchMatrix() { const run=state.run,layer=state.layer,head=state.head; if(!run)return; const cacheKey=`${run.id}:${layer}:${head}`; state.matrix=null;drawMatrix(); try { let matrix=state.matrixCache.get(cacheKey); if(!matrix){ const response=await fetch(`/api/runs/${run.id}/attention?layer=${layer}&head=${head}`); if(!response.ok){const body=await response.json();throw new Error(body.detail);} matrix=new Float32Array(await response.arrayBuffer()); state.matrixCache.set(cacheKey,matrix); if(state.matrixCache.size>12)state.matrixCache.delete(state.matrixCache.keys().next().value); } if(state.run!==run||state.layer!==layer||state.head!==head)return; state.matrix=matrix;drawMatrix();renderEdge(state.query,state.key); }catch(exc){if(state.run===run&&state.layer===layer&&state.head===head)error(exc.message);} } let matrixGeometry=null; function drawMatrix() { const canvas=$('#attention-matrix');canvas.setAttribute('aria-busy',!state.matrix); if(!state.run||!canvas.clientWidth)return; const size=Math.round(canvas.clientWidth),dpr=window.devicePixelRatio||1; canvas.width=Math.round(size*dpr);canvas.height=Math.round(size*dpr); const ctx=canvas.getContext('2d');ctx.scale(dpr,dpr); const left=39,top=34,pad=8,plot=size-left-pad,n=state.run.tokens.length,cell=plot/n; matrixGeometry={left,top,plot:Math.min(plot,size-top-pad),cell:Math.min(plot,size-top-pad)/n,size}; const g=matrixGeometry; ctx.fillStyle='#fff';ctx.fillRect(0,0,size,size); ctx.font='10px ui-monospace, monospace';ctx.textAlign='center';ctx.fillStyle='#526859'; ctx.fillText('KEY POSITION →',left+g.plot/2,11); ctx.save();ctx.translate(7,top+g.plot/2);ctx.rotate(-Math.PI/2);ctx.fillText('QUERY POSITION →',0,0);ctx.restore(); if(!state.matrix){ctx.fillStyle='#93928b';ctx.fillText('Loading matrix…',size/2,size/2);return;} const relative=$('#relative-scale').checked; // Rasterize at one pixel per token pair, then scale. This also handles long contexts efficiently. const off=document.createElement('canvas');off.width=n;off.height=n; const offctx=off.getContext('2d'),pixels=offctx.createImageData(n,n); for(let q=0;q#${query} ${escape(visible(state.run.tokens[query].text))}
↓
#${key} ${escape(visible(state.run.tokens[key].text))}
${p==null?'—':percent(p)}${maskReason(query,key)?`${maskReason(query,key)}`:''}`; } function matrixPoint(event) { if(!matrixGeometry||!state.matrix)return null; const rect=$('#attention-matrix').getBoundingClientRect(),g=matrixGeometry; const x=(event.clientX-rect.left)/rect.width*g.size,y=(event.clientY-rect.top)/rect.height*g.size; if(x=g.left+g.plot||y>=g.top+g.plot)return null; return {query:Math.floor((y-g.top)/g.cell),key:Math.floor((x-g.left)/g.cell)}; } $('#messages').addEventListener('input',event=>{ if(event.target.matches('textarea')){state.messages[Number(event.target.dataset.messageIndex)].content=event.target.value;lastEditor=event.target;changed();} }); $('#messages').addEventListener('change',event=>{ if(event.target.dataset.roleIndex!==undefined){const i=Number(event.target.dataset.roleIndex);state.messages[i].role=event.target.value;event.target.closest('.message').dataset.role=event.target.value;changed();} }); $('#messages').addEventListener('click',event=>{ const button=event.target.closest('[data-remove-message]');if(!button)return; const i=Number(button.dataset.removeMessage);$('#messages').innerHTML='';state.messages.splice(i,1);renderEditor();changed(); }); document.addEventListener('focusin',event=>{if(event.target.matches('#raw-text, #messages textarea, #append-text'))lastEditor=event.target;}); $('#raw-text').addEventListener('input',event=>{state.raw=event.target.value;lastEditor=event.target;changed();}); $('#add-message').addEventListener('click',()=>{$('#messages').innerHTML='';state.messages.push({role:state.messages.at(-1)?.role==='user'?'assistant':'user',content:''});renderEditor();changed();$('#messages .message:last-child textarea')?.focus();}); $('#thinking').addEventListener('change',event=>{state.thinking=event.target.checked;$('#thinking-note').textContent=state.thinking?'The model can start a block':'An empty block precedes the answer';changed();}); $('#template').addEventListener('change',event=>loadTemplate(event.target.value)); $('#model-select').addEventListener('change',event=>switchModel(event.target.value)); $('#reset').addEventListener('click',()=>{$('#template').value='question';loadTemplate('question');}); $$('[data-mode]').forEach(button=>button.addEventListener('click',()=>switchMode(button.dataset.mode))); $('#edit-raw').addEventListener('click',async()=>{await switchMode('raw');$('#raw-text').focus();}); document.addEventListener('click',event=>{const button=event.target.closest('[data-insert-token]');if(button)insertToken(button.dataset.insertToken);}); $('#analyze').addEventListener('click',()=>analyze()); document.addEventListener('keydown',event=>{if((event.metaKey||event.ctrlKey)&&event.key==='Enter'){event.preventDefault();if(event.target.closest('#append-form')){if(!$('#append-custom').disabled)$('#append-form').requestSubmit();}else if(!$('#analyze').disabled)analyze();}}); $('#context-tokens').addEventListener('click',event=>{const token=event.target.closest('[data-query]');if(token)selectPosition(Number(token.dataset.query),state.layer,{reveal:false});}); $('#lens-table').addEventListener('click',event=>{ if(event.target.closest('#append-next')){if(state.run)analyze(`/api/runs/${state.run.id}/continue`,{});return;} if(event.target.closest('#write-token')){openAppendForm();return;} const column=event.target.closest('[data-column]'); if(column){selectPosition(Number(column.dataset.column),state.layer,{reveal:false});return;} const cell=event.target.closest('[data-cell]'); if(cell){const [layer,query]=cell.dataset.cell.split(':').map(Number);selectPosition(query,layer);} }); $$('[data-lens]').forEach(button=>button.addEventListener('click',()=>{state.lens=button.dataset.lens;$$('[data-lens]').forEach(b=>{b.classList.toggle('active',b===button);b.setAttribute('aria-pressed',b===button);});if(state.run)renderLens();})); $$('[data-lens-color]').forEach(button=>button.addEventListener('click',()=>setLensColor(button.dataset.lensColor))); $('#lens-head').addEventListener('change',event=>setHead(event.target.value)); $('#lens-relative').addEventListener('change',()=>renderLens()); $('#lens-peak').addEventListener('click',()=>{ const peak=layerAttentionPeak(); if(peak)revealLensPosition(peak.layer,peak.key,{vertical:true}); }); $('#prev-token').addEventListener('click',()=>selectPosition(state.query-1));$('#next-token').addEventListener('click',()=>selectPosition(state.query+1)); $('#layer-select').addEventListener('change',event=>selectPosition(state.query,Number(event.target.value))); $('#head-select').addEventListener('change',event=>setHead(event.target.value)); $('#heads-grid').addEventListener('click',event=>{const head=event.target.closest('[data-head]');if(head)setHead(Number(head.dataset.head));}); $('#relative-scale').addEventListener('change',()=>{if(state.selection)renderAttention();}); $('#attention-tokens').addEventListener('click',event=>{const token=event.target.closest('[data-attention-key]');if(token){state.key=Number(token.dataset.attentionKey);drawMatrix();renderEdge(state.query,state.key);}}); $('#attention-tokens').addEventListener('mouseover',event=>{const token=event.target.closest('[data-attention-key]');if(token)renderEdge(state.query,Number(token.dataset.attentionKey));}); $('#attention-matrix').addEventListener('mousemove',event=>{const point=matrixPoint(event);if(point)renderEdge(point.query,point.key);}); $('#attention-matrix').addEventListener('mouseleave',()=>renderEdge(state.query,state.key)); $('#attention-matrix').addEventListener('click',event=>{const point=matrixPoint(event);if(point){state.key=point.key;selectPosition(point.query);}}); $('#attention-matrix').addEventListener('keydown',event=>{ if(!state.run||!['ArrowUp','ArrowDown','ArrowLeft','ArrowRight'].includes(event.key))return; event.preventDefault(); if(event.key==='ArrowUp'||event.key==='ArrowDown')selectPosition(state.query+(event.key==='ArrowUp'?-1:1)); else{state.key=Math.max(0,Math.min(state.run.tokens.length-1,state.key+(event.key==='ArrowLeft'?-1:1)));drawMatrix();renderEdge(state.query,state.key);} }); $('#append-text').addEventListener('input',queueAppendPreview); $('#append-cancel').addEventListener('click',()=>{closeAppendForm();$('#write-token')?.focus({preventScroll:true});}); $('#append-form').addEventListener('submit',event=>{ event.preventDefault(); if(!$('#append-custom').disabled)analyze(`/api/runs/${state.run.id}/continue`,{text:$('#append-text').value}); }); $('#export').addEventListener('click',()=>{ if(!state.run)return; const artifact={...state.run,layer_view:{color_by:state.lensColor,query:state.query,head:state.lensHead,attention:state.lensColor==='attention'?state.layerAttention?.weights:null},selected:{layer:state.layer+1,head:state.head,query:state.query,key:state.key,top:state.selection?.top,attention:state.selection?.attention,head_matrix:state.matrix?Array.from({length:state.run.tokens.length},(_,q)=>Array.from(state.matrix.slice(q*state.run.tokens.length,(q+1)*state.run.tokens.length))):null},method:'softmax(lm_head(final_rmsnorm(layer_output))); final layer normalized once',checkpoint_revision:state.run.revision}; const url=URL.createObjectURL(new Blob([JSON.stringify(artifact,null,2)],{type:'application/json'})); const anchor=document.createElement('a');anchor.href=url;anchor.download=`${state.run.model===QWEN_ID?'qwen':'gemma'}-lab-${state.run.id.slice(0,8)}.json`;anchor.click();setTimeout(()=>URL.revokeObjectURL(url),1000); }); new ResizeObserver(()=>drawMatrix()).observe($('.matrix-wrap')); async function pollStatus() { clearTimeout(statusTimer); const model=state.model; try { const previous=state.status; const result=await api(`/api/status?model=${encodeURIComponent(model)}`); if(model!==state.model)return; state.status=result; renderModel(result); $('#model-status').textContent=state.status.message; $('#model-status').className=`load-status ${state.status.phase==='ready'?'ready':state.status.phase==='error'?'error':''}`; $('#runtime-note').textContent=state.status.phase==='ready'?`${state.status.device.toUpperCase()} · float32 · eager attention`:'Checkpoint · loading…'; $('#limit-note').textContent=state.status.max_tokens; if(state.status.active&&state.busy)$('#busy-text').textContent=state.status.active; if(state.status.tokenizer_ready&&!previous?.tokenizer_ready){renderSpecialTokens(state.status.special_tokens);await prepare();} updateRunButton(); maybeAutoStart(); }catch(exc){if(model===state.model)$('#model-status').textContent=`Cannot reach the server: ${exc.message}`;} if(model===state.model)statusTimer=setTimeout(pollStatus,state.busy||state.status?.phase!=='ready'?1500:5000); } loadTemplate('question');pollStatus();