const $ = (selector) => document.querySelector(selector);
const $$ = (selector) => [...document.querySelectorAll(selector)];
const escape = (text) => String(text).replace(/[&<>"']/g, c => ({'&':'&','<':'<','>':'>','"':'"',"'":'''}[c]));
const visible = (text) => String(text).replace(/ /g, '·').replace(/\n/g, '↵').replace(/\t/g, '⇥').replace(/\r/g, '␍') || '∅';
const percent = (p) => `${(p * 100).toFixed(p >= .1 ? 1 : p >= .001 ? 2 : 3)}%`;
const colorRGB = (value) => {
const p = Math.max(0, Math.min(1, value));
return [Math.round(245-238*p),Math.round(251-160*p),Math.round(247-195*p)];
};
const color = value => `rgb(${colorRGB(value).join(',')})`;
const ink = value => {
const linear=colorRGB(value).map(v=>{const s=v/255;return s<=.04045?s/12.92:((s+.055)/1.055)**2.4;});
return .2126*linear[0]+.7152*linear[1]+.0722*linear[2]>.179?'#000000':'#ffffff';
};
const QWEN_ID = 'Qwen/Qwen3-0.6B';
const state = {
model:QWEN_ID,
mode:'chat', messages:[], thinking:false, raw:'', revision:0, prepared:null,
run:null, runRevision:null, layer:27, head:0, query:0, key:0, lens:'token', busy:false,
appendPreview:null, appendVersion:0,
status:null, selection:null, matrix:null, matrixCache:new Map(), selectionVersion:0,
lensColor:'probability', lensHead:'mean', layerAttention:null, layerAttentionCache:new Map(), layerAttentionVersion:0,
};
let prepareTimer, prepareVersion = 0, lastEditor = null, autoStarted = false, busyTimer, statusTimer, appendTimer;
const descriptions = {
'<|im_start|>':'Starts a message, followed by the role (system, user, or assistant) and a newline.',
'<|im_end|>':'Ends a message; also marks the end of an assistant response.',
'<|endoftext|>':'Text boundary. Chat messages normally use im_end.',
'':'Starts a reasoning block. Added token ID 151667, special=false.',
'':'Ends a reasoning block. Added token ID 151668, special=false.',
'':'Starts a tool call in the Qwen format.',
'':'Ends a tool call.',
'':'Starts a tool response.',
'':'Ends a tool response.',
'':'Beginning of the Gemma input sequence.',
'':'End-of-sequence marker.',
'':'Starts a Gemma message, followed by user or model and a newline.',
'':'Ends a Gemma turn; also stops this checkpoint’s response.',
'':'Padding token. Normally excluded from the visible conversation.',
'':'Unknown-token marker.',
};
const primaryTokens = ['<|im_start|>','<|im_end|>','<|endoftext|>','','','','','','','','','',''];
const supportsThinking = () => state.model === QWEN_ID;
const kvHead = head => Math.floor(head / (state.run.heads / state.run.kv_heads));
function maskReason(query,key,layer=state.layer) {
if(key>query)return 'Hidden by the causal mask';
if(state.run?.layer_types?.[layer]==='sliding_attention' && query-key>=state.run.sliding_window)return 'Outside the sliding attention window';
return '';
}
function renderModel(meta) {
$('#model-pill').textContent=meta.model===QWEN_ID?'Qwen3-0.6B':'Gemma3-270M';
$('#model-name').textContent=meta.name;
$('#footer-model').textContent=`Model laboratory · ${meta.name}`;
$('#checkpoint-link').href=`https://huggingface.co/${meta.model}`;
$('#empty-model-note').textContent=`Computed from the actual ${meta.name} weights`;
if(!meta.layers)return;
const group=meta.heads/meta.kv_heads;
$('#intro-description').innerHTML=`One context. ${meta.layers} layers. ${meta.layers*meta.heads} attention heads. Trace how the prediction at each token changes.`;
$('#model-shape').textContent=`${meta.layers} LAYERS / ${meta.heads} Q HEADS / ${meta.kv_heads} KV HEADS`;
$('#model-description').textContent=`${meta.layers} layers · ${meta.heads} heads · ${meta.native_thinking?'native reasoning switch':'learned Reasoning / Solution format'}`;
$('#diagram-layers').textContent=`${meta.layers} layers`;
$('#metric-next-label').textContent=`Next token · layer ${meta.layers}`;
$('#lens-footnote').textContent=`Input → L01 → L${String(meta.layers).padStart(2,'0')} ↑ · positions start at 0 · predictions refer to the next token`;
$('#heads-heading').textContent=`All ${meta.heads} heads in this layer`;
$('#attention-description').textContent=`Color shows how much attention the selected position assigns to each token: softmax(QKᵀ / √${meta.attention_scale} + attention mask). Choose one head or the arithmetic mean of all ${meta.heads} query heads after softmax.`;
const windowNote=meta.sliding_window?` Sliding layers use a ${meta.sliding_window}-token window; layers ${meta.layer_types.map((type,i)=>type==='full_attention'?i+1:null).filter(Boolean).join(', ')} use full causal attention.`:'';
$('#method-model').textContent=`${meta.name} has ${meta.layers} decoder layers, ${meta.heads} query heads, and ${meta.kv_heads} KV heads. With GQA, query head h uses KV head ⌊h / ${group}⌋.${windowNote}`;
$('#method-lens').textContent=`After each layer, apply the final RMSNorm and shared language model head, then softmax over all ${meta.vocab_size.toLocaleString('en-US')} tokens. This is a logit lens: early layers were not trained to predict text independently. The final layer gives the standard model prediction.`;
$('#method-reasoning').textContent=meta.native_thinking?'Chat uses the checkpoint’s apply_chat_template with enable_thinking. Disabling reasoning adds an empty … block. In Raw mode, you supply the control tokens and prefix. is an added token with special=false.':'Chat uses the checkpoint’s standard Gemma template with user and model. This TinyGSM fine-tune generates ordinary Reasoning / Solution text. It has no special token or native reasoning on/off switch. A system instruction is folded into the first user turn.';
$('#attention-formula').textContent=`A⁽ˡ,ʰ⁾ = softmax(QKᵀ / √${meta.attention_scale} + attention mask)`;
if($('#layer-select').options.length!==meta.layers)$('#layer-select').innerHTML=Array.from({length:meta.layers},(_,i)=>``).join('');
if($('#head-select').options.length!==meta.heads+1){
$('#head-select').innerHTML=``+Array.from({length:meta.heads},(_,i)=>``).join('');
$('#head-select').value=state.head;
}
if($('#lens-head').options.length!==meta.heads+1){
$('#lens-head').innerHTML=``+Array.from({length:meta.heads},(_,i)=>``).join('');
$('#lens-head').value=state.lensHead;
}
}
async function switchModel(model) {
if(state.busy||model===state.model)return;
clearTimeout(statusTimer);clearTimeout(prepareTimer);
state.model=model;state.status=null;state.run=null;state.prepared=null;
closeAppendForm();state.runRevision=null;
state.selection=null;state.matrix=null;state.matrixCache.clear();state.selectionVersion++;prepareVersion++;
state.layerAttention=null;state.layerAttentionCache.clear();state.layerAttentionVersion++;state.lensHead='mean';
state.layer=0;state.head=0;state.query=0;state.key=0;autoStarted=false;
if(!supportsThinking())state.thinking=false;
$('#run-content').classList.add('hidden');$('#empty-state').classList.remove('hidden');
$('#stale-notice').classList.add('hidden');$('#export').disabled=true;
for(const id of ['metric-tokens','metric-next','metric-time'])$(`#${id}`).textContent='—';
$('#metric-input').textContent='Exact model tokenization';$('#metric-prob').textContent='Probability over the full vocabulary';
$('#metric-runtime').textContent='Loading model…';$('#model-description').textContent='Loading model configuration…';
$('#prompt-preview').textContent='Loading tokenizer…';$('#token-count').textContent='— tokens';
$('#special-tokens').replaceChildren();$('#extra-tokens').replaceChildren();
$('#layer-select').replaceChildren();$('#head-select').replaceChildren();
error();renderEditor();updateRunButton();await pollStatus();
}
async function api(path, data) {
const response = await fetch(path, data === undefined ? {} : {method:'POST', headers:{'Content-Type':'application/json'}, body:JSON.stringify(data)});
if (!response.ok) {
let detail;
try { detail = (await response.json()).detail; } catch { detail = response.statusText; }
throw new Error(typeof detail === 'string' ? detail : 'Check the context format and length.');
}
return response.json();
}
function payload() {
return {model:state.model, mode:state.mode, messages:state.messages, text:state.raw, thinking:supportsThinking()&&state.thinking, max_new_tokens:0};
}
function error(message='') { $('#input-error').textContent=message; $('#input-error').classList.toggle('hidden',!message); }
function changed() {
state.revision++;
if (state.run) $('#stale-notice').classList.remove('hidden');
error();
clearTimeout(prepareTimer);
// Invalidate an earlier response immediately, before the debounce starts.
prepareVersion++;
state.prepared=null;
$('#token-count').textContent='Counting tokens…';
updateRunButton();
prepareTimer=setTimeout(prepare,300);
}
function updateRunButton() {
$('#model-select').disabled=state.busy;
$('#analyze').disabled=state.busy || state.status?.phase!=='ready' || !state.prepared?.fits;
$('#analyze span:first-child').textContent=state.busy?'Computing…':'Inspect context';
updateAppendControls();
}
function maybeAutoStart() {
if(autoStarted||state.status?.phase!=='ready'||!state.prepared?.fits||state.busy||state.run)return;
autoStarted=true;
analyze();
}
async function prepare() {
if (!state.status?.tokenizer_ready) return;
const version=++prepareVersion;
try {
const result=await api('/api/prepare',payload());
if(version!==prepareVersion)return;
state.prepared=result;
$('#prompt-preview').textContent=result.text;
$('#token-count').textContent=`${result.count} / ${result.max_tokens} tokens`;
if(!result.fits) error(`Context exceeds the ${result.max_tokens}-token limit. Input has not been truncated.`);
} catch(exc) {
if(version!==prepareVersion)return;
state.prepared=null;
$('#token-count').textContent='Could not prepare input';
$('#prompt-preview').textContent=exc.message;
error(exc.message);
}
updateRunButton();
maybeAutoStart();
}
function renderEditor() {
$$('#messages textarea').forEach((el,i)=>{if(state.messages[i])state.messages[i].content=el.value;});
$('#chat-editor').classList.toggle('hidden',state.mode!=='chat');
$('#raw-editor').classList.toggle('hidden',state.mode!=='raw');
$('#raw-text').value=state.raw;
$$('[data-mode]').forEach(button=>{const active=button.dataset.mode===state.mode;button.classList.toggle('active',active);button.setAttribute('aria-pressed',active);});
$('#format-label').textContent=state.mode==='chat'?'Official chat template':'Exact text, no wrapper';
$('#thinking').checked=state.thinking;
$('#thinking').disabled=state.mode==='raw'||!supportsThinking();
$('#reasoning-switch').classList.toggle('hidden',!supportsThinking());
$('#thinking-note').textContent=!supportsThinking()?'Learned Reasoning / Solution text. No native on/off switch.':state.mode==='raw'?'Controlled by the Raw prefix':state.thinking?'The model can start a block':'An empty block precedes the answer';
$('#messages').innerHTML=state.messages.map((message,i)=>`
`).join('');
lastEditor=null;
}
function loadTemplate(name) {
// Remove old editors before changing message state.
$('#messages').innerHTML='';
state.mode='chat';state.thinking=false;state.raw='';
const templates={
question:[{role:'user',content:'What is the capital of France? Answer with one word.'}],
math:[{role:'user',content:'Anna has 3 apples. She buys twice as many apples as she originally had. How many apples does Anna have now?'}],
system:[{role:'system',content:'Answer in English using one word.'},{role:'user',content:'Which city is the capital of France?'}],
dialogue:[{role:'user',content:'My name is Anna. I have a cat named Mars.'},{role:'assistant',content:'Nice to meet you, Anna!'},{role:'user',content:'What is the name of my cat?'}],
completion:[],empty:[{role:'user',content:''}],
};
state.messages=structuredClone(templates[name]??templates.question);
if(name==='math')state.thinking=supportsThinking();
if(name==='completion'){state.mode='raw';state.raw='The capital of France is';}
renderEditor();changed();
}
async function switchMode(mode) {
if(mode===state.mode)return;
if(mode==='raw'){
try {const prepared=await api('/api/prepare',payload());state.raw=prepared.text;}catch {state.raw=state.messages.map(m=>m.content).join('\n');}
}
state.mode=mode;renderEditor();changed();
}
function insertToken(token) {
const editor=lastEditor?.isConnected?lastEditor:state.mode==='raw'?$('#raw-text'):$('#messages .message:last-child textarea');
if(!editor||editor.disabled)return;
editor.focus();
editor.setRangeText(token,editor.selectionStart,editor.selectionEnd,'end');
editor.dispatchEvent(new Event('input',{bubbles:true}));
}
function renderSpecialTokens(tokens) {
const button=t=>``;
$('#special-tokens').innerHTML=primaryTokens.map(text=>tokens.find(t=>t.text===text)).filter(Boolean).map(button).join('');
$('#extra-tokens').innerHTML=tokens.filter(t=>!primaryTokens.includes(t.text)).map(button).join('');
}
function appendError(message='') {
$('#append-error').textContent=message;
$('#append-error').classList.toggle('hidden',!message);
}
function updateAppendControls() {
const blocked=state.busy||!state.run||state.run.tokens.length>=(state.run.max_tokens??512);
for(const id of ['append-next','write-token']){const button=$(`#${id}`);if(button)button.disabled=blocked;}
const preview=state.appendPreview;
$('#append-text').disabled=state.busy;
$('#append-cancel').disabled=state.busy;
$('#append-custom').disabled=blocked||!preview?.fits||preview.runId!==state.run?.id||preview.text!==$('#append-text').value;
$('#append-custom').textContent='Append first token →';
}
function closeAppendForm() {
clearTimeout(appendTimer);state.appendVersion++;state.appendPreview=null;
$('#append-form').classList.add('hidden');$('#append-text').value='';
$('#append-preview').replaceChildren();appendError();
if(lastEditor?.id==='append-text')lastEditor=null;
$('#write-token')?.setAttribute('aria-expanded','false');
updateAppendControls();
}
function openAppendForm() {
if(!state.run||state.busy)return;
$('#append-form').classList.remove('hidden');
$('#write-token')?.setAttribute('aria-expanded','true');
queueAppendPreview();
$('#append-text').focus();
$('#append-form').scrollIntoView({block:'nearest'});
}
function queueAppendPreview() {
clearTimeout(appendTimer);
const version=++state.appendVersion,run=state.run,text=$('#append-text').value;
state.appendPreview=null;appendError();$('#append-preview').replaceChildren();
$('#append-count').textContent=text?'Finding the first token…':'Only the first tokenizer token will be appended. Spaces are preserved.';
updateAppendControls();
if(!run||!text)return;
appendTimer=setTimeout(async()=>{
try {
const result=await api(`/api/runs/${run.id}/append-preview`,{text});
if(version!==state.appendVersion||state.run!==run)return;
state.appendPreview={...result,text,runId:run.id};
$('#append-count').textContent=`First token only · ${result.total} / ${result.max_tokens} after append`;
$('#append-preview').innerHTML=result.tokens.map(token=>`${escape(visible(token.text))}`).join('');
if(!result.fits)appendError(`The context has reached the ${result.max_tokens}-token limit.`);
} catch(exc){
if(version!==state.appendVersion||state.run!==run)return;
$('#append-count').textContent='Could not tokenize this text';appendError(exc.message);
}
updateAppendControls();
},200);
}
async function analyze(endpoint='/api/analyze', customPayload=null) {
if(state.busy)return;
error();state.busy=true;updateRunButton();
const revision=endpoint==='/api/analyze'?state.revision:state.runRevision;
const started=performance.now();
$('#busy-notice').classList.remove('hidden');
$('#busy-text').textContent='Running the forward pass…';
$('#busy-time').textContent='0 s';
$('#results').setAttribute('aria-busy','true');
busyTimer=setInterval(()=>{$('#busy-time').textContent=`${((performance.now()-started)/1000).toFixed(0)} s`;},1000);
try {
const result=await api(endpoint,customPayload??payload());
state.run=result;state.runRevision=revision;state.query=result.tokens.length-1;state.layer=result.layers-1;state.key=state.query;state.head=state.head==='mean'?'mean':Math.min(state.head,result.heads-1);
state.selectionVersion++;state.selection=null;state.matrix=null;state.matrixCache.clear();
state.layerAttention=null;state.layerAttentionCache.clear();state.layerAttentionVersion++;
closeAppendForm();
renderRun();
$('#stale-notice').classList.toggle('hidden',state.revision===revision);
selectPosition(state.query,state.layer);
} catch(exc) {error(exc.message);if(!$('#append-form').classList.contains('hidden'))appendError(exc.message);}
finally {
state.busy=false;clearInterval(busyTimer);$('#busy-notice').classList.add('hidden');
$('#results').setAttribute('aria-busy','false');updateRunButton();
}
}
function tokenTitle(token,index) {
const source=state.run?.token_sources?.[index];
return `Position ${index} · ID ${token.id}\nText: ${token.text}\nBPE: ${token.piece}${token.special?'\nSpecial token':''}${source==='manual'?'\nManually appended':source==='predicted'?'\nAppended from the model prediction':''}`;
}
function renderRun() {
const run=state.run;
$('#empty-state').classList.add('hidden');$('#run-content').classList.remove('hidden');
$('#export').disabled=false;
$('#run-label').textContent=`ACTUAL FORWARD PASS / ${run.device.toUpperCase()} / FLOAT32`;
$('#metric-tokens').textContent=run.tokens.length;
$('#metric-input').textContent=`${run.input_count} input${run.added_count?` + ${run.added_count} added`:''}`;
const next=run.predictions.at(-1).ids.at(-1), probability=run.predictions.at(-1).probabilities.at(-1);
$('#metric-next').textContent=visible(run.vocabulary[next].text);
$('#metric-next').title=`ID ${next} · ${run.vocabulary[next].piece}`;
$('#metric-prob').textContent=`${percent(probability)} · no sampling temperature`;
$('#metric-time').textContent=`${run.elapsed.toFixed(1)} s`;
$('#metric-runtime').textContent=`${run.device.toUpperCase()} · eager attention`;
$('#context-tokens').innerHTML=run.tokens.map((token,index)=>``).join('');
$('#generated-legend').classList.toggle('hidden',!run.added_count);
$('#generation-details').classList.toggle('hidden',!run.added_count);
$('#generated-text').textContent=run.continuation_text;
$('#generation-note').textContent=`${run.generated_count} predicted · ${run.manual_count} manually added${run.stopped_on_eos?` · ends with ${run.stop_token}`:''}`;
renderLens();
if(state.lensColor==='attention')fetchLayerAttention();
}
function renderLens() {
const run=state.run;
if(!run)return;
const attention=state.lensColor==='attention',ready=!!state.layerAttention;
const amplified=attention&&$('#lens-relative').checked;
const focused=document.activeElement;
const focusSelector=focused?.dataset.cell?`[data-cell="${focused.dataset.cell}"]`:focused?.dataset.column!==undefined?`[data-column="${focused.dataset.column}"]`:['append-next','write-token'].includes(focused?.id)?`#${focused.id}`:null;
updateLensControls();
let html='';
for(let layer=run.layers-1;layer>=0;layer--){
const row=attention?state.layerAttention?.weights[layer]:null;
const scale=amplified&&row?Math.max(...row,1e-9):1;
html+=`