serving: manual load + GenerationMixin + use_cache=False + streaming

index.html

@@ -134,14 +134,24 @@ const out=document.getElementById('out'), btn=document.getElementById('go'), sta
 async function gen(){
   const prompt=document.getElementById('prompt').value.trim();
   if(!prompt){return}
-  btn.disabled=true; out.textContent='…'; stat.textContent='generating (model may be loading on first call)…';
+  btn.disabled=true; out.textContent=''; stat.textContent='generating (model may be loading on first call)…';
+  const t0=performance.now();
   try{
-    const r=await fetch('/api/generate',{method:'POST',headers:{'Content-Type':'application/json'},
+    const r=await fetch('/api/stream',{method:'POST',headers:{'Content-Type':'application/json'},
       body:JSON.stringify({prompt:prompt,max_new_tokens:200,temperature:0.7})});
-    const j=await r.json();
-    if(j.ok){ out.textContent=j.completion||'(empty)';
-      stat.textContent=`${j.tokens} tokens · ${j.seconds}s · ${j.tok_per_s||'–'} tok/s · ${j.device}`;
-    } else { out.textContent=j.message||'unavailable'; stat.textContent=j.status||''; }
+    if(!r.body){ out.textContent=await r.text(); stat.textContent=''; btn.disabled=false; return; }
+    const reader=r.body.getReader(), dec=new TextDecoder();
+    let text='';
+    while(true){
+      const {done,value}=await reader.read();
+      if(done) break;
+      text+=dec.decode(value,{stream:true});
+      out.textContent=text;
+      stat.textContent=`streaming… ${((performance.now()-t0)/1000).toFixed(1)}s`;
+    }
+    const secs=((performance.now()-t0)/1000).toFixed(1);
+    out.textContent=text||'(empty)';
+    stat.textContent = r.status>=400 ? '' : `done · ${secs}s · live token streaming`;
   }catch(e){ out.textContent='Request failed: '+e; stat.textContent=''; }
   btn.disabled=false;
 }