feat: deep diagnostics + uncapped tokens (up to 512)

Three diagnostic panels:
- Glossolalia Agent Diagnostics: per-token perplexity, vocab coverage,
deficit weights, entropy, top-3 per agent
- Standard Sampling Diagnostics: per-token perplexity, confidence,
vocab coverage, top-3 predictions
- Layer Health: per-layer hidden state norms and residual contribution
through all 32 transformer layers

Token limit raised to 512 (slider) / 8192 (server). Zero cost --
all Aether, no API calls.

aether-server.mjs

@@ -131,13 +131,17 @@ function forwardPass(prompt) {
   // Returns a function that generates one token at a time
   return {
     inputTokens,
-    step(allToks, kvC) {
+    step(allToks, kvC, diag) {
       const pos = allToks.length - 1;
       const tid = allToks[allToks.length - 1];
       const x0 = model.tokenEmbd.slice(tid*C.hiddenDim,(tid+1)*C.hiddenDim);
       let x = x0;
+      const layerNorms = diag ? [] : null;
+      const attnEntropies = diag ? [] : null;
+
       for (let l=0;l<C.numLayers;l++) {
         const ly=model.layers[l];
+        const xPrev = x;
         const normed=o.rmsNorm(x,ly.an,C.rmsNormEps);
         const q=o.matVec(ly.qw,normed,C.hiddenDim,C.hiddenDim);
         const k=o.matVec(ly.kw,normed,kvDim,C.hiddenDim);
@@ -146,16 +150,29 @@ function forwardPass(prompt) {
         for(let h=0;h<C.numKvHeads;h++)applyRoPE(k.subarray(h*C.headDim,(h+1)*C.headDim),C.headDim,pos,C.ropeTheta);
         kvC[l].k.push(new Float32Array(k));kvC[l].v.push(new Float32Array(v));
         const seqLen=kvC[l].k.length;const attnOut=new Float32Array(C.hiddenDim);
+        const headEntropies = diag ? [] : null;
         for(let h=0;h<C.numHeads;h++){const kvH=Math.floor(h/gqaRatio);const qH=q.subarray(h*C.headDim,(h+1)*C.headDim);const scores=new Float32Array(seqLen);
           for(let s=0;s<seqLen;s++){const kH=kvC[l].k[s].subarray(kvH*C.headDim,(kvH+1)*C.headDim);let dot=0;for(let d=0;d<C.headDim;d++)dot+=qH[d]*kH[d];scores[s]=dot/Math.sqrt(C.headDim);}
-          const w=softmaxJS(scores);for(let s=0;s<seqLen;s++){const vH=kvC[l].v[s].subarray(kvH*C.headDim,(kvH+1)*C.headDim);const wt=w[s];for(let d=0;d<C.headDim;d++)attnOut[h*C.headDim+d]+=wt*vH[d];}}
+          const w=softmaxJS(scores);
+          // Attention entropy per head
+          if (diag) { let he=0; for(let s=0;s<seqLen;s++) if(w[s]>1e-10) he-=w[s]*Math.log(w[s]); headEntropies.push(Math.round(he*1000)/1000); }
+          for(let s=0;s<seqLen;s++){const vH=kvC[l].v[s].subarray(kvH*C.headDim,(kvH+1)*C.headDim);const wt=w[s];for(let d=0;d<C.headDim;d++)attnOut[h*C.headDim+d]+=wt*vH[d];}}
+        if (diag) attnEntropies.push(headEntropies);
         const projected=o.matVec(ly.ow,attnOut,C.hiddenDim,C.hiddenDim);const postAttn=o.add(x,projected);
         const ffnIn=o.rmsNorm(postAttn,ly.fn,C.rmsNormEps);const gate=o.matVec(ly.gw,ffnIn,C.intermediateSize,C.hiddenDim);
         const up=o.matVec(ly.uw,ffnIn,C.intermediateSize,C.hiddenDim);const activated=o.fusedSiluMul(gate,up);
         const down=o.matVec(ly.dw,activated,C.hiddenDim,C.intermediateSize);x=o.add(postAttn,down);
+
+        // Layer norms + residual contribution
+        if (diag) {
+          let norm=0, delta=0, prevNorm=0;
+          for(let i=0;i<C.hiddenDim;i++) { norm+=x[i]*x[i]; delta+=(x[i]-xPrev[i])**2; prevNorm+=xPrev[i]*xPrev[i]; }
+          layerNorms.push({ norm: Math.round(Math.sqrt(norm)*100)/100, residual: prevNorm>0 ? Math.round(Math.sqrt(delta/prevNorm)*1000)/1000 : 0 });
+        }
       }
       const finalNormed=o.rmsNorm(x,model.outNorm,C.rmsNormEps);
-      return o.matVec(model.outWeight,finalNormed,C.vocabSize,C.hiddenDim);
+      const logits = o.matVec(model.outWeight,finalNormed,C.vocabSize,C.hiddenDim);
+      return { logits, layerNorms, attnEntropies };
     }
   };
 }
@@ -230,7 +247,7 @@ function sampleGlossolalia(logits) {
 
 // ─── Generation Loops ───────────────────────────────────────────────────────
 
-function generateStandard(prompt, maxTokens = 48) {
+function generateStandard(prompt, maxTokens = 8192) {
   const t0 = performance.now();
   const fwd = forwardPass(prompt);
   const allTokens = [...fwd.inputTokens];
@@ -239,15 +256,35 @@ function generateStandard(prompt, maxTokens = 48) {
 
   // Prefill
   for (let i = 0; i < fwd.inputTokens.length; i++) {
-    fwd.step(allTokens.slice(0, i+1), kvC);
+    fwd.step(allTokens.slice(0, i+1), kvC, false);
   }
 
+  const perTokenInfo = [];
+
   // Decode
   for (let i = 0; i < maxTokens; i++) {
     const ts = performance.now();
-    const logits = fwd.step(allTokens, kvC);
+    const { logits, layerNorms, attnEntropies } = fwd.step(allTokens, kvC, true);
+    const o2 = op();
+    const scaled = new Float32Array(logits.length);
+    for (let j = 0; j < logits.length; j++) scaled[j] = logits[j] / 0.7;
+    const probs = o2.softmax(scaled);
+
     const chosen = sampleStandard(logits);
+    const chosenProb = probs[chosen];
+    const perplexity = chosenProb > 0 ? -Math.log2(chosenProb) : 99;
+
+    // Vocab coverage: tokens with >0.1% probability
+    let vocabCoverage = 0;
+    for (let j = 0; j < probs.length; j++) if (probs[j] > 0.001) vocabCoverage++;
+
+    // Top-5
+    const top5 = Array.from(probs).map((p,j)=>({p,i:j})).sort((a,b)=>b.p-a.p).slice(0,5)
+      .map(t => ({ token: model.tokenizer.decode([t.i]), prob: Math.round(t.p*1000)/1000 }));
+
     tokenTimes.push(performance.now() - ts);
+    perTokenInfo.push({ perplexity: Math.round(perplexity*100)/100, chosenProb: Math.round(chosenProb*1000)/1000, vocabCoverage, top5, layerNorms, attnEntropies });
+
     if (chosen === C.eosToken) break;
     allTokens.push(chosen);
   }
@@ -260,10 +297,11 @@ function generateStandard(prompt, maxTokens = 48) {
     text: model.tokenizer.decode(genTokens), tokens: genTokens.length,
     totalTimeMs: Math.round(totalTime), avgTokenMs: Math.round(avgMs),
     mode: 'standard', temperature: 0.7, topP: 0.9,
+    tokenDiagnostics: perTokenInfo,
   };
 }
 
-function generateGlossolalia(prompt, maxTokens = 48) {
+function generateGlossolalia(prompt, maxTokens = 8192) {
   const t0 = performance.now();
   const fwd = forwardPass(prompt);
   const allTokens = [...fwd.inputTokens];
@@ -273,25 +311,35 @@ function generateGlossolalia(prompt, maxTokens = 48) {
 
   // Prefill
   for (let i = 0; i < fwd.inputTokens.length; i++) {
-    fwd.step(allTokens.slice(0, i+1), kvC);
+    fwd.step(allTokens.slice(0, i+1), kvC, false);
   }
 
   // Decode with Glossolalia
   for (let i = 0; i < maxTokens; i++) {
     const ts = performance.now();
-    const logits = fwd.step(allTokens, kvC);
+    const { logits, layerNorms, attnEntropies } = fwd.step(allTokens, kvC, true);
     const { tokenId, agents } = sampleGlossolalia(logits);
+
+    // Token-level perplexity from merged distribution
+    const { merged } = glossolaliaMerge(logits);
+    const chosenProb = merged[tokenId] || 0;
+    const perplexity = chosenProb > 0 ? -Math.log2(chosenProb) : 99;
+    let vocabCoverage = 0;
+    for (let j = 0; j < merged.length; j++) if (merged[j] > 0.001) vocabCoverage++;
+
     tokenTimes.push(performance.now() - ts);
 
-    perTokenDiag.push(agents.map(a => ({
-      tau: a.tau,
-      entropy: Math.round(a.entropy * 1000) / 1000,
-      weight: Math.round(a.weight * 1000) / 1000,
-      top3: a.top5.slice(0, 3).map(t => ({
-        token: model.tokenizer.decode([t.i]),
-        prob: Math.round(t.p * 1000) / 1000,
+    perTokenDiag.push({
+      agents: agents.map(a => ({
+        tau: a.tau, entropy: Math.round(a.entropy*1000)/1000, weight: Math.round(a.weight*1000)/1000,
+        top3: a.top5.slice(0,3).map(t => ({ token: model.tokenizer.decode([t.i]), prob: Math.round(t.p*1000)/1000 })),
       })),
-    })));
+      perplexity: Math.round(perplexity*100)/100,
+      chosenProb: Math.round(chosenProb*1000)/1000,
+      vocabCoverage,
+      layerNorms,
+      attnEntropies,
+    });
 
     if (tokenId === C.eosToken) break;
     allTokens.push(tokenId);
@@ -317,7 +365,7 @@ const server = createServer((req, res) => {
     req.on('end', () => {
       try {
         const { prompt, max_tokens } = JSON.parse(body);
-        const result = genFn(prompt, max_tokens || 48);
+        const result = genFn(prompt, max_tokens || 256);
         res.writeHead(200, { 'Content-Type': 'application/json' });
         res.end(JSON.stringify(result));
       } catch (e) {

app.py

@@ -2,9 +2,9 @@
 Glossolalia -- Semiotic Ensemble Inference
 Act 2: Fork/Race/Fold at the decoder level.
 
-Standard sampling vs Glossolalia (temperature-ensemble MOA with
-deficit-weighted complement merge). Same model, same forward pass,
-different decoder. All inference via Aether WASM-SIMD engine.
+Standard sampling vs Glossolalia. Same forward pass, different decoder.
+All inference via Aether WASM-SIMD engine. Deep diagnostics: per-token
+perplexity, per-layer norms, attention entropy, vocabulary coverage.
 """
 
 import gradio as gr
@@ -44,14 +44,14 @@ else:
     print("[Glossolalia] WARNING: Aether not ready after 180s", flush=True)
 
 
-def call_aether(endpoint, prompt, max_tokens=48):
+def call_aether(endpoint, prompt, max_tokens=256):
     try:
         data = json.dumps({"prompt": prompt, "max_tokens": max_tokens}).encode()
         req = urllib.request.Request(
             f"http://127.0.0.1:7861/{endpoint}", data=data,
             headers={"Content-Type": "application/json"},
         )
-        resp = urllib.request.urlopen(req, timeout=300)
+        resp = urllib.request.urlopen(req, timeout=600)
         return json.loads(resp.read())
     except urllib.error.HTTPError as e:
         body = e.read().decode() if e.fp else str(e)
@@ -62,24 +62,73 @@ def call_aether(endpoint, prompt, max_tokens=48):
         return {"error": str(e), "text": f"[Error: {e}]", "tokens": 0, "totalTimeMs": 0, "avgTokenMs": 0}
 
 
-def format_diagnostics(diag_list):
+# ─── Diagnostic Formatters ───────────────────────────────────────────────────
+
+def format_glossolalia_diag(diag_list):
     if not diag_list:
-        return "No diagnostics available."
-    lines = ["PER-TOKEN AGENT DIAGNOSTICS", "=" * 60, "",
-             "Three agents at tau={0.4, 0.7, 1.0}. Weight = 1 - H/log(V):",
-             "low entropy => high confidence => high weight in fold.", ""]
-    for step, agents in enumerate(diag_list[:20]):  # limit to 20 tokens
-        lines.append(f"--- Token {step + 1} ---")
-        for a in agents:
+        return "No diagnostics."
+    lines = ["GLOSSOLALIA DEEP DIAGNOSTICS", "=" * 60, "",
+             "tau={0.4, 0.7, 1.0}  |  w = 1-H/log(V)  |  ppl = -log2(P)", ""]
+    for step, d in enumerate(diag_list):
+        if not isinstance(d, dict):
+            continue
+        ppl = d.get("perplexity", "?")
+        prob = d.get("chosenProb", "?")
+        vc = d.get("vocabCoverage", "?")
+        lines.append(f"--- Token {step+1}  |  ppl={ppl}  |  P={prob}  |  vocabCov={vc} ---")
+        for a in d.get("agents", []):
+            if not isinstance(a, dict):
+                continue
             top_str = ", ".join(f"'{t['token']}' ({t['prob']:.3f})" for t in a.get("top3", []))
-            lines.append(f"  tau={a['tau']:.1f} | H={a['entropy']:.3f} | w={a['weight']:.3f} | {top_str}")
+            lines.append(f"  tau={a.get('tau','?'):.1f} | H={a.get('entropy',0):.3f} | w={a.get('weight',0):.3f} | {top_str}")
         lines.append("")
     return "\n".join(lines)
 
 
+def format_standard_diag(token_diag):
+    if not token_diag:
+        return "No diagnostics."
+    lines = ["STANDARD SAMPLING DIAGNOSTICS", "=" * 60,
+             "ppl = -log2(P(chosen))  |  vocabCov = tokens with P>0.1%", ""]
+    for step, d in enumerate(token_diag):
+        if not isinstance(d, dict):
+            continue
+        ppl = d.get("perplexity", "?")
+        prob = d.get("chosenProb", "?")
+        vc = d.get("vocabCoverage", "?")
+        top5 = d.get("top5", [])
+        top_str = ", ".join(f"'{t['token']}' ({t['prob']:.3f})" for t in top5[:3])
+        lines.append(f"Token {step+1} | ppl={ppl} | P={prob} | vc={vc} | {top_str}")
+    return "\n".join(lines)
+
+
+def format_layer_health(diag_list):
+    if not diag_list:
+        return "No layer data."
+    last = diag_list[-1] if diag_list else {}
+    if not isinstance(last, dict):
+        return "No layer data."
+    norms = last.get("layerNorms", [])
+    if not norms:
+        return "No layer norms."
+    lines = ["LAYER HEALTH (last generated token)", "=" * 60,
+             "Layer | Hidden Norm | Residual Contribution", "-" * 50]
+    for i, n in enumerate(norms):
+        if not isinstance(n, dict):
+            continue
+        norm_val = n.get("norm", 0)
+        res_val = n.get("residual", 0)
+        bar = "#" * min(int(res_val * 40), 40)
+        lines.append(f"  {i:2d}   | {norm_val:9.2f}  | {res_val:.4f} {bar}")
+    return "\n".join(lines)
+
+
+# ─── Compare Function ────────────────────────────────────────────────────────
+
 def compare(prompt, max_tokens):
+    empty = ("", "", "", "", "", "", "")
     if not prompt or not prompt.strip():
-        yield "", "", "", "", ""
+        yield empty
         return
 
     max_tokens = int(max_tokens)
@@ -91,33 +140,34 @@ def compare(prompt, max_tokens):
     def run_glo():
         glo_result[0] = call_aether("generate-glossolalia", prompt, max_tokens)
 
+    def fmt_stats(r):
+        if not r:
+            return "running..."
+        return f"{r['tokens']} tokens in {r['totalTimeMs']/1000:.1f}s ({r['avgTokenMs']}ms/tok)"
+
+    def build():
+        sr, gr_ = std_result[0], glo_result[0]
+        return (
+            sr["text"] if sr else "generating...",
+            gr_["text"] if gr_ else "generating...",
+            fmt_stats(sr),
+            fmt_stats(gr_),
+            format_glossolalia_diag(gr_.get("diagnostics", [])) if gr_ else "",
+            format_standard_diag(sr.get("tokenDiagnostics", [])) if sr else "",
+            format_layer_health(gr_.get("diagnostics", [])) if gr_ else "",
+        )
+
     with ThreadPoolExecutor(max_workers=2) as pool:
         futures = {pool.submit(run_std): "std", pool.submit(run_glo): "glo"}
         for future in as_completed(futures):
-            name = futures[future]
             future.result()
-            if name == "std" and std_result[0]:
-                r = std_result[0]
-                st = f"{r['tokens']} tokens in {r['totalTimeMs']/1000:.1f}s ({r['avgTokenMs']}ms/tok)"
-                gt = glo_result[0]["text"] if glo_result[0] else "generating..."
-                gs = f"{glo_result[0]['tokens']} tokens in {glo_result[0]['totalTimeMs']/1000:.1f}s ({glo_result[0]['avgTokenMs']}ms/tok)" if glo_result[0] else "running..."
-                diag = format_diagnostics(glo_result[0].get("diagnostics", [])) if glo_result[0] else ""
-                yield r["text"], gt, st, gs, diag
-            elif name == "glo" and glo_result[0]:
-                r = glo_result[0]
-                gs = f"{r['tokens']} tokens in {r['totalTimeMs']/1000:.1f}s ({r['avgTokenMs']}ms/tok)"
-                bt = std_result[0]["text"] if std_result[0] else "generating..."
-                bs = f"{std_result[0]['tokens']} tokens in {std_result[0]['totalTimeMs']/1000:.1f}s ({std_result[0]['avgTokenMs']}ms/tok)" if std_result[0] else "running..."
-                yield bt, r["text"], bs, gs, format_diagnostics(r.get("diagnostics", []))
-
-    if std_result[0] and glo_result[0]:
-        sr, gr_ = std_result[0], glo_result[0]
-        yield (sr["text"], gr_["text"],
-               f"{sr['tokens']} tokens in {sr['totalTimeMs']/1000:.1f}s ({sr['avgTokenMs']}ms/tok)",
-               f"{gr_['tokens']} tokens in {gr_['totalTimeMs']/1000:.1f}s ({gr_['avgTokenMs']}ms/tok)",
-               format_diagnostics(gr_.get("diagnostics", [])))
+            yield build()
+
+    yield build()
 
 
+# ─── CSS ─────────────────────────────────────────────────────────────────────
+
 CSS = """
 .gradio-container { max-width: 1060px !important; margin: 0 auto !important; }
 .gradio-container, .dark { background: #09090b !important; }
@@ -144,20 +194,22 @@ footer.svelte-1ax1toq { display: none !important; }
 .built-with { display: none !important; }
 """
 
+# ─── Gradio UI ───────────────────────────────────────────────────────────────
+
 with gr.Blocks(css=CSS, theme=gr.themes.Base(primary_hue="purple", neutral_hue="zinc"), title="Glossolalia") as demo:
 
     gr.HTML("""
     <div id="hero">
         <h1><span class="accent">Glossolalia</span></h1>
-        <p class="subtitle">Semiotic ensemble inference. Same model, same forward pass, different decoder.<br/>
-        Left: standard top-p sampling. Right: fork/race/fold with deficit-weighted complement merge.<br/>
-        All inference via Aether -- pure JS + 14KB WASM SIMD. Zero ML dependencies.</p>
+        <p class="subtitle">Semiotic ensemble inference. Same forward pass, different decoder.<br/>
+        Left: standard top-p. Right: fork/race/fold with deficit-weighted complement merge.<br/>
+        All inference via Aether. Zero ML dependencies. Full diagnostic visibility.</p>
     </div>
     """)
 
     with gr.Row():
         prompt = gr.Textbox(elem_id="prompt-input", placeholder="What is the shape of failure?", lines=2, label="Prompt", show_label=False, interactive=True, scale=4)
-        max_tok = gr.Slider(minimum=8, maximum=64, value=48, step=1, label="Max tokens", scale=1)
+        max_tok = gr.Slider(minimum=8, maximum=512, value=64, step=1, label="Max tokens", scale=1)
 
     btn = gr.Button("Generate", elem_id="gen-btn", variant="primary")
 
@@ -173,28 +225,36 @@ with gr.Blocks(css=CSS, theme=gr.themes.Base(primary_hue="purple", neutral_hue="
             glo_out = gr.Textbox(lines=10, show_label=False, interactive=False, elem_classes=["response-card"])
             glo_stats = gr.HTML('<p class="stats-text">--</p>')
 
-    with gr.Accordion("Agent Diagnostics (per-token)", open=False):
-        diag_out = gr.Textbox(lines=15, show_label=False, interactive=False, )
+    with gr.Accordion("Glossolalia Agent Diagnostics", open=False):
+        glo_diag = gr.Textbox(lines=18, show_label=False, interactive=False)
+
+    with gr.Accordion("Standard Sampling Diagnostics", open=False):
+        std_diag = gr.Textbox(lines=12, show_label=False, interactive=False)
+
+    with gr.Accordion("Layer Health (32 transformer layers)", open=False):
+        layer_health = gr.Textbox(lines=18, show_label=False, interactive=False)
+
+    outputs = [std_out, glo_out, std_stats, glo_stats, glo_diag, std_diag, layer_health]
 
     def run(prompt_text, max_tokens):
-        for st, gt, ss, gs, diag in compare(prompt_text, max_tokens):
-            yield st, gt, f'<p class="stats-text">{ss}</p>', f'<p class="stats-text">{gs}</p>', diag
+        for vals in compare(prompt_text, max_tokens):
+            st, gt, ss, gs, gd, sd, lh = vals
+            yield st, gt, f'<p class="stats-text">{ss}</p>', f'<p class="stats-text">{gs}</p>', gd, sd, lh
 
-    btn.click(run, [prompt, max_tok], [std_out, glo_out, std_stats, glo_stats, diag_out])
-    prompt.submit(run, [prompt, max_tok], [std_out, glo_out, std_stats, glo_stats, diag_out])
+    btn.click(run, [prompt, max_tok], outputs)
+    prompt.submit(run, [prompt, max_tok], outputs)
 
     gr.HTML('<p style="color:#52525b; font-size:0.8rem; margin-top:1.5rem; margin-bottom:0.5rem;">Try these:</p>')
     with gr.Row():
         for p in ["What is the shape of failure?", "The theory of everything begins with", "If silence had a color", "Write a haiku about parallel universes"]:
             gr.Button(p, size="sm", elem_classes=["prompt-chip"]).click(
                 fn=lambda x=p: x, outputs=[prompt]
-            ).then(fn=run, inputs=[prompt, max_tok], outputs=[std_out, glo_out, std_stats, glo_stats, diag_out])
+            ).then(fn=run, inputs=[prompt, max_tok], outputs=outputs)
 
     gr.HTML("""
     <div id="footer">
         <p style="color:#a1a1aa; font-size:0.85rem; margin-bottom:0.5rem;">
-            SmolLM2-360M-Instruct &middot; Q8_0 GGUF &middot; Aether WASM-SIMD &middot;
-            THM-SEMIOTIC-ERASURE (Lean 4)
+            SmolLM2-360M-Instruct &middot; Q8_0 GGUF &middot; Aether WASM-SIMD &middot; THM-SEMIOTIC-ERASURE (Lean 4)
         </p>
         <p>
             <a href="https://forkracefold.com/">Whitepaper</a> &middot;