Diagnostics: timeouts + structured logging on all model/media calls

Root cause of 'stuck at loading': none of the image/voice/skill/text fetches had a
timeout, so a slow/hung backend spun forever. Added web/diag.js — diagFetch (timeout +
timing + console log + ring buffer) and diagStream (connect + idle timeouts for SSE):
- image /portrait: 120s ceiling
- voice /qwen-tts + /voxcpm (synth + clone): 90s
- skill + text /text/generate/stream: 30s connect, 60s idle-stall
Calls now FAIL LOUDLY with a clear timing-stamped error instead of hanging. Every call is
logged ([tinyarmy:kind] ✓/✗ in Ns) and recorded; dump with tinyDiag() in the console, or
open Settings → Diagnostics for a live readout. Skill-forge icon failures now warn too.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

web/codingModel.js

@@ -49,7 +49,7 @@ export async function streamCoding(system, user, { maxTokens = 512, temperature
     temperature,
     think,
   }, {
-    signal,
+    signal, kind: 'skill', label: (get(_sel) && get(_sel).label) || _sel,
     onEvent(evt, parsed) {
       if (evt !== 'delta') return
       const piece = parsed?.content || ''

web/diag.js

@@ -0,0 +1,71 @@
+// Call diagnostics for the model/media backends (image, voice, skill/text generation).
+// Every call gets: a timeout (so a hung backend FAILS LOUDLY instead of spinning forever),
+// timing, and a structured console log + a ring buffer you can dump with `tinyDiag()` in the
+// devtools console. This is the troubleshooting surface for "stuck at loading" / silent failures.
+
+const RING = []
+const MAX = 80
+const listeners = new Set()
+const nowMs = () => (typeof performance !== 'undefined' && performance.now ? performance.now() : Date.now())
+const fmt = (ms) => (ms >= 1000 ? (ms / 1000).toFixed(1) + 's' : Math.round(ms) + 'ms')
+
+export function onDiag(fn) { listeners.add(fn); return () => listeners.delete(fn) }
+export function recentCalls() { return RING.slice() }
+
+function record(e) {
+  RING.push(e); if (RING.length > MAX) RING.shift()
+  for (const fn of listeners) { try { fn(e) } catch { /* ignore */ } }
+}
+
+const tagOf = (kind) => `[tinyarmy:${kind}]`
+
+// Wrap a non-streaming fetch (image / voice) with a timeout + logging. Returns the Response.
+// Throws a clear, timing-stamped error on failure/timeout so callers can surface it.
+export async function diagFetch(kind, url, opts = {}, { timeoutMs = 90000, label } = {}) {
+  const ctrl = new AbortController()
+  if (opts.signal) { if (opts.signal.aborted) ctrl.abort(); else opts.signal.addEventListener('abort', () => ctrl.abort(), { once: true }) }
+  const timer = setTimeout(() => ctrl.abort(new DOMException('timeout', 'TimeoutError')), timeoutMs)
+  const t0 = nowMs(); const tag = tagOf(kind); const what = `${opts.method || 'GET'} ${url}${label ? ` · ${label}` : ''}`
+  console.info(`${tag} → ${what}`)
+  try {
+    const res = await fetch(url, { ...opts, signal: ctrl.signal })
+    const ms = nowMs() - t0
+    if (!res.ok) {
+      console.warn(`${tag} ✗ ${res.status} in ${fmt(ms)} · ${url}`)
+      record({ kind, url, label, ok: false, status: res.status, ms, ts: Date.now() })
+    } else {
+      console.info(`${tag} ✓ ${res.status} in ${fmt(ms)} · ${url}`)
+      record({ kind, url, label, ok: true, status: res.status, ms, ts: Date.now() })
+    }
+    return res
+  } catch (e) {
+    const ms = nowMs() - t0
+    const timedOut = e && (e.name === 'TimeoutError' || e.name === 'AbortError')
+    const msg = timedOut ? `TIMEOUT after ${fmt(ms)}` : (e && e.message ? e.message : String(e))
+    console.error(`${tag} ✗ ${msg} · ${url}`)
+    record({ kind, url, label, ok: false, error: msg, timedOut, ms, ts: Date.now() })
+    if (timedOut) throw new Error(`${kind} timed out after ${fmt(ms)} — the backend (${url}) is slow or unresponsive`)
+    throw e
+  } finally { clearTimeout(timer) }
+}
+
+// Lifecycle logging for a streaming call (SSE). Returns handles the streamer calls as it goes.
+// Logs start / first-token latency / done / error with timing, and records the outcome.
+export function diagStream(kind, url, label) {
+  const t0 = nowMs(); const tag = tagOf(kind); const what = `${url}${label ? ` · ${label}` : ''}`
+  let first = null
+  console.info(`${tag} ⇢ stream ${what}`)
+  return {
+    firstToken() { if (first == null) { first = nowMs() - t0; console.info(`${tag} · first token in ${fmt(first)} · ${url}`) } },
+    done(extra) { const ms = nowMs() - t0; console.info(`${tag} ✓ stream done in ${fmt(ms)}${extra ? ' · ' + extra : ''} · ${url}`); record({ kind, url, label, ok: true, ms, firstMs: first, ts: Date.now() }) },
+    fail(e, timedOut) { const ms = nowMs() - t0; const msg = timedOut ? `TIMEOUT after ${fmt(ms)}` : (e && e.message ? e.message : String(e)); console.error(`${tag} ✗ stream ${msg} · ${url}`); record({ kind, url, label, ok: false, error: msg, timedOut, ms, firstMs: first, ts: Date.now() }) },
+  }
+}
+
+// Dump recent calls to the console (call `tinyDiag()` in devtools to troubleshoot).
+function dump() {
+  const rows = RING.map((e) => ({ when: new Date(e.ts).toLocaleTimeString(), kind: e.kind, ok: e.ok ? '✓' : '✗', status: e.status || (e.timedOut ? 'TIMEOUT' : (e.error ? 'ERR' : '')), ms: fmt(e.ms || 0), detail: e.label || e.error || e.url }))
+  try { console.table(rows) } catch { console.log(rows) }
+  return RING.slice()
+}
+try { if (typeof window !== 'undefined') window.tinyDiag = dump } catch { /* ignore */ }

web/engineServer.js

@@ -22,7 +22,7 @@ async function stream(id, system, user, { maxTokens = 200, temperature = 0.8, on
     max_tokens: maxTokens,
     temperature,
   }, {
-    signal,
+    signal, kind: 'text', label: m.id,
     onEvent(evt, parsed) {
       if (evt !== 'delta') return
       const piece = parsed?.content || ''

web/imagenServer.js

@@ -2,15 +2,18 @@
 //   • zimage-local — Z-Image-Turbo on YOUR GPU (TINY_IMAGE_MODE=local). Localhost only.
 //   • flux / flux-dev — FLUX via NVIDIA NIM (or HF Inference) in the cloud.
 // Mirrors ttsQwen3.js (engine + engineLocal). generate() returns a PNG Blob.
+import { diagFetch } from '/web/diag.js'
 
 export const isLocalhost = () => {
   try { return /^(localhost|127\.0\.0\.1|\[?::1\]?|0\.0\.0\.0)$/i.test(location.hostname) } catch { return false }
 }
 
+// 2-minute ceiling: image gen on a cold ZeroGPU can take a while, but past this it's hung —
+// fail loudly (logged via diag) instead of spinning the loader forever.
 async function postPortrait(body) {
-  const resp = await fetch('/portrait', {
+  const resp = await diagFetch('image', '/portrait', {
     method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify(body),
-  })
+  }, { timeoutMs: 120000, label: `${body.engine || ''}${body.provider ? '/' + body.provider : ''}` })
   if (!resp.ok) throw new Error(`portrait ${resp.status}: ${(await resp.text()).slice(0, 160)}`)
   return resp.blob() // image/png
 }

web/settingsPanel.js

@@ -12,6 +12,7 @@ import { mountImagenBar } from '/web/imagenBar.js'
 import { mountPersonaPromptBar } from '/web/personaPromptBar.js'
 import { mountQualityBar } from '/web/qualityBar.js'
 import { mountCodingModelBar } from '/web/codingModelBar.js'
+import { recentCalls, onDiag } from '/web/diag.js'
 
 function el(tag, props = {}, kids = []) {
   const n = document.createElement(tag)
@@ -47,6 +48,28 @@ function mountGameplayBar(host) {
   host.append(lab)
 }
 
+// Diagnostics: a live readout of recent model/media calls (image, voice, skill, text) with
+// status + timing — for troubleshooting "stuck loading" / silent failures without devtools.
+function mountDiagnosticsBar(host) {
+  const fmt = (ms) => (ms >= 1000 ? (ms / 1000).toFixed(1) + 's' : Math.round(ms || 0) + 'ms')
+  const list = el('div', { style: 'font:11px ui-monospace,monospace;max-height:240px;overflow:auto;border:1px solid #2a3340;border-radius:8px;padding:8px;background:rgba(20,24,33,.5)' })
+  const render = () => {
+    const calls = recentCalls().slice(-25).reverse()
+    list.replaceChildren()
+    if (!calls.length) { list.append(el('div', { style: 'color:#9aa4b2' }, 'No model/media calls yet — generate a portrait, voice or skill.')); return }
+    for (const c of calls) {
+      const mark = c.ok ? '✓' : (c.timedOut ? '⏱ TIMEOUT' : '✗')
+      const detail = c.label || c.error || (c.status ? 'HTTP ' + c.status : '') || c.url || ''
+      const row = el('div', { style: `padding:3px 0;white-space:nowrap;overflow:hidden;text-overflow:ellipsis;color:${c.ok ? '#8fe388' : '#ff8a8a'}` },
+        `${new Date(c.ts).toLocaleTimeString()}  ${c.kind}  ${mark}  ${fmt(c.ms)}  ${detail}`)
+      list.append(row)
+    }
+  }
+  const hint = el('p', { class: 'tac-set-intro', style: 'margin:0 0 6px' }, 'Tip: also run tinyDiag() in the browser console for the full table.')
+  host.append(hint, list)
+  render(); onDiag(render)
+}
+
 export function mountSettingsPanel() {
   const tryInject = () => {
     const sample = [...document.querySelectorAll('.banner-wrap')].find((e) => /Display Theme/i.test(e.textContent))
@@ -74,6 +97,9 @@ export function mountSettingsPanel() {
       'Nemotron 3 Nano (NVIDIA) runs via NVIDIA NIM; Mellum2 (JetBrains) runs as a ' +
       'ZeroGPU sidecar and falls back to Nemotron (NIM) if its sidecar is unavailable.',
       mountCodingModelBar)
+    injectSection(sample, 'tac-diagnostics-settings', 'Diagnostics',
+      'Recent model/media calls (image, voice, skill, text) with status + timing. Use this to ' +
+      'troubleshoot generation that is stuck loading or failing — timeouts now surface as ⏱.', mountDiagnosticsBar)
     injectSection(sample, 'tac-gameplay-settings', 'Gameplay',
       'On-map flourishes during the Game. Emotes pop little reaction bubbles above heroes ' +
       'and enemies when they take a hit, get afflicted, drop low, or score a kill.', mountGameplayBar)

web/skillForge.js

@@ -86,7 +86,7 @@ export async function forgeSkillForHero(personaId, request, { onStatus, onToken,
   try {
     iconBlob = await generatePortrait(iconPrompt(p, skill), { seed: skill.id })
     if (iconBlob) await putSkillIcon(skill.id, iconBlob)
-  } catch { /* art is optional */ }
+  } catch (e) { console.warn('[tinyarmy:skill] icon generation failed (skill still learned):', (e && e.message) || e) } // logged via diag; art is optional
 
   const fresh = getPersona(p.id) || p
   patchPersona(p.id, {

web/sseText.js

@@ -2,37 +2,55 @@
 // Parses the `event:`/`data:` wire format and dispatches each event to onEvent.
 // Used by both engineServer.js (the model-bar "Server / ZeroGPU" engine) and
 // codingModel.js (the Skill Forge coding-model picker) so the parser lives once.
-export async function streamSse(url, body, { onEvent, signal } = {}) {
-  const res = await fetch(url, {
-    method: 'POST',
-    headers: { 'Content-Type': 'application/json' },
-    body: JSON.stringify(body),
-    signal,
-  })
-  if (!res.ok || !res.body) throw new Error(`HTTP ${res.status}`)
-  const reader = res.body.getReader()
-  const decoder = new TextDecoder()
-  let buf = ''
-  while (true) {
-    const { value, done } = await reader.read()
-    if (done) break
-    buf += decoder.decode(value, { stream: true })
-    const events = buf.split(/\n\n/)
-    buf = events.pop() ?? ''
-    for (const evChunk of events) {
-      const lines = evChunk.split('\n')
-      let evt = 'message'
-      const dataLines = []
-      for (const line of lines) {
-        if (line.startsWith('event:')) evt = line.slice(6).trim()
-        else if (line.startsWith('data:')) dataLines.push(line.slice(5).trimStart())
+//
+// Hardened with a CONNECT timeout (no response headers) + an IDLE timeout (stream stalls with
+// no new chunk), so a hung/unresponsive backend FAILS LOUDLY instead of spinning forever, and
+// every call is logged (see diag.js / `tinyDiag()`).
+import { diagStream } from '/web/diag.js'
+
+export async function streamSse(url, body, { onEvent, signal, kind = 'text', label, connectMs = 30000, idleMs = 60000 } = {}) {
+  const ctrl = new AbortController()
+  if (signal) { if (signal.aborted) ctrl.abort(); else signal.addEventListener('abort', () => ctrl.abort(), { once: true }) }
+  let timedOut = false
+  let timer = setTimeout(() => { timedOut = true; ctrl.abort() }, connectMs)
+  const arm = (ms) => { clearTimeout(timer); timer = setTimeout(() => { timedOut = true; ctrl.abort() }, ms) }
+  const log = diagStream(kind, url, label)
+  try {
+    const res = await fetch(url, { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify(body), signal: ctrl.signal })
+    if (!res.ok || !res.body) { const txt = res.body ? await res.text().catch(() => '') : ''; throw new Error(`HTTP ${res.status}${txt ? ': ' + txt.slice(0, 140) : ''}`) }
+    arm(idleMs) // connected → switch to the idle (stall) timeout, reset on every chunk
+    const reader = res.body.getReader()
+    const decoder = new TextDecoder()
+    let buf = ''
+    while (true) {
+      const { value, done } = await reader.read()
+      if (done) break
+      arm(idleMs)
+      buf += decoder.decode(value, { stream: true })
+      const events = buf.split(/\n\n/)
+      buf = events.pop() ?? ''
+      for (const evChunk of events) {
+        const lines = evChunk.split('\n')
+        let evt = 'message'
+        const dataLines = []
+        for (const line of lines) {
+          if (line.startsWith('event:')) evt = line.slice(6).trim()
+          else if (line.startsWith('data:')) dataLines.push(line.slice(5).trimStart())
+        }
+        const data = dataLines.join('\n')
+        if (!data) continue
+        let parsed = null
+        try { parsed = JSON.parse(data) } catch { /* ignore */ }
+        if (evt === 'error') throw new Error(parsed?.error || data)
+        if (evt === 'delta') log.firstToken()
+        onEvent?.(evt, parsed, data)
       }
-      const data = dataLines.join('\n')
-      if (!data) continue
-      let parsed = null
-      try { parsed = JSON.parse(data) } catch { /* ignore */ }
-      if (evt === 'error') throw new Error(parsed?.error || data)
-      onEvent?.(evt, parsed, data)
     }
+    clearTimeout(timer); log.done()
+  } catch (e) {
+    clearTimeout(timer)
+    log.fail(e, timedOut)
+    if (timedOut) throw new Error(`${kind} timed out — the generation backend is slow or unresponsive`)
+    throw e
   }
 }

web/ttsQwen3.js

@@ -3,6 +3,7 @@
 // language DESCRIPTION (the persona's `voice` field, or a preset), used as DashScope's
 // `voice_prompt`. NETWORKED — not local-first (clearly labeled). mode 'pcm'.
 import { decodeAudio } from '/web/ttsAudio.js'
+import { diagFetch } from '/web/diag.js'
 
 // Endpoint: default is our Space backend (/qwen-tts → DashScope). A `?tts=<base>` query
 // param (persisted to localStorage) points it at a self-run local server instead —
@@ -38,10 +39,10 @@ export const isLocalhost = () => {
 // POST to `${base}/qwen-tts` → raw WAV ArrayBuffer. base '' = same-origin.
 async function postSynthWav(base, text, voiceId) {
   const instruct = (get(voiceId).desc() || '').trim()
-  const resp = await fetch(`${base}/qwen-tts`, {
+  const resp = await diagFetch('voice', `${base}/qwen-tts`, {
     method: 'POST', headers: { 'Content-Type': 'application/json' },
     body: JSON.stringify({ text, instruct, language: 'English' }),
-  })
+  }, { timeoutMs: 90000, label: 'qwen3 synth' })
   if (!resp.ok) throw new Error(`Qwen3-TTS ${resp.status}: ${(await resp.text()).slice(0, 140)}`)
   return resp.arrayBuffer()
 }
@@ -55,12 +56,12 @@ function abToB64(ab) {
   return btoa(s)
 }
 async function postClone(base, text, refAb, refText, instruct) {
-  const resp = await fetch(`${base}/qwen-tts`, {
+  const resp = await diagFetch('voice', `${base}/qwen-tts`, {
     method: 'POST', headers: { 'Content-Type': 'application/json' },
     // instruct lets prod (DashScope, no clone model) gracefully re-design from the
     // description instead of cloning; local mode uses ref_audio to clone the timbre.
     body: JSON.stringify({ text, ref_audio: abToB64(refAb), ref_text: refText || '', instruct: instruct || '', language: 'English' }),
-  })
+  }, { timeoutMs: 90000, label: 'qwen3 clone' })
   if (!resp.ok) throw new Error(`Qwen3-TTS clone ${resp.status}: ${(await resp.text()).slice(0, 140)}`)
   return resp.arrayBuffer()
 }

web/ttsVoxcpm.js

@@ -2,6 +2,7 @@
 // tokens and sidecar details stay server-side. Like Qwen3, this designs a voice
 // from each hero's free-form voice description.
 import { decodeAudio } from '/web/ttsAudio.js'
+import { diagFetch } from '/web/diag.js'
 
 let _desc = ''
 const VOICES = [
@@ -15,10 +16,10 @@ const get = (id) => VOICES.find((v) => v.id === id) || VOICES[0]
 
 async function postSynthWav(text, voiceId) {
   const instruct = (get(voiceId).desc() || '').trim()
-  const resp = await fetch('/voxcpm-tts', {
+  const resp = await diagFetch('voice', '/voxcpm-tts', {
     method: 'POST', headers: { 'Content-Type': 'application/json' },
     body: JSON.stringify({ text, instruct, language: 'English' }),
-  })
+  }, { timeoutMs: 90000, label: 'voxcpm synth' })
   if (!resp.ok) throw new Error(`VoxCPM ${resp.status}: ${(await resp.text()).slice(0, 140)}`)
   return resp.arrayBuffer()
 }
@@ -30,10 +31,10 @@ function abToB64(ab) {
   return btoa(s)
 }
 async function postCloneWav(text, refAb, refText, instruct) {
-  const resp = await fetch('/voxcpm-clone', {
+  const resp = await diagFetch('voice', '/voxcpm-clone', {
     method: 'POST', headers: { 'Content-Type': 'application/json' },
     body: JSON.stringify({ text, ref_audio: abToB64(refAb), ref_text: refText || '', instruct: instruct || '', language: 'English' }),
-  })
+  }, { timeoutMs: 90000, label: 'voxcpm clone' })
   if (!resp.ok) throw new Error(`VoxCPM clone ${resp.status}: ${(await resp.text()).slice(0, 140)}`)
   return resp.arrayBuffer()
 }